From 3d57554e720afe4cfd29b2a6347006902d0d18c9 Mon Sep 17 00:00:00 2001 From: Lukas Piwowarski Date: Wed, 19 Aug 2026 14:34:08 +0200 Subject: [PATCH] Migrate to operator-sdk 1.42.3 Migrate to operator-sdk 1.42.3 by generating a new bare operator repository with the target operator-sdk version and porting the existing code into it. The aim was to preserve the existing functionality while adopting the "features" of the newly generated code as much as possible. Things worth pointing out: - The Makefile was reorganized so that our custom targets now live at the bottom in the @Custom section. This makes future migrations easier. The Makefile no longer supports the TAG variable (the new version dropped it); we use $(VERSION) and v$(VERSION) instead. - The .golangci.yml was produced by merging the config generated by operator-sdk with the one we already had from openstack-k8s-operators/openstack-operator [1]. The generated one is more bulletproof and does reasonable ignoring for test files (e.g. gosec is disabled for them), so we kept it as the base and folded in the linters/settings we relied on before. Note that previously we had confusingly two files in the repo (.golanci.yaml and .golanci.yml). - We now use --metrics-cert-path to configure TLS for the /metrics endpoint. We still support TLS for that endpoint the same way as before; the internal logic just relies on the code generated by operator-sdk [2]. - The metrics Service was renamed to controller-manager-metrics. The scaffold's default name combined with the namePrefix overflowed the 63-character DNS label limit, so the name had to be shortened. [1] dde4f04e0a194b13d7beed47eb488b0a452939cd [2] 4e0116a54c6d0496cd0fdd4d18d669c31b954618 Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build-and-push.yaml | 2 +- .github/workflows/verify-generation.yaml | 2 +- .golangci.yaml | 18 - .golangci.yml | 19 +- Dockerfile | 5 +- Makefile | 293 +++++++++------- PROJECT | 1 - bundle.Dockerfile | 2 +- ...ed.openstack.org_openstacklightspeeds.yaml | 2 +- ...ontroller-manager-metrics_v1_service.yaml} | 5 +- ...ole_rbac.authorization.k8s.io_v1_role.yaml | 61 ++++ ...c.authorization.k8s.io_v1_rolebinding.yaml | 16 - ...c.authorization.k8s.io_v1_clusterrole.yaml | 21 ++ ...tspeed-operator.clusterserviceversion.yaml | 23 +- bundle/metadata/annotations.yaml | 2 +- bundle/tests/scorecard/config.yaml | 12 +- cmd/main.go | 119 +++++-- ...ed.openstack.org_openstacklightspeeds.yaml | 2 +- config/crd/kustomization.yaml | 6 - .../default/cert_metrics_manager_patch.yaml | 32 ++ config/default/kustomization.yaml | 294 ++++++++++------ config/default/metrics_service.yaml | 13 +- config/manager/kustomization.yaml | 6 - config/manager/manager.yaml | 25 +- .../network-policy/allow-metrics-traffic.yaml | 27 ++ config/network-policy/kustomization.yaml | 2 + config/prometheus/kustomization.yaml | 9 + config/prometheus/monitor.yaml | 13 +- config/prometheus/monitor_tls_patch.yaml | 19 + config/rbac/kustomization.yaml | 5 +- .../rbac/openstacklightspeed_admin_role.yaml | 27 ++ .../rbac/openstacklightspeed_editor_role.yaml | 8 +- .../rbac/openstacklightspeed_viewer_role.yaml | 8 +- config/samples/kustomization.yaml | 2 +- ...ghtspeed_v1beta1_openstacklightspeed.yaml} | 0 config/scorecard/patches/basic.config.yaml | 2 +- config/scorecard/patches/olm.config.yaml | 10 +- internal/controller/common.go | 18 +- internal/controller/lcore_config.go | 2 + internal/controller/llama_stack_config.go | 1 + test/e2e/e2e_suite_test.go | 71 +++- test/e2e/e2e_test.go | 329 ++++++++++++++---- test/utils/utils.go | 155 +++++++-- 43 files changed, 1217 insertions(+), 472 deletions(-) delete mode 100644 .golangci.yaml rename bundle/manifests/{openstack-lightspeed-operator-metrics_v1_service.yaml => openstack-lightspeed-operator-controller-manager-metrics_v1_service.yaml} (66%) create mode 100644 bundle/manifests/openstack-lightspeed-operator-manager-role_rbac.authorization.k8s.io_v1_role.yaml delete mode 100644 bundle/manifests/openstack-lightspeed-operator-manager-rolebinding_rbac.authorization.k8s.io_v1_rolebinding.yaml create mode 100644 bundle/manifests/openstack-lightspeed-operator-openstacklightspeed-admin-role_rbac.authorization.k8s.io_v1_clusterrole.yaml create mode 100644 config/default/cert_metrics_manager_patch.yaml create mode 100644 config/network-policy/allow-metrics-traffic.yaml create mode 100644 config/network-policy/kustomization.yaml create mode 100644 config/prometheus/monitor_tls_patch.yaml create mode 100644 config/rbac/openstacklightspeed_admin_role.yaml rename config/samples/{api_v1beta1_openstacklightspeed.yaml => lightspeed_v1beta1_openstacklightspeed.yaml} (100%) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index deb2924..e0515cd 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -60,7 +60,7 @@ jobs: uses: redhat-actions/openshift-tools-installer@e3b32c2ab7b064b2ce189121c4a49f509a99a7b4 # v3 with: source: github - operator-sdk: 1.38.0 + operator-sdk: 1.42.3 - name: Log in to Quay uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 diff --git a/.github/workflows/verify-generation.yaml b/.github/workflows/verify-generation.yaml index 6f026e1..6bbcab5 100644 --- a/.github/workflows/verify-generation.yaml +++ b/.github/workflows/verify-generation.yaml @@ -23,7 +23,7 @@ jobs: uses: redhat-actions/openshift-tools-installer@e3b32c2ab7b064b2ce189121c4a49f509a99a7b4 # v3 with: source: github - operator-sdk: 1.38.0 + operator-sdk: 1.42.3 - name: Verify generated files are up to date and fail if anything changed run: | diff --git a/.golangci.yaml b/.golangci.yaml deleted file mode 100644 index a5e4920..0000000 --- a/.golangci.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -version: 2 - -linters: - # Enable specific linter - # https://golangci-lint.run/usage/linters/#enabled-by-default - enable: - - errorlint - - revive - - ginkgolinter - - govet - -formatters: - enable: - - gofmt - -run: - timeout: 5m diff --git a/.golangci.yml b/.golangci.yml index a67edde..2d6cc61 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,27 +4,16 @@ run: linters: default: none enable: - - dupl - - errcheck + - errorlint + - revive - ginkgolinter - - goconst - - gosec - - gocyclo - govet - - ineffassign - - lll - - misspell - - nakedret - - prealloc - - revive - - staticcheck - - unconvert - - unparam - - unused + - gosec settings: revive: rules: - name: comment-spacings + - name: import-shadowing exclusions: generated: lax rules: diff --git a/Dockerfile b/Dockerfile index 0d3641e..40e8367 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN go mod download # Copy the go source COPY cmd/main.go cmd/main.go COPY api/ api/ -COPY internal/controller/ internal/controller/ +COPY internal/ internal/ # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command @@ -26,7 +26,8 @@ RUN GOMAXPROCS=${GOMAXPROCS} CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARG # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.4 +# FROM gcr.io/distroless/static:nonroot +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.6 WORKDIR / COPY --from=builder /workspace/manager . USER 65532:65532 diff --git a/Makefile b/Makefile index cf00dab..200715e 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -ifeq ($(origin VERSION), undefined) -TAG ?= latest -VERSION := 0.0.1 -else -TAG ?= v$(VERSION) -endif +VERSION ?= 0.0.1 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -33,22 +28,12 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# lightspeed.openstack.org/openstack-lightspeed-operator-bundle:$TAG and lightspeed.openstack.org/openstack-lightspeed-operator-catalog:$TAG. +# openstack.org/lightspeed-operator-bundle:$VERSION and openstack.org/lightspeed-operator-catalog:$VERSION. IMAGE_TAG_BASE ?= quay.io/openstack-lightspeed/operator -# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). -CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:$(TAG) - -CATALOG_NAME ?= openstack-lightspeed-catalog -CATALOG_CHANNEL ?= alpha - -# OpenShift internal registry support for local development/testing. -OCP_REGISTRY_NAMESPACE ?= openstack-lightspeed -OCP_INTERNAL_REGISTRY ?= image-registry.openshift-image-registry.svc:5000 - # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(TAG) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) # BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) @@ -63,13 +48,9 @@ endif # Set the Operator SDK version to use. By default, what is installed on the system is used. # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. -OPERATOR_SDK_VERSION ?= v1.38.0-ocp +OPERATOR_SDK_VERSION ?= v1.42.3 # Image URL to use all building/pushing image targets IMG ?= $(IMAGE_TAG_BASE):latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.30.0 - -BRANCH ?= main # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -119,38 +100,48 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." -.PHONY: gowork -gowork: ## Generate go.work file - test -f go.work || go work init - go work use . - go work sync - -.PHONY: force-bump -force-bump: ## Force bump after tagging - for dep in $$(cat go.mod | grep openstack-k8s-operators | grep -vE -- 'indirect|lightspeed-operator|^replace|^//' | awk '{print $$1}'); do \ - go get $$dep@$(BRANCH) ; \ - done - .PHONY: fmt fmt: ## Run go fmt against code. go fmt ./... -.PHONY: tidy -tidy: ## Run go mod tidy on every mod file in the repo - go mod tidy - .PHONY: vet vet: ## Run go vet against code. go vet ./... .PHONY: test -test: manifests generate fmt vet envtest ## Run tests. +test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. -.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: - go test ./test/e2e/ -v -ginkgo.v +# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# CertManager is installed by default; skip with: +# - CERT_MANAGER_INSTALL_SKIP=true +KIND_CLUSTER ?= lightspeed-operator-test-e2e + +.PHONY: setup-test-e2e +setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @case "$$($(KIND) get clusters)" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + esac + +.PHONY: test-e2e +test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + @status=0; \ + KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v || status=$$?; \ + $(MAKE) cleanup-test-e2e; \ + exit $$status + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) .PHONY: lint lint: golangci-lint ## Run golangci-lint linter @@ -160,17 +151,15 @@ lint: golangci-lint ## Run golangci-lint linter lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run --fix -##@ Security - -.PHONY: govulncheck -govulncheck: govulncheck-install ## Run govulncheck vulnerability scanner with ignore list. - @GOVULNCHECK_BIN="$(GOVULNCHECK)" ./hack/govulncheck-wrapper.sh +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + $(GOLANGCI_LINT) config verify ##@ Build .PHONY: build build: manifests generate fmt vet ## Build manager binary. - GOMAXPROCS=$(GOMAXPROCS) go build -o bin/manager cmd/main.go + go build -o bin/manager cmd/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -181,7 +170,7 @@ run: manifests generate fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build --build-arg GOMAXPROCS=$(GOMAXPROCS) -t ${IMG} . + $(CONTAINER_TOOL) build -t ${IMG} . .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -198,10 +187,10 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le docker-buildx: ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name openstack-lightspeed-operator-builder - $(CONTAINER_TOOL) buildx use openstack-lightspeed-operator-builder + - $(CONTAINER_TOOL) buildx create --name lightspeed-operator-builder + $(CONTAINER_TOOL) buildx use lightspeed-operator-builder - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm openstack-lightspeed-operator-builder + - $(CONTAINER_TOOL) buildx rm lightspeed-operator-builder rm Dockerfile.cross .PHONY: build-installer @@ -233,27 +222,6 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - -# Deploy using the catalog image. -.PHONY: openstack-lightspeed-deploy -openstack-lightspeed-deploy: export OUTPUT_DIR = out -openstack-lightspeed-deploy: ## Deploy using a catalog image. - bash scripts/gen-catalog.sh $(CATALOG_IMG) $(CATALOG_NAME) - oc apply -f $(OUTPUT_DIR)/catalog - bash scripts/gen-rhosls.sh $(CATALOG_NAME) $(CATALOG_CHANNEL) - oc apply -f $(OUTPUT_DIR)/rhosls - bash scripts/confirm-rhosls-running.sh - -# Undeploy using the catalog image. -# Remove OpenStackLightspeds so the namespace deletion doesn't get stuck -.PHONY: openstack-lightspeed-undeploy -openstack-lightspeed-undeploy: export OUTPUT_DIR = out -openstack-lightspeed-undeploy: ## Undeploy using a catalog image. - oc delete openstacklightspeed --all -n openstack-lightspeed --ignore-not-found=true --timeout=120s - find out/{catalog,rhosls} -name "*.yaml" -printf " -f %p" | xargs oc delete --ignore-not-found=true - -CATALOG_NAME ?= openstack-lightspeed-catalog -CATALOG_CHANNEL ?= alpha - ##@ Dependencies ## Location to install dependencies to @@ -263,20 +231,20 @@ $(LOCALBIN): ## Tool Binaries KUBECTL ?= kubectl +KIND ?= kind KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest GOLANGCI_LINT = $(LOCALBIN)/golangci-lint -KUTTL ?= $(LOCALBIN)/kubectl-kuttl -GOVULNCHECK ?= $(LOCALBIN)/govulncheck ## Tool Versions -KUSTOMIZE_VERSION ?= v5.4.2 -CONTROLLER_TOOLS_VERSION ?= v0.16.5 -ENVTEST_VERSION ?= release-0.22 +KUSTOMIZE_VERSION ?= v5.6.0 +CONTROLLER_TOOLS_VERSION ?= v0.18.0 +#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) +ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') GOLANGCI_LINT_VERSION ?= v2.12.2 -KUTTL_VERSION ?= 0.22.0 -GOVULNCHECK_VERSION ?= v1.6.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -288,6 +256,14 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } + .PHONY: envtest envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) @@ -298,53 +274,6 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(LOCALBIN) $(GOLANGCI_LINT_VERSION) -.PHONY: kuttl -kuttl: $(KUTTL) ## Download kubectl-kuttl locally if necessary. -$(KUTTL): $(LOCALBIN) - test -s $(LOCALBIN)/kubectl-kuttl || curl -L -o $(LOCALBIN)/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v$(KUTTL_VERSION)/kubectl-kuttl_$(KUTTL_VERSION)_linux_x86_64 - chmod +x $(LOCALBIN)/kubectl-kuttl - -.PHONY: kuttl-test -kuttl-test: kuttl ## Run kuttl tests - @command -v diff >/dev/null 2>&1 || { echo "ERROR: 'diff' command is required for KUTTL tests but not found in PATH" >&2; exit 1; } - @command -v oc >/dev/null 2>&1 || { echo "ERROR: 'oc' command is required for KUTTL tests but not found in PATH" >&2; exit 1; } - $(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests $(KUTTL_ARGS) - -.PHONY: govulncheck-install -govulncheck-install: $(LOCALBIN) ## Download govulncheck locally if necessary. - $(call go-install-tool,$(GOVULNCHECK),golang.org/x/vuln/cmd/govulncheck,$(GOVULNCHECK_VERSION)) - -.PHONY: kuttl-test-run -kuttl-test-run: kuttl openstack-lightspeed-deploy kuttl-test openstack-lightspeed-undeploy - -.PHONY: ocp-registry-push -ocp-registry-push: ## Push images to the OpenShift internal registry. - bash scripts/ocp-registry-push.sh $(CONTAINER_TOOL) $(OCP_REGISTRY_NAMESPACE) $(IMG) $(CATALOG_IMG) - -.PHONY: ocp-catalog-build -ocp-catalog-build: opm ## Build a catalog image for the OpenShift internal registry. - bash scripts/ocp-catalog-build.sh $(CONTAINER_TOOL) $(BUNDLE_IMG) $(CATALOG_IMG) $(OPM) - -.PHONY: kuttl-test-ocp -kuttl-test-ocp: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest -kuttl-test-ocp: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:$(TAG) -kuttl-test-ocp: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:$(TAG) -kuttl-test-ocp: docker-build bundle bundle-build ocp-catalog-build ocp-registry-push kuttl-test-run - -.PHONY: ocp-deploy -ocp-deploy: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest -ocp-deploy: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:$(TAG) -ocp-deploy: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:$(TAG) -ocp-deploy: docker-build bundle bundle-build ocp-catalog-build ocp-registry-push openstack-lightspeed-deploy ## Build, push, and deploy the operator on an OCP cluster. - -.PHONY: ocp-deploy-cleanup -ocp-deploy-cleanup: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest -ocp-deploy-cleanup: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:$(TAG) -ocp-deploy-cleanup: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:$(TAG) -ocp-deploy-cleanup: openstack-lightspeed-undeploy ## Clean up everything created by ocp-deploy. - oc delete imagestreamtag operator-catalog:$(TAG) -n openshift-marketplace --ignore-not-found=true - oc delete namespace $(OCP_REGISTRY_NAMESPACE) --ignore-not-found=true --wait - # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed @@ -402,7 +331,7 @@ ifeq (,$(shell which opm 2>/dev/null)) set -e ;\ mkdir -p $(dir $(OPM)) ;\ OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.23.0/$${OS}-$${ARCH}-opm ;\ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.55.0/$${OS}-$${ARCH}-opm ;\ chmod +x $(OPM) ;\ } else @@ -414,6 +343,9 @@ endif # These images MUST exist in a registry and be pull-able. BUNDLE_IMGS ?= $(BUNDLE_IMG) +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + # Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. ifneq ($(origin CATALOG_BASE_IMG), undefined) FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) @@ -430,3 +362,104 @@ catalog-build: opm ## Build a catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. $(MAKE) docker-push IMG=$(CATALOG_IMG) + +##@ Custom (not generated using operator-sdk) + +# OpenShift internal registry support for local development/testing. +OCP_REGISTRY_NAMESPACE ?= openstack-lightspeed +OCP_INTERNAL_REGISTRY ?= image-registry.openshift-image-registry.svc:5000 +KUTTL ?= $(LOCALBIN)/kubectl-kuttl +KUTTL_VERSION ?= 0.22.0 +GOVULNCHECK_VERSION ?= v1.6.0 +GOVULNCHECK ?= $(LOCALBIN)/govulncheck + +# Branch used by force-bump when re-resolving openstack-k8s-operators dependencies. +BRANCH ?= main +# Catalog name/channel used by the openstack-lightspeed-deploy/undeploy targets. +CATALOG_NAME ?= openstack-lightspeed-catalog +CATALOG_CHANNEL ?= alpha + +.PHONY: gowork +gowork: ## Generate go.work file + test -f go.work || go work init + go work use . + go work sync + +.PHONY: force-bump +force-bump: ## Force bump after tagging + for dep in $$(cat go.mod | grep openstack-k8s-operators | grep -vE -- 'indirect|lightspeed-operator|^replace|^//' | awk '{print $$1}'); do \ + go get $$dep@$(BRANCH) ; \ + done + +.PHONY: tidy +tidy: ## Run go mod tidy on every mod file in the repo + go mod tidy + +.PHONY: govulncheck +govulncheck: govulncheck-install ## Run govulncheck vulnerability scanner with ignore list. + @GOVULNCHECK_BIN="$(GOVULNCHECK)" ./hack/govulncheck-wrapper.sh + + +.PHONY: govulncheck-install +govulncheck-install: $(LOCALBIN) ## Download govulncheck locally if necessary. + $(call go-install-tool,$(GOVULNCHECK),golang.org/x/vuln/cmd/govulncheck,$(GOVULNCHECK_VERSION)) + +.PHONY: openstack-lightspeed-deploy +openstack-lightspeed-deploy: export OUTPUT_DIR = out +openstack-lightspeed-deploy: ## Deploy using a catalog image. + bash scripts/gen-catalog.sh $(CATALOG_IMG) $(CATALOG_NAME) + oc apply -f $(OUTPUT_DIR)/catalog + bash scripts/gen-rhosls.sh $(CATALOG_NAME) $(CATALOG_CHANNEL) + oc apply -f $(OUTPUT_DIR)/rhosls + bash scripts/confirm-rhosls-running.sh + +# Undeploy using the catalog image. +# Remove OpenStackLightspeds so the namespace deletion doesn't get stuck +.PHONY: openstack-lightspeed-undeploy +openstack-lightspeed-undeploy: export OUTPUT_DIR = out +openstack-lightspeed-undeploy: ## Undeploy using a catalog image. + oc delete openstacklightspeed --all -n openstack-lightspeed --ignore-not-found=true --timeout=120s + find out/catalog out/rhosls -name "*.yaml" -printf " -f %p" | xargs oc delete --ignore-not-found=true + +.PHONY: kuttl +kuttl: $(KUTTL) ## Download kubectl-kuttl locally if necessary. +$(KUTTL): $(LOCALBIN) + test -s $(LOCALBIN)/kubectl-kuttl || curl -L -o $(LOCALBIN)/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v$(KUTTL_VERSION)/kubectl-kuttl_$(KUTTL_VERSION)_linux_x86_64 + chmod +x $(LOCALBIN)/kubectl-kuttl + +.PHONY: kuttl-test +kuttl-test: kuttl ## Run kuttl tests + @command -v diff >/dev/null 2>&1 || { echo "ERROR: 'diff' command is required for KUTTL tests but not found in PATH" >&2; exit 1; } + @command -v oc >/dev/null 2>&1 || { echo "ERROR: 'oc' command is required for KUTTL tests but not found in PATH" >&2; exit 1; } + $(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests $(KUTTL_ARGS) + +.PHONY: ocp-registry-push +ocp-registry-push: ## Push images to the OpenShift internal registry. + bash scripts/ocp-registry-push.sh $(CONTAINER_TOOL) $(OCP_REGISTRY_NAMESPACE) $(IMG) $(CATALOG_IMG) + +.PHONY: ocp-catalog-build +ocp-catalog-build: opm ## Build a catalog image for the OpenShift internal registry. + bash scripts/ocp-catalog-build.sh $(CONTAINER_TOOL) $(BUNDLE_IMG) $(CATALOG_IMG) $(OPM) + +.PHONY: kuttl-test-run +kuttl-test-run: kuttl openstack-lightspeed-deploy kuttl-test openstack-lightspeed-undeploy + +.PHONY: kuttl-test-ocp +kuttl-test-ocp: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest +kuttl-test-ocp: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:v$(VERSION) +kuttl-test-ocp: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:v$(VERSION) +kuttl-test-ocp: docker-build bundle bundle-build ocp-catalog-build ocp-registry-push kuttl-test-run ## Run kuttl tests against locally built catalog image + +.PHONY: ocp-deploy +ocp-deploy: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest +ocp-deploy: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:v$(VERSION) +ocp-deploy: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:v$(VERSION) +ocp-deploy: docker-build bundle bundle-build ocp-catalog-build ocp-registry-push openstack-lightspeed-deploy ## Build, push, and deploy the operator on an OCP cluster. + +.PHONY: ocp-deploy-cleanup +ocp-deploy-cleanup: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest +ocp-deploy-cleanup: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:v$(VERSION) +ocp-deploy-cleanup: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:v$(VERSION) +ocp-deploy-cleanup: openstack-lightspeed-undeploy ## Clean up everything created by ocp-deploy. + oc delete imagestreamtag operator-catalog:v$(VERSION) -n openshift-marketplace --ignore-not-found=true + oc delete namespace $(OCP_REGISTRY_NAMESPACE) --ignore-not-found=true --wait diff --git a/PROJECT b/PROJECT index 3a7f081..086da6a 100644 --- a/PROJECT +++ b/PROJECT @@ -8,7 +8,6 @@ layout: plugins: manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} - sdk.x-openshift.io/v1: {} projectName: openstack-lightspeed-operator repo: github.com/openstack-k8s-operators/lightspeed-operator resources: diff --git a/bundle.Dockerfile b/bundle.Dockerfile index 3b92ba1..8b58d1d 100644 --- a/bundle.Dockerfile +++ b/bundle.Dockerfile @@ -6,7 +6,7 @@ LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/ LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/ LABEL operators.operatorframework.io.bundle.package.v1=openstack-lightspeed-operator LABEL operators.operatorframework.io.bundle.channels.v1=alpha -LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.38.0 +LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.42.3 LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 LABEL operators.operatorframework.io.metrics.project_layout=go.kubebuilder.io/v4 diff --git a/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml b/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml index f107cfb..fc335e7 100644 --- a/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml +++ b/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.18.0 creationTimestamp: null name: openstacklightspeeds.lightspeed.openstack.org spec: diff --git a/bundle/manifests/openstack-lightspeed-operator-metrics_v1_service.yaml b/bundle/manifests/openstack-lightspeed-operator-controller-manager-metrics_v1_service.yaml similarity index 66% rename from bundle/manifests/openstack-lightspeed-operator-metrics_v1_service.yaml rename to bundle/manifests/openstack-lightspeed-operator-controller-manager-metrics_v1_service.yaml index 6236f5b..6ba7607 100644 --- a/bundle/manifests/openstack-lightspeed-operator-metrics_v1_service.yaml +++ b/bundle/manifests/openstack-lightspeed-operator-controller-manager-metrics_v1_service.yaml @@ -2,13 +2,13 @@ apiVersion: v1 kind: Service metadata: annotations: - service.beta.openshift.io/serving-cert-secret-name: operator-metrics-tls + service.beta.openshift.io/serving-cert-secret-name: metrics-server-cert creationTimestamp: null labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: openstack-lightspeed-operator control-plane: controller-manager - name: openstack-lightspeed-operator-metrics + name: openstack-lightspeed-operator-controller-manager-metrics spec: ports: - name: https @@ -16,6 +16,7 @@ spec: protocol: TCP targetPort: 8443 selector: + app.kubernetes.io/name: openstack-lightspeed-operator control-plane: controller-manager status: loadBalancer: {} diff --git a/bundle/manifests/openstack-lightspeed-operator-manager-role_rbac.authorization.k8s.io_v1_role.yaml b/bundle/manifests/openstack-lightspeed-operator-manager-role_rbac.authorization.k8s.io_v1_role.yaml new file mode 100644 index 0000000..4f3f056 --- /dev/null +++ b/bundle/manifests/openstack-lightspeed-operator-manager-role_rbac.authorization.k8s.io_v1_role.yaml @@ -0,0 +1,61 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + name: openstack-lightspeed-operator-manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - serviceaccounts + - services + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - get + - list + - patch + - watch diff --git a/bundle/manifests/openstack-lightspeed-operator-manager-rolebinding_rbac.authorization.k8s.io_v1_rolebinding.yaml b/bundle/manifests/openstack-lightspeed-operator-manager-rolebinding_rbac.authorization.k8s.io_v1_rolebinding.yaml deleted file mode 100644 index 341f0e2..0000000 --- a/bundle/manifests/openstack-lightspeed-operator-manager-rolebinding_rbac.authorization.k8s.io_v1_rolebinding.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: openstack-lightspeed-operator - name: openstack-lightspeed-operator-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: manager-role -subjects: -- kind: ServiceAccount - name: openstack-lightspeed-operator-controller-manager - namespace: openstack-lightspeed-operator-system diff --git a/bundle/manifests/openstack-lightspeed-operator-openstacklightspeed-admin-role_rbac.authorization.k8s.io_v1_clusterrole.yaml b/bundle/manifests/openstack-lightspeed-operator-openstacklightspeed-admin-role_rbac.authorization.k8s.io_v1_clusterrole.yaml new file mode 100644 index 0000000..3052f5f --- /dev/null +++ b/bundle/manifests/openstack-lightspeed-operator-openstacklightspeed-admin-role_rbac.authorization.k8s.io_v1_clusterrole.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: openstack-lightspeed-operator + name: openstack-lightspeed-operator-openstacklightspeed-admin-role +rules: +- apiGroups: + - lightspeed.openstack.org + resources: + - openstacklightspeeds + verbs: + - '*' +- apiGroups: + - lightspeed.openstack.org + resources: + - openstacklightspeeds/status + verbs: + - get diff --git a/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml b/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml index 0246f4e..d713125 100644 --- a/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml +++ b/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml @@ -25,7 +25,7 @@ metadata: ] capabilities: Basic Install categories: AI/Machine Learning - createdAt: "2026-08-18T12:58:24Z" + createdAt: "2026-08-20T15:05:40Z" description: AI-powered virtual assistant for Red Hat OpenStack Services on OpenShift features.operators.openshift.io/cnf: "false" features.operators.openshift.io/cni: "false" @@ -38,7 +38,7 @@ metadata: features.operators.openshift.io/token-auth-azure: "false" features.operators.openshift.io/token-auth-gcp: "false" operatorframework.io/suggested-namespace: openstack-lightspeed - operators.operatorframework.io/builder: operator-sdk-v1.38.0 + operators.operatorframework.io/builder: operator-sdk-v1.42.3 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 repository: https://github.com/openstack-k8s-operators/lightspeed-operator name: openstack-lightspeed-operator.v0.0.1 @@ -319,6 +319,7 @@ spec: replicas: 1 selector: matchLabels: + app.kubernetes.io/name: openstack-lightspeed-operator control-plane: controller-manager strategy: {} template: @@ -326,6 +327,7 @@ spec: annotations: kubectl.kubernetes.io/default-container: manager labels: + app.kubernetes.io/name: openstack-lightspeed-operator control-plane: controller-manager spec: containers: @@ -333,6 +335,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs command: - /manager env: @@ -383,17 +386,25 @@ spec: drop: - ALL volumeMounts: - - mountPath: /tmp/k8s-metrics-server/serving-certs - name: cert + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs readOnly: true securityContext: runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: openstack-lightspeed-operator-controller-manager terminationGracePeriodSeconds: 10 volumes: - - name: cert + - name: metrics-certs secret: - secretName: operator-metrics-tls + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false + secretName: metrics-server-cert permissions: - rules: - apiGroups: diff --git a/bundle/metadata/annotations.yaml b/bundle/metadata/annotations.yaml index 828f4f1..17ff894 100644 --- a/bundle/metadata/annotations.yaml +++ b/bundle/metadata/annotations.yaml @@ -5,7 +5,7 @@ annotations: operators.operatorframework.io.bundle.metadata.v1: metadata/ operators.operatorframework.io.bundle.package.v1: openstack-lightspeed-operator operators.operatorframework.io.bundle.channels.v1: alpha - operators.operatorframework.io.metrics.builder: operator-sdk-v1.38.0 + operators.operatorframework.io.metrics.builder: operator-sdk-v1.42.3 operators.operatorframework.io.metrics.mediatype.v1: metrics+v1 operators.operatorframework.io.metrics.project_layout: go.kubebuilder.io/v4 diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml index ea6b807..b2761e6 100644 --- a/bundle/tests/scorecard/config.yaml +++ b/bundle/tests/scorecard/config.yaml @@ -8,7 +8,7 @@ stages: - entrypoint: - scorecard-test - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: basic test: basic-check-spec-test @@ -18,7 +18,7 @@ stages: - entrypoint: - scorecard-test - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-bundle-validation-test @@ -28,7 +28,7 @@ stages: - entrypoint: - scorecard-test - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-crds-have-validation-test @@ -38,7 +38,7 @@ stages: - entrypoint: - scorecard-test - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-crds-have-resources-test @@ -48,7 +48,7 @@ stages: - entrypoint: - scorecard-test - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-spec-descriptors-test @@ -58,7 +58,7 @@ stages: - entrypoint: - scorecard-test - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-status-descriptors-test diff --git a/cmd/main.go b/cmd/main.go index e059532..63ea237 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,5 +1,5 @@ /* -Copyright 2025. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "strings" "sync/atomic" @@ -37,6 +38,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -47,7 +49,7 @@ import ( openshiftv1 "github.com/openshift/api/operator/v1" operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" - apiv1beta1 "github.com/openstack-k8s-operators/lightspeed-operator/api/v1beta1" + lightspeedv1beta1 "github.com/openstack-k8s-operators/lightspeed-operator/api/v1beta1" "github.com/openstack-k8s-operators/lightspeed-operator/internal/controller" // +kubebuilder:scaffold:imports ) @@ -62,7 +64,7 @@ func init() { utilruntime.Must(operatorsv1alpha1.AddToScheme(scheme)) - utilruntime.Must(apiv1beta1.AddToScheme(scheme)) + utilruntime.Must(lightspeedv1beta1.AddToScheme(scheme)) utilruntime.Must(consolev1.AddToScheme(scheme)) @@ -72,15 +74,15 @@ func init() { // +kubebuilder:scaffold:scheme } +// nolint:gocyclo func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool var enableHTTP2 bool - var certDir string - var certName string - var keyName string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -90,14 +92,15 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") - flag.StringVar(&certDir, "cert-dir", "/tmp/k8s-metrics-server/serving-certs", - "The directory where the TLS certificates are stored.") - flag.StringVar(&certName, "cert-name", "tls.crt", - "The name of the TLS certificate file.") - flag.StringVar(&keyName, "key-name", "tls.key", - "The name of the TLS key file.") opts := zap.Options{ // Development: false uses JSON encoding and omits stack traces / DPanic behaviour. // For local development use: make run ARGS="--zap-devel" @@ -114,40 +117,94 @@ func main() { // Rapid Reset CVEs. For more information see: // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + tlsOpts = append(tlsOpts, func(c *tls.Config) { setupLog.Info("enforcing minimum TLS version 1.3") c.MinVersion = tls.VersionTLS13 - if !enableHTTP2 { - setupLog.Info("disabling http/2") - c.NextProtos = []string{"http/1.1"} - } }) + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + // Create watchers for metrics and webhooks certificates + var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + + if len(webhookCertPath) > 0 { + setupLog.Info("Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) + + var err error + webhookCertWatcher, err = certwatcher.New( + filepath.Join(webhookCertPath, webhookCertName), + filepath.Join(webhookCertPath, webhookCertKey), + ) + if err != nil { + setupLog.Error(err, "Failed to initialize webhook certificate watcher") + os.Exit(1) + } + + webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { + config.GetCertificate = webhookCertWatcher.GetCertificate + }) + } + webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: tlsOpts, + TLSOpts: webhookTLSOpts, }) // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: - // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html metricsServerOptions := metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - CertDir: certDir, - CertName: certName, - KeyName: keyName, } if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: - // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization } + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) + } + watchNamespaces, err := getWatchNamespaces() if err != nil { setupLog.Error(err, "unable to get WatchNamespace, "+ @@ -197,7 +254,7 @@ func main() { } // Defaults for OpenStackLightspeed - apiv1beta1.SetupDefaults() + lightspeedv1beta1.SetupDefaults() dynamicWatchCRDs := getDynamicWatchCRDs() @@ -213,6 +270,22 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + + if webhookCertWatcher != nil { + setupLog.Info("Adding webhook certificate watcher to manager") + if err := mgr.Add(webhookCertWatcher); err != nil { + setupLog.Error(err, "unable to add webhook certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml b/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml index 59837dc..5e1e59e 100644 --- a/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml +++ b/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.18.0 name: openstacklightspeeds.lightspeed.openstack.org spec: group: lightspeed.openstack.org diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 47705b8..847d6e2 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -10,13 +10,7 @@ patches: # patches here are for enabling the conversion webhook for each CRD # +kubebuilder:scaffold:crdkustomizewebhookpatch -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -#- path: patches/cainjection_in_openstacklightspeeds.yaml -# +kubebuilder:scaffold:crdkustomizecainjectionpatch - # [WEBHOOK] To enable webhook, uncomment the following section # the following config is for teaching kustomize how to do kustomization for CRDs. - #configurations: #- kustomizeconfig.yaml diff --git a/config/default/cert_metrics_manager_patch.yaml b/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 0000000..158ed62 --- /dev/null +++ b/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,32 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +# NOTE(lpiwowar): The secret is generated by the OpenShift service-ca operator +# (see the annotation on the metrics Service). service-ca serving-cert secrets +# contain only tls.crt and tls.key -- there is no ca.crt key -- so we must not +# request a ca.crt item here (optional: false would fail the mount otherwise). +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index d331a90..de271c1 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -27,8 +27,13 @@ resources: #- ../prometheus # [METRICS] Expose the controller manager metrics service. - metrics_service.yaml +# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. +# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. +# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will +# be able to communicate with the Webhook Server. +#- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -36,111 +41,196 @@ patches: target: kind: Deployment +# [METRICS-WITH-CERTS] Protect the metrics endpoint with a TLS serving cert. +# The cert is provided by the OpenShift service-ca operator (see the +# service.beta.openshift.io/serving-cert-secret-name annotation in +# metrics_service.yaml), NOT cert-manager -- so the cert-manager `replacements:` +# block below stays commented. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- path: manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -#- path: webhookcainjection_patch.yaml +# target: +# kind: Deployment # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: -# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: # Add cert-manager annotation to the webhook Service -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have any webhook +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # Name of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # Namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionns +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml index fe32bde..896e8bf 100644 --- a/config/default/metrics_service.yaml +++ b/config/default/metrics_service.yaml @@ -6,8 +6,16 @@ metadata: app.kubernetes.io/name: openstack-lightspeed-operator app.kubernetes.io/managed-by: kustomize annotations: - service.beta.openshift.io/serving-cert-secret-name: operator-metrics-tls - name: metrics + service.beta.openshift.io/serving-cert-secret-name: metrics-server-cert + + # NOTE(lpiwowar): Do NOT restore the operator-sdk scaffold default name + # "controller-manager-metrics-service". Service names must be a valid RFC 1035 + # DNS label (<= 63 chars), and kustomize prepends the namePrefix + # "openstack-lightspeed-operator-" (30 chars). The scaffold default (34 chars) + # would total 64 chars and be rejected on apply, blocking the deploy. Keeping + # this shortened to "controller-manager-metrics" (26 chars) stays under the + # limit (56 chars total). Re-check this after any operator-sdk regeneration. + name: controller-manager-metrics namespace: system spec: ports: @@ -17,3 +25,4 @@ spec: targetPort: 8443 selector: control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index ebefbe3..5ff6ba4 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -6,9 +6,3 @@ images: - name: controller newName: quay.io/openstack-lightspeed/operator newTag: latest -patches: -- patch: '[{"op": "replace", "path": "/spec/template/spec/containers/0/env/0/value", - "value": "latest"}]' - target: - kind: Deployment - name: controller-manager diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e210e0b..26ff7c3 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -20,6 +20,7 @@ spec: selector: matchLabels: control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator replicas: 1 template: metadata: @@ -27,6 +28,7 @@ spec: kubectl.kubernetes.io/default-container: manager labels: control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator spec: # TODO(user): Uncomment the following code to configure the nodeAffinity expression # according to the platforms which are supported by your solution. @@ -49,14 +51,12 @@ spec: # values: # - linux securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted runAsNonRoot: true - # TODO(user): For common cases that do not require escalating privileges - # it is recommended to ensure that all your Pods/Containers are restrictive. - # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). - # seccompProfile: - # type: RuntimeDefault + seccompProfile: + type: RuntimeDefault containers: - command: - /manager @@ -65,6 +65,7 @@ spec: - --health-probe-bind-address=:8081 image: controller:latest name: manager + ports: [] env: - name: WATCH_NAMESPACE valueFrom: @@ -115,13 +116,7 @@ spec: requests: cpu: 10m memory: 128Mi - volumeMounts: - - name: cert - mountPath: /tmp/k8s-metrics-server/serving-certs - readOnly: true + volumeMounts: [] + volumes: [] serviceAccountName: controller-manager - volumes: - - name: cert - secret: - secretName: operator-metrics-tls terminationGracePeriodSeconds: 10 diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 0000000..f08ec9e --- /dev/null +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: openstack-lightspeed-operator + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml new file mode 100644 index 0000000..ec0fb5e --- /dev/null +++ b/config/network-policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- allow-metrics-traffic.yaml diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed13716..fdc5481 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index a02a048..ebf80cc 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -16,15 +16,12 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator diff --git a/config/prometheus/monitor_tls_patch.yaml b/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 0000000..5bf84ce --- /dev/null +++ b/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,19 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +- op: replace + path: /spec/endpoints/0/tlsConfig + value: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 4c75b00..120be39 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -19,10 +19,11 @@ resources: - metrics_auth_role.yaml - metrics_auth_role_binding.yaml - metrics_reader_role.yaml -# For each CRD, "Editor" and "Viewer" roles are scaffolded by +# For each CRD, "Admin", "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are -# not used by the Project itself. You can comment the following lines +# not used by the openstack-lightspeed-operator itself. You can comment the following lines # if you do not want those helpers be installed with your Project. +- openstacklightspeed_admin_role.yaml - openstacklightspeed_editor_role.yaml - openstacklightspeed_viewer_role.yaml diff --git a/config/rbac/openstacklightspeed_admin_role.yaml b/config/rbac/openstacklightspeed_admin_role.yaml new file mode 100644 index 0000000..33abafd --- /dev/null +++ b/config/rbac/openstacklightspeed_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project openstack-lightspeed-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over lightspeed.openstack.org. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: openstack-lightspeed-operator + app.kubernetes.io/managed-by: kustomize + name: openstacklightspeed-admin-role +rules: +- apiGroups: + - lightspeed.openstack.org + resources: + - openstacklightspeeds + verbs: + - '*' +- apiGroups: + - lightspeed.openstack.org + resources: + - openstacklightspeeds/status + verbs: + - get diff --git a/config/rbac/openstacklightspeed_editor_role.yaml b/config/rbac/openstacklightspeed_editor_role.yaml index eb54717..3ad4c61 100644 --- a/config/rbac/openstacklightspeed_editor_role.yaml +++ b/config/rbac/openstacklightspeed_editor_role.yaml @@ -1,4 +1,10 @@ -# permissions for end users to edit openstacklightspeeds. +# This rule is not used by the project openstack-lightspeed-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the lightspeed.openstack.org. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/config/rbac/openstacklightspeed_viewer_role.yaml b/config/rbac/openstacklightspeed_viewer_role.yaml index 558b1df..44dbb3e 100644 --- a/config/rbac/openstacklightspeed_viewer_role.yaml +++ b/config/rbac/openstacklightspeed_viewer_role.yaml @@ -1,4 +1,10 @@ -# permissions for end users to view openstacklightspeeds. +# This rule is not used by the project openstack-lightspeed-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to lightspeed.openstack.org resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index bc7a348..103692a 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,4 +1,4 @@ ## Append samples of your project ## resources: -- api_v1beta1_openstacklightspeed.yaml +- lightspeed_v1beta1_openstacklightspeed.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/api_v1beta1_openstacklightspeed.yaml b/config/samples/lightspeed_v1beta1_openstacklightspeed.yaml similarity index 100% rename from config/samples/api_v1beta1_openstacklightspeed.yaml rename to config/samples/lightspeed_v1beta1_openstacklightspeed.yaml diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml index 84683cf..e2afe91 100644 --- a/config/scorecard/patches/basic.config.yaml +++ b/config/scorecard/patches/basic.config.yaml @@ -4,7 +4,7 @@ entrypoint: - scorecard-test - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: basic test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml index 43f40a8..687c864 100644 --- a/config/scorecard/patches/olm.config.yaml +++ b/config/scorecard/patches/olm.config.yaml @@ -4,7 +4,7 @@ entrypoint: - scorecard-test - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-bundle-validation-test @@ -14,7 +14,7 @@ entrypoint: - scorecard-test - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-crds-have-validation-test @@ -24,7 +24,7 @@ entrypoint: - scorecard-test - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-crds-have-resources-test @@ -34,7 +34,7 @@ entrypoint: - scorecard-test - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-spec-descriptors-test @@ -44,7 +44,7 @@ entrypoint: - scorecard-test - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-status-descriptors-test diff --git a/internal/controller/common.go b/internal/controller/common.go index 4d45c07..032de12 100644 --- a/internal/controller/common.go +++ b/internal/controller/common.go @@ -157,29 +157,29 @@ func generateOKPSelectorLabels() map[string]string { // parseDevConfig unmarshals the Dev RawExtension into a DevSpec. // Returns a zero-value DevSpec and an error on malformed input. func parseDevConfig(instance *apiv1beta1.OpenStackLightspeed) (apiv1beta1.DevSpec, error) { - var config apiv1beta1.DevSpec + var devConfig apiv1beta1.DevSpec if len(instance.Spec.Dev.Raw) > 0 { - if err := json.Unmarshal(instance.Spec.Dev.Raw, &config); err != nil { - return config, err + if err := json.Unmarshal(instance.Spec.Dev.Raw, &devConfig); err != nil { + return devConfig, err } } - return config, nil + return devConfig, nil } // isRHOSOMCPEnabled returns true if the "rhoso_mcps" feature flag is present in the dev config. func isRHOSOMCPEnabled(instance *apiv1beta1.OpenStackLightspeed) (bool, error) { - config, err := parseDevConfig(instance) + devConfig, err := parseDevConfig(instance) if err != nil { return false, err } - return slices.Contains(config.FeatureFlags, "rhoso_mcps"), nil + return slices.Contains(devConfig.FeatureFlags, "rhoso_mcps"), nil } // getOKPChunkFilterQuery returns the chunk filter query from the dev config, or a version-aware default. func getOKPChunkFilterQuery(ctx context.Context, h *common_helper.Helper, instance *apiv1beta1.OpenStackLightspeed) string { - config, _ := parseDevConfig(instance) - if config.OKPChunkFilterQuery != "" { - return config.OKPChunkFilterQuery + devConfig, _ := parseDevConfig(instance) + if devConfig.OKPChunkFilterQuery != "" { + return devConfig.OKPChunkFilterQuery } logger := h.GetLogger() diff --git a/internal/controller/lcore_config.go b/internal/controller/lcore_config.go index 951c923..e4fd6ab 100644 --- a/internal/controller/lcore_config.go +++ b/internal/controller/lcore_config.go @@ -133,6 +133,7 @@ func buildLCoreInferenceConfig(_ *common_helper.Helper, instance *apiv1beta1.Ope // buildLCoreDatabaseConfig configures persistent database storage (PostgreSQL) func buildLCoreDatabaseConfig(h *common_helper.Helper, _ *apiv1beta1.OpenStackLightspeed) map[string]interface{} { return map[string]interface{}{ + // #nosec G101 -- values are env-var substitution placeholders, not hardcoded credentials "postgres": map[string]interface{}{ "host": PostgresServiceName + "." + h.GetBeforeObject().GetNamespace() + ".svc", "port": PostgresServicePort, @@ -165,6 +166,7 @@ func buildLCoreCustomizationConfig() map[string]interface{} { func buildLCoreConversationCacheConfig(h *common_helper.Helper, _ *apiv1beta1.OpenStackLightspeed) map[string]interface{} { return map[string]interface{}{ "type": "postgres", + // #nosec G101 -- values are env-var substitution placeholders, not hardcoded credentials "postgres": map[string]interface{}{ "host": PostgresServiceName + "." + h.GetBeforeObject().GetNamespace() + ".svc", "port": PostgresServicePort, diff --git a/internal/controller/llama_stack_config.go b/internal/controller/llama_stack_config.go index e5339c3..2321ec6 100644 --- a/internal/controller/llama_stack_config.go +++ b/internal/controller/llama_stack_config.go @@ -317,6 +317,7 @@ func buildLlamaStackStorage(_ *common_helper.Helper, instance *apiv1beta1.OpenSt "type": "kv_sqlite", "db_path": "/tmp/llama-stack/kv_store.db", }, + // #nosec G101 -- values are env-var substitution placeholders, not hardcoded credentials "postgres_backend": map[string]interface{}{ "type": "sql_postgres", "host": fmt.Sprintf("lightspeed-postgres-server.%s.svc", instance.GetNamespace()), diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 3079c7a..30c95d6 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2025. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,15 +18,72 @@ package e2e import ( "fmt" + "os" + "os/exec" "testing" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openstack-k8s-operators/lightspeed-operator/test/utils" +) + +var ( + // Optional Environment Variables: + // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. + // These variables are useful if CertManager is already installed, avoiding + // re-installation and conflicts. + skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" + // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster + isCertManagerAlreadyInstalled = false + + // projectImage is the name of the image which will be build and loaded + // with the code source changes to be tested. + projectImage = "example.com/lightspeed-operator:v0.0.1" ) -// Run e2e tests using the Ginkgo runner. +// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, +// temporary environment to validate project changes with the purposed to be used in CI jobs. +// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs +// CertManager. func TestE2E(t *testing.T) { - gomega.RegisterFailHandler(ginkgo.Fail) - _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "Starting openstack-lightspeed-operator suite\n") - ginkgo.RunSpecs(t, "e2e suite") + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting lightspeed-operator integration test suite\n") + RunSpecs(t, "e2e suite") } + +var _ = BeforeSuite(func() { + By("building the manager(Operator) image") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage)) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") + + // TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is + // built and available before running the tests. Also, remove the following block. + By("loading the manager(Operator) image on Kind") + err = utils.LoadImageToKindClusterWithName(projectImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") + + // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing. + // To prevent errors when tests run in environments with CertManager already installed, + // we check for its presence before execution. + // Setup CertManager before the suite if not skipped and if not already installed + if !skipCertManagerInstall { + By("checking if cert manager is installed already") + isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled() + if !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") + Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") + } + } +}) + +var _ = AfterSuite(func() { + // Teardown CertManager after the suite if not skipped and if it was not already installed + if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") + utils.UninstallCertManager() + } +}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d3f1b31..19695db 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,5 +1,5 @@ /* -Copyright 2025. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,75 +17,132 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/openstack-k8s-operators/lightspeed-operator/test/utils" ) -const namespace = "openstack-lightspeed-operator-system" +// namespace where the project is deployed in +const namespace = "lightspeed-operator-system" -var _ = ginkgo.Describe("controller", ginkgo.Ordered, func() { - ginkgo.BeforeAll(func() { - ginkgo.By("installing prometheus operator") - gomega.Expect(utils.InstallPrometheusOperator()).To(gomega.Succeed()) +// serviceAccountName created for the project +const serviceAccountName = "lightspeed-operator-controller-manager" - ginkgo.By("installing the cert-manager") - gomega.Expect(utils.InstallCertManager()).To(gomega.Succeed()) +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "openstack-lightspeed-operator-controller-manager-metrics" - ginkgo.By("creating manager namespace") +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "lightspeed-operator-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func() { + By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - }) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") - ginkgo.AfterAll(func() { - ginkgo.By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") - ginkgo.By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") - ginkgo.By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) - ginkgo.Context("Operator", func() { - ginkgo.It("should run successfully", func() { - var controllerPodName string - var err error + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. + AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) - // projectimage stores the name of the image used in the example - var projectimage = "example.com/openstack-lightspeed-operator:v0.0.1" + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) - ginkgo.By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + }) - ginkgo.By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } - ginkgo.By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } - ginkgo.By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } - ginkgo.By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { - // Get pod name + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) - cmd = exec.Command("kubectl", "get", + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("Manager", func() { + It("should run successfully", func() { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ @@ -95,28 +152,180 @@ var _ = ginkgo.Describe("controller", ginkgo.Ordered, func() { ) podOutput, err := utils.Run(cmd) - gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) - podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) - } + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - gomega.ExpectWithOffset(2, controllerPodName).Should(gomega.ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) - // Validate pod status + // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=lightspeed-operator-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": ["curl -i -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccount": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + // Run directly instead of utils.Run: the --overrides payload embeds the + // bearer token, and utils.Run logs every command's args to GinkgoWriter. + _, err = cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") } - gomega.EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(gomega.Succeed()) + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/test/utils/utils.go b/test/utils/utils.go index 8e7f831..448055b 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2025. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,57 +14,58 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package utils contains utility function for testing package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" "strings" - "github.com/onsi/ginkgo/v2" //nolint:staticcheck + . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck ) const ( - prometheusOperatorVersion = "v0.72.0" + prometheusOperatorVersion = "v0.77.1" prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + "releases/download/%s/bundle.yaml" - certmanagerVersion = "v1.14.4" - certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" + certmanagerVersion = "v1.16.3" + certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" ) func warnError(err error) { - _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "warning: %v\n", err) -} - -// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) - return err + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) } // Run executes the provided command within this context -func Run(cmd *exec.Cmd) ([]byte, error) { +func Run(cmd *exec.Cmd) (string, error) { dir, _ := GetProjectDir() cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { - _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "chdir dir: %s\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) } cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") - _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s\n", command) + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) output, err := cmd.CombinedOutput() if err != nil { - return output, fmt.Errorf("%s failed with error: (%w) %s", command, err, string(output)) + return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) } - return output, nil + return string(output), nil +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator() error { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "create", "-f", url) + _, err := Run(cmd) + return err } // UninstallPrometheusOperator uninstalls the prometheus @@ -76,6 +77,33 @@ func UninstallPrometheusOperator() { } } +// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed +// by verifying the existence of key CRDs related to Prometheus. +func IsPrometheusCRDsInstalled() bool { + // List of common Prometheus CRDs + prometheusCRDs := []string{ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "prometheusagents.monitoring.coreos.com", + } + + cmd := exec.Command("kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") + output, err := Run(cmd) + if err != nil { + return false + } + crdList := GetNonEmptyLines(output) + for _, crd := range prometheusCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + // UninstallCertManager uninstalls the cert manager func UninstallCertManager() { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) @@ -104,6 +132,39 @@ func InstallCertManager() error { return err } +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + // LoadImageToKindClusterWithName loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error { cluster := "kind" @@ -134,8 +195,60 @@ func GetNonEmptyLines(output string) []string { func GetProjectDir() (string, error) { wd, err := os.Getwd() if err != nil { - return wd, err + return wd, fmt.Errorf("failed to get current working directory: %w", err) } wd = strings.ReplaceAll(wd, "/test/e2e", "") return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read file %q: %w", filename, err) + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %q to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + } + + if _, err = out.Write(content[idx+len(target):]); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + // false positive + // nolint:gosec + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write file %q: %w", filename, err) + } + + return nil +}