diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index deb2924..e0515cd 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -60,7 +60,7 @@ jobs: uses: redhat-actions/openshift-tools-installer@e3b32c2ab7b064b2ce189121c4a49f509a99a7b4 # v3 with: source: github - operator-sdk: 1.38.0 + operator-sdk: 1.42.3 - name: Log in to Quay uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 diff --git a/.github/workflows/verify-generation.yaml b/.github/workflows/verify-generation.yaml index 6f026e1..6bbcab5 100644 --- a/.github/workflows/verify-generation.yaml +++ b/.github/workflows/verify-generation.yaml @@ -23,7 +23,7 @@ jobs: uses: redhat-actions/openshift-tools-installer@e3b32c2ab7b064b2ce189121c4a49f509a99a7b4 # v3 with: source: github - operator-sdk: 1.38.0 + operator-sdk: 1.42.3 - name: Verify generated files are up to date and fail if anything changed run: | diff --git a/.golangci.yaml b/.golangci.yaml deleted file mode 100644 index a5e4920..0000000 --- a/.golangci.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -version: 2 - -linters: - # Enable specific linter - # https://golangci-lint.run/usage/linters/#enabled-by-default - enable: - - errorlint - - revive - - ginkgolinter - - govet - -formatters: - enable: - - gofmt - -run: - timeout: 5m diff --git a/.golangci.yml b/.golangci.yml index a67edde..2d6cc61 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,27 +4,16 @@ run: linters: default: none enable: - - dupl - - errcheck + - errorlint + - revive - ginkgolinter - - goconst - - gosec - - gocyclo - govet - - ineffassign - - lll - - misspell - - nakedret - - prealloc - - revive - - staticcheck - - unconvert - - unparam - - unused + - gosec settings: revive: rules: - name: comment-spacings + - name: import-shadowing exclusions: generated: lax rules: diff --git a/Dockerfile b/Dockerfile index 0d3641e..40e8367 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN go mod download # Copy the go source COPY cmd/main.go cmd/main.go COPY api/ api/ -COPY internal/controller/ internal/controller/ +COPY internal/ internal/ # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command @@ -26,7 +26,8 @@ RUN GOMAXPROCS=${GOMAXPROCS} CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARG # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.4 +# FROM gcr.io/distroless/static:nonroot +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.6 WORKDIR / COPY --from=builder /workspace/manager . USER 65532:65532 diff --git a/Makefile b/Makefile index cf00dab..200715e 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -ifeq ($(origin VERSION), undefined) -TAG ?= latest -VERSION := 0.0.1 -else -TAG ?= v$(VERSION) -endif +VERSION ?= 0.0.1 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -33,22 +28,12 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# lightspeed.openstack.org/openstack-lightspeed-operator-bundle:$TAG and lightspeed.openstack.org/openstack-lightspeed-operator-catalog:$TAG. +# openstack.org/lightspeed-operator-bundle:$VERSION and openstack.org/lightspeed-operator-catalog:$VERSION. IMAGE_TAG_BASE ?= quay.io/openstack-lightspeed/operator -# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). -CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:$(TAG) - -CATALOG_NAME ?= openstack-lightspeed-catalog -CATALOG_CHANNEL ?= alpha - -# OpenShift internal registry support for local development/testing. -OCP_REGISTRY_NAMESPACE ?= openstack-lightspeed -OCP_INTERNAL_REGISTRY ?= image-registry.openshift-image-registry.svc:5000 - # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(TAG) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) # BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) @@ -63,13 +48,9 @@ endif # Set the Operator SDK version to use. By default, what is installed on the system is used. # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. -OPERATOR_SDK_VERSION ?= v1.38.0-ocp +OPERATOR_SDK_VERSION ?= v1.42.3 # Image URL to use all building/pushing image targets IMG ?= $(IMAGE_TAG_BASE):latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.30.0 - -BRANCH ?= main # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -119,38 +100,48 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." -.PHONY: gowork -gowork: ## Generate go.work file - test -f go.work || go work init - go work use . - go work sync - -.PHONY: force-bump -force-bump: ## Force bump after tagging - for dep in $$(cat go.mod | grep openstack-k8s-operators | grep -vE -- 'indirect|lightspeed-operator|^replace|^//' | awk '{print $$1}'); do \ - go get $$dep@$(BRANCH) ; \ - done - .PHONY: fmt fmt: ## Run go fmt against code. go fmt ./... -.PHONY: tidy -tidy: ## Run go mod tidy on every mod file in the repo - go mod tidy - .PHONY: vet vet: ## Run go vet against code. go vet ./... .PHONY: test -test: manifests generate fmt vet envtest ## Run tests. +test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. -.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: - go test ./test/e2e/ -v -ginkgo.v +# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# CertManager is installed by default; skip with: +# - CERT_MANAGER_INSTALL_SKIP=true +KIND_CLUSTER ?= lightspeed-operator-test-e2e + +.PHONY: setup-test-e2e +setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @case "$$($(KIND) get clusters)" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + esac + +.PHONY: test-e2e +test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + @status=0; \ + KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v || status=$$?; \ + $(MAKE) cleanup-test-e2e; \ + exit $$status + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) .PHONY: lint lint: golangci-lint ## Run golangci-lint linter @@ -160,17 +151,15 @@ lint: golangci-lint ## Run golangci-lint linter lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run --fix -##@ Security - -.PHONY: govulncheck -govulncheck: govulncheck-install ## Run govulncheck vulnerability scanner with ignore list. - @GOVULNCHECK_BIN="$(GOVULNCHECK)" ./hack/govulncheck-wrapper.sh +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + $(GOLANGCI_LINT) config verify ##@ Build .PHONY: build build: manifests generate fmt vet ## Build manager binary. - GOMAXPROCS=$(GOMAXPROCS) go build -o bin/manager cmd/main.go + go build -o bin/manager cmd/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -181,7 +170,7 @@ run: manifests generate fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build --build-arg GOMAXPROCS=$(GOMAXPROCS) -t ${IMG} . + $(CONTAINER_TOOL) build -t ${IMG} . .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -198,10 +187,10 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le docker-buildx: ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name openstack-lightspeed-operator-builder - $(CONTAINER_TOOL) buildx use openstack-lightspeed-operator-builder + - $(CONTAINER_TOOL) buildx create --name lightspeed-operator-builder + $(CONTAINER_TOOL) buildx use lightspeed-operator-builder - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm openstack-lightspeed-operator-builder + - $(CONTAINER_TOOL) buildx rm lightspeed-operator-builder rm Dockerfile.cross .PHONY: build-installer @@ -233,27 +222,6 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - -# Deploy using the catalog image. -.PHONY: openstack-lightspeed-deploy -openstack-lightspeed-deploy: export OUTPUT_DIR = out -openstack-lightspeed-deploy: ## Deploy using a catalog image. - bash scripts/gen-catalog.sh $(CATALOG_IMG) $(CATALOG_NAME) - oc apply -f $(OUTPUT_DIR)/catalog - bash scripts/gen-rhosls.sh $(CATALOG_NAME) $(CATALOG_CHANNEL) - oc apply -f $(OUTPUT_DIR)/rhosls - bash scripts/confirm-rhosls-running.sh - -# Undeploy using the catalog image. -# Remove OpenStackLightspeds so the namespace deletion doesn't get stuck -.PHONY: openstack-lightspeed-undeploy -openstack-lightspeed-undeploy: export OUTPUT_DIR = out -openstack-lightspeed-undeploy: ## Undeploy using a catalog image. - oc delete openstacklightspeed --all -n openstack-lightspeed --ignore-not-found=true --timeout=120s - find out/{catalog,rhosls} -name "*.yaml" -printf " -f %p" | xargs oc delete --ignore-not-found=true - -CATALOG_NAME ?= openstack-lightspeed-catalog -CATALOG_CHANNEL ?= alpha - ##@ Dependencies ## Location to install dependencies to @@ -263,20 +231,20 @@ $(LOCALBIN): ## Tool Binaries KUBECTL ?= kubectl +KIND ?= kind KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest GOLANGCI_LINT = $(LOCALBIN)/golangci-lint -KUTTL ?= $(LOCALBIN)/kubectl-kuttl -GOVULNCHECK ?= $(LOCALBIN)/govulncheck ## Tool Versions -KUSTOMIZE_VERSION ?= v5.4.2 -CONTROLLER_TOOLS_VERSION ?= v0.16.5 -ENVTEST_VERSION ?= release-0.22 +KUSTOMIZE_VERSION ?= v5.6.0 +CONTROLLER_TOOLS_VERSION ?= v0.18.0 +#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) +ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') GOLANGCI_LINT_VERSION ?= v2.12.2 -KUTTL_VERSION ?= 0.22.0 -GOVULNCHECK_VERSION ?= v1.6.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -288,6 +256,14 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } + .PHONY: envtest envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) @@ -298,53 +274,6 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(LOCALBIN) $(GOLANGCI_LINT_VERSION) -.PHONY: kuttl -kuttl: $(KUTTL) ## Download kubectl-kuttl locally if necessary. -$(KUTTL): $(LOCALBIN) - test -s $(LOCALBIN)/kubectl-kuttl || curl -L -o $(LOCALBIN)/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v$(KUTTL_VERSION)/kubectl-kuttl_$(KUTTL_VERSION)_linux_x86_64 - chmod +x $(LOCALBIN)/kubectl-kuttl - -.PHONY: kuttl-test -kuttl-test: kuttl ## Run kuttl tests - @command -v diff >/dev/null 2>&1 || { echo "ERROR: 'diff' command is required for KUTTL tests but not found in PATH" >&2; exit 1; } - @command -v oc >/dev/null 2>&1 || { echo "ERROR: 'oc' command is required for KUTTL tests but not found in PATH" >&2; exit 1; } - $(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests $(KUTTL_ARGS) - -.PHONY: govulncheck-install -govulncheck-install: $(LOCALBIN) ## Download govulncheck locally if necessary. - $(call go-install-tool,$(GOVULNCHECK),golang.org/x/vuln/cmd/govulncheck,$(GOVULNCHECK_VERSION)) - -.PHONY: kuttl-test-run -kuttl-test-run: kuttl openstack-lightspeed-deploy kuttl-test openstack-lightspeed-undeploy - -.PHONY: ocp-registry-push -ocp-registry-push: ## Push images to the OpenShift internal registry. - bash scripts/ocp-registry-push.sh $(CONTAINER_TOOL) $(OCP_REGISTRY_NAMESPACE) $(IMG) $(CATALOG_IMG) - -.PHONY: ocp-catalog-build -ocp-catalog-build: opm ## Build a catalog image for the OpenShift internal registry. - bash scripts/ocp-catalog-build.sh $(CONTAINER_TOOL) $(BUNDLE_IMG) $(CATALOG_IMG) $(OPM) - -.PHONY: kuttl-test-ocp -kuttl-test-ocp: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest -kuttl-test-ocp: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:$(TAG) -kuttl-test-ocp: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:$(TAG) -kuttl-test-ocp: docker-build bundle bundle-build ocp-catalog-build ocp-registry-push kuttl-test-run - -.PHONY: ocp-deploy -ocp-deploy: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest -ocp-deploy: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:$(TAG) -ocp-deploy: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:$(TAG) -ocp-deploy: docker-build bundle bundle-build ocp-catalog-build ocp-registry-push openstack-lightspeed-deploy ## Build, push, and deploy the operator on an OCP cluster. - -.PHONY: ocp-deploy-cleanup -ocp-deploy-cleanup: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest -ocp-deploy-cleanup: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:$(TAG) -ocp-deploy-cleanup: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:$(TAG) -ocp-deploy-cleanup: openstack-lightspeed-undeploy ## Clean up everything created by ocp-deploy. - oc delete imagestreamtag operator-catalog:$(TAG) -n openshift-marketplace --ignore-not-found=true - oc delete namespace $(OCP_REGISTRY_NAMESPACE) --ignore-not-found=true --wait - # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed @@ -402,7 +331,7 @@ ifeq (,$(shell which opm 2>/dev/null)) set -e ;\ mkdir -p $(dir $(OPM)) ;\ OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.23.0/$${OS}-$${ARCH}-opm ;\ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.55.0/$${OS}-$${ARCH}-opm ;\ chmod +x $(OPM) ;\ } else @@ -414,6 +343,9 @@ endif # These images MUST exist in a registry and be pull-able. BUNDLE_IMGS ?= $(BUNDLE_IMG) +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + # Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. ifneq ($(origin CATALOG_BASE_IMG), undefined) FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) @@ -430,3 +362,104 @@ catalog-build: opm ## Build a catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. $(MAKE) docker-push IMG=$(CATALOG_IMG) + +##@ Custom (not generated using operator-sdk) + +# OpenShift internal registry support for local development/testing. +OCP_REGISTRY_NAMESPACE ?= openstack-lightspeed +OCP_INTERNAL_REGISTRY ?= image-registry.openshift-image-registry.svc:5000 +KUTTL ?= $(LOCALBIN)/kubectl-kuttl +KUTTL_VERSION ?= 0.22.0 +GOVULNCHECK_VERSION ?= v1.6.0 +GOVULNCHECK ?= $(LOCALBIN)/govulncheck + +# Branch used by force-bump when re-resolving openstack-k8s-operators dependencies. +BRANCH ?= main +# Catalog name/channel used by the openstack-lightspeed-deploy/undeploy targets. +CATALOG_NAME ?= openstack-lightspeed-catalog +CATALOG_CHANNEL ?= alpha + +.PHONY: gowork +gowork: ## Generate go.work file + test -f go.work || go work init + go work use . + go work sync + +.PHONY: force-bump +force-bump: ## Force bump after tagging + for dep in $$(cat go.mod | grep openstack-k8s-operators | grep -vE -- 'indirect|lightspeed-operator|^replace|^//' | awk '{print $$1}'); do \ + go get $$dep@$(BRANCH) ; \ + done + +.PHONY: tidy +tidy: ## Run go mod tidy on every mod file in the repo + go mod tidy + +.PHONY: govulncheck +govulncheck: govulncheck-install ## Run govulncheck vulnerability scanner with ignore list. + @GOVULNCHECK_BIN="$(GOVULNCHECK)" ./hack/govulncheck-wrapper.sh + + +.PHONY: govulncheck-install +govulncheck-install: $(LOCALBIN) ## Download govulncheck locally if necessary. + $(call go-install-tool,$(GOVULNCHECK),golang.org/x/vuln/cmd/govulncheck,$(GOVULNCHECK_VERSION)) + +.PHONY: openstack-lightspeed-deploy +openstack-lightspeed-deploy: export OUTPUT_DIR = out +openstack-lightspeed-deploy: ## Deploy using a catalog image. + bash scripts/gen-catalog.sh $(CATALOG_IMG) $(CATALOG_NAME) + oc apply -f $(OUTPUT_DIR)/catalog + bash scripts/gen-rhosls.sh $(CATALOG_NAME) $(CATALOG_CHANNEL) + oc apply -f $(OUTPUT_DIR)/rhosls + bash scripts/confirm-rhosls-running.sh + +# Undeploy using the catalog image. +# Remove OpenStackLightspeds so the namespace deletion doesn't get stuck +.PHONY: openstack-lightspeed-undeploy +openstack-lightspeed-undeploy: export OUTPUT_DIR = out +openstack-lightspeed-undeploy: ## Undeploy using a catalog image. + oc delete openstacklightspeed --all -n openstack-lightspeed --ignore-not-found=true --timeout=120s + find out/catalog out/rhosls -name "*.yaml" -printf " -f %p" | xargs oc delete --ignore-not-found=true + +.PHONY: kuttl +kuttl: $(KUTTL) ## Download kubectl-kuttl locally if necessary. +$(KUTTL): $(LOCALBIN) + test -s $(LOCALBIN)/kubectl-kuttl || curl -L -o $(LOCALBIN)/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v$(KUTTL_VERSION)/kubectl-kuttl_$(KUTTL_VERSION)_linux_x86_64 + chmod +x $(LOCALBIN)/kubectl-kuttl + +.PHONY: kuttl-test +kuttl-test: kuttl ## Run kuttl tests + @command -v diff >/dev/null 2>&1 || { echo "ERROR: 'diff' command is required for KUTTL tests but not found in PATH" >&2; exit 1; } + @command -v oc >/dev/null 2>&1 || { echo "ERROR: 'oc' command is required for KUTTL tests but not found in PATH" >&2; exit 1; } + $(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests $(KUTTL_ARGS) + +.PHONY: ocp-registry-push +ocp-registry-push: ## Push images to the OpenShift internal registry. + bash scripts/ocp-registry-push.sh $(CONTAINER_TOOL) $(OCP_REGISTRY_NAMESPACE) $(IMG) $(CATALOG_IMG) + +.PHONY: ocp-catalog-build +ocp-catalog-build: opm ## Build a catalog image for the OpenShift internal registry. + bash scripts/ocp-catalog-build.sh $(CONTAINER_TOOL) $(BUNDLE_IMG) $(CATALOG_IMG) $(OPM) + +.PHONY: kuttl-test-run +kuttl-test-run: kuttl openstack-lightspeed-deploy kuttl-test openstack-lightspeed-undeploy + +.PHONY: kuttl-test-ocp +kuttl-test-ocp: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest +kuttl-test-ocp: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:v$(VERSION) +kuttl-test-ocp: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:v$(VERSION) +kuttl-test-ocp: docker-build bundle bundle-build ocp-catalog-build ocp-registry-push kuttl-test-run ## Run kuttl tests against locally built catalog image + +.PHONY: ocp-deploy +ocp-deploy: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest +ocp-deploy: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:v$(VERSION) +ocp-deploy: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:v$(VERSION) +ocp-deploy: docker-build bundle bundle-build ocp-catalog-build ocp-registry-push openstack-lightspeed-deploy ## Build, push, and deploy the operator on an OCP cluster. + +.PHONY: ocp-deploy-cleanup +ocp-deploy-cleanup: IMG = $(OCP_INTERNAL_REGISTRY)/$(OCP_REGISTRY_NAMESPACE)/operator:latest +ocp-deploy-cleanup: BUNDLE_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-bundle:v$(VERSION) +ocp-deploy-cleanup: CATALOG_IMG = $(OCP_INTERNAL_REGISTRY)/openshift-marketplace/operator-catalog:v$(VERSION) +ocp-deploy-cleanup: openstack-lightspeed-undeploy ## Clean up everything created by ocp-deploy. + oc delete imagestreamtag operator-catalog:v$(VERSION) -n openshift-marketplace --ignore-not-found=true + oc delete namespace $(OCP_REGISTRY_NAMESPACE) --ignore-not-found=true --wait diff --git a/PROJECT b/PROJECT index 3a7f081..086da6a 100644 --- a/PROJECT +++ b/PROJECT @@ -8,7 +8,6 @@ layout: plugins: manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} - sdk.x-openshift.io/v1: {} projectName: openstack-lightspeed-operator repo: github.com/openstack-k8s-operators/lightspeed-operator resources: diff --git a/bundle.Dockerfile b/bundle.Dockerfile index 3b92ba1..8b58d1d 100644 --- a/bundle.Dockerfile +++ b/bundle.Dockerfile @@ -6,7 +6,7 @@ LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/ LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/ LABEL operators.operatorframework.io.bundle.package.v1=openstack-lightspeed-operator LABEL operators.operatorframework.io.bundle.channels.v1=alpha -LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.38.0 +LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.42.3 LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 LABEL operators.operatorframework.io.metrics.project_layout=go.kubebuilder.io/v4 diff --git a/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml b/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml index f107cfb..fc335e7 100644 --- a/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml +++ b/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.18.0 creationTimestamp: null name: openstacklightspeeds.lightspeed.openstack.org spec: diff --git a/bundle/manifests/openstack-lightspeed-operator-metrics_v1_service.yaml b/bundle/manifests/openstack-lightspeed-operator-controller-manager-metrics_v1_service.yaml similarity index 66% rename from bundle/manifests/openstack-lightspeed-operator-metrics_v1_service.yaml rename to bundle/manifests/openstack-lightspeed-operator-controller-manager-metrics_v1_service.yaml index 6236f5b..6ba7607 100644 --- a/bundle/manifests/openstack-lightspeed-operator-metrics_v1_service.yaml +++ b/bundle/manifests/openstack-lightspeed-operator-controller-manager-metrics_v1_service.yaml @@ -2,13 +2,13 @@ apiVersion: v1 kind: Service metadata: annotations: - service.beta.openshift.io/serving-cert-secret-name: operator-metrics-tls + service.beta.openshift.io/serving-cert-secret-name: metrics-server-cert creationTimestamp: null labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: openstack-lightspeed-operator control-plane: controller-manager - name: openstack-lightspeed-operator-metrics + name: openstack-lightspeed-operator-controller-manager-metrics spec: ports: - name: https @@ -16,6 +16,7 @@ spec: protocol: TCP targetPort: 8443 selector: + app.kubernetes.io/name: openstack-lightspeed-operator control-plane: controller-manager status: loadBalancer: {} diff --git a/bundle/manifests/openstack-lightspeed-operator-manager-role_rbac.authorization.k8s.io_v1_role.yaml b/bundle/manifests/openstack-lightspeed-operator-manager-role_rbac.authorization.k8s.io_v1_role.yaml new file mode 100644 index 0000000..4f3f056 --- /dev/null +++ b/bundle/manifests/openstack-lightspeed-operator-manager-role_rbac.authorization.k8s.io_v1_role.yaml @@ -0,0 +1,61 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + name: openstack-lightspeed-operator-manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - serviceaccounts + - services + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - get + - list + - patch + - watch diff --git a/bundle/manifests/openstack-lightspeed-operator-manager-rolebinding_rbac.authorization.k8s.io_v1_rolebinding.yaml b/bundle/manifests/openstack-lightspeed-operator-manager-rolebinding_rbac.authorization.k8s.io_v1_rolebinding.yaml deleted file mode 100644 index 341f0e2..0000000 --- a/bundle/manifests/openstack-lightspeed-operator-manager-rolebinding_rbac.authorization.k8s.io_v1_rolebinding.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: openstack-lightspeed-operator - name: openstack-lightspeed-operator-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: manager-role -subjects: -- kind: ServiceAccount - name: openstack-lightspeed-operator-controller-manager - namespace: openstack-lightspeed-operator-system diff --git a/bundle/manifests/openstack-lightspeed-operator-openstacklightspeed-admin-role_rbac.authorization.k8s.io_v1_clusterrole.yaml b/bundle/manifests/openstack-lightspeed-operator-openstacklightspeed-admin-role_rbac.authorization.k8s.io_v1_clusterrole.yaml new file mode 100644 index 0000000..3052f5f --- /dev/null +++ b/bundle/manifests/openstack-lightspeed-operator-openstacklightspeed-admin-role_rbac.authorization.k8s.io_v1_clusterrole.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: openstack-lightspeed-operator + name: openstack-lightspeed-operator-openstacklightspeed-admin-role +rules: +- apiGroups: + - lightspeed.openstack.org + resources: + - openstacklightspeeds + verbs: + - '*' +- apiGroups: + - lightspeed.openstack.org + resources: + - openstacklightspeeds/status + verbs: + - get diff --git a/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml b/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml index 0246f4e..d713125 100644 --- a/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml +++ b/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml @@ -25,7 +25,7 @@ metadata: ] capabilities: Basic Install categories: AI/Machine Learning - createdAt: "2026-08-18T12:58:24Z" + createdAt: "2026-08-20T15:05:40Z" description: AI-powered virtual assistant for Red Hat OpenStack Services on OpenShift features.operators.openshift.io/cnf: "false" features.operators.openshift.io/cni: "false" @@ -38,7 +38,7 @@ metadata: features.operators.openshift.io/token-auth-azure: "false" features.operators.openshift.io/token-auth-gcp: "false" operatorframework.io/suggested-namespace: openstack-lightspeed - operators.operatorframework.io/builder: operator-sdk-v1.38.0 + operators.operatorframework.io/builder: operator-sdk-v1.42.3 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 repository: https://github.com/openstack-k8s-operators/lightspeed-operator name: openstack-lightspeed-operator.v0.0.1 @@ -319,6 +319,7 @@ spec: replicas: 1 selector: matchLabels: + app.kubernetes.io/name: openstack-lightspeed-operator control-plane: controller-manager strategy: {} template: @@ -326,6 +327,7 @@ spec: annotations: kubectl.kubernetes.io/default-container: manager labels: + app.kubernetes.io/name: openstack-lightspeed-operator control-plane: controller-manager spec: containers: @@ -333,6 +335,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs command: - /manager env: @@ -383,17 +386,25 @@ spec: drop: - ALL volumeMounts: - - mountPath: /tmp/k8s-metrics-server/serving-certs - name: cert + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs readOnly: true securityContext: runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: openstack-lightspeed-operator-controller-manager terminationGracePeriodSeconds: 10 volumes: - - name: cert + - name: metrics-certs secret: - secretName: operator-metrics-tls + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false + secretName: metrics-server-cert permissions: - rules: - apiGroups: diff --git a/bundle/metadata/annotations.yaml b/bundle/metadata/annotations.yaml index 828f4f1..17ff894 100644 --- a/bundle/metadata/annotations.yaml +++ b/bundle/metadata/annotations.yaml @@ -5,7 +5,7 @@ annotations: operators.operatorframework.io.bundle.metadata.v1: metadata/ operators.operatorframework.io.bundle.package.v1: openstack-lightspeed-operator operators.operatorframework.io.bundle.channels.v1: alpha - operators.operatorframework.io.metrics.builder: operator-sdk-v1.38.0 + operators.operatorframework.io.metrics.builder: operator-sdk-v1.42.3 operators.operatorframework.io.metrics.mediatype.v1: metrics+v1 operators.operatorframework.io.metrics.project_layout: go.kubebuilder.io/v4 diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml index ea6b807..b2761e6 100644 --- a/bundle/tests/scorecard/config.yaml +++ b/bundle/tests/scorecard/config.yaml @@ -8,7 +8,7 @@ stages: - entrypoint: - scorecard-test - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: basic test: basic-check-spec-test @@ -18,7 +18,7 @@ stages: - entrypoint: - scorecard-test - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-bundle-validation-test @@ -28,7 +28,7 @@ stages: - entrypoint: - scorecard-test - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-crds-have-validation-test @@ -38,7 +38,7 @@ stages: - entrypoint: - scorecard-test - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-crds-have-resources-test @@ -48,7 +48,7 @@ stages: - entrypoint: - scorecard-test - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-spec-descriptors-test @@ -58,7 +58,7 @@ stages: - entrypoint: - scorecard-test - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-status-descriptors-test diff --git a/cmd/main.go b/cmd/main.go index e059532..63ea237 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,5 +1,5 @@ /* -Copyright 2025. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "strings" "sync/atomic" @@ -37,6 +38,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -47,7 +49,7 @@ import ( openshiftv1 "github.com/openshift/api/operator/v1" operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" - apiv1beta1 "github.com/openstack-k8s-operators/lightspeed-operator/api/v1beta1" + lightspeedv1beta1 "github.com/openstack-k8s-operators/lightspeed-operator/api/v1beta1" "github.com/openstack-k8s-operators/lightspeed-operator/internal/controller" // +kubebuilder:scaffold:imports ) @@ -62,7 +64,7 @@ func init() { utilruntime.Must(operatorsv1alpha1.AddToScheme(scheme)) - utilruntime.Must(apiv1beta1.AddToScheme(scheme)) + utilruntime.Must(lightspeedv1beta1.AddToScheme(scheme)) utilruntime.Must(consolev1.AddToScheme(scheme)) @@ -72,15 +74,15 @@ func init() { // +kubebuilder:scaffold:scheme } +// nolint:gocyclo func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool var enableHTTP2 bool - var certDir string - var certName string - var keyName string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -90,14 +92,15 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") - flag.StringVar(&certDir, "cert-dir", "/tmp/k8s-metrics-server/serving-certs", - "The directory where the TLS certificates are stored.") - flag.StringVar(&certName, "cert-name", "tls.crt", - "The name of the TLS certificate file.") - flag.StringVar(&keyName, "key-name", "tls.key", - "The name of the TLS key file.") opts := zap.Options{ // Development: false uses JSON encoding and omits stack traces / DPanic behaviour. // For local development use: make run ARGS="--zap-devel" @@ -114,40 +117,94 @@ func main() { // Rapid Reset CVEs. For more information see: // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + tlsOpts = append(tlsOpts, func(c *tls.Config) { setupLog.Info("enforcing minimum TLS version 1.3") c.MinVersion = tls.VersionTLS13 - if !enableHTTP2 { - setupLog.Info("disabling http/2") - c.NextProtos = []string{"http/1.1"} - } }) + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + // Create watchers for metrics and webhooks certificates + var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + + if len(webhookCertPath) > 0 { + setupLog.Info("Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) + + var err error + webhookCertWatcher, err = certwatcher.New( + filepath.Join(webhookCertPath, webhookCertName), + filepath.Join(webhookCertPath, webhookCertKey), + ) + if err != nil { + setupLog.Error(err, "Failed to initialize webhook certificate watcher") + os.Exit(1) + } + + webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { + config.GetCertificate = webhookCertWatcher.GetCertificate + }) + } + webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: tlsOpts, + TLSOpts: webhookTLSOpts, }) // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: - // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html metricsServerOptions := metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - CertDir: certDir, - CertName: certName, - KeyName: keyName, } if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: - // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization } + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) + } + watchNamespaces, err := getWatchNamespaces() if err != nil { setupLog.Error(err, "unable to get WatchNamespace, "+ @@ -197,7 +254,7 @@ func main() { } // Defaults for OpenStackLightspeed - apiv1beta1.SetupDefaults() + lightspeedv1beta1.SetupDefaults() dynamicWatchCRDs := getDynamicWatchCRDs() @@ -213,6 +270,22 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + + if webhookCertWatcher != nil { + setupLog.Info("Adding webhook certificate watcher to manager") + if err := mgr.Add(webhookCertWatcher); err != nil { + setupLog.Error(err, "unable to add webhook certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml b/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml index 59837dc..5e1e59e 100644 --- a/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml +++ b/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.18.0 name: openstacklightspeeds.lightspeed.openstack.org spec: group: lightspeed.openstack.org diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 47705b8..847d6e2 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -10,13 +10,7 @@ patches: # patches here are for enabling the conversion webhook for each CRD # +kubebuilder:scaffold:crdkustomizewebhookpatch -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -#- path: patches/cainjection_in_openstacklightspeeds.yaml -# +kubebuilder:scaffold:crdkustomizecainjectionpatch - # [WEBHOOK] To enable webhook, uncomment the following section # the following config is for teaching kustomize how to do kustomization for CRDs. - #configurations: #- kustomizeconfig.yaml diff --git a/config/default/cert_metrics_manager_patch.yaml b/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 0000000..158ed62 --- /dev/null +++ b/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,32 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +# NOTE(lpiwowar): The secret is generated by the OpenShift service-ca operator +# (see the annotation on the metrics Service). service-ca serving-cert secrets +# contain only tls.crt and tls.key -- there is no ca.crt key -- so we must not +# request a ca.crt item here (optional: false would fail the mount otherwise). +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index d331a90..de271c1 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -27,8 +27,13 @@ resources: #- ../prometheus # [METRICS] Expose the controller manager metrics service. - metrics_service.yaml +# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. +# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. +# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will +# be able to communicate with the Webhook Server. +#- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -36,111 +41,196 @@ patches: target: kind: Deployment +# [METRICS-WITH-CERTS] Protect the metrics endpoint with a TLS serving cert. +# The cert is provided by the OpenShift service-ca operator (see the +# service.beta.openshift.io/serving-cert-secret-name annotation in +# metrics_service.yaml), NOT cert-manager -- so the cert-manager `replacements:` +# block below stays commented. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- path: manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -#- path: webhookcainjection_patch.yaml +# target: +# kind: Deployment # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: -# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: # Add cert-manager annotation to the webhook Service -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have any webhook +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # Name of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # Namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionns +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml index fe32bde..896e8bf 100644 --- a/config/default/metrics_service.yaml +++ b/config/default/metrics_service.yaml @@ -6,8 +6,16 @@ metadata: app.kubernetes.io/name: openstack-lightspeed-operator app.kubernetes.io/managed-by: kustomize annotations: - service.beta.openshift.io/serving-cert-secret-name: operator-metrics-tls - name: metrics + service.beta.openshift.io/serving-cert-secret-name: metrics-server-cert + + # NOTE(lpiwowar): Do NOT restore the operator-sdk scaffold default name + # "controller-manager-metrics-service". Service names must be a valid RFC 1035 + # DNS label (<= 63 chars), and kustomize prepends the namePrefix + # "openstack-lightspeed-operator-" (30 chars). The scaffold default (34 chars) + # would total 64 chars and be rejected on apply, blocking the deploy. Keeping + # this shortened to "controller-manager-metrics" (26 chars) stays under the + # limit (56 chars total). Re-check this after any operator-sdk regeneration. + name: controller-manager-metrics namespace: system spec: ports: @@ -17,3 +25,4 @@ spec: targetPort: 8443 selector: control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index ebefbe3..5ff6ba4 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -6,9 +6,3 @@ images: - name: controller newName: quay.io/openstack-lightspeed/operator newTag: latest -patches: -- patch: '[{"op": "replace", "path": "/spec/template/spec/containers/0/env/0/value", - "value": "latest"}]' - target: - kind: Deployment - name: controller-manager diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e210e0b..26ff7c3 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -20,6 +20,7 @@ spec: selector: matchLabels: control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator replicas: 1 template: metadata: @@ -27,6 +28,7 @@ spec: kubectl.kubernetes.io/default-container: manager labels: control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator spec: # TODO(user): Uncomment the following code to configure the nodeAffinity expression # according to the platforms which are supported by your solution. @@ -49,14 +51,12 @@ spec: # values: # - linux securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted runAsNonRoot: true - # TODO(user): For common cases that do not require escalating privileges - # it is recommended to ensure that all your Pods/Containers are restrictive. - # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). - # seccompProfile: - # type: RuntimeDefault + seccompProfile: + type: RuntimeDefault containers: - command: - /manager @@ -65,6 +65,7 @@ spec: - --health-probe-bind-address=:8081 image: controller:latest name: manager + ports: [] env: - name: WATCH_NAMESPACE valueFrom: @@ -115,13 +116,7 @@ spec: requests: cpu: 10m memory: 128Mi - volumeMounts: - - name: cert - mountPath: /tmp/k8s-metrics-server/serving-certs - readOnly: true + volumeMounts: [] + volumes: [] serviceAccountName: controller-manager - volumes: - - name: cert - secret: - secretName: operator-metrics-tls terminationGracePeriodSeconds: 10 diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 0000000..f08ec9e --- /dev/null +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: openstack-lightspeed-operator + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml new file mode 100644 index 0000000..ec0fb5e --- /dev/null +++ b/config/network-policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- allow-metrics-traffic.yaml diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed13716..fdc5481 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index a02a048..ebf80cc 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -16,15 +16,12 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager + app.kubernetes.io/name: openstack-lightspeed-operator diff --git a/config/prometheus/monitor_tls_patch.yaml b/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 0000000..5bf84ce --- /dev/null +++ b/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,19 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +- op: replace + path: /spec/endpoints/0/tlsConfig + value: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 4c75b00..120be39 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -19,10 +19,11 @@ resources: - metrics_auth_role.yaml - metrics_auth_role_binding.yaml - metrics_reader_role.yaml -# For each CRD, "Editor" and "Viewer" roles are scaffolded by +# For each CRD, "Admin", "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are -# not used by the Project itself. You can comment the following lines +# not used by the openstack-lightspeed-operator itself. You can comment the following lines # if you do not want those helpers be installed with your Project. +- openstacklightspeed_admin_role.yaml - openstacklightspeed_editor_role.yaml - openstacklightspeed_viewer_role.yaml diff --git a/config/rbac/openstacklightspeed_admin_role.yaml b/config/rbac/openstacklightspeed_admin_role.yaml new file mode 100644 index 0000000..33abafd --- /dev/null +++ b/config/rbac/openstacklightspeed_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project openstack-lightspeed-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over lightspeed.openstack.org. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: openstack-lightspeed-operator + app.kubernetes.io/managed-by: kustomize + name: openstacklightspeed-admin-role +rules: +- apiGroups: + - lightspeed.openstack.org + resources: + - openstacklightspeeds + verbs: + - '*' +- apiGroups: + - lightspeed.openstack.org + resources: + - openstacklightspeeds/status + verbs: + - get diff --git a/config/rbac/openstacklightspeed_editor_role.yaml b/config/rbac/openstacklightspeed_editor_role.yaml index eb54717..3ad4c61 100644 --- a/config/rbac/openstacklightspeed_editor_role.yaml +++ b/config/rbac/openstacklightspeed_editor_role.yaml @@ -1,4 +1,10 @@ -# permissions for end users to edit openstacklightspeeds. +# This rule is not used by the project openstack-lightspeed-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the lightspeed.openstack.org. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/config/rbac/openstacklightspeed_viewer_role.yaml b/config/rbac/openstacklightspeed_viewer_role.yaml index 558b1df..44dbb3e 100644 --- a/config/rbac/openstacklightspeed_viewer_role.yaml +++ b/config/rbac/openstacklightspeed_viewer_role.yaml @@ -1,4 +1,10 @@ -# permissions for end users to view openstacklightspeeds. +# This rule is not used by the project openstack-lightspeed-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to lightspeed.openstack.org resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index bc7a348..103692a 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,4 +1,4 @@ ## Append samples of your project ## resources: -- api_v1beta1_openstacklightspeed.yaml +- lightspeed_v1beta1_openstacklightspeed.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/api_v1beta1_openstacklightspeed.yaml b/config/samples/lightspeed_v1beta1_openstacklightspeed.yaml similarity index 100% rename from config/samples/api_v1beta1_openstacklightspeed.yaml rename to config/samples/lightspeed_v1beta1_openstacklightspeed.yaml diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml index 84683cf..e2afe91 100644 --- a/config/scorecard/patches/basic.config.yaml +++ b/config/scorecard/patches/basic.config.yaml @@ -4,7 +4,7 @@ entrypoint: - scorecard-test - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: basic test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml index 43f40a8..687c864 100644 --- a/config/scorecard/patches/olm.config.yaml +++ b/config/scorecard/patches/olm.config.yaml @@ -4,7 +4,7 @@ entrypoint: - scorecard-test - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-bundle-validation-test @@ -14,7 +14,7 @@ entrypoint: - scorecard-test - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-crds-have-validation-test @@ -24,7 +24,7 @@ entrypoint: - scorecard-test - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-crds-have-resources-test @@ -34,7 +34,7 @@ entrypoint: - scorecard-test - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-spec-descriptors-test @@ -44,7 +44,7 @@ entrypoint: - scorecard-test - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.38.0 + image: quay.io/operator-framework/scorecard-test:v1.42.3 labels: suite: olm test: olm-status-descriptors-test diff --git a/internal/controller/common.go b/internal/controller/common.go index 4d45c07..032de12 100644 --- a/internal/controller/common.go +++ b/internal/controller/common.go @@ -157,29 +157,29 @@ func generateOKPSelectorLabels() map[string]string { // parseDevConfig unmarshals the Dev RawExtension into a DevSpec. // Returns a zero-value DevSpec and an error on malformed input. func parseDevConfig(instance *apiv1beta1.OpenStackLightspeed) (apiv1beta1.DevSpec, error) { - var config apiv1beta1.DevSpec + var devConfig apiv1beta1.DevSpec if len(instance.Spec.Dev.Raw) > 0 { - if err := json.Unmarshal(instance.Spec.Dev.Raw, &config); err != nil { - return config, err + if err := json.Unmarshal(instance.Spec.Dev.Raw, &devConfig); err != nil { + return devConfig, err } } - return config, nil + return devConfig, nil } // isRHOSOMCPEnabled returns true if the "rhoso_mcps" feature flag is present in the dev config. func isRHOSOMCPEnabled(instance *apiv1beta1.OpenStackLightspeed) (bool, error) { - config, err := parseDevConfig(instance) + devConfig, err := parseDevConfig(instance) if err != nil { return false, err } - return slices.Contains(config.FeatureFlags, "rhoso_mcps"), nil + return slices.Contains(devConfig.FeatureFlags, "rhoso_mcps"), nil } // getOKPChunkFilterQuery returns the chunk filter query from the dev config, or a version-aware default. func getOKPChunkFilterQuery(ctx context.Context, h *common_helper.Helper, instance *apiv1beta1.OpenStackLightspeed) string { - config, _ := parseDevConfig(instance) - if config.OKPChunkFilterQuery != "" { - return config.OKPChunkFilterQuery + devConfig, _ := parseDevConfig(instance) + if devConfig.OKPChunkFilterQuery != "" { + return devConfig.OKPChunkFilterQuery } logger := h.GetLogger() diff --git a/internal/controller/lcore_config.go b/internal/controller/lcore_config.go index 951c923..e4fd6ab 100644 --- a/internal/controller/lcore_config.go +++ b/internal/controller/lcore_config.go @@ -133,6 +133,7 @@ func buildLCoreInferenceConfig(_ *common_helper.Helper, instance *apiv1beta1.Ope // buildLCoreDatabaseConfig configures persistent database storage (PostgreSQL) func buildLCoreDatabaseConfig(h *common_helper.Helper, _ *apiv1beta1.OpenStackLightspeed) map[string]interface{} { return map[string]interface{}{ + // #nosec G101 -- values are env-var substitution placeholders, not hardcoded credentials "postgres": map[string]interface{}{ "host": PostgresServiceName + "." + h.GetBeforeObject().GetNamespace() + ".svc", "port": PostgresServicePort, @@ -165,6 +166,7 @@ func buildLCoreCustomizationConfig() map[string]interface{} { func buildLCoreConversationCacheConfig(h *common_helper.Helper, _ *apiv1beta1.OpenStackLightspeed) map[string]interface{} { return map[string]interface{}{ "type": "postgres", + // #nosec G101 -- values are env-var substitution placeholders, not hardcoded credentials "postgres": map[string]interface{}{ "host": PostgresServiceName + "." + h.GetBeforeObject().GetNamespace() + ".svc", "port": PostgresServicePort, diff --git a/internal/controller/llama_stack_config.go b/internal/controller/llama_stack_config.go index e5339c3..2321ec6 100644 --- a/internal/controller/llama_stack_config.go +++ b/internal/controller/llama_stack_config.go @@ -317,6 +317,7 @@ func buildLlamaStackStorage(_ *common_helper.Helper, instance *apiv1beta1.OpenSt "type": "kv_sqlite", "db_path": "/tmp/llama-stack/kv_store.db", }, + // #nosec G101 -- values are env-var substitution placeholders, not hardcoded credentials "postgres_backend": map[string]interface{}{ "type": "sql_postgres", "host": fmt.Sprintf("lightspeed-postgres-server.%s.svc", instance.GetNamespace()), diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 3079c7a..30c95d6 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2025. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,15 +18,72 @@ package e2e import ( "fmt" + "os" + "os/exec" "testing" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openstack-k8s-operators/lightspeed-operator/test/utils" +) + +var ( + // Optional Environment Variables: + // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. + // These variables are useful if CertManager is already installed, avoiding + // re-installation and conflicts. + skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" + // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster + isCertManagerAlreadyInstalled = false + + // projectImage is the name of the image which will be build and loaded + // with the code source changes to be tested. + projectImage = "example.com/lightspeed-operator:v0.0.1" ) -// Run e2e tests using the Ginkgo runner. +// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, +// temporary environment to validate project changes with the purposed to be used in CI jobs. +// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs +// CertManager. func TestE2E(t *testing.T) { - gomega.RegisterFailHandler(ginkgo.Fail) - _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "Starting openstack-lightspeed-operator suite\n") - ginkgo.RunSpecs(t, "e2e suite") + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting lightspeed-operator integration test suite\n") + RunSpecs(t, "e2e suite") } + +var _ = BeforeSuite(func() { + By("building the manager(Operator) image") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage)) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") + + // TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is + // built and available before running the tests. Also, remove the following block. + By("loading the manager(Operator) image on Kind") + err = utils.LoadImageToKindClusterWithName(projectImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") + + // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing. + // To prevent errors when tests run in environments with CertManager already installed, + // we check for its presence before execution. + // Setup CertManager before the suite if not skipped and if not already installed + if !skipCertManagerInstall { + By("checking if cert manager is installed already") + isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled() + if !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") + Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") + } + } +}) + +var _ = AfterSuite(func() { + // Teardown CertManager after the suite if not skipped and if it was not already installed + if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") + utils.UninstallCertManager() + } +}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d3f1b31..19695db 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,5 +1,5 @@ /* -Copyright 2025. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,75 +17,132 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/openstack-k8s-operators/lightspeed-operator/test/utils" ) -const namespace = "openstack-lightspeed-operator-system" +// namespace where the project is deployed in +const namespace = "lightspeed-operator-system" -var _ = ginkgo.Describe("controller", ginkgo.Ordered, func() { - ginkgo.BeforeAll(func() { - ginkgo.By("installing prometheus operator") - gomega.Expect(utils.InstallPrometheusOperator()).To(gomega.Succeed()) +// serviceAccountName created for the project +const serviceAccountName = "lightspeed-operator-controller-manager" - ginkgo.By("installing the cert-manager") - gomega.Expect(utils.InstallCertManager()).To(gomega.Succeed()) +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "openstack-lightspeed-operator-controller-manager-metrics" - ginkgo.By("creating manager namespace") +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "lightspeed-operator-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func() { + By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - }) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") - ginkgo.AfterAll(func() { - ginkgo.By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") - ginkgo.By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") - ginkgo.By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) - ginkgo.Context("Operator", func() { - ginkgo.It("should run successfully", func() { - var controllerPodName string - var err error + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. + AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) - // projectimage stores the name of the image used in the example - var projectimage = "example.com/openstack-lightspeed-operator:v0.0.1" + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) - ginkgo.By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + }) - ginkgo.By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } - ginkgo.By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } - ginkgo.By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } - ginkgo.By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { - // Get pod name + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) - cmd = exec.Command("kubectl", "get", + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("Manager", func() { + It("should run successfully", func() { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ @@ -95,28 +152,180 @@ var _ = ginkgo.Describe("controller", ginkgo.Ordered, func() { ) podOutput, err := utils.Run(cmd) - gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) - podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) - } + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - gomega.ExpectWithOffset(2, controllerPodName).Should(gomega.ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) - // Validate pod status + // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=lightspeed-operator-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": ["curl -i -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccount": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + // Run directly instead of utils.Run: the --overrides payload embeds the + // bearer token, and utils.Run logs every command's args to GinkgoWriter. + _, err = cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") } - gomega.EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(gomega.Succeed()) + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/test/utils/utils.go b/test/utils/utils.go index 8e7f831..448055b 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2025. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,57 +14,58 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package utils contains utility function for testing package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" "strings" - "github.com/onsi/ginkgo/v2" //nolint:staticcheck + . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck ) const ( - prometheusOperatorVersion = "v0.72.0" + prometheusOperatorVersion = "v0.77.1" prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + "releases/download/%s/bundle.yaml" - certmanagerVersion = "v1.14.4" - certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" + certmanagerVersion = "v1.16.3" + certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" ) func warnError(err error) { - _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "warning: %v\n", err) -} - -// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) - return err + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) } // Run executes the provided command within this context -func Run(cmd *exec.Cmd) ([]byte, error) { +func Run(cmd *exec.Cmd) (string, error) { dir, _ := GetProjectDir() cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { - _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "chdir dir: %s\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) } cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") - _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s\n", command) + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) output, err := cmd.CombinedOutput() if err != nil { - return output, fmt.Errorf("%s failed with error: (%w) %s", command, err, string(output)) + return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) } - return output, nil + return string(output), nil +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator() error { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "create", "-f", url) + _, err := Run(cmd) + return err } // UninstallPrometheusOperator uninstalls the prometheus @@ -76,6 +77,33 @@ func UninstallPrometheusOperator() { } } +// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed +// by verifying the existence of key CRDs related to Prometheus. +func IsPrometheusCRDsInstalled() bool { + // List of common Prometheus CRDs + prometheusCRDs := []string{ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "prometheusagents.monitoring.coreos.com", + } + + cmd := exec.Command("kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") + output, err := Run(cmd) + if err != nil { + return false + } + crdList := GetNonEmptyLines(output) + for _, crd := range prometheusCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + // UninstallCertManager uninstalls the cert manager func UninstallCertManager() { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) @@ -104,6 +132,39 @@ func InstallCertManager() error { return err } +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + // LoadImageToKindClusterWithName loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error { cluster := "kind" @@ -134,8 +195,60 @@ func GetNonEmptyLines(output string) []string { func GetProjectDir() (string, error) { wd, err := os.Getwd() if err != nil { - return wd, err + return wd, fmt.Errorf("failed to get current working directory: %w", err) } wd = strings.ReplaceAll(wd, "/test/e2e", "") return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read file %q: %w", filename, err) + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %q to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + } + + if _, err = out.Write(content[idx+len(target):]); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + // false positive + // nolint:gosec + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write file %q: %w", filename, err) + } + + return nil +}