[DNM] Debug KUTTL tests (0.7.0) migration

Makefile

@@ -277,6 +277,7 @@ ENVTEST_VERSION ?= release-0.22
 GOLANGCI_LINT_VERSION ?= v2.12.2
 KUTTL_VERSION ?= 0.22.0
 GOVULNCHECK_VERSION ?= v1.6.0
+GO_VERSION := $(shell go version | cut -d' ' -f3)
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -308,7 +309,12 @@ $(KUTTL): $(LOCALBIN)
 kuttl-test: kuttl ## Run kuttl tests
 	@command -v diff >/dev/null 2>&1 || { echo "ERROR: 'diff' command is required for KUTTL tests but not found in PATH" >&2; exit 1; }
 	@command -v oc >/dev/null 2>&1 || { echo "ERROR: 'oc' command is required for KUTTL tests but not found in PATH" >&2; exit 1; }
-	$(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests $(KUTTL_ARGS)
+	$(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests --test application-credentials $(KUTTL_ARGS)
+	$(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests --test basic-openstack-lightspeed-configuration $(KUTTL_ARGS)
+	$(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests --test dynamic-crd-watch-recovery $(KUTTL_ARGS)
+	$(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests --test persistent-database $(KUTTL_ARGS)
+	$(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests --test rhoso-mcps-configuration $(KUTTL_ARGS)
+	$(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests --test update-openstacklightspeed $(KUTTL_ARGS)
 
 .PHONY: govulncheck-install
 govulncheck-install: $(LOCALBIN) ## Download govulncheck locally if necessary.
@@ -350,15 +356,15 @@ ocp-deploy-cleanup: openstack-lightspeed-undeploy ## Clean up everything created
 # $2 - package url which can be installed
 # $3 - specific version of package
 define go-install-tool
-@[ -f "$(1)-$(3)" ] || { \
+@[ -f "$(1)-$(3)-$(GO_VERSION)" ] || { \
 set -e; \
 package=$(2)@$(3) ;\
 echo "Downloading $${package}" ;\
-rm -f $(1) || true ;\
-GOBIN=$(LOCALBIN) go install $${package} ;\
-mv $(1) $(1)-$(3) ;\
+rm -f $(1) $(1)-$(3)-* || true ;\
+GOBIN=$(LOCALBIN) GOTOOLCHAIN=$(GO_VERSION) go install $${package} ;\
+mv $(1) $(1)-$(3)-$(GO_VERSION) ;\
 } ;\
-ln -sf $(1)-$(3) $(1)
+ln -sf $(1)-$(3)-$(GO_VERSION) $(1)
 endef
 
 .PHONY: operator-sdk

api/v1beta1/openstacklightspeed_types.go

@@ -30,7 +30,7 @@ const (
 	OpenStackLightspeedContainerImage = "quay.io/openstack-lightspeed/rag-content:os-docs-2026.1-ogx"
 
 	// LCoreContainerImage is the fall-back container image for LCore
-	LCoreContainerImage = "quay.io/lightspeed-core/lightspeed-stack:latest"
+	LCoreContainerImage = "quay.io/lightspeed-core/lightspeed-stack:dev-latest"
 
 	// ExporterContainerImage is the fall-back container image for the Dataverse Exporter
 	ExporterContainerImage = "quay.io/lightspeed-core/lightspeed-to-dataverse-exporter:latest"

bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml

@@ -343,7 +343,7 @@ spec:
                 - name: RELATED_IMAGE_OPENSTACK_LIGHTSPEED_IMAGE_URL_DEFAULT
                   value: quay.io/openstack-lightspeed/rag-content:os-docs-2026.1-ogx
                 - name: RELATED_IMAGE_LCORE_IMAGE_URL_DEFAULT
-                  value: quay.io/lightspeed-core/lightspeed-stack:latest
+                  value: quay.io/lightspeed-core/lightspeed-stack:dev-latest
                 - name: RELATED_IMAGE_EXPORTER_IMAGE_URL_DEFAULT
                   value: quay.io/lightspeed-core/lightspeed-to-dataverse-exporter:latest
                 - name: RELATED_IMAGE_POSTGRES_IMAGE_URL_DEFAULT
@@ -516,7 +516,7 @@ spec:
   relatedImages:
   - image: quay.io/openstack-lightspeed/rag-content:os-docs-2026.1-ogx
     name: openstack-lightspeed-image-url-default
-  - image: quay.io/lightspeed-core/lightspeed-stack:latest
+  - image: quay.io/lightspeed-core/lightspeed-stack:dev-latest
     name: lcore-image-url-default
   - image: quay.io/lightspeed-core/lightspeed-to-dataverse-exporter:latest
     name: exporter-image-url-default

config/manager/manager.yaml

@@ -76,7 +76,7 @@ spec:
           - name: RELATED_IMAGE_OPENSTACK_LIGHTSPEED_IMAGE_URL_DEFAULT
             value: quay.io/openstack-lightspeed/rag-content:os-docs-2026.1-ogx
           - name: RELATED_IMAGE_LCORE_IMAGE_URL_DEFAULT
-            value: quay.io/lightspeed-core/lightspeed-stack:latest
+            value: quay.io/lightspeed-core/lightspeed-stack:dev-latest
           - name: RELATED_IMAGE_EXPORTER_IMAGE_URL_DEFAULT
             value: quay.io/lightspeed-core/lightspeed-to-dataverse-exporter:latest
           - name: RELATED_IMAGE_POSTGRES_IMAGE_URL_DEFAULT

hack/env.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-export RELATED_IMAGE_LCORE_IMAGE_URL_DEFAULT="quay.io/lightspeed-core/lightspeed-stack:latest"
+export RELATED_IMAGE_LCORE_IMAGE_URL_DEFAULT="quay.io/lightspeed-core/lightspeed-stack:dev-latest"
 export RELATED_IMAGE_EXPORTER_IMAGE_URL_DEFAULT="quay.io/lightspeed-core/lightspeed-to-dataverse-exporter:latest"
 export RELATED_IMAGE_POSTGRES_IMAGE_URL_DEFAULT="registry.redhat.io/rhel9/postgresql-16:latest"
 # TODO(lpiwowar): Replace this with a stable (non-alpha) image version once

internal/controller/assets/postgres_bootstrap.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
-# This script prepares databases for lightspeed-stack and llama-stack (OGX) with
+# This script prepares databases for lightspeed-stack and OGX (old llama-stack) with
 # postgres_bootstrap.sql.
 #
 # Note:
 # - lightspeed-stack database: Auto-created by container image via POSTGRESQL_DATABASE.
-# - llama-stack database: Explicitly created by this script via POSTGRESQL_LLAMA_STACK_DATABASE.
+# - ogx database: Explicitly created by this script via POSTGRESQL_LLAMA_STACK_DATABASE.
 # - POSTGRESQL_ADMIN_PASSWORD is intentionally not set. The postgres superuser has no password
 #   by default, which restricts it to local connections only — a deliberate security improvement.
 #   Setting POSTGRESQL_ADMIN_PASSWORD would enable remote login for the postgres account.

internal/controller/assets/vector_database_build.py

@@ -185,7 +185,13 @@ def ogx_process(ogx_config_source_path: Path, ogx_config_target: dict[str, Any])
     add_unique(tgt_models, src_model, "model_id")
 
     # Populate registered_resources.vector_stores
-    embedding_model = f"{src_model['provider_id']}/{embedding_model_dir}"
+    # OGX 1.0.2+ uses provider_id/model_id as the model identifier
+    model_id = src_model["model_id"]
+    provider_id = src_model["provider_id"]
+    if model_id.startswith(f"{provider_id}/"):
+        embedding_model = model_id
+    else:
+        embedding_model = f"{provider_id}/{model_id}"
     src_vstore = ogx_config_source["registered_resources"]["vector_stores"][0].copy()
     src_vstore["embedding_model"] = embedding_model
     tgt_vstores = ogx_config_target["registered_resources"]["vector_stores"]

internal/controller/constants.go

@@ -62,7 +62,7 @@ const (
 	PostgresNetworkPolicyName                    = "lightspeed-postgres-server"
 	PostgresServicePort                          = int32(5432)
 	PostgresLightspeedStackDbName                = "lightspeed-stack"
-	PostgresLlamaStackDbName                     = "llamastack"
+	PostgresLlamaStackDbName                     = "ogx"
 	PostgresSharedBuffers                        = "256MB"
 	PostgresMaxConnections                       = 100
 	OpenStackLightspeedComponentPasswordFileName = "password"
@@ -81,8 +81,8 @@ const (
 	PostgresVarRunVolumeMountPath                = "/var/run/postgresql"
 
 	// PostgresSQLUsername is non-admin user that should be used by lightspeed-stack and
-	// llama-stack (OGX) to access the PostgreSQL database. This user gets created by the
-	// PostgreSQL container by setting the POSTGRESQL_USER and POSTGRESQL_PASSWORD environment
+	// (OGX) to access the PostgreSQL database. This user gets created by the PostgreSQL
+	// container by setting the POSTGRESQL_USER and POSTGRESQL_PASSWORD environment
 	// variable.
 	PostgresSQLUsername = "lightspeed-app-user"
 
@@ -113,7 +113,7 @@ const (
 	// -- LCore specific ---------------------------------------------------------
 
 	LlamaStackContainerPort  = int32(8321)
-	LlamaStackConfigCmName   = "llama-stack-config"
+	LlamaStackConfigCmName   = "ogx-config"
 	LCoreConfigCmName        = "lightspeed-stack-config"
 	LCoreDeploymentName      = "lightspeed-stack-deployment"
 	LCoreConfigMountPath     = "/app-root/lightspeed-stack.yaml"
@@ -122,7 +122,7 @@ const (
 
 	// ---------------------------------------------------------------------------
 
-	// -- Health probe settings for the llama-stack/OGX container. ---------------
+	// -- Health probe settings for the stack/OGX container. ---------------------
 
 	// The startup probe allows up to 30 failures (300s) for the slow initialization,
 	// while liveness and readiness probes use a tighter threshold of 3 failures.
@@ -314,10 +314,6 @@ const (
 	// script is stored in the ConfigMap containing vector database init scripts.
 	VectorDBBuildScriptKey = "vector_database_build.py"
 
-	// LlamaStartupWrapperKey is the ConfigMap key for the startup wrapper script
-	// that monkey-patches the asyncpg event loop bug fix. Remove with PR #5837 backport.
-	LlamaStartupWrapperKey = "llama_startup_wrapper.py" // #nosec G101 -- ConfigMap key, not a credential
-
 	// -- Resource Version Annotation --------------------------------------------
 
 	// These constants define annotation keys used to track the resource versions of specific ConfigMaps.
@@ -449,13 +445,6 @@ var vectorDatabaseCollectScript string
 //go:embed assets/vector_database_build.py
 var vectorDatabaseBuildScript string
 
-// llamaStartupWrapperScript is a Python monkey-patch that fixes the asyncpg
-// event loop bug (ogx-ai/ogx#5978) by resetting SQL engines after StackApp
-// initialization. Remove when the container image includes upstream PR #5837.
-//
-//go:embed assets/llama_startup_wrapper.py
-var llamaStartupWrapperScript string
-
 //go:embed assets/console_nginx.conf.tmpl
 var consoleNginxConfigTemplate string
 

internal/controller/lcore_config.go

@@ -273,8 +273,9 @@ func buildLCoreConfigYAML(ctx context.Context, h *common_helper.Helper, instance
 
 	// Build the complete config as a map
 	config := map[string]interface{}{
-		"name":                 "Lightspeed Core Service (LCS)",
-		"service":              buildLCoreServiceConfig(h, instance),
+		"name":    "Lightspeed Core Service (LCS)",
+		"service": buildLCoreServiceConfig(h, instance),
+		// This has not been renamed in the LCORE config to "ogx"
 		"llama_stack":          buildLCoreLlamaStackConfig(),
 		"user_data_collection": buildLCoreUserDataCollectionConfig(h, instance),
 		"authentication":       buildLCoreAuthenticationConfig(h, instance),

internal/controller/lcore_deployment.go

@@ -55,25 +55,20 @@ func buildLCorePodTemplateSpec(ctx context.Context, h *common_helper.Helper, ins
 	// Build env vars
 	llamaEnvVars, err := buildLlamaStackEnvVars(ctx, h, instance)
 	if err != nil {
-		return corev1.PodTemplateSpec{}, fmt.Errorf("failed to build llama-stack env vars: %w", err)
+		return corev1.PodTemplateSpec{}, fmt.Errorf("failed to build ogx env vars: %w", err)
 	}
 	lsEnvVars := buildLightspeedStackEnvVars(instance)
 
 	// Llama Stack container mounts: its config + shared + cache + vector_store_db data
 	llamaStackMounts := []corev1.VolumeMount{}
 	llamaStackMounts = append(llamaStackMounts, sharedMounts...)
 	llamaStackMounts = append(llamaStackMounts, llamaCacheMounts...)
-	llamaStackMounts = append(llamaStackMounts, corev1.VolumeMount{
-		Name:      VectorDBScriptsVolumeName,
-		MountPath: VectorDBScriptsMountPath,
-		ReadOnly:  true,
-	})
 
 	llamaStackContainer := corev1.Container{
-		Name:         "llama-stack",
+		Name:         "ogx",
 		Image:        apiv1beta1.OpenStackLightspeedDefaultValues.LCoreImageURL,
-		Command:      []string{"python3", VectorDBScriptsMountPath + "/" + LlamaStartupWrapperKey, "stack", "run", VectorDBVolumeOGXConfigPath},
-		Ports:        []corev1.ContainerPort{{Name: "llama-stack", ContainerPort: LlamaStackContainerPort}},
+		Command:      []string{"ogx", "run", VectorDBVolumeOGXConfigPath},
+		Ports:        []corev1.ContainerPort{{Name: "ogx", ContainerPort: LlamaStackContainerPort}},
 		VolumeMounts: llamaStackMounts,
 		Env:          llamaEnvVars,
 		StartupProbe: &corev1.Probe{
@@ -413,17 +408,17 @@ func addTLSVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMou
 	})
 }
 
-// addLlamaCacheVolumesAndMounts adds an emptydir volume for llama-stack cache.
+// addLlamaCacheVolumesAndMounts adds an emptydir volume for ogx cache.
 func addLlamaCacheVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMount) {
 	*volumes = append(*volumes, corev1.Volume{
-		Name: "llama-cache",
+		Name: "ogx-cache",
 		VolumeSource: corev1.VolumeSource{
 			EmptyDir: &corev1.EmptyDirVolumeSource{},
 		},
 	})
 	*mounts = append(*mounts, corev1.VolumeMount{
-		Name:      "llama-cache",
-		MountPath: "/tmp/llama-stack",
+		Name:      "ogx-cache",
+		MountPath: "/tmp/ogx",
 	})
 }
 
@@ -526,7 +521,7 @@ func addCABundleVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.Volu
 	})
 }
 
-// buildLlamaStackEnvVars builds environment variables for llama-stack,
+// buildLlamaStackEnvVars builds environment variables for ogx
 // primarily provider API keys read from Kubernetes secrets.
 func buildLlamaStackEnvVars(ctx context.Context, h *common_helper.Helper, instance *apiv1beta1.OpenStackLightspeed) ([]corev1.EnvVar, error) {
 	envVars := []corev1.EnvVar{}
@@ -628,7 +623,7 @@ func buildLlamaStackEnvVars(ctx context.Context, h *common_helper.Helper, instan
 	}
 
 	// Postgres credentials for ${env.POSTGRESQL_PASSWORD} and ${env.POSTGRESQL_USER}
-	// substitution in llama-stack config
+	// substitution in ogx config
 	envVars = append(envVars, buildPostgresCredsEnvVars()...)
 
 	// PostgreSQL SSL configuration for OGX (llama-stack).
@@ -644,12 +639,8 @@ func buildLlamaStackEnvVars(ctx context.Context, h *common_helper.Helper, instan
 		Value: CABundleMountPath,
 	})
 
-	// Logging configuration - set both for compatibility with llama-stack and OGX
+	// Logging configuration - only for OGX, incompatible with llama-stack
 	ogxLogLevel := getOGXLogLevel(instance)
-	envVars = append(envVars, corev1.EnvVar{
-		Name:  "LLAMA_STACK_LOGGING",
-		Value: ogxLogLevel,
-	})
 	envVars = append(envVars, corev1.EnvVar{
 		Name:  "OGX_LOGGING",
 		Value: ogxLogLevel,
@@ -709,6 +700,10 @@ func buildLightspeedStackEnvVars(instance *apiv1beta1.OpenStackLightspeed) []cor
 		Name:  "RH_SERVER_OKP",
 		Value: fmt.Sprintf("http://%s.%s.svc:%d", OKPServiceName, instance.GetNamespace(), OKPServicePort),
 	})
+	envVars = append(envVars, corev1.EnvVar{
+		Name:  "OTEL_SDK_DISABLED",
+		Value: "true",
+	})
 	envVars = append(envVars, buildPostgresCredsEnvVars()...)
 	return envVars
 }
@@ -761,7 +756,7 @@ func buildLightspeedStackReadinessProbe() *corev1.Probe {
 	}
 }
 
-// getOGXLogLevel returns the log level for OGX/llama-stack container.
+// getOGXLogLevel returns the log level for OGX container.
 // Supports either standard levels (INFO, DEBUG, WARNING, ERROR, CRITICAL) or fine-grained control.
 // Examples: "INFO" -> "all=info", "DEBUG" -> "all=debug", "core=debug,providers=info" -> "core=debug,providers=info"
 // Defaults to "all=info" if not specified.

internal/controller/lcore_reconciler.go

@@ -297,7 +297,6 @@ func reconcileVectorDBScriptsConfigMap(ctx context.Context, h *common_helper.Hel
 		cm.Data = map[string]string{
 			VectorDBCollectScriptKey: vectorDatabaseCollectScript,
 			VectorDBBuildScriptKey:   vectorDatabaseBuildScript,
-			LlamaStartupWrapperKey:   llamaStartupWrapperScript,
 		}
 
 		return controllerutil.SetControllerReference(h.GetBeforeObject(), cm, h.GetScheme())