diff --git a/e2e/framework/runenv/dependencies.go b/e2e/framework/runenv/dependencies.go index 59e6d21488..849e80f98b 100644 --- a/e2e/framework/runenv/dependencies.go +++ b/e2e/framework/runenv/dependencies.go @@ -24,11 +24,12 @@ const ( // Command describes one subprocess. Env is an overlay applied to the parent // environment of that process only; nothing mutates the global environment. type Command struct { - Path string - Args []string - Env []string // NAME=value overlay; replaces same-named parent entries - Dir string - Timeout time.Duration + Path string + Args []string + Env []string // NAME=value overlay; replaces same-named parent entries + Dir string + Timeout time.Duration + CancelProcessGroup bool // Stdout and Stderr optionally receive output live, in addition to the // tail-capped capture returned in CommandResult. Stdout io.Writer diff --git a/e2e/framework/runenv/exec.go b/e2e/framework/runenv/exec.go index b8d9efbeb2..e2439f24ab 100644 --- a/e2e/framework/runenv/exec.go +++ b/e2e/framework/runenv/exec.go @@ -12,6 +12,8 @@ import ( "strings" "sync" "time" + + "github.com/devsy-org/devsy/pkg/docker" ) const ( @@ -39,6 +41,9 @@ func (ExecRunner) Run(ctx context.Context, c Command) (CommandResult, error) { stdout, stderr := newTailBuffer(maxCaptureBytes), newTailBuffer(maxCaptureBytes) //nolint:gosec // G204: ExecRunner runs the command its caller specified, by design cmd := exec.CommandContext(ctx, c.Path, c.Args...) + if c.CancelProcessGroup { + docker.PrepareForGroupCancellation(cmd) + } cmd.Dir = c.Dir cmd.Env = overlayEnv(os.Environ(), c.Env, runtime.GOOS == "windows") cmd.WaitDelay = waitDelay diff --git a/e2e/tests/up/provider_kubernetes_image_config_file.go b/e2e/tests/up/provider_kubernetes_image_config_file.go new file mode 100644 index 0000000000..b0cc97f202 --- /dev/null +++ b/e2e/tests/up/provider_kubernetes_image_config_file.go @@ -0,0 +1,861 @@ +package up + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/e2e/framework/runenv" + "github.com/devsy-org/devsy/pkg/driver/kubernetes" + devsyencoding "github.com/devsy-org/devsy/pkg/encoding" + "github.com/devsy-org/devsy/pkg/image" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "golang.org/x/crypto/bcrypt" + corev1 "k8s.io/api/core/v1" +) + +const ( + imageConfigContainerUser = "containerUser" + imageConfigRemoteUser = "remoteUser" + imageConfigUIDFlag = "--uid" + imageConfigKubectl = "kubectl" + imageConfigLogMarker = "image-config-fixture-log-marker" + imageConfigDaemonSecretPrefix = kubernetes.DevContainerName + "-daemon-secret-" + kubernetes.DevContainerName + "-" +) + +const imageConfigRegistryImage = "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373" + +type imageConfigFixture struct { + f *framework.Framework + admin, namespace, registry, authority, image, file, dir, id, node, root, password string + uid, controlUID, configDigest string +} + +type imageConfigOutput struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (o *imageConfigOutput) Write(data []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + return o.buffer.Write(data) +} + +type imageConfigLogObserver struct { + mu sync.Mutex + tail []byte + seen bool + cancel context.CancelFunc +} + +func (o *imageConfigLogObserver) Write(data []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.seen { + return len(data), nil + } + o.tail = append(o.tail, data...) + window := o.tail + if bytes.Contains(window, []byte(imageConfigLogMarker)) { + o.seen = true + o.cancel() + } + keep := len(imageConfigLogMarker) - 1 + if len(window) > keep { + window = window[len(window)-keep:] + } + o.tail = append(o.tail[:0], window...) + return len(data), nil +} + +func (x *imageConfigFixture) observeLogs(ctx context.Context) { + logsCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + observer := &imageConfigLogObserver{cancel: cancel} + result, err := (runenv.ExecRunner{}).Run(logsCtx, runenv.Command{ + Path: filepath.Join(x.f.DevsyBinDir, x.f.DevsyBinName), + Args: []string{cmdWorkspace, "logs", x.id}, + Stdout: observer, + CancelProcessGroup: true, + }) + observer.mu.Lock() + seen := observer.seen + observer.mu.Unlock() + gomega.Expect(seen).To(gomega.BeTrue(), "workspace logs must contain the fixture marker") + gomega.Expect(logsCtx.Err()).To(gomega.MatchError(context.Canceled)) + gomega.Expect(result.TimedOut).To(gomega.BeFalse()) + gomega.Expect(result.ExitCode != 0 || errors.Is(err, context.Canceled)).To(gomega.BeTrue()) +} + +func imageConfigCommand( + ctx context.Context, env []string, program string, args ...string, +) (string, error) { + switch program { + case testDockerCommand, imageConfigKubectl, "kind": + default: + return "", fmt.Errorf("unsupported fixture command %q", program) + } + output := &imageConfigOutput{} + result, err := (runenv.ExecRunner{}).Run(ctx, runenv.Command{ + Path: program, Args: args, Env: env, Stdout: output, Stderr: output, + }) + out := strings.TrimSpace(output.buffer.String()) + if err != nil { + return out, fmt.Errorf("run %s: %w", program, err) + } + if result.TimedOut || result.ExitCode != 0 { + return out, fmt.Errorf("%s exited with code %d: %s", program, result.ExitCode, out) + } + return out, nil +} + +func imageConfigMust(ctx context.Context, env []string, program string, args ...string) string { + out, err := imageConfigCommand(ctx, env, program, args...) + framework.ExpectNoError(err) + return out +} + +func (x *imageConfigFixture) kubectl(ctx context.Context, args ...string) string { + env := append(os.Environ(), "KUBECONFIG="+x.admin) + return imageConfigMust(ctx, env, imageConfigKubectl, args...) +} + +func imageConfigWrite(path string, value any) { + data, err := json.Marshal(value) + framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(path, data, 0o600)) +} + +func newImageConfigFixture(ctx context.Context) *imageConfigFixture { + root := ginkgo.GinkgoT().TempDir() + cwd, err := os.Getwd() + framework.ExpectNoError(err) + x := &imageConfigFixture{ + f: framework.NewDefaultFramework(filepath.Join(cwd, "bin")), + admin: os.Getenv("KUBECONFIG"), + id: fmt.Sprintf("cfg%d", time.Now().UnixNano()), + } + if x.admin == "" { + x.admin = filepath.Join(os.Getenv("HOME"), ".kube", "config") + } + x.root = root + x.uid = devsyencoding.CreateNewUID("", x.id) + x.controlUID = devsyencoding.CreateNewUID("", x.id+"control") + x.namespace = x.id + x.registry = "devsy-registry-" + x.id + clusterContext, arch := x.cluster(ctx) + x.startRegistry(ctx) + x.publishImage(ctx, arch) + x.provisionIdentity(ctx) + x.configureClient(ctx, clusterContext) + x.configureProvider(ctx, arch) + return x +} + +func (x *imageConfigFixture) cluster(ctx context.Context) (string, string) { + clusterContext := x.kubectl(ctx, "config", "current-context") + gomega.Expect(clusterContext).To(gomega.HavePrefix("kind-")) + nodes := strings.Fields( + imageConfigMust( + ctx, + nil, + "kind", + "get", + "nodes", + "--name", + strings.TrimPrefix(clusterContext, "kind-"), + ), + ) + gomega.Expect(nodes).To(gomega.HaveLen(1)) + x.node = nodes[0] + arch := x.kubectl(ctx, "get", "node", x.node, "-o", "jsonpath={.status.nodeInfo.architecture}") + gomega.Expect(arch).To(gomega.Equal(runtime.GOARCH)) + return clusterContext, arch +} + +func (x *imageConfigFixture) startRegistry(ctx context.Context) { + root := x.root + password := "private-" + x.id + x.password = password + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + framework.ExpectNoError(err) + authDir := filepath.Join(root, "auth") + framework.ExpectNoError(os.MkdirAll(authDir, 0o700)) + framework.ExpectNoError( + os.WriteFile( + filepath.Join(authDir, "htpasswd"), + []byte("fixture:"+string(hash)+"\n"), + 0o600, + ), + ) + imageConfigMust( + ctx, + nil, + testDockerCommand, + "run", + "-d", + "--name", + x.registry, + "--network", + "kind", + "-p", + "127.0.0.1::5000", + "-v", + authDir+":/auth:ro", + "-e", + "REGISTRY_AUTH=htpasswd", + "-e", + "REGISTRY_AUTH_HTPASSWD_REALM=fixture", + "-e", + "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd", + imageConfigRegistryImage, + ) + ginkgo.DeferCleanup( + func(c ginkgo.SpecContext) { + _, _ = imageConfigCommand(c, nil, testDockerCommand, "rm", "-f", x.registry) + }, + ) + x.authority = imageConfigMust(ctx, nil, testDockerCommand, "port", x.registry, "5000/tcp") + x.authority = "localhost:" + strings.TrimPrefix(x.authority, "127.0.0.1:") + x.configureNodeRegistry(ctx) +} + +func (x *imageConfigFixture) configureNodeRegistry(ctx context.Context) { + root := x.root + hostDir := "/etc/containerd/certs.d/" + x.authority + _, existingErr := imageConfigCommand( + ctx, + nil, + testDockerCommand, + "exec", + x.node, + "test", + "-e", + hostDir, + ) + gomega.Expect(existingErr). + To(gomega.HaveOccurred(), "fixture must not overwrite an existing containerd host configuration") + imageConfigMust(ctx, nil, testDockerCommand, "exec", x.node, "mkdir", "-p", hostDir) + ginkgo.DeferCleanup(func(c ginkgo.SpecContext) { + _, _ = imageConfigCommand(c, nil, testDockerCommand, "exec", x.node, "rm", "-rf", hostDir) + }) + hosts := filepath.Join(root, "hosts.toml") + framework.ExpectNoError( + os.WriteFile( + hosts, + []byte( + fmt.Sprintf( + "server = \"http://%s:5000\"\n[host.\"http://%s:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n", + x.registry, + x.registry, + ), + ), + 0o600, + ), + ) + imageConfigMust(ctx, nil, testDockerCommand, "cp", hosts, x.node+":"+hostDir+"/hosts.toml") +} + +func (x *imageConfigFixture) publishImage(ctx context.Context, arch string) { + root, password := x.root, x.password + img, err := image.GetImageForArch(ctx, "ghcr.io/devsy-org/test-images/base:alpine", arch) + framework.ExpectNoError(err) + cfg, err := img.ConfigFile() + framework.ExpectNoError(err) + cfg.Config.User = "nobody" + cfg.Config.Entrypoint = []string{ + "/bin/sh", + "-c", + "echo " + imageConfigLogMarker + "; echo entrypoint > /tmp/image-config-entrypoint; exec \"$@\"", + "--", + } + cfg.Config.Cmd = []string{"sleep", "36000"} + cfg.Config.Env = append(cfg.Config.Env, "IMAGE_CONFIG_VALUE=a=b", "IMAGE_CONFIG_OVERRIDE=image") + cfg.Config.Labels = map[string]string{ + "devsy-e2e-nonce": x.id, + "devcontainer.metadata": `[{"containerEnv":{"IMAGE_CONFIG_METADATA":"metadata"},` + + `"postCreateCommand":"echo metadata > /tmp/image-config-lifecycle"}]`, + } + img, err = mutate.ConfigFile(img, cfg) + framework.ExpectNoError(err) + configDigest, err := img.ConfigName() + framework.ExpectNoError(err) + x.configDigest = configDigest.String() + digest, err := img.Digest() + framework.ExpectNoError(err) + x.image = x.authority + "/fixture/workspace@" + digest.String() + tag, err := name.NewTag(x.authority+"/fixture/workspace:fixture", name.Insecure) + framework.ExpectNoError(err) + gomega.Eventually(func() error { + return remote.Write( + tag, + img, + remote.WithContext(ctx), + remote.WithAuth(&authn.Basic{Username: "fixture", Password: password}), + ) + }, 30*time.Second, time.Second).Should(gomega.Succeed()) + x.file = filepath.Join(root, "image.json") + imageConfigWrite( + x.file, + map[string]any{ + "schemaVersion": 1, + compatibilityImageCommand: x.image, + "os": "linux", + "architecture": arch, + "config": map[string]any{ + "User": cfg.Config.User, + "Env": cfg.Config.Env, + "Labels": cfg.Config.Labels, + "Entrypoint": cfg.Config.Entrypoint, + "Cmd": cfg.Config.Cmd, + }, + }, + ) +} + +func (x *imageConfigFixture) provisionIdentity(ctx context.Context) { + password := x.password + x.kubectl(ctx, "create", "namespace", x.namespace) + ginkgo.DeferCleanup(func(c ginkgo.SpecContext) { + _, _ = imageConfigCommand( + c, + append(os.Environ(), "KUBECONFIG="+x.admin), + imageConfigKubectl, + "delete", + "namespace", + x.namespace, + "--ignore-not-found", + "--wait=false", + ) + }) + x.kubectl( + ctx, + "-n", + x.namespace, + "create", + "secret", + "docker-registry", + "registry-admin", + "--docker-server="+x.authority, + "--docker-username=fixture", + "--docker-password="+password, + ) + x.kubectl(ctx, "-n", x.namespace, "create", "serviceaccount", cmdWorkspace) + x.kubectl( + ctx, + "-n", + x.namespace, + "patch", + "serviceaccount", + cmdWorkspace, + "--type=merge", + "-p", + `{"imagePullSecrets":[{"name":"registry-admin"}]}`, + ) + x.kubectl(ctx, "-n", x.namespace, "create", "serviceaccount", "client") + x.createClientRole(ctx) +} + +func (x *imageConfigFixture) createClientRole(ctx context.Context) { + root := x.root + role := fmt.Sprintf(`apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: client, namespace: %s} +rules: +- apiGroups: [""] + resources: [pods, pods/exec, pods/log, persistentvolumeclaims, serviceaccounts, events, services, configmaps] + verbs: ["*"] +- apiGroups: [""] + resources: [secrets] + verbs: [create] +- apiGroups: [""] + resources: [secrets] + resourceNames: [%q, %q] + verbs: [get, update, patch, delete] +`, x.namespace, imageConfigDaemonSecretPrefix+x.uid, imageConfigDaemonSecretPrefix+x.controlUID) + rolePath := filepath.Join(root, "role.yaml") + framework.ExpectNoError(os.WriteFile(rolePath, []byte(role), 0o600)) + x.kubectl(ctx, "apply", "-f", rolePath) + x.kubectl( + ctx, + "-n", + x.namespace, + "create", + "rolebinding", + "client", + "--role=client", + "--serviceaccount="+x.namespace+":client", + ) +} + +func (x *imageConfigFixture) configureClient(ctx context.Context, clusterContext string) { + root := x.root + token := x.kubectl(ctx, "-n", x.namespace, "create", "token", "client", "--duration=1h") + kube := filepath.Join(root, "client.kubeconfig") + raw := x.kubectl(ctx, "config", "view", "--raw", "--minify", "--flatten") + framework.ExpectNoError(os.WriteFile(kube, []byte(raw), 0o600)) + env := append(os.Environ(), "KUBECONFIG="+kube) + imageConfigMust( + ctx, + env, + imageConfigKubectl, + "config", + "set-credentials", + "image-config-client", + "--token="+token, + ) + imageConfigMust( + ctx, + env, + imageConfigKubectl, + "config", + "set-context", + clusterContext, + "--user=image-config-client", + "--namespace="+x.namespace, + ) + framework.SetEnvForSpec("KUBECONFIG", kube) + x.configureCredentialSpy() + x.checkClientPermissions(ctx) +} + +func (x *imageConfigFixture) configureCredentialSpy() { + root := x.root + framework.SetEnvForSpec("DOCKER_CONFIG", filepath.Join(root, "empty-docker")) + helperDir := filepath.Join(root, "helpers") + framework.ExpectNoError(os.MkdirAll(helperDir, 0o700)) + framework.SetEnvForSpec("IMAGE_CONFIG_HELPER_LOG", filepath.Join(root, "helper-calls")) + for _, helper := range []string{ + "docker-credential-desktop", "docker-credential-osxkeychain", + "docker-credential-pass", "docker-credential-secretservice", + } { + script := []byte("#!/bin/sh\necho called >> \"$IMAGE_CONFIG_HELPER_LOG\"\n" + + "echo 'credentials not found in native keychain'\nexit 1\n") + framework.ExpectNoError( + (runenv.OsFileStore{}).WriteFile(filepath.Join(helperDir, helper), script, 0o700), + ) + } + framework.SetEnvForSpec("PATH", helperDir+string(os.PathListSeparator)+os.Getenv("PATH")) + framework.ExpectNoError(os.MkdirAll(filepath.Join(root, "empty-docker"), 0o700)) + imageConfigWrite( + filepath.Join(root, "empty-docker", "config.json"), + map[string]string{"credsStore": "pass"}, + ) + framework.SetEnvForSpec("DEVSY_HOME", filepath.Join(root, "devsy")) + framework.SetEnvForSpec("DEVSY_SECRETS_BACKEND", "file") + framework.SetEnvForSpec("DEVSY_SECRETS_PASSPHRASE", "fixture") +} + +func (x *imageConfigFixture) checkClientPermissions(ctx context.Context) { + out, err := imageConfigCommand( + ctx, + nil, + imageConfigKubectl, + "auth", + "can-i", + "get", + "secret/registry-admin", + "-n", + x.namespace, + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(out).To(gomega.ContainSubstring("no")) + out, err = imageConfigCommand(ctx, nil, imageConfigKubectl, "auth", "can-i", "list", "nodes") + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(out).To(gomega.ContainSubstring("no")) + out, err = imageConfigCommand(ctx, nil, imageConfigKubectl, "auth", "can-i", "get", + "secret/"+imageConfigDaemonSecretPrefix+x.uid, "-n", x.namespace) + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("yes")) +} + +func (x *imageConfigFixture) configureProvider(ctx context.Context, arch string) { + root := x.root + _, stderr, err := x.f.ExecCommandCapture( + ctx, + []string{ + "context", + microsandboxSecretSetCommand, + microsandboxResourceOption, + "SSH_INJECT_DOCKER_CREDENTIALS=false", + microsandboxResourceOption, + "SSH_ADD_PRIVATE_KEYS=false", + }, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr) + framework.ExpectNoError( + x.f.DevsyProviderAdd( + ctx, + "kubernetes", + "--name", + x.id, + "-o", + "KUBERNETES_NAMESPACE="+x.namespace, + "-o", + "CREATE_NAMESPACE=false", + "-o", + "SERVICE_ACCOUNT=workspace", + "-o", + "ARCHITECTURE="+arch, + "-o", + "KUBERNETES_PULL_SECRETS_ENABLED=false", + "-o", + "POD_TIMEOUT=2m", + "-o", + "INACTIVITY_TIMEOUT=0", + ), + ) + ginkgo.DeferCleanup( + func(c ginkgo.SpecContext) { _ = x.f.CleanupWorkspace(c, x.id); _ = x.f.DevsyProviderDelete(c, x.id) }, + ) + x.dir = filepath.Join(root, cmdWorkspace) + framework.ExpectNoError(os.MkdirAll(filepath.Join(x.dir, ".devcontainer"), 0o700)) + imageConfigWrite( + filepath.Join(x.dir, ".devcontainer", "devcontainer.json"), + map[string]any{ + compatibilityImageCommand: x.image, + "workspaceFolder": "/workspaces/project", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/project,type=bind", + "overrideCommand": false, + imageConfigContainerUser: microsandboxRootUser, + imageConfigRemoteUser: microsandboxRootUser, + "containerEnv": map[string]string{"IMAGE_CONFIG_OVERRIDE": "override"}, + }, + ) +} + +func (x *imageConfigFixture) up(ctx context.Context, args ...string) (string, string, error) { + command := []string{ + cmdWorkspace, + "up", + x.dir, + "--provider", + x.id, + "--id", + x.id, + imageConfigUIDFlag, + x.uid, + flagIDEOption, + kubernetesIgnoreIDE, + "--provider-option", + "ARCHITECTURE=" + runtime.GOARCH, + "--provider-option", + "POD_TIMEOUT=2m", + } + return x.f.ExecCommandCapture(ctx, append(command, args...)) +} + +func (x *imageConfigFixture) pod(ctx context.Context) corev1.Pod { + raw := x.kubectl( + ctx, + "-n", + x.namespace, + "get", + "pods", + "-l", + "devsy.sh/created=true", + "-o", + "json", + ) + var list corev1.PodList + framework.ExpectNoError(json.Unmarshal([]byte(raw), &list)) + gomega.Expect(list.Items).To(gomega.HaveLen(1)) + return list.Items[0] +} + +var _ = ginkgo.Describe( + "Kubernetes supplied image config", + ginkgo.Label("up-provider-kubernetes", "up-provider-kubernetes-image-config-file"), + func() { + ginkgo.It( + "uses cluster registry authorization without client inspection and preserves workspace data", + func(ctx ginkgo.SpecContext) { + x := newImageConfigFixture(ctx) + negativeStarted := time.Now().UTC().Format(time.RFC3339Nano) + _, stderr, err := x.up(ctx) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(strings.ToLower(stderr)). + To(gomega.Or(gomega.ContainSubstring("unauthorized"), gomega.ContainSubstring("401"))) + negativeLogs := imageConfigMust(ctx, nil, testDockerCommand, + "logs", "--since", negativeStarted, x.registry) + gomega.Expect(negativeLogs).To(gomega.ContainSubstring(" 401 ")) + gomega.Expect(negativeLogs).To(gomega.ContainSubstring("/manifests/")) + _, err = os.Stat(filepath.Join(x.root, "helper-calls")) + framework.ExpectNoError(err) + _, stderr, err = x.up( + ctx, + "--provider-option", + "IMAGE_CONFIG_FILE="+filepath.Join(x.root, "missing.json"), + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stderr).To(gomega.ContainSubstring("IMAGE_CONFIG_FILE")) + _, stderr, err = x.up( + ctx, + "--provider-option", + "IMAGE_CONFIG_FILE="+x.file, + "--provider-option", + "ARCHITECTURE=wrong", + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stderr).To(gomega.ContainSubstring("ARCHITECTURE")) + pods := x.kubectl( + ctx, + "-n", + x.namespace, + "get", + "pods", + "-l", + "devsy.sh/created=true", + "-o", + "json", + ) + var empty corev1.PodList + framework.ExpectNoError(json.Unmarshal([]byte(pods), &empty)) + gomega.Expect(empty.Items).To(gomega.BeEmpty()) + valid, err := os.ReadFile(x.file) + framework.ExpectNoError(err) + var mismatch map[string]any + framework.ExpectNoError(json.Unmarshal(valid, &mismatch)) + mismatch[compatibilityImageCommand] = x.authority + "/fixture/workspace@sha256:" + strings.Repeat( + "0", + 64, + ) + imageConfigWrite(x.file, mismatch) + _, mismatchError, err := x.up(ctx, "--provider-option", "IMAGE_CONFIG_FILE="+x.file) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(mismatchError).To(gomega.ContainSubstring("IMAGE_CONFIG_FILE")) + gomega.Expect(x.kubectl( + ctx, + "-n", + x.namespace, + "get", + "pods", + "-l", + "devsy.sh/created=true", + "-o", + "jsonpath={.items[*].metadata.name}", + )).To(gomega.BeEmpty()) + framework.ExpectNoError(os.WriteFile(x.file, valid, 0o600)) + x.kubectl( + ctx, + "-n", + x.namespace, + "patch", + "serviceaccount", + cmdWorkspace, + "--type=merge", + "-p", + `{"imagePullSecrets":[]}`, + ) + _, pullError, err := x.up( + ctx, + "--provider-option", + "IMAGE_CONFIG_FILE="+x.file, + "--provider-option", + "POD_TIMEOUT=30s", + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + events := x.kubectl(ctx, "-n", x.namespace, "get", "events", "-o", "json") + gomega.Expect(strings.ToLower(pullError + events)). + To(gomega.Or(gomega.ContainSubstring("imagepullbackoff"), gomega.ContainSubstring("pull image"))) + framework.ExpectNoError(x.f.CleanupWorkspace(ctx, x.id)) + x.kubectl( + ctx, + "-n", + x.namespace, + "patch", + "serviceaccount", + cmdWorkspace, + "--type=merge", + "-p", + `{"imagePullSecrets":[{"name":"registry-admin"}]}`, + ) + helperLog := filepath.Join(x.root, "helper-calls") + if err := os.Remove(helperLog); err != nil && !os.IsNotExist(err) { + framework.ExpectNoError(err) + } + launchStarted := time.Now().UTC().Format(time.RFC3339Nano) + _, stderr, err = x.up(ctx, "--provider-option", "IMAGE_CONFIG_FILE="+x.file) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr) + pod := x.pod(ctx) + gomega.Expect(pod.Spec.ServiceAccountName).To(gomega.Equal(cmdWorkspace)) + gomega.Expect(pod.Spec.Containers[0].Image).To(gomega.Equal(x.image)) + gomega.Expect(pod.Status.ContainerStatuses[0].ImageID). + To(gomega.ContainSubstring(strings.Split(x.image, "@")[1])) + delta := imageConfigMust( + ctx, + nil, + testDockerCommand, + "logs", + "--since", + launchStarted, + x.registry, + ) + gomega.Expect(strings.ToLower(delta)).To(gomega.ContainSubstring("containerd")) + nodeIP := imageConfigMust( + ctx, + nil, + testDockerCommand, + "inspect", + "-f", + "{{(index .NetworkSettings.Networks \"kind\").IPAddress}}", + x.node, + ) + nodeRequests := 0 + configFetched := false + for line := range strings.SplitSeq(delta, "\n") { + if strings.Contains(line, "http.request.remoteaddr") { + gomega.Expect(line).To(gomega.ContainSubstring(nodeIP)) + nodeRequests++ + if strings.Contains(line, "http.request.method=GET") && + strings.Contains(line, "/blobs/"+x.configDigest) && + strings.Contains(line, "http.response.status=200") { + configFetched = true + } + } + } + gomega.Expect(nodeRequests).To(gomega.BeNumerically(">", 0)) + gomega.Expect(configFetched). + To(gomega.BeTrue(), "node must fetch the unique image config") + _, err = os.Stat(helperLog) + gomega.Expect(err).To(gomega.MatchError(os.ErrNotExist)) + observe := "printf '%s|%s|%s|%s' \"$(id -un)\" \"$IMAGE_CONFIG_VALUE\" " + + "\"$IMAGE_CONFIG_OVERRIDE\" \"$IMAGE_CONFIG_METADATA\" && " + + "test -f /tmp/image-config-lifecycle && test -f /tmp/image-config-entrypoint && " + + "tr '\\0' ' ' imageConfigLimit { + return nil, fmt.Errorf("image config file exceeds 1 MiB") + } + if !utf8.Valid(data) { + return nil, fmt.Errorf("image config file is not UTF-8") + } + return data, nil +} + +func parseImageConfig(data []byte) (imageConfigFile, config.ImageDetailsConfig, error) { + var file imageConfigFile + var payload config.ImageDetailsConfig + if err := decodeObject(data, &file); err != nil || !validWrapper(data) { + return file, payload, fmt.Errorf("invalid wrapper") + } + if file.SchemaVersion != 1 { + return file, payload, fmt.Errorf("unsupported schemaVersion") + } + if err := decodeObject(file.Config, &payload); err != nil || !validPayload(file.Config) { + return file, payload, fmt.Errorf("invalid config object") + } + return file, payload, nil +} + +func matchImageReferences(imageName, fileImage string) error { + requested, err := pinnedReference(imageName) + if err != nil { + return fmt.Errorf("requested image must be a fully qualified digest reference") + } + supplied, err := pinnedReference(fileImage) + if err != nil { + return fmt.Errorf("image must be a fully qualified digest reference") + } + if requested.Name() != supplied.Name() { + return fmt.Errorf("image repository or digest does not match requested image") + } + return nil +} + +func imageConfigPlatform(file imageConfigFile, architecture string) (ocispec.Platform, error) { + platform := platforms.Normalize( + ocispec.Platform{OS: file.OS, Architecture: file.Architecture, Variant: file.Variant}, + ) + target := platforms.Normalize(ocispec.Platform{OS: "linux", Architecture: architecture}) + if file.OS == "" || platform.OS != "linux" || platform.Architecture == "" || + platform.Architecture != target.Architecture || !validVariant(platform.Architecture, platform.Variant) { + return platform, fmt.Errorf("image platform does not match Linux ARCHITECTURE") + } + return platform, nil +} + +func pinnedReference(ref string) (name.Digest, error) { + parts := strings.SplitN(ref, "/", 2) + if len(parts) != 2 || (!strings.ContainsAny(parts[0], ".:") && parts[0] != "localhost") { + return name.Digest{}, fmt.Errorf("missing registry") + } + return name.NewDigest(ref, name.StrictValidation) +} + +func decodeObject(data []byte, out any) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || data[0] != '{' { + return fmt.Errorf("expected object") + } + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(out); err != nil { + return err + } + var extra any + if err := dec.Decode(&extra); err != io.EOF { + return fmt.Errorf("trailing data") + } + return nil +} + +func validateImageMetadata(raw string) error { + if raw == "" { + return nil + } + var value any + if json.Unmarshal([]byte(raw), &value) != nil || value == nil { + return fmt.Errorf("invalid %s label", metadata.ImageMetadataLabel) + } + entries, ok := value.([]any) + if !ok { + entries = []any{value} + } + for _, entry := range entries { + if !validMetadataEntry(entry) { + return fmt.Errorf("invalid %s label", metadata.ImageMetadataLabel) + } + } + return nil +} + +func validMetadataEntry(entry any) bool { + if _, ok := entry.(map[string]any); !ok { + return false + } + b, err := json.Marshal(entry) + if err != nil { + return false + } + var item config.ImageMetadata + return json.Unmarshal(b, &item) == nil +} + +func validWrapper(data []byte) bool { + if !uniqueJSONKeys(data) { + return false + } + var fields map[string]json.RawMessage + if json.Unmarshal(data, &fields) != nil { + return false + } + for key, value := range fields { + if !validWrapperField(key, value) { + return false + } + } + for _, key := range []string{"schemaVersion", "image", "os", "architecture", "config"} { + if _, ok := fields[key]; !ok { + return false + } + } + return true +} + +func validWrapperField(key string, value json.RawMessage) bool { + switch key { + case "schemaVersion": + var version int + return !bytes.Equal(bytes.TrimSpace(value), []byte("null")) && + json.Unmarshal(value, &version) == nil + case "image", "os", "architecture", "variant": + return isJSONString(value) + case "config": + return startsWith(value, '{') + default: + return false + } +} + +func validPayload(data []byte) bool { + var fields map[string]json.RawMessage + if json.Unmarshal(data, &fields) != nil { + return false + } + for key, value := range fields { + if !validPayloadField(key, value) { + return false + } + } + return true +} + +func validPayloadField(key string, value json.RawMessage) bool { + switch key { + case "User": + return isJSONString(value) + case "Env", "Entrypoint", "Cmd": + return isStringArray(value) + case "Labels": + return isStringMap(value) + default: + return false + } +} + +func startsWith(value json.RawMessage, prefix byte) bool { + value = bytes.TrimSpace(value) + return len(value) > 0 && value[0] == prefix +} + +func isStringArray(value json.RawMessage) bool { + var values []json.RawMessage + if !startsWith(value, '[') || json.Unmarshal(value, &values) != nil { + return false + } + for _, v := range values { + if !isJSONString(v) { + return false + } + } + return true +} + +func isStringMap(value json.RawMessage) bool { + var values map[string]json.RawMessage + if !startsWith(value, '{') || json.Unmarshal(value, &values) != nil { + return false + } + for _, v := range values { + if !isJSONString(v) { + return false + } + } + return true +} + +func isJSONString(value json.RawMessage) bool { + var s string + return len(bytes.TrimSpace(value)) > 0 && bytes.TrimSpace(value)[0] == '"' && + json.Unmarshal(value, &s) == nil +} + +func uniqueJSONKeys(data []byte) bool { + dec := json.NewDecoder(bytes.NewReader(data)) + return uniqueJSONValue(dec) == nil +} + +func uniqueJSONValue(dec *json.Decoder) error { + token, err := dec.Token() + if err != nil { + return err + } + delim, ok := token.(json.Delim) + if !ok { + return nil + } + switch delim { + case '{': + err = uniqueJSONObject(dec) + case '[': + err = uniqueJSONArray(dec) + default: + return fmt.Errorf("invalid JSON") + } + if err != nil { + return err + } + _, err = dec.Token() + return err +} + +func uniqueJSONObject(dec *json.Decoder) error { + seen := map[string]bool{} + for dec.More() { + token, err := dec.Token() + if err != nil { + return err + } + key, ok := token.(string) + if !ok || seen[key] { + return fmt.Errorf("duplicate key") + } + seen[key] = true + if err := uniqueJSONValue(dec); err != nil { + return err + } + } + return nil +} + +func uniqueJSONArray(dec *json.Decoder) error { + for dec.More() { + if err := uniqueJSONValue(dec); err != nil { + return err + } + } + return nil +} + +func validVariant(arch, variant string) bool { + if variant == "" { + return true + } + switch arch { + case "amd64": + return variantIn(variant, "v2", "v3", "v4") + case "arm": + return variantIn(variant, "v5", "v6", "v7", "v8") + case "arm64": + return arm64VariantPattern.MatchString(variant) + default: + return true + } +} + +func variantIn(variant string, allowed ...string) bool { + return slices.Contains(allowed, variant) +} diff --git a/pkg/driver/kubernetes/image_details_test.go b/pkg/driver/kubernetes/image_details_test.go new file mode 100644 index 0000000000..14afdd6638 --- /dev/null +++ b/pkg/driver/kubernetes/image_details_test.go @@ -0,0 +1,195 @@ +package kubernetes + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/devsy-org/devsy/pkg/driver" + provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/require" +) + +const testImage = "registry.example.com/team/image@sha256:" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +const testArchitecture = "amd64" + +type imageDetailsTestFixture struct { + driver *KubernetesDriver + path string + request driver.ImageDetailsRequest + wrapper func(string) string + write func(string) +} + +func imageDetailsFixture(t *testing.T) imageDetailsTestFixture { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + d := &KubernetesDriver{ + options: &provider2.ProviderKubernetesDriverConfig{ + ImageConfigFile: path, + Architecture: testArchitecture, + }, + } + request := driver.ImageDetailsRequest{ImageName: testImage} + wrapper := func(payload string) string { + return `{"schemaVersion":1,"image":"` + testImage + `","os":"linux","architecture":"amd64","config":` + payload + `}` + } + write := func(data string) { t.Helper(); require.NoError(t, os.WriteFile(path, []byte(data), 0o600)) } + return imageDetailsTestFixture{ + driver: d, + path: path, + request: request, + wrapper: wrapper, + write: write, + } +} + +func TestImageDetailsFile(t *testing.T) { + f := imageDetailsFixture(t) + d, path, request, wrapper, write := f.driver, f.path, f.request, f.wrapper, f.write + d.options.ImageConfigFile = "" + details, err := d.ImageDetails(request) + require.NoError(t, err) + require.Nil(t, details) + d.options.ImageConfigFile = path + _, err = d.ImageDetails(driver.ImageDetailsRequest{UnsupportedReason: "Features"}) + require.ErrorContains(t, err, "Features") + write( + wrapper( + `{"User":"vscode","Env":["A=b=c"],"Labels":{"devcontainer.metadata":"[]"},"Entrypoint":["a b"],"Cmd":["$x"]}`, + ), + ) + details, err = d.ImageDetails(request) + require.NoError(t, err) + require.Equal(t, testImage, details.ID) + require.Equal(t, []string{"A=b=c"}, details.Config.Env) + require.Equal(t, []string{"a b"}, details.Config.Entrypoint) + require.Equal(t, []string{"$x"}, details.Config.Cmd) + write(wrapper(`{}`)) + _, err = d.ImageDetails(request) + require.NoError(t, err) + exact := wrapper(`{}`) + write(exact + strings.Repeat(" ", imageConfigLimit-len(exact))) + _, err = d.ImageDetails(request) + require.NoError(t, err) + _, err = d.ImageDetails( + driver.ImageDetailsRequest{ImageName: "registry.example.com/team/image:latest"}, + ) + require.Error(t, err) +} + +func invalidImageDetailsFiles(wrapper func(string) string) map[string]string { + return map[string]string{ + "missing config": `{"schemaVersion":1}`, + "null config": wrapper(`null`), + "unknown config": wrapper(`{"Bad":1}`), + "trailing": wrapper(`{}`) + ` {}`, + "schema": strings.Replace( + wrapper(`{}`), + `"schemaVersion":1`, + `"schemaVersion":2`, + 1, + ), + "wrong image": strings.Replace(wrapper(`{}`), "/team/image@", "/team/other@", 1), + "wrong platform": strings.Replace(wrapper(`{}`), `"amd64"`, `"arm64"`, 1), + "metadata": wrapper(`{"Labels":{"devcontainer.metadata":"[null]"}}`), + "invalid utf8": wrapper(`{}`) + string([]byte{0xff}), + "oversize": wrapper( + `{}`, + ) + strings.Repeat( + " ", + imageConfigLimit+1-len(wrapper(`{}`)), + ), + "wrong digest": strings.Replace( + wrapper(`{}`), + strings.Repeat("a", 64), + strings.Repeat("b", 64), + 1, + ), + "tagged file": strings.Replace( + wrapper(`{}`), + testImage, + "registry.example.com/team/image:latest", + 1, + ), + "non linux": strings.Replace(wrapper(`{}`), `"linux"`, `"windows"`, 1), + "null architecture": strings.Replace(wrapper(`{}`), `"amd64"`, `null`, 1), + "array config": wrapper(`[]`), + "wrong env type": wrapper(`{"Env":[null]}`), + "unknown wrapper": strings.Replace( + wrapper(`{}`), + `"schemaVersion":1`, + `"schemaVersion":1,"extra":true`, + 1, + ), + "case alias": strings.Replace(wrapper(`{}`), `"schemaVersion"`, `"SchemaVersion"`, 1), + "duplicate": strings.Replace( + wrapper(`{}`), + `"schemaVersion":1`, + `"schemaVersion":1,"schemaVersion":1`, + 1, + ), + "contradictory variant": strings.Replace( + wrapper(`{}`), + `"architecture":"amd64"`, + `"architecture":"amd64","variant":"v8"`, + 1, + ), + } +} + +func TestImageDetailsInvalidFiles(t *testing.T) { + f := imageDetailsFixture(t) + d, request, wrapper, write := f.driver, f.request, f.wrapper, f.write + cases := invalidImageDetailsFiles(wrapper) + for name, data := range cases { + t.Run( + name, + func(t *testing.T) { write(data); _, err := d.ImageDetails(request); require.Error(t, err) }, + ) + } +} + +func TestImageDetailsMetadataAndPaths(t *testing.T) { + f := imageDetailsFixture(t) + d, path, request, wrapper, write := f.driver, f.path, f.request, f.wrapper, f.write + b, err := json.Marshal(map[string]string{"devcontainer.metadata": `{"remoteUser":1}`}) + require.NoError(t, err) + write(wrapper(`{"Labels":` + string(b) + `}`)) + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "devcontainer.metadata") + write( + strings.Replace( + wrapper( + `{"Labels":{"devcontainer.metadata":"[{\"customizations\":{\"example\":null}}]"}}`, + ), + `"architecture":"amd64"`, + `"architecture":"aarch64","variant":"v8"`, + 1, + ), + ) + d.options.Architecture = "arm64" + details, err := d.ImageDetails(request) + require.NoError(t, err) + require.Equal(t, "arm64", details.Architecture) + require.Empty(t, details.Variant) + d.options.Architecture = testArchitecture + d.options.ImageConfigFile = filepath.Join(filepath.Dir(path), "missing") + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "read image config file") + d.options.ImageConfigFile = filepath.Dir(path) + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "regular") + d.options.ImageConfigFile = "relative" + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "absolute") + d.options.ImageConfigFile = path + d.options.Architecture = "" + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "ARCHITECTURE") +} diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go index b93b568c12..9b8ce270db 100644 --- a/pkg/driver/kubernetes/pod_lifecycle_test.go +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -5,13 +5,17 @@ import ( "encoding/json" "errors" "os" + "path/filepath" + goruntime "runtime" "sync/atomic" "testing" "time" + "github.com/devsy-org/devsy/e2e/framework/runenv" pkgconfig "github.com/devsy-org/devsy/pkg/config" agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/dockercredentials" "github.com/devsy-org/devsy/pkg/driver" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/stretchr/testify/assert" @@ -221,6 +225,43 @@ func TestReconcileReplacesPodWhenOptionsChanged(t *testing.T) { assert.False(t, c.podExists(t)) } +func TestReconcileReusesPodWhenOnlyImageConfigFileChanged(t *testing.T) { + c := newFakeCluster(t) + c.k.options.ImageConfigFile = "/new/image-config.json" + c.putPod(t, c.pod(t, podSpec{ + uid: "u1", + phase: corev1.PodRunning, + options: &provider2.ProviderKubernetesDriverConfig{ + PodTimeout: "2s", + DiskSize: "1Gi", + ImageConfigFile: "/old/image-config.json", + }, + devsy: &corev1.ContainerStatus{ + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + })) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.True(t, reused) + assert.Zero(t, c.deleteCalls()) +} + +func TestWithoutLocalOptionsDropsImageConfigFile(t *testing.T) { + options := &provider2.ProviderKubernetesDriverConfig{ + DiskSize: "1Gi", + ImageConfigFile: "/home/me/image-config.json", + } + + raw, err := json.Marshal(withoutLocalOptions(options)) + + require.NoError(t, err) + assert.NotContains(t, string(raw), "imageConfigFile") + assert.Contains(t, string(raw), "1Gi") + assert.Equal(t, "/home/me/image-config.json", options.ImageConfigFile) +} + func TestReconcilePendingPodWaitsForReadiness(t *testing.T) { c := newFakeCluster(t) c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending})) @@ -1178,3 +1219,77 @@ func assertOnlyReads(t *testing.T, c *fakeCluster) { func writeFile(path, content string) error { return os.WriteFile(path, []byte(content), 0o600) } + +func credentialHelperMarker(t *testing.T) string { + t.Helper() + if goruntime.GOOS == "windows" { + t.Skip("credential helper spy requires a Unix shell") + } + dir := t.TempDir() + marker := filepath.Join(dir, "called") + helper := filepath.Join(dir, "docker-credential-devsyspy") + script := "#!/bin/sh\nprintf called > \"$DEVSY_TEST_CREDENTIAL_MARKER\"\n" + + "printf '{\"Username\":\"test\",\"Secret\":\"test\"}\\n'\n" + require.NoError(t, (runenv.OsFileStore{}).WriteFile(helper, []byte(script), 0o700)) + require.NoError( + t, + os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"credsStore":"devsyspy"}`), 0o600), + ) + t.Setenv("DOCKER_CONFIG", dir) + t.Setenv("DEVSY_TEST_CREDENTIAL_MARKER", marker) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + _, err := dockercredentials.GetAuthConfig("registry.example.com") + require.NoError(t, err) + require.FileExists(t, marker) + require.NoError(t, os.Remove(marker)) + return marker +} + +func TestImageConfigPreservesClusterPullCredentials(t *testing.T) { + marker := credentialHelperMarker(t) + const adminSecret = "admin-pull" + account := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-account", Namespace: testNamespace}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: adminSecret}}, + } + c := newFakeCluster(t, account) + c.k.options.ImageConfigFile = "/local/image.json" + c.k.options.ServiceAccount = account.Name + c.k.options.KubernetesPullSecretsEnabled = pkgconfig.BoolFalse + c.k.agentConfig.InjectDockerCredentials = pkgconfig.BoolFalse + template := filepath.Join(t.TempDir(), "pod.yaml") + require.NoError(t, writeFile(template, "spec:\n imagePullSecrets:\n - name: template-pull\n")) + c.k.options.PodManifestTemplate = template + reads := 0 + c.cs.PrependReactor( + "get", + "secrets", + func(action clienttesting.Action) (bool, runtime.Object, error) { + if action.(clienttesting.GetAction).GetName() != adminSecret { + return false, nil, nil + } + reads++ + return true, nil, kerrors.NewForbidden( + schema.GroupResource{Resource: "secrets"}, + adminSecret, + errors.New("denied"), + ) + }, + ) + options := runOptionsForWorkspace("1m") + options.Image = testImage + pod, err := c.k.buildPod(context.Background(), testPodID, options, false) + require.NoError(t, err) + assert.Equal(t, account.Name, pod.Spec.ServiceAccountName) + assert.Equal( + t, + []corev1.LocalObjectReference{{Name: "template-pull"}}, + pod.Spec.ImagePullSecrets, + ) + saved, err := c.cs.CoreV1().ServiceAccounts(testNamespace). + Get(context.Background(), account.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, account.ImagePullSecrets, saved.ImagePullSecrets) + assert.Zero(t, reads) + assert.NoFileExists(t, marker) +} diff --git a/pkg/driver/kubernetes/run.go b/pkg/driver/kubernetes/run.go index 6833d17f8c..c1bbceb4af 100644 --- a/pkg/driver/kubernetes/run.go +++ b/pkg/driver/kubernetes/run.go @@ -555,7 +555,7 @@ func (k *KubernetesDriver) runPod( var err error // set configuration before creating the pod - lastAppliedConfigRaw, err := json.Marshal(k.options) + lastAppliedConfigRaw, err := json.Marshal(withoutLocalOptions(k.options)) if err != nil { return fmt.Errorf("marshal last applied config: %w", err) } @@ -750,13 +750,13 @@ func getDaemonSecretName(workspaceID string) string { func optionsEqual(a, b *provider2.ProviderKubernetesDriverConfig) bool { // copy a and b and the compare them without the context, config, namespace and podTimeout - aCopy := *a + aCopy := withoutLocalOptions(a) aCopy.KubernetesContext = "" aCopy.KubernetesConfig = "" aCopy.KubernetesNamespace = "" aCopy.PodTimeout = "" - bCopy := *b + bCopy := withoutLocalOptions(b) bCopy.KubernetesContext = "" bCopy.KubernetesConfig = "" bCopy.KubernetesNamespace = "" @@ -764,6 +764,14 @@ func optionsEqual(a, b *provider2.ProviderKubernetesDriverConfig) bool { return aCopy == bCopy } +func withoutLocalOptions( + options *provider2.ProviderKubernetesDriverConfig, +) provider2.ProviderKubernetesDriverConfig { + c := *options + c.ImageConfigFile = "" + return c +} + func (k *KubernetesDriver) createNamespace(ctx context.Context) error { _, err := k.client.Client().CoreV1().Namespaces().Get(ctx, k.namespace, metav1.GetOptions{}) if kerrors.IsNotFound(err) || kerrors.IsForbidden(err) { diff --git a/pkg/driver/types.go b/pkg/driver/types.go index b961acc283..afc94f4249 100644 --- a/pkg/driver/types.go +++ b/pkg/driver/types.go @@ -54,6 +54,15 @@ type WorkspaceChowner interface { RequiresWorkspaceChown() bool } +type ImageDetailsRequest struct { + ImageName string + UnsupportedReason string +} + +type ImageDetailsDriver interface { + ImageDetails(request ImageDetailsRequest) (*config.ImageDetails, error) +} + // ManagedLifecycleDriver identifies drivers that can run the container's main // process as the Devsy daemon. Actual ownership also depends on the run topology. type ManagedLifecycleDriver interface { diff --git a/pkg/options/resolve.go b/pkg/options/resolve.go index 9a56470012..3883c4dc00 100644 --- a/pkg/options/resolve.go +++ b/pkg/options/resolve.go @@ -337,6 +337,7 @@ func resolveAgentKubernetesConfig( k8s.KubernetesConfig = resolver.ResolveDefaultValue(k8s.KubernetesConfig, options) k8s.KubernetesNamespace = resolver.ResolveDefaultValue(k8s.KubernetesNamespace, options) k8s.Architecture = resolver.ResolveDefaultValue(k8s.Architecture, options) + k8s.ImageConfigFile = resolver.ResolveDefaultValue(k8s.ImageConfigFile, options) k8s.InactivityTimeout = resolver.ResolveDefaultValue(k8s.InactivityTimeout, options) k8s.StorageClass = resolver.ResolveDefaultValue(k8s.StorageClass, options) k8s.PvcAccessMode = resolver.ResolveDefaultValue(k8s.PvcAccessMode, options) diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 5dbffebf00..e86217ea8d 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -452,7 +452,8 @@ type ProviderKubernetesDriverConfig struct { ClusterRole string `json:"clusterRole,omitempty"` ServiceAccount string `json:"serviceAccount,omitempty"` - Architecture string `json:"architecture,omitempty"` + Architecture string `json:"architecture,omitempty"` + ImageConfigFile string `json:"imageConfigFile,omitempty"` // InactivityTimeout is a deprecated alias of Agent.ContainerTimeout, used only // when the canonical field is empty. InactivityTimeout string `json:"inactivityTimeout,omitempty"` diff --git a/providers/kubernetes/provider.yaml b/providers/kubernetes/provider.yaml index a4873121fe..29d863ced5 100644 --- a/providers/kubernetes/provider.yaml +++ b/providers/kubernetes/provider.yaml @@ -31,6 +31,7 @@ optionGroups: - AGENT_SECURITY_CONTEXT - AGENT_INSTALL_PATH - KUBERNETES_USER_NAMESPACES + - IMAGE_CONFIG_FILE name: "Advanced Options" options: DISK_SIZE: @@ -117,6 +118,9 @@ options: ARCHITECTURE: description: The cpu architecture to use for the workspace pod, e.g. amd64, arm64, etc. type: string + IMAGE_CONFIG_FILE: + description: Absolute local path to a version 1 image config file for a digest-pinned prebuilt image without Features. Requires matching ARCHITECTURE. + type: string agent: containerInactivityTimeout: ${INACTIVITY_TIMEOUT} local: true @@ -135,6 +139,7 @@ agent: clusterRole: ${CLUSTER_ROLE} serviceAccount: ${SERVICE_ACCOUNT} architecture: ${ARCHITECTURE} + imageConfigFile: ${IMAGE_CONFIG_FILE} storageClass: ${STORAGE_CLASS} pvcAccessMode: ${PVC_ACCESS_MODE} pvcAnnotations: ${PVC_ANNOTATIONS} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx index ef1e91dae9..0a4f5b9d35 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx @@ -53,6 +53,7 @@ The allowed options for the Kubernetes driver are: - **clusterRole**: if defined, Devsy will create a role binding for the given cluster role for the workspace container. This is useful if you need Kubernetes access within the workspace container - **serviceAccount**: if defined, Devsy will use the given service account for the dev container - **architecture**: the CPU architecture to use for the workspace pod, e.g. `amd64`, `arm64`. If empty, Devsy inspects the cluster's nodes to auto-detect it (and errors out if the cluster has mixed architectures). +- **imageConfigFile**: path to a versioned JSON wrapper containing the digest-pinned image reference, platform and OCI config for the devcontainer.json `image`. When set, Devsy reads the user, env, entrypoint, cmd and `devcontainer.metadata` label from the file instead of fetching the image config from the registry on the client. See [Starting workspaces without registry access](#starting-workspaces-without-registry-access). - **inactivityTimeout**: after how much time to automatically stop the pod due to inactivity - **storageClass**: the storage class to use to create the persistent volume claim - **diskSize**: the default size for the persistent volume to use, e.g. `10Gi` @@ -81,6 +82,12 @@ container cannot write the default `/usr/local/bin/devsy`. Devsy builds images inside Kubernetes without Docker through the `dockerless` build path (`agent.dockerless.disabled` / `agent.dockerless.image`). +### Starting workspaces without registry access + +The Kubernetes driver can use a local image configuration file for a prebuilt `image` in `devcontainer.json`. Devsy reads the user, environment, entrypoint, command and `devcontainer.metadata` label from this file instead of requesting image configuration from the registry on the client. See [Kubernetes registry access](../managing-providers/kubernetes-registry-access.mdx) for the file format, setup and limitations. + +When a driver supports supplied image details, its `ImageDetails` capability method accepts an `ImageDetailsRequest` with `ImageName` and `UnsupportedReason`, then returns the validated `ImageDetails` or an error. The result is `nil, nil` only when the option is unset. Configured input must be validated for the requested image before use. Unsupported combinations and invalid files return errors. They must not fall back to registry inspection. + ### Example Kubernetes Provider Example Kubernetes provider that uses local kubectl to run a workspace in the current kube context: diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/kubernetes-registry-access.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/kubernetes-registry-access.mdx new file mode 100644 index 0000000000..bd94fb391b --- /dev/null +++ b/sites/docs-devsy-sh/content/docs/managing-providers/kubernetes-registry-access.mdx @@ -0,0 +1,67 @@ +--- +title: Kubernetes registry access without client credentials +sidebar_label: Kubernetes registry access +--- + +When a Kubernetes workspace uses a prebuilt `image` from a private registry, the cluster may already have permission to pull it through a node identity or a service account `imagePullSecret`. Devsy normally also reads the image configuration on the client to determine the container user, environment, entrypoint, command and `devcontainer.metadata`. The Kubernetes `IMAGE_CONFIG_FILE` option supplies those details locally, so the client does not need registry read credentials for this inspection. + +This option applies to a digest-pinned `image` with no Features. It does not support Dockerfile builds, Compose, recovery, prebuilds or workspace image builds. The file is trusted input. Its digest association prevents accidental mismatch when used correctly, but does not prove that the file contents came from the image. + +## Create the image details file + +Use a fully qualified digest reference and export the config for the selected Linux platform. Convert the OCI config to Devsy's versioned wrapper: + +```sh +IMAGE='registry.example.com/team/dev-image@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' +crane config --platform linux/amd64 "$IMAGE" | jq --arg image "$IMAGE" '{schemaVersion: 1, image: $image, os: "linux", architecture: "amd64", config: (.config | {User: (.User // ""), Env: (.Env // []), Labels: (.Labels // {}), Entrypoint: (.Entrypoint // []), Cmd: (.Cmd // [])})}' > image-details.json +``` + +The complete wrapper has this shape: + +```json +{ + "schemaVersion": 1, + "image": "registry.example.com/team/dev-image@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "os": "linux", + "architecture": "amd64", + "variant": "", + "config": { + "User": "1000:1000", + "Env": ["PATH=/usr/local/bin:/usr/bin"], + "Labels": {"devcontainer.metadata": "[]"}, + "Entrypoint": ["/usr/local/bin/start"], + "Cmd": ["serve"] + } +} +``` + +`variant` is optional. The file must be a regular UTF-8 file no larger than 1 MiB. It must contain one version 1 wrapper, a fully qualified digest reference matching the requested image, a Linux platform matching `ARCHITECTURE`, and a config object with valid fields. Missing or invalid input is a hard error, with no registry inspection fallback. + +Keep the image reference in `devcontainer.json` pinned to the same digest. Generate a separate file for each platform when publishing a multi-platform image. + +`ARCHITECTURE` selects the config platform and avoids listing cluster nodes. It does not schedule a pod onto a matching node. Use `NODE_SELECTOR` as well when the cluster has mixed architectures. + +Publish or distribute the image and its details file as separate artifacts. Importing the details file into Devsy does not upload it alongside the image or send it to the cluster. + +## Configure the Kubernetes provider + +The registry pull authorization must already be available to Kubernetes. Configure the workspace service account or pod template with the admin-managed `imagePullSecrets`, or use the node's registry identity. The Devsy client still needs its normal namespace permissions and access to the daemon-configuration Secrets it uses. It does not need permission to read the registry pull Secret. + +For cluster-managed pull credentials without client Docker credential reads, disable both client credential forwarding and Devsy pull-secret creation: + +```sh +devsy context set [context-name] --option SSH_INJECT_DOCKER_CREDENTIALS=false +devsy provider set kubernetes --option KUBERNETES_PULL_SECRETS_ENABLED=false +``` + +Set the file and architecture per workspace, using a stable absolute path: + +```sh +devsy workspace up ./project --provider kubernetes --ide none \ + --provider-option IMAGE_CONFIG_FILE="$PWD/image-details.json" \ + --provider-option ARCHITECTURE=amd64 +``` + +In Devsy Desktop, set `IMAGE_CONFIG_FILE` to an absolute path on the machine running Desktop. Keep that file at the same path while using the provider. Moving it does not update an existing workspace's saved image configuration. A normal `up` validates the configured file but keeps the workspace's saved launch recipe. Changing image metadata takes effect when you explicitly recreate the workspace with the updated file. A local path change alone does not replace a running pod. If you move to another machine or workspace host, copy the file there and configure its new absolute local path. Once a workspace exists, the file is not needed for status, logs, stop or delete operations. + +If cluster pull authorization is missing, Kubernetes reports an image pull failure such as `ImagePullBackOff`. Configure the service account, pod template or node identity that performs the pull. Adding credentials to the client's Docker config does not grant the cluster permission to pull the image. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx index a9f0702810..6b839494db 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx @@ -112,6 +112,10 @@ Change one later: devsy provider set --option KEY=VALUE ``` +## Kubernetes registry access + +For digest-pinned prebuilt images on Kubernetes, [configure cluster registry access without client credentials](./kubernetes-registry-access.mdx). + ## Docker provider Devsy follows the Docker CLI's rules for contexts and daemon endpoints. It keeps your existing Docker `config.json` and contexts, so contexts from Docker Desktop, OrbStack, or a remote endpoint keep working. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/meta.json b/sites/docs-devsy-sh/content/docs/managing-providers/meta.json index 775e246cff..e26c174546 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/meta.json +++ b/sites/docs-devsy-sh/content/docs/managing-providers/meta.json @@ -1,4 +1,4 @@ { "title": "Managing your Providers", - "pages": ["what-are-providers", "manage-providers"] + "pages": ["what-are-providers", "manage-providers", "kubernetes-registry-access"] }