From f81f6c75223a6a20d5cc319974157cf61fa57d51 Mon Sep 17 00:00:00 2001 From: wsmoak Date: Thu, 29 Oct 2026 19:17:12 -0400 Subject: [PATCH 01/10] feat(kubernetes): read prebuilt image config from IMAGE_DETAILS file Add an IMAGE_DETAILS option to the kubernetes provider. When set, the image config of the devcontainer.json image (user, env, entrypoint, cmd and the devcontainer.metadata label) is read from a local JSON file instead of the registry, so the client needs no registry credentials when the cluster can pull the image itself. The file can be an OCI image config (e.g. from `crane config`), a docker image inspect object, or config.ImageDetails JSON. A missing or invalid file is an error; there is no fallback to the registry. Co-Authored-By: Claude Opus 5.5 --- pkg/devcontainer/build.go | 18 +++- pkg/devcontainer/image_details_test.go | 89 +++++++++++++++++ pkg/driver/kubernetes/driver.go | 1 + pkg/driver/kubernetes/image_details.go | 63 ++++++++++++ pkg/driver/kubernetes/image_details_test.go | 98 +++++++++++++++++++ pkg/driver/types.go | 9 ++ pkg/options/resolve.go | 1 + pkg/provider/provider.go | 3 + providers/kubernetes/provider.yaml | 5 + .../docs/developing-providers/driver.mdx | 18 ++++ 10 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 pkg/devcontainer/image_details_test.go create mode 100644 pkg/driver/kubernetes/image_details.go create mode 100644 pkg/driver/kubernetes/image_details_test.go diff --git a/pkg/devcontainer/build.go b/pkg/devcontainer/build.go index 08e1a512ee..0c15922ba2 100644 --- a/pkg/devcontainer/build.go +++ b/pkg/devcontainer/build.go @@ -252,13 +252,29 @@ func (r *runner) getImageBuildInfoFromImage( substitutionContext *config.SubstitutionContext, imageName string, ) (*config.ImageBuildInfo, error) { - imageDetails, err := r.inspectImage(ctx, imageName) + imageDetails, err := r.configuredImageDetails(imageName) if err != nil { return nil, err } + if imageDetails == nil { + imageDetails, err = r.inspectImage(ctx, imageName) + if err != nil { + return nil, err + } + } return imageBuildInfoFromDetails(substitutionContext, imageDetails) } +// configuredImageDetails returns image details the driver was configured with, +// or nil when the image has to be inspected. +func (r *runner) configuredImageDetails(imageName string) (*config.ImageDetails, error) { + d, ok := r.driver.(driver.ImageDetailsDriver) + if !ok { + return nil, nil + } + return d.ImageDetails(imageName) +} + func imageBuildInfoFromDetails( substitutionContext *config.SubstitutionContext, imageDetails *config.ImageDetails, diff --git a/pkg/devcontainer/image_details_test.go b/pkg/devcontainer/image_details_test.go new file mode 100644 index 0000000000..14dcc23161 --- /dev/null +++ b/pkg/devcontainer/image_details_test.go @@ -0,0 +1,89 @@ +package devcontainer + +import ( + "context" + "errors" + "testing" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/stretchr/testify/suite" +) + +const testImageUser = "vscode" + +// imageDetailsDriver supplies image details and fails TargetArchitecture, so +// any fallback to the registry inspect makes the test fail. +type imageDetailsDriver struct { + failingArchitectureDriver + details *config.ImageDetails + err error +} + +func (d *imageDetailsDriver) ImageDetails(string) (*config.ImageDetails, error) { + return d.details, d.err +} + +type ImageDetailsSuite struct { + suite.Suite +} + +func TestImageDetailsSuite(t *testing.T) { + suite.Run(t, new(ImageDetailsSuite)) +} + +func (s *ImageDetailsSuite) TestUsesConfiguredDetailsWithoutInspect() { + d := &imageDetailsDriver{ + failingArchitectureDriver: failingArchitectureDriver{err: errors.New("inspect called")}, + details: &config.ImageDetails{ + ID: "registry.example.com/team/image@sha256:abc", + Config: config.ImageDetailsConfig{ + User: testImageUser, + Labels: map[string]string{ + "devcontainer.metadata": `[{"remoteUser":"dev","containerEnv":{"FOO":"bar"}}]`, + }, + }, + }, + } + r := newTestRunner(d) + + info, err := r.getImageBuildInfoFromImage( + context.Background(), + &config.SubstitutionContext{}, + "registry.example.com/team/image@sha256:abc", + ) + s.Require().NoError(err) + s.Equal(testImageUser, info.User) + s.Same(d.details, info.ImageDetails) + s.Require().Len(info.Metadata.Config, 1) + s.Equal("dev", info.Metadata.Config[0].RemoteUser) + s.Equal("bar", info.Metadata.Config[0].ContainerEnv["FOO"]) +} + +func (s *ImageDetailsSuite) TestReturnsConfiguredDetailsError() { + d := &imageDetailsDriver{ + failingArchitectureDriver: failingArchitectureDriver{err: errors.New("inspect called")}, + err: errors.New("bad image details"), + } + r := newTestRunner(d) + + _, err := r.getImageBuildInfoFromImage( + context.Background(), + &config.SubstitutionContext{}, + "image", + ) + s.Require().EqualError(err, "bad image details") +} + +func (s *ImageDetailsSuite) TestFallsBackToInspectWhenNotConfigured() { + d := &imageDetailsDriver{ + failingArchitectureDriver: failingArchitectureDriver{err: errors.New("inspect called")}, + } + r := newTestRunner(d) + + _, err := r.getImageBuildInfoFromImage( + context.Background(), + &config.SubstitutionContext{}, + "image", + ) + s.Require().ErrorContains(err, "inspect called") +} diff --git a/pkg/driver/kubernetes/driver.go b/pkg/driver/kubernetes/driver.go index 3cbb0e2bb0..045514940e 100644 --- a/pkg/driver/kubernetes/driver.go +++ b/pkg/driver/kubernetes/driver.go @@ -56,6 +56,7 @@ var ( _ driver.ReprovisioningDriver = (*KubernetesDriver)(nil) _ driver.Preflighter = (*KubernetesDriver)(nil) + _ driver.ImageDetailsDriver = (*KubernetesDriver)(nil) ) // Preflight verifies the cluster's API server is reachable. A remote cluster diff --git a/pkg/driver/kubernetes/image_details.go b/pkg/driver/kubernetes/image_details.go new file mode 100644 index 0000000000..d445abc2a4 --- /dev/null +++ b/pkg/driver/kubernetes/image_details.go @@ -0,0 +1,63 @@ +package kubernetes + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/log" +) + +// ImageDetails reads the image config from the file set in the IMAGE_DETAILS +// option. It returns nil when the option is empty, so callers fall back to +// reading the config from the registry. +func (k *KubernetesDriver) ImageDetails(imageName string) (*config.ImageDetails, error) { + if k.options.ImageDetails == "" { + return nil, nil + } + + log.Debugf("Reading image details for %q from %q", imageName, k.options.ImageDetails) + return loadImageDetails(k.options.ImageDetails, imageName) +} + +// loadImageDetails parses an image details file. It accepts the JSON of +// config.ImageDetails, an OCI image config (e.g. from `crane config`) or a +// single `docker image inspect` object, since field names match +// case-insensitively and unknown fields are ignored. +func loadImageDetails(path, imageName string) (*config.ImageDetails, error) { + // #nosec G304 -- path comes from the operator-controlled IMAGE_DETAILS provider option + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read image details file %q: %w", path, err) + } + + data = bytes.TrimSpace(data) + if len(data) == 0 || data[0] != '{' { + return nil, fmt.Errorf("image details file %q must contain a JSON object", path) + } + + details := &config.ImageDetails{} + if err := json.Unmarshal(data, details); err != nil { + return nil, fmt.Errorf("parse image details file %q: %w", path, err) + } + + if isEmptyImageConfig(details.Config) { + return nil, fmt.Errorf( + "image details file %q has no image config (User, Env, Labels, Entrypoint or Cmd)", + path, + ) + } + + if details.ID == "" { + details.ID = imageName + } + + return details, nil +} + +func isEmptyImageConfig(c config.ImageDetailsConfig) bool { + return c.User == "" && len(c.Env) == 0 && len(c.Labels) == 0 && + len(c.Entrypoint) == 0 && len(c.Cmd) == 0 +} diff --git a/pkg/driver/kubernetes/image_details_test.go b/pkg/driver/kubernetes/image_details_test.go new file mode 100644 index 0000000000..2e5b25d326 --- /dev/null +++ b/pkg/driver/kubernetes/image_details_test.go @@ -0,0 +1,98 @@ +package kubernetes + +import ( + "os" + "path/filepath" + "testing" + + provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/suite" +) + +const testImage = "registry.example.com/team/image@sha256:abc" + +type ImageDetailsSuite struct { + suite.Suite +} + +func TestImageDetailsSuite(t *testing.T) { + suite.Run(t, new(ImageDetailsSuite)) +} + +func (s *ImageDetailsSuite) TestNotConfigured() { + details, err := s.driver("").ImageDetails(testImage) + s.Require().NoError(err) + s.Nil(details) +} + +func (s *ImageDetailsSuite) TestImageDetailsJSON() { + path := s.writeFile(`{ + "ID": "custom-id", + "Config": { + "User": "vscode", + "Env": ["PATH=/usr/bin"], + "Labels": {"devcontainer.metadata": "[]"}, + "Entrypoint": ["/entrypoint.sh"], + "Cmd": ["sleep", "infinity"] + } + }`) + + details, err := s.driver(path).ImageDetails(testImage) + s.Require().NoError(err) + s.Equal("custom-id", details.ID) + s.Equal("vscode", details.Config.User) + s.Equal([]string{"PATH=/usr/bin"}, details.Config.Env) + s.Equal(map[string]string{"devcontainer.metadata": "[]"}, details.Config.Labels) + s.Equal([]string{"/entrypoint.sh"}, details.Config.Entrypoint) + s.Equal([]string{"sleep", "infinity"}, details.Config.Cmd) +} + +func (s *ImageDetailsSuite) TestOCIImageConfig() { + // Output shape of `crane config `. + path := s.writeFile(`{ + "architecture": "amd64", + "os": "linux", + "config": {"User": "vscode", "Env": ["PATH=/usr/bin"]}, + "rootfs": {"type": "layers", "diff_ids": []} + }`) + + details, err := s.driver(path).ImageDetails(testImage) + s.Require().NoError(err) + s.Equal(testImage, details.ID) + s.Equal("amd64", details.Architecture) + s.Equal("linux", details.OS) + s.Equal("vscode", details.Config.User) +} + +func (s *ImageDetailsSuite) TestMissingFile() { + path := filepath.Join(s.T().TempDir(), "missing.json") + _, err := s.driver(path).ImageDetails(testImage) + s.Require().ErrorContains(err, "read image details file") +} + +func (s *ImageDetailsSuite) TestInvalidContent() { + tests := map[string]string{ + "empty": "", + "array": `[{"Config": {"User": "vscode"}}]`, + "bad json": `{"Config": `, + "no config": `{"schemaVersion": 2, "layers": []}`, + } + for name, content := range tests { + s.Run(name, func() { + _, err := s.driver(s.writeFile(content)).ImageDetails(testImage) + s.Error(err) + }) + } +} + +func (s *ImageDetailsSuite) writeFile(content string) string { + path := filepath.Join(s.T().TempDir(), "image-details.json") + s.Require().NoError(os.WriteFile(path, []byte(content), 0o600)) + return path +} + +func (s *ImageDetailsSuite) driver(path string) *KubernetesDriver { + return &KubernetesDriver{ + options: &provider2.ProviderKubernetesDriverConfig{ImageDetails: path}, + } +} diff --git a/pkg/driver/types.go b/pkg/driver/types.go index b961acc283..3f0dcbb30f 100644 --- a/pkg/driver/types.go +++ b/pkg/driver/types.go @@ -54,6 +54,15 @@ type WorkspaceChowner interface { RequiresWorkspaceChown() bool } +// ImageDetailsDriver is a capability interface for drivers that can supply the +// image config of a prebuilt image themselves, so the client does not need to +// read it from the registry. +type ImageDetailsDriver interface { + // ImageDetails returns the configured details for imageName, or nil when + // none are configured. + ImageDetails(imageName string) (*config.ImageDetails, error) +} + // ManagedLifecycleDriver identifies drivers that can run the container's main // process as the Devsy daemon. Actual ownership also depends on the run topology. type ManagedLifecycleDriver interface { diff --git a/pkg/options/resolve.go b/pkg/options/resolve.go index 9a56470012..90d672f982 100644 --- a/pkg/options/resolve.go +++ b/pkg/options/resolve.go @@ -337,6 +337,7 @@ func resolveAgentKubernetesConfig( k8s.KubernetesConfig = resolver.ResolveDefaultValue(k8s.KubernetesConfig, options) k8s.KubernetesNamespace = resolver.ResolveDefaultValue(k8s.KubernetesNamespace, options) k8s.Architecture = resolver.ResolveDefaultValue(k8s.Architecture, options) + k8s.ImageDetails = resolver.ResolveDefaultValue(k8s.ImageDetails, options) k8s.InactivityTimeout = resolver.ResolveDefaultValue(k8s.InactivityTimeout, options) k8s.StorageClass = resolver.ResolveDefaultValue(k8s.StorageClass, options) k8s.PvcAccessMode = resolver.ResolveDefaultValue(k8s.PvcAccessMode, options) diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 5dbffebf00..844ed0f70a 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -453,6 +453,9 @@ type ProviderKubernetesDriverConfig struct { ServiceAccount string `json:"serviceAccount,omitempty"` Architecture string `json:"architecture,omitempty"` + // ImageDetails is a path to a JSON file with the image config of a prebuilt + // image, used instead of reading it from the registry on the client. + ImageDetails string `json:"imageDetails,omitempty"` // InactivityTimeout is a deprecated alias of Agent.ContainerTimeout, used only // when the canonical field is empty. InactivityTimeout string `json:"inactivityTimeout,omitempty"` diff --git a/providers/kubernetes/provider.yaml b/providers/kubernetes/provider.yaml index a4873121fe..870e7a7559 100644 --- a/providers/kubernetes/provider.yaml +++ b/providers/kubernetes/provider.yaml @@ -31,6 +31,7 @@ optionGroups: - AGENT_SECURITY_CONTEXT - AGENT_INSTALL_PATH - KUBERNETES_USER_NAMESPACES + - IMAGE_DETAILS name: "Advanced Options" options: DISK_SIZE: @@ -117,6 +118,9 @@ options: ARCHITECTURE: description: The cpu architecture to use for the workspace pod, e.g. amd64, arm64, etc. type: string + IMAGE_DETAILS: + description: Path to a JSON file with the image config of the devcontainer's prebuilt image (e.g. the output of `crane config `). If set, Devsy reads the user, env, entrypoint, cmd and devcontainer.metadata label from this file instead of fetching the image config from the registry, so the machine running Devsy needs no registry credentials. Devsy fails if the file is missing or invalid. Pin the image by digest so the file matches the image the cluster pulls. Applies to the devcontainer.json image only; Dockerfile builds still read their base image from the registry. + type: string agent: containerInactivityTimeout: ${INACTIVITY_TIMEOUT} local: true @@ -135,6 +139,7 @@ agent: clusterRole: ${CLUSTER_ROLE} serviceAccount: ${SERVICE_ACCOUNT} architecture: ${ARCHITECTURE} + imageDetails: ${IMAGE_DETAILS} storageClass: ${STORAGE_CLASS} pvcAccessMode: ${PVC_ACCESS_MODE} pvcAnnotations: ${PVC_ANNOTATIONS} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx index ef1e91dae9..f0b6488b48 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx @@ -53,6 +53,7 @@ The allowed options for the Kubernetes driver are: - **clusterRole**: if defined, Devsy will create a role binding for the given cluster role for the workspace container. This is useful if you need Kubernetes access within the workspace container - **serviceAccount**: if defined, Devsy will use the given service account for the dev container - **architecture**: the CPU architecture to use for the workspace pod, e.g. `amd64`, `arm64`. If empty, Devsy inspects the cluster's nodes to auto-detect it (and errors out if the cluster has mixed architectures). +- **imageDetails**: path to a JSON file with the image config of the devcontainer.json `image` (for example the output of `crane config `). When set, Devsy reads the user, env, entrypoint, cmd and `devcontainer.metadata` label from the file instead of fetching the image config from the registry on the client. See [Starting workspaces without registry access](#starting-workspaces-without-registry-access). - **inactivityTimeout**: after how much time to automatically stop the pod due to inactivity - **storageClass**: the storage class to use to create the persistent volume claim - **diskSize**: the default size for the persistent volume to use, e.g. `10Gi` @@ -81,6 +82,23 @@ container cannot write the default `/usr/local/bin/devsy`. Devsy builds images inside Kubernetes without Docker through the `dockerless` build path (`agent.dockerless.disabled` / `agent.dockerless.image`). +### Starting workspaces without registry access + +For a devcontainer.json with a prebuilt `image`, Devsy reads the image config (user, env, entrypoint, cmd and the `devcontainer.metadata` label) from the registry on the machine that runs Devsy. In clusters that already have their own registry credentials, the person starting the workspace would otherwise need registry read access and a local `docker login` just for this step. + +To skip it, produce the image config next to the image, for example in CI, and point the `IMAGE_DETAILS` option at the file: + +```bash +crane config registry.example.com/team/image@sha256: > image-details.json +devsy provider set kubernetes -o IMAGE_DETAILS=$PWD/image-details.json +``` + +The file can be an OCI image config (as printed by `crane config`), a single `docker image inspect` object, or the JSON of Devsy's image details. Devsy never contacts the registry for this step when the option is set, and it fails if the file is missing or invalid. Notes: + +- Pin the image by digest in devcontainer.json, so the file always matches the image the cluster pulls. +- This applies to the devcontainer.json `image`. Dockerfile builds still read their base image from the registry, and `features` are built from the image as usual. +- Also set `ARCHITECTURE` (for example `amd64`). Otherwise Devsy lists the cluster's nodes to detect it, which needs cluster-scoped permissions. + ### Example Kubernetes Provider Example Kubernetes provider that uses local kubectl to run a workspace in the current kube context: From a06eb6951c854502f2b4b034704a4eac44c85ca5 Mon Sep 17 00:00:00 2001 From: wsmoak Date: Thu, 29 Oct 2026 19:24:06 -0400 Subject: [PATCH 02/10] test(e2e): cover kubernetes IMAGE_DETAILS option Add specs under the up-provider-kubernetes suite: one checks that the devcontainer.metadata label from the IMAGE_DETAILS file is applied in the workspace, and one checks that a missing file fails up with a clear error. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DyaWKyMtqZa9xiyeW71JuS --- .../up/provider_kubernetes_image_details.go | 80 +++++++++++++++++++ hack/ci/suites.yaml | 2 + 2 files changed, 82 insertions(+) create mode 100644 e2e/tests/up/provider_kubernetes_image_details.go diff --git a/e2e/tests/up/provider_kubernetes_image_details.go b/e2e/tests/up/provider_kubernetes_image_details.go new file mode 100644 index 0000000000..99786e4664 --- /dev/null +++ b/e2e/tests/up/provider_kubernetes_image_details.go @@ -0,0 +1,80 @@ +package up + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +const ( + imageDetailsMarkerEnv = "IMAGE_DETAILS_MARKER" + imageDetailsMarkerValue = "from-image-details-file" +) + +// writeImageDetailsFile writes an OCI-style image config whose +// devcontainer.metadata label sets a marker env var the real image lacks, so +// seeing the marker in the workspace proves the file was used instead of the +// registry. +func writeImageDetailsFile() string { + metadata, err := json.Marshal([]map[string]any{ + {"containerEnv": map[string]string{imageDetailsMarkerEnv: imageDetailsMarkerValue}}, + }) + framework.ExpectNoError(err) + + content, err := json.Marshal(map[string]any{ + "config": map[string]any{ + "Env": []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, + "Labels": map[string]string{"devcontainer.metadata": string(metadata)}, + }, + }) + framework.ExpectNoError(err) + + path := filepath.Join(ginkgo.GinkgoT().TempDir(), "image-details.json") + framework.ExpectNoError(os.WriteFile(path, content, 0o600)) + return path +} + +var _ = ginkgo.Describe( + "kubernetes IMAGE_DETAILS option", + ginkgo.Label("up-provider-kubernetes", "up-provider-kubernetes-image-details"), + func() { + var initialDir string + + ginkgo.BeforeEach(func() { + var err error + initialDir, err = os.Getwd() + framework.ExpectNoError(err) + }) + + ginkgo.It( + "applies the image config from the IMAGE_DETAILS file", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "IMAGE_DETAILS="+writeImageDetailsFile()) + + framework.ExpectNoError(e.up(ctx)) + gomega.Expect(e.ssh(ctx, "echo $"+imageDetailsMarkerEnv)). + To(gomega.Equal(imageDetailsMarkerValue)) + }, + ginkgo.SpecTimeout(framework.TimeoutShort()), + ) + + ginkgo.It( + "fails clearly when the IMAGE_DETAILS file is missing", + func(ctx context.Context) { + missing := filepath.Join(ginkgo.GinkgoT().TempDir(), "missing.json") + e := newIdleEnv(ctx, initialDir, "IMAGE_DETAILS="+missing) + + _, stderr, err := e.f.ExecCommandCapture(ctx, + []string{cmdWorkspace, "up", e.dir, flagIDEOption, kubernetesIgnoreIDE}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stderr).To(gomega.ContainSubstring("read image details file")) + }, + ginkgo.SpecTimeout(framework.TimeoutShort()), + ) + }, +) diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml index 169cc47b1d..2087ccb32b 100644 --- a/hack/ci/suites.yaml +++ b/hack/ci/suites.yaml @@ -172,6 +172,8 @@ e2e: reason: sub-label that always accompanies the up-provider-kubernetes suite label - label: up-provider-kubernetes-ignore reason: sub-label that always accompanies the up-provider-kubernetes suite label + - label: up-provider-kubernetes-image-details + reason: sub-label that always accompanies the up-provider-kubernetes suite label suites: # --- unprivileged --- From c294380fa2de48464b375268143532025ddbc063 Mon Sep 17 00:00:00 2001 From: wsmoak Date: Thu, 29 Oct 2026 20:53:18 -0400 Subject: [PATCH 03/10] refactor(kubernetes): rename IMAGE_DETAILS option to IMAGE_CONFIG_FILE IMAGE_DETAILS came from the Go type name. IMAGE_CONFIG_FILE says what to pass: a path to the image config file. The driver field becomes imageConfigFile, error messages say "image config file", and the e2e spec and CI sub-label are renamed to match. Co-Authored-By: Claude Opus 5.5 --- ... provider_kubernetes_image_config_file.go} | 30 +++++++++---------- hack/ci/suites.yaml | 2 +- pkg/driver/kubernetes/image_details.go | 20 ++++++------- pkg/driver/kubernetes/image_details_test.go | 6 ++-- pkg/options/resolve.go | 2 +- pkg/provider/provider.go | 4 +-- providers/kubernetes/provider.yaml | 6 ++-- .../docs/developing-providers/driver.mdx | 8 ++--- 8 files changed, 39 insertions(+), 39 deletions(-) rename e2e/tests/up/{provider_kubernetes_image_details.go => provider_kubernetes_image_config_file.go} (65%) diff --git a/e2e/tests/up/provider_kubernetes_image_details.go b/e2e/tests/up/provider_kubernetes_image_config_file.go similarity index 65% rename from e2e/tests/up/provider_kubernetes_image_details.go rename to e2e/tests/up/provider_kubernetes_image_config_file.go index 99786e4664..8fea346656 100644 --- a/e2e/tests/up/provider_kubernetes_image_details.go +++ b/e2e/tests/up/provider_kubernetes_image_config_file.go @@ -12,17 +12,17 @@ import ( ) const ( - imageDetailsMarkerEnv = "IMAGE_DETAILS_MARKER" - imageDetailsMarkerValue = "from-image-details-file" + imageConfigFileMarkerEnv = "IMAGE_CONFIG_FILE_MARKER" + imageConfigFileMarkerValue = "from-image-config-file" ) -// writeImageDetailsFile writes an OCI-style image config whose +// writeImageConfigFile writes an OCI-style image config whose // devcontainer.metadata label sets a marker env var the real image lacks, so // seeing the marker in the workspace proves the file was used instead of the // registry. -func writeImageDetailsFile() string { +func writeImageConfigFile() string { metadata, err := json.Marshal([]map[string]any{ - {"containerEnv": map[string]string{imageDetailsMarkerEnv: imageDetailsMarkerValue}}, + {"containerEnv": map[string]string{imageConfigFileMarkerEnv: imageConfigFileMarkerValue}}, }) framework.ExpectNoError(err) @@ -34,14 +34,14 @@ func writeImageDetailsFile() string { }) framework.ExpectNoError(err) - path := filepath.Join(ginkgo.GinkgoT().TempDir(), "image-details.json") + path := filepath.Join(ginkgo.GinkgoT().TempDir(), "image-config.json") framework.ExpectNoError(os.WriteFile(path, content, 0o600)) return path } var _ = ginkgo.Describe( - "kubernetes IMAGE_DETAILS option", - ginkgo.Label("up-provider-kubernetes", "up-provider-kubernetes-image-details"), + "kubernetes IMAGE_CONFIG_FILE option", + ginkgo.Label("up-provider-kubernetes", "up-provider-kubernetes-image-config-file"), func() { var initialDir string @@ -52,27 +52,27 @@ var _ = ginkgo.Describe( }) ginkgo.It( - "applies the image config from the IMAGE_DETAILS file", + "applies the image config from the IMAGE_CONFIG_FILE file", func(ctx context.Context) { - e := newIdleEnv(ctx, initialDir, "IMAGE_DETAILS="+writeImageDetailsFile()) + e := newIdleEnv(ctx, initialDir, "IMAGE_CONFIG_FILE="+writeImageConfigFile()) framework.ExpectNoError(e.up(ctx)) - gomega.Expect(e.ssh(ctx, "echo $"+imageDetailsMarkerEnv)). - To(gomega.Equal(imageDetailsMarkerValue)) + gomega.Expect(e.ssh(ctx, "echo $"+imageConfigFileMarkerEnv)). + To(gomega.Equal(imageConfigFileMarkerValue)) }, ginkgo.SpecTimeout(framework.TimeoutShort()), ) ginkgo.It( - "fails clearly when the IMAGE_DETAILS file is missing", + "fails clearly when the IMAGE_CONFIG_FILE file is missing", func(ctx context.Context) { missing := filepath.Join(ginkgo.GinkgoT().TempDir(), "missing.json") - e := newIdleEnv(ctx, initialDir, "IMAGE_DETAILS="+missing) + e := newIdleEnv(ctx, initialDir, "IMAGE_CONFIG_FILE="+missing) _, stderr, err := e.f.ExecCommandCapture(ctx, []string{cmdWorkspace, "up", e.dir, flagIDEOption, kubernetesIgnoreIDE}) gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(stderr).To(gomega.ContainSubstring("read image details file")) + gomega.Expect(stderr).To(gomega.ContainSubstring("read image config file")) }, ginkgo.SpecTimeout(framework.TimeoutShort()), ) diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml index 2087ccb32b..f85cef8525 100644 --- a/hack/ci/suites.yaml +++ b/hack/ci/suites.yaml @@ -172,7 +172,7 @@ e2e: reason: sub-label that always accompanies the up-provider-kubernetes suite label - label: up-provider-kubernetes-ignore reason: sub-label that always accompanies the up-provider-kubernetes suite label - - label: up-provider-kubernetes-image-details + - label: up-provider-kubernetes-image-config-file reason: sub-label that always accompanies the up-provider-kubernetes suite label suites: diff --git a/pkg/driver/kubernetes/image_details.go b/pkg/driver/kubernetes/image_details.go index d445abc2a4..322f9e2e75 100644 --- a/pkg/driver/kubernetes/image_details.go +++ b/pkg/driver/kubernetes/image_details.go @@ -10,42 +10,42 @@ import ( "github.com/devsy-org/devsy/pkg/log" ) -// ImageDetails reads the image config from the file set in the IMAGE_DETAILS +// ImageDetails reads the image config from the file set in the IMAGE_CONFIG_FILE // option. It returns nil when the option is empty, so callers fall back to // reading the config from the registry. func (k *KubernetesDriver) ImageDetails(imageName string) (*config.ImageDetails, error) { - if k.options.ImageDetails == "" { + if k.options.ImageConfigFile == "" { return nil, nil } - log.Debugf("Reading image details for %q from %q", imageName, k.options.ImageDetails) - return loadImageDetails(k.options.ImageDetails, imageName) + log.Debugf("Reading image config for %q from %q", imageName, k.options.ImageConfigFile) + return loadImageDetails(k.options.ImageConfigFile, imageName) } -// loadImageDetails parses an image details file. It accepts the JSON of +// loadImageDetails parses an image config file. It accepts the JSON of // config.ImageDetails, an OCI image config (e.g. from `crane config`) or a // single `docker image inspect` object, since field names match // case-insensitively and unknown fields are ignored. func loadImageDetails(path, imageName string) (*config.ImageDetails, error) { - // #nosec G304 -- path comes from the operator-controlled IMAGE_DETAILS provider option + // #nosec G304 -- path comes from the operator-controlled IMAGE_CONFIG_FILE provider option data, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("read image details file %q: %w", path, err) + return nil, fmt.Errorf("read image config file %q: %w", path, err) } data = bytes.TrimSpace(data) if len(data) == 0 || data[0] != '{' { - return nil, fmt.Errorf("image details file %q must contain a JSON object", path) + return nil, fmt.Errorf("image config file %q must contain a JSON object", path) } details := &config.ImageDetails{} if err := json.Unmarshal(data, details); err != nil { - return nil, fmt.Errorf("parse image details file %q: %w", path, err) + return nil, fmt.Errorf("parse image config file %q: %w", path, err) } if isEmptyImageConfig(details.Config) { return nil, fmt.Errorf( - "image details file %q has no image config (User, Env, Labels, Entrypoint or Cmd)", + "image config file %q has no image config (User, Env, Labels, Entrypoint or Cmd)", path, ) } diff --git a/pkg/driver/kubernetes/image_details_test.go b/pkg/driver/kubernetes/image_details_test.go index 2e5b25d326..be4242a6f5 100644 --- a/pkg/driver/kubernetes/image_details_test.go +++ b/pkg/driver/kubernetes/image_details_test.go @@ -67,7 +67,7 @@ func (s *ImageDetailsSuite) TestOCIImageConfig() { func (s *ImageDetailsSuite) TestMissingFile() { path := filepath.Join(s.T().TempDir(), "missing.json") _, err := s.driver(path).ImageDetails(testImage) - s.Require().ErrorContains(err, "read image details file") + s.Require().ErrorContains(err, "read image config file") } func (s *ImageDetailsSuite) TestInvalidContent() { @@ -86,13 +86,13 @@ func (s *ImageDetailsSuite) TestInvalidContent() { } func (s *ImageDetailsSuite) writeFile(content string) string { - path := filepath.Join(s.T().TempDir(), "image-details.json") + path := filepath.Join(s.T().TempDir(), "image-config.json") s.Require().NoError(os.WriteFile(path, []byte(content), 0o600)) return path } func (s *ImageDetailsSuite) driver(path string) *KubernetesDriver { return &KubernetesDriver{ - options: &provider2.ProviderKubernetesDriverConfig{ImageDetails: path}, + options: &provider2.ProviderKubernetesDriverConfig{ImageConfigFile: path}, } } diff --git a/pkg/options/resolve.go b/pkg/options/resolve.go index 90d672f982..3883c4dc00 100644 --- a/pkg/options/resolve.go +++ b/pkg/options/resolve.go @@ -337,7 +337,7 @@ func resolveAgentKubernetesConfig( k8s.KubernetesConfig = resolver.ResolveDefaultValue(k8s.KubernetesConfig, options) k8s.KubernetesNamespace = resolver.ResolveDefaultValue(k8s.KubernetesNamespace, options) k8s.Architecture = resolver.ResolveDefaultValue(k8s.Architecture, options) - k8s.ImageDetails = resolver.ResolveDefaultValue(k8s.ImageDetails, options) + k8s.ImageConfigFile = resolver.ResolveDefaultValue(k8s.ImageConfigFile, options) k8s.InactivityTimeout = resolver.ResolveDefaultValue(k8s.InactivityTimeout, options) k8s.StorageClass = resolver.ResolveDefaultValue(k8s.StorageClass, options) k8s.PvcAccessMode = resolver.ResolveDefaultValue(k8s.PvcAccessMode, options) diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 844ed0f70a..7b29100079 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -453,9 +453,9 @@ type ProviderKubernetesDriverConfig struct { ServiceAccount string `json:"serviceAccount,omitempty"` Architecture string `json:"architecture,omitempty"` - // ImageDetails is a path to a JSON file with the image config of a prebuilt + // ImageConfigFile is a path to a JSON file with the image config of a prebuilt // image, used instead of reading it from the registry on the client. - ImageDetails string `json:"imageDetails,omitempty"` + ImageConfigFile string `json:"imageConfigFile,omitempty"` // InactivityTimeout is a deprecated alias of Agent.ContainerTimeout, used only // when the canonical field is empty. InactivityTimeout string `json:"inactivityTimeout,omitempty"` diff --git a/providers/kubernetes/provider.yaml b/providers/kubernetes/provider.yaml index 870e7a7559..b61c570610 100644 --- a/providers/kubernetes/provider.yaml +++ b/providers/kubernetes/provider.yaml @@ -31,7 +31,7 @@ optionGroups: - AGENT_SECURITY_CONTEXT - AGENT_INSTALL_PATH - KUBERNETES_USER_NAMESPACES - - IMAGE_DETAILS + - IMAGE_CONFIG_FILE name: "Advanced Options" options: DISK_SIZE: @@ -118,7 +118,7 @@ options: ARCHITECTURE: description: The cpu architecture to use for the workspace pod, e.g. amd64, arm64, etc. type: string - IMAGE_DETAILS: + IMAGE_CONFIG_FILE: description: Path to a JSON file with the image config of the devcontainer's prebuilt image (e.g. the output of `crane config `). If set, Devsy reads the user, env, entrypoint, cmd and devcontainer.metadata label from this file instead of fetching the image config from the registry, so the machine running Devsy needs no registry credentials. Devsy fails if the file is missing or invalid. Pin the image by digest so the file matches the image the cluster pulls. Applies to the devcontainer.json image only; Dockerfile builds still read their base image from the registry. type: string agent: @@ -139,7 +139,7 @@ agent: clusterRole: ${CLUSTER_ROLE} serviceAccount: ${SERVICE_ACCOUNT} architecture: ${ARCHITECTURE} - imageDetails: ${IMAGE_DETAILS} + imageConfigFile: ${IMAGE_CONFIG_FILE} storageClass: ${STORAGE_CLASS} pvcAccessMode: ${PVC_ACCESS_MODE} pvcAnnotations: ${PVC_ANNOTATIONS} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx index f0b6488b48..149b9e282f 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx @@ -53,7 +53,7 @@ The allowed options for the Kubernetes driver are: - **clusterRole**: if defined, Devsy will create a role binding for the given cluster role for the workspace container. This is useful if you need Kubernetes access within the workspace container - **serviceAccount**: if defined, Devsy will use the given service account for the dev container - **architecture**: the CPU architecture to use for the workspace pod, e.g. `amd64`, `arm64`. If empty, Devsy inspects the cluster's nodes to auto-detect it (and errors out if the cluster has mixed architectures). -- **imageDetails**: path to a JSON file with the image config of the devcontainer.json `image` (for example the output of `crane config `). When set, Devsy reads the user, env, entrypoint, cmd and `devcontainer.metadata` label from the file instead of fetching the image config from the registry on the client. See [Starting workspaces without registry access](#starting-workspaces-without-registry-access). +- **imageConfigFile**: path to a JSON file with the image config of the devcontainer.json `image` (for example the output of `crane config `). When set, Devsy reads the user, env, entrypoint, cmd and `devcontainer.metadata` label from the file instead of fetching the image config from the registry on the client. See [Starting workspaces without registry access](#starting-workspaces-without-registry-access). - **inactivityTimeout**: after how much time to automatically stop the pod due to inactivity - **storageClass**: the storage class to use to create the persistent volume claim - **diskSize**: the default size for the persistent volume to use, e.g. `10Gi` @@ -86,11 +86,11 @@ Devsy builds images inside Kubernetes without Docker through the `dockerless` bu For a devcontainer.json with a prebuilt `image`, Devsy reads the image config (user, env, entrypoint, cmd and the `devcontainer.metadata` label) from the registry on the machine that runs Devsy. In clusters that already have their own registry credentials, the person starting the workspace would otherwise need registry read access and a local `docker login` just for this step. -To skip it, produce the image config next to the image, for example in CI, and point the `IMAGE_DETAILS` option at the file: +To skip it, produce the image config next to the image, for example in CI, and point the `IMAGE_CONFIG_FILE` option at the file: ```bash -crane config registry.example.com/team/image@sha256: > image-details.json -devsy provider set kubernetes -o IMAGE_DETAILS=$PWD/image-details.json +crane config registry.example.com/team/image@sha256: > image-config.json +devsy provider set kubernetes -o IMAGE_CONFIG_FILE=$PWD/image-config.json ``` The file can be an OCI image config (as printed by `crane config`), a single `docker image inspect` object, or the JSON of Devsy's image details. Devsy never contacts the registry for this step when the option is set, and it fails if the file is missing or invalid. Notes: From e8b39e4581e42afaf566cb8a0d7c94d90c4d9459 Mon Sep 17 00:00:00 2001 From: wsmoak Date: Sat, 31 Oct 2026 09:03:34 -0400 Subject: [PATCH 04/10] fix(kubernetes): keep IMAGE_CONFIG_FILE out of pod options The image config file path is local to the machine running Devsy. Leave it out of the last-applied pod annotation and out of the comparison that decides pod replacement, so moving the file does not recreate the pod. Co-Authored-By: Claude Opus 5.5 --- pkg/driver/kubernetes/pod_lifecycle_test.go | 37 +++++++++++++++++++++ pkg/driver/kubernetes/run.go | 17 ++++++++-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go index b93b568c12..48f830303e 100644 --- a/pkg/driver/kubernetes/pod_lifecycle_test.go +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -221,6 +221,43 @@ func TestReconcileReplacesPodWhenOptionsChanged(t *testing.T) { assert.False(t, c.podExists(t)) } +func TestReconcileReusesPodWhenOnlyImageConfigFileChanged(t *testing.T) { + c := newFakeCluster(t) + c.k.options.ImageConfigFile = "/new/image-config.json" + c.putPod(t, c.pod(t, podSpec{ + uid: "u1", + phase: corev1.PodRunning, + options: &provider2.ProviderKubernetesDriverConfig{ + PodTimeout: "2s", + DiskSize: "1Gi", + ImageConfigFile: "/old/image-config.json", + }, + devsy: &corev1.ContainerStatus{ + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + })) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.True(t, reused) + assert.Zero(t, c.deleteCalls()) +} + +func TestWithoutLocalOptionsDropsImageConfigFile(t *testing.T) { + options := &provider2.ProviderKubernetesDriverConfig{ + DiskSize: "1Gi", + ImageConfigFile: "/home/me/image-config.json", + } + + raw, err := json.Marshal(withoutLocalOptions(options)) + + require.NoError(t, err) + assert.NotContains(t, string(raw), "imageConfigFile") + assert.Contains(t, string(raw), "1Gi") + assert.Equal(t, "/home/me/image-config.json", options.ImageConfigFile) +} + func TestReconcilePendingPodWaitsForReadiness(t *testing.T) { c := newFakeCluster(t) c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending})) diff --git a/pkg/driver/kubernetes/run.go b/pkg/driver/kubernetes/run.go index 6833d17f8c..b25b7d620f 100644 --- a/pkg/driver/kubernetes/run.go +++ b/pkg/driver/kubernetes/run.go @@ -555,7 +555,7 @@ func (k *KubernetesDriver) runPod( var err error // set configuration before creating the pod - lastAppliedConfigRaw, err := json.Marshal(k.options) + lastAppliedConfigRaw, err := json.Marshal(withoutLocalOptions(k.options)) if err != nil { return fmt.Errorf("marshal last applied config: %w", err) } @@ -750,13 +750,13 @@ func getDaemonSecretName(workspaceID string) string { func optionsEqual(a, b *provider2.ProviderKubernetesDriverConfig) bool { // copy a and b and the compare them without the context, config, namespace and podTimeout - aCopy := *a + aCopy := withoutLocalOptions(a) aCopy.KubernetesContext = "" aCopy.KubernetesConfig = "" aCopy.KubernetesNamespace = "" aCopy.PodTimeout = "" - bCopy := *b + bCopy := withoutLocalOptions(b) bCopy.KubernetesContext = "" bCopy.KubernetesConfig = "" bCopy.KubernetesNamespace = "" @@ -764,6 +764,17 @@ func optionsEqual(a, b *provider2.ProviderKubernetesDriverConfig) bool { return aCopy == bCopy } +// withoutLocalOptions returns a copy of options without settings that only +// apply on the machine running Devsy. They must not be stored on the pod or +// decide whether it gets replaced. +func withoutLocalOptions( + options *provider2.ProviderKubernetesDriverConfig, +) provider2.ProviderKubernetesDriverConfig { + c := *options + c.ImageConfigFile = "" + return c +} + func (k *KubernetesDriver) createNamespace(ctx context.Context) error { _, err := k.client.Client().CoreV1().Namespaces().Get(ctx, k.namespace, metav1.GetOptions{}) if kerrors.IsNotFound(err) || kerrors.IsForbidden(err) { From 45507ab69bc6ca60d403a8e5732d5e0f6ce8fcb0 Mon Sep 17 00:00:00 2001 From: wsmoak Date: Sat, 31 Oct 2026 09:16:34 -0400 Subject: [PATCH 05/10] fix(kubernetes): accept an empty image config in IMAGE_CONFIG_FILE An image can have no user, env, labels, entrypoint or cmd. Check that the config object is present instead of rejecting a config whose fields are all empty. Co-Authored-By: Claude Opus 5.5 --- pkg/driver/kubernetes/image_details.go | 20 ++++++++++---------- pkg/driver/kubernetes/image_details_test.go | 20 ++++++++++++++++---- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pkg/driver/kubernetes/image_details.go b/pkg/driver/kubernetes/image_details.go index 322f9e2e75..cd886b3daf 100644 --- a/pkg/driver/kubernetes/image_details.go +++ b/pkg/driver/kubernetes/image_details.go @@ -43,11 +43,16 @@ func loadImageDetails(path, imageName string) (*config.ImageDetails, error) { return nil, fmt.Errorf("parse image config file %q: %w", path, err) } - if isEmptyImageConfig(details.Config) { - return nil, fmt.Errorf( - "image config file %q has no image config (User, Env, Labels, Entrypoint or Cmd)", - path, - ) + // An image may legitimately have an empty config, so check that the + // config object is present rather than that any of its fields are set. + var probe struct { + Config json.RawMessage `json:"config"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return nil, fmt.Errorf("parse image config file %q: %w", path, err) + } + if len(probe.Config) == 0 || probe.Config[0] != '{' { + return nil, fmt.Errorf("image config file %q has no config object", path) } if details.ID == "" { @@ -56,8 +61,3 @@ func loadImageDetails(path, imageName string) (*config.ImageDetails, error) { return details, nil } - -func isEmptyImageConfig(c config.ImageDetailsConfig) bool { - return c.User == "" && len(c.Env) == 0 && len(c.Labels) == 0 && - len(c.Entrypoint) == 0 && len(c.Cmd) == 0 -} diff --git a/pkg/driver/kubernetes/image_details_test.go b/pkg/driver/kubernetes/image_details_test.go index be4242a6f5..d434bc92be 100644 --- a/pkg/driver/kubernetes/image_details_test.go +++ b/pkg/driver/kubernetes/image_details_test.go @@ -64,6 +64,16 @@ func (s *ImageDetailsSuite) TestOCIImageConfig() { s.Equal("vscode", details.Config.User) } +func (s *ImageDetailsSuite) TestEmptyConfig() { + // An image with no user, env, labels, entrypoint or cmd. + path := s.writeFile(`{"architecture": "amd64", "os": "linux", "config": {}}`) + + details, err := s.driver(path).ImageDetails(testImage) + s.Require().NoError(err) + s.Equal(testImage, details.ID) + s.Empty(details.Config.User) +} + func (s *ImageDetailsSuite) TestMissingFile() { path := filepath.Join(s.T().TempDir(), "missing.json") _, err := s.driver(path).ImageDetails(testImage) @@ -72,10 +82,12 @@ func (s *ImageDetailsSuite) TestMissingFile() { func (s *ImageDetailsSuite) TestInvalidContent() { tests := map[string]string{ - "empty": "", - "array": `[{"Config": {"User": "vscode"}}]`, - "bad json": `{"Config": `, - "no config": `{"schemaVersion": 2, "layers": []}`, + "empty": "", + "array": `[{"Config": {"User": "vscode"}}]`, + "bad json": `{"Config": `, + "no config": `{"schemaVersion": 2, "layers": []}`, + "null config": `{"config": null}`, + "config list": `{"config": []}`, } for name, content := range tests { s.Run(name, func() { From a11b426ab48827422e3cf7931ef3f6a1e48b3543 Mon Sep 17 00:00:00 2001 From: wsmoak Date: Sat, 31 Oct 2026 09:18:38 -0400 Subject: [PATCH 06/10] fix(kubernetes): require ARCHITECTURE with IMAGE_CONFIG_FILE Without ARCHITECTURE, Devsy lists the cluster's nodes to detect it, which needs the cluster-scoped access IMAGE_CONFIG_FILE is meant to avoid. Require it, and reject a file whose architecture does not match. Co-Authored-By: Claude Opus 5.5 --- .../provider_kubernetes_image_config_file.go | 11 ++++++++-- pkg/driver/kubernetes/image_details.go | 20 +++++++++++++++++- pkg/driver/kubernetes/image_details_test.go | 21 ++++++++++++++++++- providers/kubernetes/provider.yaml | 2 +- .../docs/developing-providers/driver.mdx | 4 ++-- 5 files changed, 51 insertions(+), 7 deletions(-) diff --git a/e2e/tests/up/provider_kubernetes_image_config_file.go b/e2e/tests/up/provider_kubernetes_image_config_file.go index 8fea346656..33b526caa3 100644 --- a/e2e/tests/up/provider_kubernetes_image_config_file.go +++ b/e2e/tests/up/provider_kubernetes_image_config_file.go @@ -5,12 +5,17 @@ import ( "encoding/json" "os" "path/filepath" + "runtime" "github.com/devsy-org/devsy/e2e/framework" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) +// imageConfigArchOption sets ARCHITECTURE, which IMAGE_CONFIG_FILE requires. The +// e2e cluster runs on the host, so its nodes share the host's architecture. +const imageConfigArchOption = "ARCHITECTURE=" + runtime.GOARCH + const ( imageConfigFileMarkerEnv = "IMAGE_CONFIG_FILE_MARKER" imageConfigFileMarkerValue = "from-image-config-file" @@ -54,7 +59,8 @@ var _ = ginkgo.Describe( ginkgo.It( "applies the image config from the IMAGE_CONFIG_FILE file", func(ctx context.Context) { - e := newIdleEnv(ctx, initialDir, "IMAGE_CONFIG_FILE="+writeImageConfigFile()) + e := newIdleEnv(ctx, initialDir, + "IMAGE_CONFIG_FILE="+writeImageConfigFile(), imageConfigArchOption) framework.ExpectNoError(e.up(ctx)) gomega.Expect(e.ssh(ctx, "echo $"+imageConfigFileMarkerEnv)). @@ -67,7 +73,8 @@ var _ = ginkgo.Describe( "fails clearly when the IMAGE_CONFIG_FILE file is missing", func(ctx context.Context) { missing := filepath.Join(ginkgo.GinkgoT().TempDir(), "missing.json") - e := newIdleEnv(ctx, initialDir, "IMAGE_CONFIG_FILE="+missing) + e := newIdleEnv(ctx, initialDir, + "IMAGE_CONFIG_FILE="+missing, imageConfigArchOption) _, stderr, err := e.f.ExecCommandCapture(ctx, []string{cmdWorkspace, "up", e.dir, flagIDEOption, kubernetesIgnoreIDE}) diff --git a/pkg/driver/kubernetes/image_details.go b/pkg/driver/kubernetes/image_details.go index cd886b3daf..26a5135918 100644 --- a/pkg/driver/kubernetes/image_details.go +++ b/pkg/driver/kubernetes/image_details.go @@ -18,8 +18,26 @@ func (k *KubernetesDriver) ImageDetails(imageName string) (*config.ImageDetails, return nil, nil } + // Without ARCHITECTURE, Devsy lists the cluster's nodes to detect it, + // which needs cluster-scoped permissions this option is meant to avoid. + if k.options.Architecture == "" { + return nil, fmt.Errorf("IMAGE_CONFIG_FILE requires the ARCHITECTURE option to be set") + } + log.Debugf("Reading image config for %q from %q", imageName, k.options.ImageConfigFile) - return loadImageDetails(k.options.ImageConfigFile, imageName) + details, err := loadImageDetails(k.options.ImageConfigFile, imageName) + if err != nil { + return nil, err + } + + if details.Architecture != "" && details.Architecture != k.options.Architecture { + return nil, fmt.Errorf( + "image config file %q is for architecture %q, but ARCHITECTURE is %q", + k.options.ImageConfigFile, details.Architecture, k.options.Architecture, + ) + } + + return details, nil } // loadImageDetails parses an image config file. It accepts the JSON of diff --git a/pkg/driver/kubernetes/image_details_test.go b/pkg/driver/kubernetes/image_details_test.go index d434bc92be..99296df2dc 100644 --- a/pkg/driver/kubernetes/image_details_test.go +++ b/pkg/driver/kubernetes/image_details_test.go @@ -74,6 +74,22 @@ func (s *ImageDetailsSuite) TestEmptyConfig() { s.Empty(details.Config.User) } +func (s *ImageDetailsSuite) TestRequiresArchitecture() { + path := s.writeFile(`{"config": {}}`) + d := s.driver(path) + d.options.Architecture = "" + + _, err := d.ImageDetails(testImage) + s.Require().ErrorContains(err, "requires the ARCHITECTURE option") +} + +func (s *ImageDetailsSuite) TestArchitectureMismatch() { + path := s.writeFile(`{"architecture": "arm64", "os": "linux", "config": {}}`) + + _, err := s.driver(path).ImageDetails(testImage) + s.Require().ErrorContains(err, `is for architecture "arm64", but ARCHITECTURE is "amd64"`) +} + func (s *ImageDetailsSuite) TestMissingFile() { path := filepath.Join(s.T().TempDir(), "missing.json") _, err := s.driver(path).ImageDetails(testImage) @@ -105,6 +121,9 @@ func (s *ImageDetailsSuite) writeFile(content string) string { func (s *ImageDetailsSuite) driver(path string) *KubernetesDriver { return &KubernetesDriver{ - options: &provider2.ProviderKubernetesDriverConfig{ImageConfigFile: path}, + options: &provider2.ProviderKubernetesDriverConfig{ + ImageConfigFile: path, + Architecture: "amd64", + }, } } diff --git a/providers/kubernetes/provider.yaml b/providers/kubernetes/provider.yaml index b61c570610..4abcd11d51 100644 --- a/providers/kubernetes/provider.yaml +++ b/providers/kubernetes/provider.yaml @@ -119,7 +119,7 @@ options: description: The cpu architecture to use for the workspace pod, e.g. amd64, arm64, etc. type: string IMAGE_CONFIG_FILE: - description: Path to a JSON file with the image config of the devcontainer's prebuilt image (e.g. the output of `crane config `). If set, Devsy reads the user, env, entrypoint, cmd and devcontainer.metadata label from this file instead of fetching the image config from the registry, so the machine running Devsy needs no registry credentials. Devsy fails if the file is missing or invalid. Pin the image by digest so the file matches the image the cluster pulls. Applies to the devcontainer.json image only; Dockerfile builds still read their base image from the registry. + description: Path to a JSON file with the image config of the devcontainer's prebuilt image (e.g. the output of `crane config `). If set, Devsy reads the user, env, entrypoint, cmd and devcontainer.metadata label from this file instead of fetching the image config from the registry, so the machine running Devsy needs no registry credentials. Devsy fails if the file is missing or invalid. Requires ARCHITECTURE to be set and to match the file's architecture. Pin the image by digest so the file matches the image the cluster pulls. Applies to the devcontainer.json image only; Dockerfile builds still read their base image from the registry. type: string agent: containerInactivityTimeout: ${INACTIVITY_TIMEOUT} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx index 149b9e282f..9cdf1036aa 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx @@ -90,14 +90,14 @@ To skip it, produce the image config next to the image, for example in CI, and p ```bash crane config registry.example.com/team/image@sha256: > image-config.json -devsy provider set kubernetes -o IMAGE_CONFIG_FILE=$PWD/image-config.json +devsy provider set kubernetes -o IMAGE_CONFIG_FILE=$PWD/image-config.json -o ARCHITECTURE=amd64 ``` The file can be an OCI image config (as printed by `crane config`), a single `docker image inspect` object, or the JSON of Devsy's image details. Devsy never contacts the registry for this step when the option is set, and it fails if the file is missing or invalid. Notes: - Pin the image by digest in devcontainer.json, so the file always matches the image the cluster pulls. - This applies to the devcontainer.json `image`. Dockerfile builds still read their base image from the registry, and `features` are built from the image as usual. -- Also set `ARCHITECTURE` (for example `amd64`). Otherwise Devsy lists the cluster's nodes to detect it, which needs cluster-scoped permissions. +- `ARCHITECTURE` (for example `amd64`) is required, so Devsy does not list the cluster's nodes to detect it, which needs cluster-scoped permissions. If the file has an `architecture`, it must match. On a cluster with mixed architectures, also schedule the pod onto matching nodes, for example with `NODE_SELECTOR`. ### Example Kubernetes Provider From c6491266b460dd12f91b6e899c830e11df8c60e5 Mon Sep 17 00:00:00 2001 From: wsmoak Date: Sat, 31 Oct 2026 09:22:08 -0400 Subject: [PATCH 07/10] fix(kubernetes): reject invalid devcontainer.metadata in IMAGE_CONFIG_FILE Malformed metadata read from the registry is ignored with a warning. A file the operator supplied should fail instead of silently dropping its lifecycle commands and settings. Co-Authored-By: Claude Opus 5.5 --- pkg/driver/kubernetes/image_details.go | 39 +++++++++++++++++++++ pkg/driver/kubernetes/image_details_test.go | 36 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/pkg/driver/kubernetes/image_details.go b/pkg/driver/kubernetes/image_details.go index 26a5135918..000b12faa4 100644 --- a/pkg/driver/kubernetes/image_details.go +++ b/pkg/driver/kubernetes/image_details.go @@ -7,6 +7,7 @@ import ( "os" "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/devcontainer/metadata" "github.com/devsy-org/devsy/pkg/log" ) @@ -73,9 +74,47 @@ func loadImageDetails(path, imageName string) (*config.ImageDetails, error) { return nil, fmt.Errorf("image config file %q has no config object", path) } + if err := validateImageMetadata( + details.Config.Labels[metadata.ImageMetadataLabel], + ); err != nil { + return nil, fmt.Errorf("image config file %q: %w", path, err) + } + if details.ID == "" { details.ID = imageName } return details, nil } + +// validateImageMetadata checks that the devcontainer.metadata label is a JSON +// object or an array of objects. Metadata read from the registry is ignored +// with a warning when malformed, but a file the operator supplied should fail +// instead of silently dropping its lifecycle commands and settings. +func validateImageMetadata(raw string) error { + if raw == "" { + return nil + } + + var entries []*config.ImageMetadata + if err := json.Unmarshal([]byte(raw), &entries); err == nil { + if entries == nil { + return fmt.Errorf("%s label is null", metadata.ImageMetadataLabel) + } + for _, entry := range entries { + if entry == nil { + return fmt.Errorf("%s label contains a null entry", metadata.ImageMetadataLabel) + } + } + return nil + } + + single := &config.ImageMetadata{} + if raw[0] != '{' { + return fmt.Errorf("%s label must be a JSON object or array", metadata.ImageMetadataLabel) + } + if err := json.Unmarshal([]byte(raw), single); err != nil { + return fmt.Errorf("parse %s label: %w", metadata.ImageMetadataLabel, err) + } + return nil +} diff --git a/pkg/driver/kubernetes/image_details_test.go b/pkg/driver/kubernetes/image_details_test.go index 99296df2dc..9cda0b7c26 100644 --- a/pkg/driver/kubernetes/image_details_test.go +++ b/pkg/driver/kubernetes/image_details_test.go @@ -90,6 +90,42 @@ func (s *ImageDetailsSuite) TestArchitectureMismatch() { s.Require().ErrorContains(err, `is for architecture "arm64", but ARCHITECTURE is "amd64"`) } +func (s *ImageDetailsSuite) TestValidMetadata() { + tests := map[string]string{ + "array": `[{\"remoteUser\": \"vscode\"}]`, + "object": `{\"remoteUser\": \"vscode\"}`, + "empty": `[]`, + } + for name, label := range tests { + s.Run(name, func() { + path := s.writeFile( + `{"config": {"Labels": {"devcontainer.metadata": "` + label + `"}}}`, + ) + _, err := s.driver(path).ImageDetails(testImage) + s.NoError(err) + }) + } +} + +func (s *ImageDetailsSuite) TestInvalidMetadata() { + tests := map[string]string{ + "bad json": `[{\"remoteUser\": `, + "null entry": `[null]`, + "null": `null`, + "string": `\"vscode\"`, + "wrong type": `{\"remoteUser\": 1}`, + } + for name, label := range tests { + s.Run(name, func() { + path := s.writeFile( + `{"config": {"Labels": {"devcontainer.metadata": "` + label + `"}}}`, + ) + _, err := s.driver(path).ImageDetails(testImage) + s.ErrorContains(err, "devcontainer.metadata") + }) + } +} + func (s *ImageDetailsSuite) TestMissingFile() { path := filepath.Join(s.T().TempDir(), "missing.json") _, err := s.driver(path).ImageDetails(testImage) From ba295727a777a122d3ebecc8d2ba0fb93c3545c8 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 31 Oct 2026 15:56:57 -0600 Subject: [PATCH 08/10] feat(kubernetes): validate operator supplied image configuration --- e2e/framework/runenv/dependencies.go | 11 +- e2e/framework/runenv/exec.go | 5 + .../provider_kubernetes_image_config_file.go | 869 +++++++++++++++++- pkg/devcontainer/build.go | 17 +- pkg/devcontainer/image_details.go | 126 +++ pkg/devcontainer/image_details_test.go | 441 ++++++++- pkg/devcontainer/prebuild.go | 9 + pkg/devcontainer/run.go | 11 +- .../kubernetes/image_config_open_other.go | 9 + .../kubernetes/image_config_open_unix.go | 16 + .../kubernetes/image_config_open_unix_test.go | 33 + .../kubernetes/image_config_open_windows.go | 21 + pkg/driver/kubernetes/image_details.go | 394 ++++++-- pkg/driver/kubernetes/image_details_test.go | 306 +++--- pkg/driver/kubernetes/pod_lifecycle_test.go | 78 ++ pkg/driver/kubernetes/run.go | 3 - pkg/driver/types.go | 12 +- pkg/provider/provider.go | 4 +- providers/kubernetes/provider.yaml | 2 +- .../docs/developing-providers/driver.mdx | 17 +- .../kubernetes-registry-access.mdx | 67 ++ .../managing-providers/manage-providers.mdx | 4 + .../content/docs/managing-providers/meta.json | 2 +- 23 files changed, 2117 insertions(+), 340 deletions(-) create mode 100644 pkg/devcontainer/image_details.go create mode 100644 pkg/driver/kubernetes/image_config_open_other.go create mode 100644 pkg/driver/kubernetes/image_config_open_unix.go create mode 100644 pkg/driver/kubernetes/image_config_open_unix_test.go create mode 100644 pkg/driver/kubernetes/image_config_open_windows.go create mode 100644 sites/docs-devsy-sh/content/docs/managing-providers/kubernetes-registry-access.mdx diff --git a/e2e/framework/runenv/dependencies.go b/e2e/framework/runenv/dependencies.go index 59e6d21488..849e80f98b 100644 --- a/e2e/framework/runenv/dependencies.go +++ b/e2e/framework/runenv/dependencies.go @@ -24,11 +24,12 @@ const ( // Command describes one subprocess. Env is an overlay applied to the parent // environment of that process only; nothing mutates the global environment. type Command struct { - Path string - Args []string - Env []string // NAME=value overlay; replaces same-named parent entries - Dir string - Timeout time.Duration + Path string + Args []string + Env []string // NAME=value overlay; replaces same-named parent entries + Dir string + Timeout time.Duration + CancelProcessGroup bool // Stdout and Stderr optionally receive output live, in addition to the // tail-capped capture returned in CommandResult. Stdout io.Writer diff --git a/e2e/framework/runenv/exec.go b/e2e/framework/runenv/exec.go index b8d9efbeb2..e2439f24ab 100644 --- a/e2e/framework/runenv/exec.go +++ b/e2e/framework/runenv/exec.go @@ -12,6 +12,8 @@ import ( "strings" "sync" "time" + + "github.com/devsy-org/devsy/pkg/docker" ) const ( @@ -39,6 +41,9 @@ func (ExecRunner) Run(ctx context.Context, c Command) (CommandResult, error) { stdout, stderr := newTailBuffer(maxCaptureBytes), newTailBuffer(maxCaptureBytes) //nolint:gosec // G204: ExecRunner runs the command its caller specified, by design cmd := exec.CommandContext(ctx, c.Path, c.Args...) + if c.CancelProcessGroup { + docker.PrepareForGroupCancellation(cmd) + } cmd.Dir = c.Dir cmd.Env = overlayEnv(os.Environ(), c.Env, runtime.GOOS == "windows") cmd.WaitDelay = waitDelay diff --git a/e2e/tests/up/provider_kubernetes_image_config_file.go b/e2e/tests/up/provider_kubernetes_image_config_file.go index 33b526caa3..251a9c49fd 100644 --- a/e2e/tests/up/provider_kubernetes_image_config_file.go +++ b/e2e/tests/up/provider_kubernetes_image_config_file.go @@ -1,87 +1,854 @@ package up import ( + "bytes" "context" + "encoding/base64" "encoding/json" + "errors" + "fmt" "os" "path/filepath" "runtime" + "strings" + "sync" + "time" "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/e2e/framework/runenv" + "github.com/devsy-org/devsy/pkg/driver/kubernetes" + devsyencoding "github.com/devsy-org/devsy/pkg/encoding" + "github.com/devsy-org/devsy/pkg/image" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + "golang.org/x/crypto/bcrypt" + corev1 "k8s.io/api/core/v1" ) -// imageConfigArchOption sets ARCHITECTURE, which IMAGE_CONFIG_FILE requires. The -// e2e cluster runs on the host, so its nodes share the host's architecture. -const imageConfigArchOption = "ARCHITECTURE=" + runtime.GOARCH - const ( - imageConfigFileMarkerEnv = "IMAGE_CONFIG_FILE_MARKER" - imageConfigFileMarkerValue = "from-image-config-file" + imageConfigContainerUser = "containerUser" + imageConfigRemoteUser = "remoteUser" + imageConfigUIDFlag = "--uid" + imageConfigKubectl = "kubectl" + imageConfigLogMarker = "image-config-fixture-log-marker" + imageConfigDaemonSecretPrefix = kubernetes.DevContainerName + "-daemon-secret-" + kubernetes.DevContainerName + "-" ) -// writeImageConfigFile writes an OCI-style image config whose -// devcontainer.metadata label sets a marker env var the real image lacks, so -// seeing the marker in the workspace proves the file was used instead of the -// registry. -func writeImageConfigFile() string { - metadata, err := json.Marshal([]map[string]any{ - {"containerEnv": map[string]string{imageConfigFileMarkerEnv: imageConfigFileMarkerValue}}, +const imageConfigRegistryImage = "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373" + +type imageConfigFixture struct { + f *framework.Framework + admin, namespace, registry, authority, image, file, dir, id, node, root, password string + uid, controlUID, configDigest string +} + +type imageConfigOutput struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (o *imageConfigOutput) Write(data []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + return o.buffer.Write(data) +} + +type imageConfigLogObserver struct { + mu sync.Mutex + tail []byte + seen bool + cancel context.CancelFunc +} + +func (o *imageConfigLogObserver) Write(data []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.seen { + return len(data), nil + } + o.tail = append(o.tail, data...) + window := o.tail + if bytes.Contains(window, []byte(imageConfigLogMarker)) { + o.seen = true + o.cancel() + } + keep := len(imageConfigLogMarker) - 1 + if len(window) > keep { + window = window[len(window)-keep:] + } + o.tail = append(o.tail[:0], window...) + return len(data), nil +} + +func (x *imageConfigFixture) observeLogs(ctx context.Context) { + logsCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + observer := &imageConfigLogObserver{cancel: cancel} + result, err := (runenv.ExecRunner{}).Run(logsCtx, runenv.Command{ + Path: filepath.Join(x.f.DevsyBinDir, x.f.DevsyBinName), + Args: []string{cmdWorkspace, "logs", x.id}, + Stdout: observer, + CancelProcessGroup: true, }) + observer.mu.Lock() + seen := observer.seen + observer.mu.Unlock() + gomega.Expect(seen).To(gomega.BeTrue(), "workspace logs must contain the fixture marker") + gomega.Expect(logsCtx.Err()).To(gomega.MatchError(context.Canceled)) + gomega.Expect(result.TimedOut).To(gomega.BeFalse()) + gomega.Expect(result.ExitCode != 0 || errors.Is(err, context.Canceled)).To(gomega.BeTrue()) +} + +func imageConfigCommand( + ctx context.Context, env []string, program string, args ...string, +) (string, error) { + switch program { + case testDockerCommand, imageConfigKubectl, "kind": + default: + return "", fmt.Errorf("unsupported fixture command %q", program) + } + output := &imageConfigOutput{} + result, err := (runenv.ExecRunner{}).Run(ctx, runenv.Command{ + Path: program, Args: args, Env: env, Stdout: output, Stderr: output, + }) + out := strings.TrimSpace(output.buffer.String()) + if err != nil { + return out, fmt.Errorf("run %s: %w", program, err) + } + if result.TimedOut || result.ExitCode != 0 { + return out, fmt.Errorf("%s exited with code %d: %s", program, result.ExitCode, out) + } + return out, nil +} + +func imageConfigMust(ctx context.Context, env []string, program string, args ...string) string { + out, err := imageConfigCommand(ctx, env, program, args...) + framework.ExpectNoError(err) + return out +} + +func (x *imageConfigFixture) kubectl(ctx context.Context, args ...string) string { + env := append(os.Environ(), "KUBECONFIG="+x.admin) + return imageConfigMust(ctx, env, imageConfigKubectl, args...) +} + +func imageConfigWrite(path string, value any) { + data, err := json.Marshal(value) framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(path, data, 0o600)) +} + +func newImageConfigFixture(ctx context.Context) *imageConfigFixture { + root := ginkgo.GinkgoT().TempDir() + cwd, err := os.Getwd() + framework.ExpectNoError(err) + x := &imageConfigFixture{ + f: framework.NewDefaultFramework(filepath.Join(cwd, "bin")), + admin: os.Getenv("KUBECONFIG"), + id: fmt.Sprintf("cfg%d", time.Now().UnixNano()), + } + if x.admin == "" { + x.admin = filepath.Join(os.Getenv("HOME"), ".kube", "config") + } + x.root = root + x.uid = devsyencoding.CreateNewUID("", x.id) + x.controlUID = devsyencoding.CreateNewUID("", x.id+"control") + x.namespace = x.id + x.registry = "devsy-registry-" + x.id + clusterContext, arch := x.cluster(ctx) + x.startRegistry(ctx) + x.publishImage(ctx, arch) + x.provisionIdentity(ctx) + x.configureClient(ctx, clusterContext) + x.configureProvider(ctx, arch) + return x +} + +func (x *imageConfigFixture) cluster(ctx context.Context) (string, string) { + clusterContext := x.kubectl(ctx, "config", "current-context") + gomega.Expect(clusterContext).To(gomega.HavePrefix("kind-")) + nodes := strings.Fields( + imageConfigMust( + ctx, + nil, + "kind", + "get", + "nodes", + "--name", + strings.TrimPrefix(clusterContext, "kind-"), + ), + ) + gomega.Expect(nodes).To(gomega.HaveLen(1)) + x.node = nodes[0] + arch := x.kubectl(ctx, "get", "node", x.node, "-o", "jsonpath={.status.nodeInfo.architecture}") + gomega.Expect(arch).To(gomega.Equal(runtime.GOARCH)) + return clusterContext, arch +} - content, err := json.Marshal(map[string]any{ - "config": map[string]any{ - "Env": []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Labels": map[string]string{"devcontainer.metadata": string(metadata)}, +func (x *imageConfigFixture) startRegistry(ctx context.Context) { + root := x.root + password := "private-" + x.id + x.password = password + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + framework.ExpectNoError(err) + authDir := filepath.Join(root, "auth") + framework.ExpectNoError(os.MkdirAll(authDir, 0o700)) + framework.ExpectNoError( + os.WriteFile( + filepath.Join(authDir, "htpasswd"), + []byte("fixture:"+string(hash)+"\n"), + 0o600, + ), + ) + imageConfigMust( + ctx, + nil, + testDockerCommand, + "run", + "-d", + "--name", + x.registry, + "--network", + "kind", + "-p", + "127.0.0.1::5000", + "-v", + authDir+":/auth:ro", + "-e", + "REGISTRY_AUTH=htpasswd", + "-e", + "REGISTRY_AUTH_HTPASSWD_REALM=fixture", + "-e", + "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd", + imageConfigRegistryImage, + ) + ginkgo.DeferCleanup( + func(c ginkgo.SpecContext) { + _, _ = imageConfigCommand(c, nil, testDockerCommand, "rm", "-f", x.registry) }, + ) + x.authority = imageConfigMust(ctx, nil, testDockerCommand, "port", x.registry, "5000/tcp") + x.authority = "localhost:" + strings.TrimPrefix(x.authority, "127.0.0.1:") + x.configureNodeRegistry(ctx) +} + +func (x *imageConfigFixture) configureNodeRegistry(ctx context.Context) { + root := x.root + hostDir := "/etc/containerd/certs.d/" + x.authority + _, existingErr := imageConfigCommand( + ctx, + nil, + testDockerCommand, + "exec", + x.node, + "test", + "-e", + hostDir, + ) + gomega.Expect(existingErr). + To(gomega.HaveOccurred(), "fixture must not overwrite an existing containerd host configuration") + imageConfigMust(ctx, nil, testDockerCommand, "exec", x.node, "mkdir", "-p", hostDir) + ginkgo.DeferCleanup(func(c ginkgo.SpecContext) { + _, _ = imageConfigCommand(c, nil, testDockerCommand, "exec", x.node, "rm", "-rf", hostDir) }) + hosts := filepath.Join(root, "hosts.toml") + framework.ExpectNoError( + os.WriteFile( + hosts, + []byte( + fmt.Sprintf( + "server = \"http://%s:5000\"\n[host.\"http://%s:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n", + x.registry, + x.registry, + ), + ), + 0o600, + ), + ) + imageConfigMust(ctx, nil, testDockerCommand, "cp", hosts, x.node+":"+hostDir+"/hosts.toml") +} + +func (x *imageConfigFixture) publishImage(ctx context.Context, arch string) { + root, password := x.root, x.password + img, err := image.GetImageForArch(ctx, "ghcr.io/devsy-org/test-images/base:alpine", arch) + framework.ExpectNoError(err) + cfg, err := img.ConfigFile() + framework.ExpectNoError(err) + cfg.Config.User = "nobody" + cfg.Config.Entrypoint = []string{ + "/bin/sh", + "-c", + "echo " + imageConfigLogMarker + "; echo entrypoint > /tmp/image-config-entrypoint; exec \"$@\"", + "--", + } + cfg.Config.Cmd = []string{"sleep", "36000"} + cfg.Config.Env = append(cfg.Config.Env, "IMAGE_CONFIG_VALUE=a=b", "IMAGE_CONFIG_OVERRIDE=image") + cfg.Config.Labels = map[string]string{ + "devsy-e2e-nonce": x.id, + "devcontainer.metadata": `[{"containerEnv":{"IMAGE_CONFIG_METADATA":"metadata"},` + + `"postCreateCommand":"echo metadata > /tmp/image-config-lifecycle"}]`, + } + img, err = mutate.ConfigFile(img, cfg) + framework.ExpectNoError(err) + configDigest, err := img.ConfigName() framework.ExpectNoError(err) + x.configDigest = configDigest.String() + digest, err := img.Digest() + framework.ExpectNoError(err) + x.image = x.authority + "/fixture/workspace@" + digest.String() + tag, err := name.NewTag(x.authority+"/fixture/workspace:fixture", name.Insecure) + framework.ExpectNoError(err) + gomega.Eventually(func() error { + return remote.Write( + tag, + img, + remote.WithContext(ctx), + remote.WithAuth(&authn.Basic{Username: "fixture", Password: password}), + ) + }, 30*time.Second, time.Second).Should(gomega.Succeed()) + x.file = filepath.Join(root, "image.json") + imageConfigWrite( + x.file, + map[string]any{ + "schemaVersion": 1, + compatibilityImageCommand: x.image, + "os": "linux", + "architecture": arch, + "config": map[string]any{ + "User": cfg.Config.User, + "Env": cfg.Config.Env, + "Labels": cfg.Config.Labels, + "Entrypoint": cfg.Config.Entrypoint, + "Cmd": cfg.Config.Cmd, + }, + }, + ) +} - path := filepath.Join(ginkgo.GinkgoT().TempDir(), "image-config.json") - framework.ExpectNoError(os.WriteFile(path, content, 0o600)) - return path +func (x *imageConfigFixture) provisionIdentity(ctx context.Context) { + password := x.password + x.kubectl(ctx, "create", "namespace", x.namespace) + ginkgo.DeferCleanup(func(c ginkgo.SpecContext) { + _, _ = imageConfigCommand( + c, + append(os.Environ(), "KUBECONFIG="+x.admin), + imageConfigKubectl, + "delete", + "namespace", + x.namespace, + "--ignore-not-found", + "--wait=false", + ) + }) + x.kubectl( + ctx, + "-n", + x.namespace, + "create", + "secret", + "docker-registry", + "registry-admin", + "--docker-server="+x.authority, + "--docker-username=fixture", + "--docker-password="+password, + ) + x.kubectl(ctx, "-n", x.namespace, "create", "serviceaccount", cmdWorkspace) + x.kubectl( + ctx, + "-n", + x.namespace, + "patch", + "serviceaccount", + cmdWorkspace, + "--type=merge", + "-p", + `{"imagePullSecrets":[{"name":"registry-admin"}]}`, + ) + x.kubectl(ctx, "-n", x.namespace, "create", "serviceaccount", "client") + x.createClientRole(ctx) } -var _ = ginkgo.Describe( - "kubernetes IMAGE_CONFIG_FILE option", - ginkgo.Label("up-provider-kubernetes", "up-provider-kubernetes-image-config-file"), - func() { - var initialDir string +func (x *imageConfigFixture) createClientRole(ctx context.Context) { + root := x.root + role := fmt.Sprintf(`apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: client, namespace: %s} +rules: +- apiGroups: [""] + resources: [pods, pods/exec, pods/log, persistentvolumeclaims, serviceaccounts, events, services, configmaps] + verbs: ["*"] +- apiGroups: [""] + resources: [secrets] + verbs: [create] +- apiGroups: [""] + resources: [secrets] + resourceNames: [%q, %q] + verbs: [get, update, patch, delete] +`, x.namespace, imageConfigDaemonSecretPrefix+x.uid, imageConfigDaemonSecretPrefix+x.controlUID) + rolePath := filepath.Join(root, "role.yaml") + framework.ExpectNoError(os.WriteFile(rolePath, []byte(role), 0o600)) + x.kubectl(ctx, "apply", "-f", rolePath) + x.kubectl( + ctx, + "-n", + x.namespace, + "create", + "rolebinding", + "client", + "--role=client", + "--serviceaccount="+x.namespace+":client", + ) +} - ginkgo.BeforeEach(func() { - var err error - initialDir, err = os.Getwd() - framework.ExpectNoError(err) - }) +func (x *imageConfigFixture) configureClient(ctx context.Context, clusterContext string) { + root := x.root + token := x.kubectl(ctx, "-n", x.namespace, "create", "token", "client", "--duration=1h") + kube := filepath.Join(root, "client.kubeconfig") + raw := x.kubectl(ctx, "config", "view", "--raw", "--minify", "--flatten") + framework.ExpectNoError(os.WriteFile(kube, []byte(raw), 0o600)) + env := append(os.Environ(), "KUBECONFIG="+kube) + imageConfigMust( + ctx, + env, + imageConfigKubectl, + "config", + "set-credentials", + "image-config-client", + "--token="+token, + ) + imageConfigMust( + ctx, + env, + imageConfigKubectl, + "config", + "set-context", + clusterContext, + "--user=image-config-client", + "--namespace="+x.namespace, + ) + framework.SetEnvForSpec("KUBECONFIG", kube) + x.configureCredentialSpy() + x.checkClientPermissions(ctx) +} - ginkgo.It( - "applies the image config from the IMAGE_CONFIG_FILE file", - func(ctx context.Context) { - e := newIdleEnv(ctx, initialDir, - "IMAGE_CONFIG_FILE="+writeImageConfigFile(), imageConfigArchOption) - - framework.ExpectNoError(e.up(ctx)) - gomega.Expect(e.ssh(ctx, "echo $"+imageConfigFileMarkerEnv)). - To(gomega.Equal(imageConfigFileMarkerValue)) - }, - ginkgo.SpecTimeout(framework.TimeoutShort()), +func (x *imageConfigFixture) configureCredentialSpy() { + root := x.root + framework.SetEnvForSpec("DOCKER_CONFIG", filepath.Join(root, "empty-docker")) + helperDir := filepath.Join(root, "helpers") + framework.ExpectNoError(os.MkdirAll(helperDir, 0o700)) + framework.SetEnvForSpec("IMAGE_CONFIG_HELPER_LOG", filepath.Join(root, "helper-calls")) + for _, helper := range []string{ + "docker-credential-desktop", "docker-credential-osxkeychain", + "docker-credential-pass", "docker-credential-secretservice", + } { + script := []byte("#!/bin/sh\necho called >> \"$IMAGE_CONFIG_HELPER_LOG\"\nexit 1\n") + framework.ExpectNoError( + (runenv.OsFileStore{}).WriteFile(filepath.Join(helperDir, helper), script, 0o700), ) + } + framework.SetEnvForSpec("PATH", helperDir+string(os.PathListSeparator)+os.Getenv("PATH")) + framework.ExpectNoError(os.MkdirAll(filepath.Join(root, "empty-docker"), 0o700)) + framework.SetEnvForSpec("DEVSY_HOME", filepath.Join(root, "devsy")) + framework.SetEnvForSpec("DEVSY_SECRETS_BACKEND", "file") + framework.SetEnvForSpec("DEVSY_SECRETS_PASSPHRASE", "fixture") +} + +func (x *imageConfigFixture) checkClientPermissions(ctx context.Context) { + out, err := imageConfigCommand( + ctx, + nil, + imageConfigKubectl, + "auth", + "can-i", + "get", + "secret/registry-admin", + "-n", + x.namespace, + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(out).To(gomega.ContainSubstring("no")) + out, err = imageConfigCommand(ctx, nil, imageConfigKubectl, "auth", "can-i", "list", "nodes") + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(out).To(gomega.ContainSubstring("no")) + out, err = imageConfigCommand(ctx, nil, imageConfigKubectl, "auth", "can-i", "get", + "secret/"+imageConfigDaemonSecretPrefix+x.uid, "-n", x.namespace) + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("yes")) +} + +func (x *imageConfigFixture) configureProvider(ctx context.Context, arch string) { + root := x.root + _, stderr, err := x.f.ExecCommandCapture( + ctx, + []string{ + "context", + microsandboxSecretSetCommand, + microsandboxResourceOption, + "SSH_INJECT_DOCKER_CREDENTIALS=false", + microsandboxResourceOption, + "SSH_ADD_PRIVATE_KEYS=false", + }, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr) + framework.ExpectNoError( + x.f.DevsyProviderAdd( + ctx, + "kubernetes", + "--name", + x.id, + "-o", + "KUBERNETES_NAMESPACE="+x.namespace, + "-o", + "CREATE_NAMESPACE=false", + "-o", + "SERVICE_ACCOUNT=workspace", + "-o", + "ARCHITECTURE="+arch, + "-o", + "KUBERNETES_PULL_SECRETS_ENABLED=false", + "-o", + "POD_TIMEOUT=2m", + "-o", + "INACTIVITY_TIMEOUT=0", + ), + ) + ginkgo.DeferCleanup( + func(c ginkgo.SpecContext) { _ = x.f.CleanupWorkspace(c, x.id); _ = x.f.DevsyProviderDelete(c, x.id) }, + ) + x.dir = filepath.Join(root, cmdWorkspace) + framework.ExpectNoError(os.MkdirAll(filepath.Join(x.dir, ".devcontainer"), 0o700)) + imageConfigWrite( + filepath.Join(x.dir, ".devcontainer", "devcontainer.json"), + map[string]any{ + compatibilityImageCommand: x.image, + "workspaceFolder": "/workspaces/project", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/project,type=bind", + "overrideCommand": false, + imageConfigContainerUser: microsandboxRootUser, + imageConfigRemoteUser: microsandboxRootUser, + "containerEnv": map[string]string{"IMAGE_CONFIG_OVERRIDE": "override"}, + }, + ) +} + +func (x *imageConfigFixture) up(ctx context.Context, args ...string) (string, string, error) { + command := []string{ + cmdWorkspace, + "up", + x.dir, + "--provider", + x.id, + "--id", + x.id, + imageConfigUIDFlag, + x.uid, + flagIDEOption, + kubernetesIgnoreIDE, + "--provider-option", + "ARCHITECTURE=" + runtime.GOARCH, + "--provider-option", + "POD_TIMEOUT=2m", + } + return x.f.ExecCommandCapture(ctx, append(command, args...)) +} + +func (x *imageConfigFixture) pod(ctx context.Context) corev1.Pod { + raw := x.kubectl( + ctx, + "-n", + x.namespace, + "get", + "pods", + "-l", + "devsy.sh/created=true", + "-o", + "json", + ) + var list corev1.PodList + framework.ExpectNoError(json.Unmarshal([]byte(raw), &list)) + gomega.Expect(list.Items).To(gomega.HaveLen(1)) + return list.Items[0] +} +var _ = ginkgo.Describe( + "Kubernetes supplied image config", + ginkgo.Label("up-provider-kubernetes", "up-provider-kubernetes-image-config-file"), + func() { ginkgo.It( - "fails clearly when the IMAGE_CONFIG_FILE file is missing", - func(ctx context.Context) { - missing := filepath.Join(ginkgo.GinkgoT().TempDir(), "missing.json") - e := newIdleEnv(ctx, initialDir, - "IMAGE_CONFIG_FILE="+missing, imageConfigArchOption) - - _, stderr, err := e.f.ExecCommandCapture(ctx, - []string{cmdWorkspace, "up", e.dir, flagIDEOption, kubernetesIgnoreIDE}) + "uses cluster registry authorization without client inspection and preserves workspace data", + func(ctx ginkgo.SpecContext) { + x := newImageConfigFixture(ctx) + negativeStarted := time.Now().UTC().Format(time.RFC3339Nano) + _, stderr, err := x.up(ctx) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(strings.ToLower(stderr)). + To(gomega.Or(gomega.ContainSubstring("unauthorized"), gomega.ContainSubstring("401"))) + negativeLogs := imageConfigMust(ctx, nil, testDockerCommand, + "logs", "--since", negativeStarted, x.registry) + gomega.Expect(negativeLogs).To(gomega.ContainSubstring(" 401 ")) + gomega.Expect(negativeLogs).To(gomega.ContainSubstring("/manifests/")) + _, stderr, err = x.up( + ctx, + "--provider-option", + "IMAGE_CONFIG_FILE="+filepath.Join(x.root, "missing.json"), + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stderr).To(gomega.ContainSubstring("IMAGE_CONFIG_FILE")) + _, stderr, err = x.up( + ctx, + "--provider-option", + "IMAGE_CONFIG_FILE="+x.file, + "--provider-option", + "ARCHITECTURE=wrong", + ) gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(stderr).To(gomega.ContainSubstring("read image config file")) + gomega.Expect(stderr).To(gomega.ContainSubstring("ARCHITECTURE")) + pods := x.kubectl( + ctx, + "-n", + x.namespace, + "get", + "pods", + "-l", + "devsy.sh/created=true", + "-o", + "json", + ) + var empty corev1.PodList + framework.ExpectNoError(json.Unmarshal([]byte(pods), &empty)) + gomega.Expect(empty.Items).To(gomega.BeEmpty()) + valid, err := os.ReadFile(x.file) + framework.ExpectNoError(err) + var mismatch map[string]any + framework.ExpectNoError(json.Unmarshal(valid, &mismatch)) + mismatch[compatibilityImageCommand] = x.authority + "/fixture/workspace@sha256:" + strings.Repeat( + "0", + 64, + ) + imageConfigWrite(x.file, mismatch) + _, mismatchError, err := x.up(ctx, "--provider-option", "IMAGE_CONFIG_FILE="+x.file) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(mismatchError).To(gomega.ContainSubstring("IMAGE_CONFIG_FILE")) + gomega.Expect(x.kubectl( + ctx, + "-n", + x.namespace, + "get", + "pods", + "-l", + "devsy.sh/created=true", + "-o", + "jsonpath={.items[*].metadata.name}", + )).To(gomega.BeEmpty()) + framework.ExpectNoError(os.WriteFile(x.file, valid, 0o600)) + x.kubectl( + ctx, + "-n", + x.namespace, + "patch", + "serviceaccount", + cmdWorkspace, + "--type=merge", + "-p", + `{"imagePullSecrets":[]}`, + ) + _, pullError, err := x.up( + ctx, + "--provider-option", + "IMAGE_CONFIG_FILE="+x.file, + "--provider-option", + "POD_TIMEOUT=30s", + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + events := x.kubectl(ctx, "-n", x.namespace, "get", "events", "-o", "json") + gomega.Expect(strings.ToLower(pullError + events)). + To(gomega.Or(gomega.ContainSubstring("imagepullbackoff"), gomega.ContainSubstring("pull image"))) + framework.ExpectNoError(x.f.CleanupWorkspace(ctx, x.id)) + x.kubectl( + ctx, + "-n", + x.namespace, + "patch", + "serviceaccount", + cmdWorkspace, + "--type=merge", + "-p", + `{"imagePullSecrets":[{"name":"registry-admin"}]}`, + ) + helperLog := filepath.Join(x.root, "helper-calls") + if err := os.Remove(helperLog); err != nil && !os.IsNotExist(err) { + framework.ExpectNoError(err) + } + launchStarted := time.Now().UTC().Format(time.RFC3339Nano) + _, stderr, err = x.up(ctx, "--provider-option", "IMAGE_CONFIG_FILE="+x.file) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr) + pod := x.pod(ctx) + gomega.Expect(pod.Spec.ServiceAccountName).To(gomega.Equal(cmdWorkspace)) + gomega.Expect(pod.Spec.Containers[0].Image).To(gomega.Equal(x.image)) + gomega.Expect(pod.Status.ContainerStatuses[0].ImageID). + To(gomega.ContainSubstring(strings.Split(x.image, "@")[1])) + delta := imageConfigMust( + ctx, + nil, + testDockerCommand, + "logs", + "--since", + launchStarted, + x.registry, + ) + gomega.Expect(strings.ToLower(delta)).To(gomega.ContainSubstring("containerd")) + nodeIP := imageConfigMust( + ctx, + nil, + testDockerCommand, + "inspect", + "-f", + "{{(index .NetworkSettings.Networks \"kind\").IPAddress}}", + x.node, + ) + nodeRequests := 0 + configFetched := false + for line := range strings.SplitSeq(delta, "\n") { + if strings.Contains(line, "http.request.remoteaddr") { + gomega.Expect(line).To(gomega.ContainSubstring(nodeIP)) + nodeRequests++ + if strings.Contains(line, "http.request.method=GET") && + strings.Contains(line, "/blobs/"+x.configDigest) && + strings.Contains(line, "http.response.status=200") { + configFetched = true + } + } + } + gomega.Expect(nodeRequests).To(gomega.BeNumerically(">", 0)) + gomega.Expect(configFetched). + To(gomega.BeTrue(), "node must fetch the unique image config") + _, err = os.Stat(helperLog) + gomega.Expect(err).To(gomega.MatchError(os.ErrNotExist)) + observe := "printf '%s|%s|%s|%s' \"$(id -un)\" \"$IMAGE_CONFIG_VALUE\" " + + "\"$IMAGE_CONFIG_OVERRIDE\" \"$IMAGE_CONFIG_METADATA\" && " + + "test -f /tmp/image-config-lifecycle && test -f /tmp/image-config-entrypoint && " + + "tr '\\0' ' ' imageConfigLimit { + return nil, fmt.Errorf("image config file exceeds 1 MiB") + } + if !utf8.Valid(data) { + return nil, fmt.Errorf("image config file is not UTF-8") + } + return data, nil +} - return details, nil +func parseImageConfig(data []byte) (imageConfigFile, config.ImageDetailsConfig, error) { + var file imageConfigFile + var payload config.ImageDetailsConfig + if err := decodeObject(data, &file); err != nil || !validWrapper(data) { + return file, payload, fmt.Errorf("invalid wrapper") + } + if file.SchemaVersion != 1 { + return file, payload, fmt.Errorf("unsupported schemaVersion") + } + if err := decodeObject(file.Config, &payload); err != nil || !validPayload(file.Config) { + return file, payload, fmt.Errorf("invalid config object") + } + return file, payload, nil } -// loadImageDetails parses an image config file. It accepts the JSON of -// config.ImageDetails, an OCI image config (e.g. from `crane config`) or a -// single `docker image inspect` object, since field names match -// case-insensitively and unknown fields are ignored. -func loadImageDetails(path, imageName string) (*config.ImageDetails, error) { - // #nosec G304 -- path comes from the operator-controlled IMAGE_CONFIG_FILE provider option - data, err := os.ReadFile(path) +func matchImageReferences(imageName, fileImage string) error { + requested, err := pinnedReference(imageName) + if err != nil { + return fmt.Errorf("requested image must be a fully qualified digest reference") + } + supplied, err := pinnedReference(fileImage) if err != nil { - return nil, fmt.Errorf("read image config file %q: %w", path, err) + return fmt.Errorf("image must be a fully qualified digest reference") + } + if requested.Name() != supplied.Name() { + return fmt.Errorf("image repository or digest does not match requested image") + } + return nil +} + +func imageConfigPlatform(file imageConfigFile, architecture string) (ocispec.Platform, error) { + platform := platforms.Normalize( + ocispec.Platform{OS: file.OS, Architecture: file.Architecture, Variant: file.Variant}, + ) + target := platforms.Normalize(ocispec.Platform{OS: "linux", Architecture: architecture}) + if file.OS == "" || platform.OS != "linux" || platform.Architecture == "" || + platform.Architecture != target.Architecture || !validVariant(platform.Architecture, platform.Variant) { + return platform, fmt.Errorf("image platform does not match Linux ARCHITECTURE") } + return platform, nil +} +func pinnedReference(ref string) (name.Digest, error) { + parts := strings.SplitN(ref, "/", 2) + if len(parts) != 2 || (!strings.ContainsAny(parts[0], ".:") && parts[0] != "localhost") { + return name.Digest{}, fmt.Errorf("missing registry") + } + return name.NewDigest(ref, name.StrictValidation) +} + +func decodeObject(data []byte, out any) error { data = bytes.TrimSpace(data) if len(data) == 0 || data[0] != '{' { - return nil, fmt.Errorf("image config file %q must contain a JSON object", path) + return fmt.Errorf("expected object") } - - details := &config.ImageDetails{} - if err := json.Unmarshal(data, details); err != nil { - return nil, fmt.Errorf("parse image config file %q: %w", path, err) + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(out); err != nil { + return err } + var extra any + if err := dec.Decode(&extra); err != io.EOF { + return fmt.Errorf("trailing data") + } + return nil +} - // An image may legitimately have an empty config, so check that the - // config object is present rather than that any of its fields are set. - var probe struct { - Config json.RawMessage `json:"config"` +func validateImageMetadata(raw string) error { + if raw == "" { + return nil + } + var value any + if json.Unmarshal([]byte(raw), &value) != nil || value == nil { + return fmt.Errorf("invalid %s label", metadata.ImageMetadataLabel) } - if err := json.Unmarshal(data, &probe); err != nil { - return nil, fmt.Errorf("parse image config file %q: %w", path, err) + entries, ok := value.([]any) + if !ok { + entries = []any{value} } - if len(probe.Config) == 0 || probe.Config[0] != '{' { - return nil, fmt.Errorf("image config file %q has no config object", path) + for _, entry := range entries { + if !validMetadataEntry(entry) { + return fmt.Errorf("invalid %s label", metadata.ImageMetadataLabel) + } } + return nil +} - if err := validateImageMetadata( - details.Config.Labels[metadata.ImageMetadataLabel], - ); err != nil { - return nil, fmt.Errorf("image config file %q: %w", path, err) +func validMetadataEntry(entry any) bool { + if _, ok := entry.(map[string]any); !ok { + return false } + b, err := json.Marshal(entry) + if err != nil { + return false + } + var item config.ImageMetadata + return json.Unmarshal(b, &item) == nil +} - if details.ID == "" { - details.ID = imageName +func validWrapper(data []byte) bool { + if !uniqueJSONKeys(data) { + return false + } + var fields map[string]json.RawMessage + if json.Unmarshal(data, &fields) != nil { + return false } + for key, value := range fields { + if !validWrapperField(key, value) { + return false + } + } + for _, key := range []string{"schemaVersion", "image", "os", "architecture", "config"} { + if _, ok := fields[key]; !ok { + return false + } + } + return true +} - return details, nil +func validWrapperField(key string, value json.RawMessage) bool { + switch key { + case "schemaVersion": + var version int + return !bytes.Equal(bytes.TrimSpace(value), []byte("null")) && + json.Unmarshal(value, &version) == nil + case "image", "os", "architecture", "variant": + return isJSONString(value) + case "config": + return startsWith(value, '{') + default: + return false + } } -// validateImageMetadata checks that the devcontainer.metadata label is a JSON -// object or an array of objects. Metadata read from the registry is ignored -// with a warning when malformed, but a file the operator supplied should fail -// instead of silently dropping its lifecycle commands and settings. -func validateImageMetadata(raw string) error { - if raw == "" { - return nil +func validPayload(data []byte) bool { + var fields map[string]json.RawMessage + if json.Unmarshal(data, &fields) != nil { + return false + } + for key, value := range fields { + if !validPayloadField(key, value) { + return false + } + } + return true +} + +func validPayloadField(key string, value json.RawMessage) bool { + switch key { + case "User": + return isJSONString(value) + case "Env", "Entrypoint", "Cmd": + return isStringArray(value) + case "Labels": + return isStringMap(value) + default: + return false } +} + +func startsWith(value json.RawMessage, prefix byte) bool { + value = bytes.TrimSpace(value) + return len(value) > 0 && value[0] == prefix +} - var entries []*config.ImageMetadata - if err := json.Unmarshal([]byte(raw), &entries); err == nil { - if entries == nil { - return fmt.Errorf("%s label is null", metadata.ImageMetadataLabel) +func isStringArray(value json.RawMessage) bool { + var values []json.RawMessage + if !startsWith(value, '[') || json.Unmarshal(value, &values) != nil { + return false + } + for _, v := range values { + if !isJSONString(v) { + return false } - for _, entry := range entries { - if entry == nil { - return fmt.Errorf("%s label contains a null entry", metadata.ImageMetadataLabel) - } + } + return true +} + +func isStringMap(value json.RawMessage) bool { + var values map[string]json.RawMessage + if !startsWith(value, '{') || json.Unmarshal(value, &values) != nil { + return false + } + for _, v := range values { + if !isJSONString(v) { + return false } + } + return true +} + +func isJSONString(value json.RawMessage) bool { + var s string + return len(bytes.TrimSpace(value)) > 0 && bytes.TrimSpace(value)[0] == '"' && + json.Unmarshal(value, &s) == nil +} + +func uniqueJSONKeys(data []byte) bool { + dec := json.NewDecoder(bytes.NewReader(data)) + return uniqueJSONValue(dec) == nil +} + +func uniqueJSONValue(dec *json.Decoder) error { + token, err := dec.Token() + if err != nil { + return err + } + delim, ok := token.(json.Delim) + if !ok { return nil } + switch delim { + case '{': + err = uniqueJSONObject(dec) + case '[': + err = uniqueJSONArray(dec) + default: + return fmt.Errorf("invalid JSON") + } + if err != nil { + return err + } + _, err = dec.Token() + return err +} - single := &config.ImageMetadata{} - if raw[0] != '{' { - return fmt.Errorf("%s label must be a JSON object or array", metadata.ImageMetadataLabel) +func uniqueJSONObject(dec *json.Decoder) error { + seen := map[string]bool{} + for dec.More() { + token, err := dec.Token() + if err != nil { + return err + } + key, ok := token.(string) + if !ok || seen[key] { + return fmt.Errorf("duplicate key") + } + seen[key] = true + if err := uniqueJSONValue(dec); err != nil { + return err + } } - if err := json.Unmarshal([]byte(raw), single); err != nil { - return fmt.Errorf("parse %s label: %w", metadata.ImageMetadataLabel, err) + return nil +} + +func uniqueJSONArray(dec *json.Decoder) error { + for dec.More() { + if err := uniqueJSONValue(dec); err != nil { + return err + } } return nil } + +func validVariant(arch, variant string) bool { + if variant == "" { + return true + } + switch arch { + case "amd64": + return variantIn(variant, "v2", "v3", "v4") + case "arm": + return variantIn(variant, "v5", "v6", "v7", "v8") + case "arm64": + return arm64VariantPattern.MatchString(variant) + default: + return true + } +} + +func variantIn(variant string, allowed ...string) bool { + return slices.Contains(allowed, variant) +} diff --git a/pkg/driver/kubernetes/image_details_test.go b/pkg/driver/kubernetes/image_details_test.go index 9cda0b7c26..14afdd6638 100644 --- a/pkg/driver/kubernetes/image_details_test.go +++ b/pkg/driver/kubernetes/image_details_test.go @@ -1,165 +1,195 @@ package kubernetes import ( + "encoding/json" "os" "path/filepath" + "strings" "testing" + "github.com/devsy-org/devsy/pkg/driver" provider2 "github.com/devsy-org/devsy/pkg/provider" - "github.com/stretchr/testify/suite" + "github.com/stretchr/testify/require" ) -const testImage = "registry.example.com/team/image@sha256:abc" +const testImage = "registry.example.com/team/image@sha256:" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -type ImageDetailsSuite struct { - suite.Suite -} - -func TestImageDetailsSuite(t *testing.T) { - suite.Run(t, new(ImageDetailsSuite)) -} - -func (s *ImageDetailsSuite) TestNotConfigured() { - details, err := s.driver("").ImageDetails(testImage) - s.Require().NoError(err) - s.Nil(details) -} - -func (s *ImageDetailsSuite) TestImageDetailsJSON() { - path := s.writeFile(`{ - "ID": "custom-id", - "Config": { - "User": "vscode", - "Env": ["PATH=/usr/bin"], - "Labels": {"devcontainer.metadata": "[]"}, - "Entrypoint": ["/entrypoint.sh"], - "Cmd": ["sleep", "infinity"] - } - }`) - - details, err := s.driver(path).ImageDetails(testImage) - s.Require().NoError(err) - s.Equal("custom-id", details.ID) - s.Equal("vscode", details.Config.User) - s.Equal([]string{"PATH=/usr/bin"}, details.Config.Env) - s.Equal(map[string]string{"devcontainer.metadata": "[]"}, details.Config.Labels) - s.Equal([]string{"/entrypoint.sh"}, details.Config.Entrypoint) - s.Equal([]string{"sleep", "infinity"}, details.Config.Cmd) -} - -func (s *ImageDetailsSuite) TestOCIImageConfig() { - // Output shape of `crane config `. - path := s.writeFile(`{ - "architecture": "amd64", - "os": "linux", - "config": {"User": "vscode", "Env": ["PATH=/usr/bin"]}, - "rootfs": {"type": "layers", "diff_ids": []} - }`) - - details, err := s.driver(path).ImageDetails(testImage) - s.Require().NoError(err) - s.Equal(testImage, details.ID) - s.Equal("amd64", details.Architecture) - s.Equal("linux", details.OS) - s.Equal("vscode", details.Config.User) -} - -func (s *ImageDetailsSuite) TestEmptyConfig() { - // An image with no user, env, labels, entrypoint or cmd. - path := s.writeFile(`{"architecture": "amd64", "os": "linux", "config": {}}`) - - details, err := s.driver(path).ImageDetails(testImage) - s.Require().NoError(err) - s.Equal(testImage, details.ID) - s.Empty(details.Config.User) -} - -func (s *ImageDetailsSuite) TestRequiresArchitecture() { - path := s.writeFile(`{"config": {}}`) - d := s.driver(path) - d.options.Architecture = "" +const testArchitecture = "amd64" - _, err := d.ImageDetails(testImage) - s.Require().ErrorContains(err, "requires the ARCHITECTURE option") +type imageDetailsTestFixture struct { + driver *KubernetesDriver + path string + request driver.ImageDetailsRequest + wrapper func(string) string + write func(string) } -func (s *ImageDetailsSuite) TestArchitectureMismatch() { - path := s.writeFile(`{"architecture": "arm64", "os": "linux", "config": {}}`) - - _, err := s.driver(path).ImageDetails(testImage) - s.Require().ErrorContains(err, `is for architecture "arm64", but ARCHITECTURE is "amd64"`) -} - -func (s *ImageDetailsSuite) TestValidMetadata() { - tests := map[string]string{ - "array": `[{\"remoteUser\": \"vscode\"}]`, - "object": `{\"remoteUser\": \"vscode\"}`, - "empty": `[]`, - } - for name, label := range tests { - s.Run(name, func() { - path := s.writeFile( - `{"config": {"Labels": {"devcontainer.metadata": "` + label + `"}}}`, - ) - _, err := s.driver(path).ImageDetails(testImage) - s.NoError(err) - }) +func imageDetailsFixture(t *testing.T) imageDetailsTestFixture { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + d := &KubernetesDriver{ + options: &provider2.ProviderKubernetesDriverConfig{ + ImageConfigFile: path, + Architecture: testArchitecture, + }, } -} - -func (s *ImageDetailsSuite) TestInvalidMetadata() { - tests := map[string]string{ - "bad json": `[{\"remoteUser\": `, - "null entry": `[null]`, - "null": `null`, - "string": `\"vscode\"`, - "wrong type": `{\"remoteUser\": 1}`, + request := driver.ImageDetailsRequest{ImageName: testImage} + wrapper := func(payload string) string { + return `{"schemaVersion":1,"image":"` + testImage + `","os":"linux","architecture":"amd64","config":` + payload + `}` } - for name, label := range tests { - s.Run(name, func() { - path := s.writeFile( - `{"config": {"Labels": {"devcontainer.metadata": "` + label + `"}}}`, - ) - _, err := s.driver(path).ImageDetails(testImage) - s.ErrorContains(err, "devcontainer.metadata") - }) + write := func(data string) { t.Helper(); require.NoError(t, os.WriteFile(path, []byte(data), 0o600)) } + return imageDetailsTestFixture{ + driver: d, + path: path, + request: request, + wrapper: wrapper, + write: write, } } -func (s *ImageDetailsSuite) TestMissingFile() { - path := filepath.Join(s.T().TempDir(), "missing.json") - _, err := s.driver(path).ImageDetails(testImage) - s.Require().ErrorContains(err, "read image config file") +func TestImageDetailsFile(t *testing.T) { + f := imageDetailsFixture(t) + d, path, request, wrapper, write := f.driver, f.path, f.request, f.wrapper, f.write + d.options.ImageConfigFile = "" + details, err := d.ImageDetails(request) + require.NoError(t, err) + require.Nil(t, details) + d.options.ImageConfigFile = path + _, err = d.ImageDetails(driver.ImageDetailsRequest{UnsupportedReason: "Features"}) + require.ErrorContains(t, err, "Features") + write( + wrapper( + `{"User":"vscode","Env":["A=b=c"],"Labels":{"devcontainer.metadata":"[]"},"Entrypoint":["a b"],"Cmd":["$x"]}`, + ), + ) + details, err = d.ImageDetails(request) + require.NoError(t, err) + require.Equal(t, testImage, details.ID) + require.Equal(t, []string{"A=b=c"}, details.Config.Env) + require.Equal(t, []string{"a b"}, details.Config.Entrypoint) + require.Equal(t, []string{"$x"}, details.Config.Cmd) + write(wrapper(`{}`)) + _, err = d.ImageDetails(request) + require.NoError(t, err) + exact := wrapper(`{}`) + write(exact + strings.Repeat(" ", imageConfigLimit-len(exact))) + _, err = d.ImageDetails(request) + require.NoError(t, err) + _, err = d.ImageDetails( + driver.ImageDetailsRequest{ImageName: "registry.example.com/team/image:latest"}, + ) + require.Error(t, err) } -func (s *ImageDetailsSuite) TestInvalidContent() { - tests := map[string]string{ - "empty": "", - "array": `[{"Config": {"User": "vscode"}}]`, - "bad json": `{"Config": `, - "no config": `{"schemaVersion": 2, "layers": []}`, - "null config": `{"config": null}`, - "config list": `{"config": []}`, - } - for name, content := range tests { - s.Run(name, func() { - _, err := s.driver(s.writeFile(content)).ImageDetails(testImage) - s.Error(err) - }) +func invalidImageDetailsFiles(wrapper func(string) string) map[string]string { + return map[string]string{ + "missing config": `{"schemaVersion":1}`, + "null config": wrapper(`null`), + "unknown config": wrapper(`{"Bad":1}`), + "trailing": wrapper(`{}`) + ` {}`, + "schema": strings.Replace( + wrapper(`{}`), + `"schemaVersion":1`, + `"schemaVersion":2`, + 1, + ), + "wrong image": strings.Replace(wrapper(`{}`), "/team/image@", "/team/other@", 1), + "wrong platform": strings.Replace(wrapper(`{}`), `"amd64"`, `"arm64"`, 1), + "metadata": wrapper(`{"Labels":{"devcontainer.metadata":"[null]"}}`), + "invalid utf8": wrapper(`{}`) + string([]byte{0xff}), + "oversize": wrapper( + `{}`, + ) + strings.Repeat( + " ", + imageConfigLimit+1-len(wrapper(`{}`)), + ), + "wrong digest": strings.Replace( + wrapper(`{}`), + strings.Repeat("a", 64), + strings.Repeat("b", 64), + 1, + ), + "tagged file": strings.Replace( + wrapper(`{}`), + testImage, + "registry.example.com/team/image:latest", + 1, + ), + "non linux": strings.Replace(wrapper(`{}`), `"linux"`, `"windows"`, 1), + "null architecture": strings.Replace(wrapper(`{}`), `"amd64"`, `null`, 1), + "array config": wrapper(`[]`), + "wrong env type": wrapper(`{"Env":[null]}`), + "unknown wrapper": strings.Replace( + wrapper(`{}`), + `"schemaVersion":1`, + `"schemaVersion":1,"extra":true`, + 1, + ), + "case alias": strings.Replace(wrapper(`{}`), `"schemaVersion"`, `"SchemaVersion"`, 1), + "duplicate": strings.Replace( + wrapper(`{}`), + `"schemaVersion":1`, + `"schemaVersion":1,"schemaVersion":1`, + 1, + ), + "contradictory variant": strings.Replace( + wrapper(`{}`), + `"architecture":"amd64"`, + `"architecture":"amd64","variant":"v8"`, + 1, + ), } } -func (s *ImageDetailsSuite) writeFile(content string) string { - path := filepath.Join(s.T().TempDir(), "image-config.json") - s.Require().NoError(os.WriteFile(path, []byte(content), 0o600)) - return path +func TestImageDetailsInvalidFiles(t *testing.T) { + f := imageDetailsFixture(t) + d, request, wrapper, write := f.driver, f.request, f.wrapper, f.write + cases := invalidImageDetailsFiles(wrapper) + for name, data := range cases { + t.Run( + name, + func(t *testing.T) { write(data); _, err := d.ImageDetails(request); require.Error(t, err) }, + ) + } } -func (s *ImageDetailsSuite) driver(path string) *KubernetesDriver { - return &KubernetesDriver{ - options: &provider2.ProviderKubernetesDriverConfig{ - ImageConfigFile: path, - Architecture: "amd64", - }, - } +func TestImageDetailsMetadataAndPaths(t *testing.T) { + f := imageDetailsFixture(t) + d, path, request, wrapper, write := f.driver, f.path, f.request, f.wrapper, f.write + b, err := json.Marshal(map[string]string{"devcontainer.metadata": `{"remoteUser":1}`}) + require.NoError(t, err) + write(wrapper(`{"Labels":` + string(b) + `}`)) + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "devcontainer.metadata") + write( + strings.Replace( + wrapper( + `{"Labels":{"devcontainer.metadata":"[{\"customizations\":{\"example\":null}}]"}}`, + ), + `"architecture":"amd64"`, + `"architecture":"aarch64","variant":"v8"`, + 1, + ), + ) + d.options.Architecture = "arm64" + details, err := d.ImageDetails(request) + require.NoError(t, err) + require.Equal(t, "arm64", details.Architecture) + require.Empty(t, details.Variant) + d.options.Architecture = testArchitecture + d.options.ImageConfigFile = filepath.Join(filepath.Dir(path), "missing") + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "read image config file") + d.options.ImageConfigFile = filepath.Dir(path) + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "regular") + d.options.ImageConfigFile = "relative" + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "absolute") + d.options.ImageConfigFile = path + d.options.Architecture = "" + _, err = d.ImageDetails(request) + require.ErrorContains(t, err, "ARCHITECTURE") } diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go index 48f830303e..9b8ce270db 100644 --- a/pkg/driver/kubernetes/pod_lifecycle_test.go +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -5,13 +5,17 @@ import ( "encoding/json" "errors" "os" + "path/filepath" + goruntime "runtime" "sync/atomic" "testing" "time" + "github.com/devsy-org/devsy/e2e/framework/runenv" pkgconfig "github.com/devsy-org/devsy/pkg/config" agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/dockercredentials" "github.com/devsy-org/devsy/pkg/driver" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/stretchr/testify/assert" @@ -1215,3 +1219,77 @@ func assertOnlyReads(t *testing.T, c *fakeCluster) { func writeFile(path, content string) error { return os.WriteFile(path, []byte(content), 0o600) } + +func credentialHelperMarker(t *testing.T) string { + t.Helper() + if goruntime.GOOS == "windows" { + t.Skip("credential helper spy requires a Unix shell") + } + dir := t.TempDir() + marker := filepath.Join(dir, "called") + helper := filepath.Join(dir, "docker-credential-devsyspy") + script := "#!/bin/sh\nprintf called > \"$DEVSY_TEST_CREDENTIAL_MARKER\"\n" + + "printf '{\"Username\":\"test\",\"Secret\":\"test\"}\\n'\n" + require.NoError(t, (runenv.OsFileStore{}).WriteFile(helper, []byte(script), 0o700)) + require.NoError( + t, + os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"credsStore":"devsyspy"}`), 0o600), + ) + t.Setenv("DOCKER_CONFIG", dir) + t.Setenv("DEVSY_TEST_CREDENTIAL_MARKER", marker) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + _, err := dockercredentials.GetAuthConfig("registry.example.com") + require.NoError(t, err) + require.FileExists(t, marker) + require.NoError(t, os.Remove(marker)) + return marker +} + +func TestImageConfigPreservesClusterPullCredentials(t *testing.T) { + marker := credentialHelperMarker(t) + const adminSecret = "admin-pull" + account := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-account", Namespace: testNamespace}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: adminSecret}}, + } + c := newFakeCluster(t, account) + c.k.options.ImageConfigFile = "/local/image.json" + c.k.options.ServiceAccount = account.Name + c.k.options.KubernetesPullSecretsEnabled = pkgconfig.BoolFalse + c.k.agentConfig.InjectDockerCredentials = pkgconfig.BoolFalse + template := filepath.Join(t.TempDir(), "pod.yaml") + require.NoError(t, writeFile(template, "spec:\n imagePullSecrets:\n - name: template-pull\n")) + c.k.options.PodManifestTemplate = template + reads := 0 + c.cs.PrependReactor( + "get", + "secrets", + func(action clienttesting.Action) (bool, runtime.Object, error) { + if action.(clienttesting.GetAction).GetName() != adminSecret { + return false, nil, nil + } + reads++ + return true, nil, kerrors.NewForbidden( + schema.GroupResource{Resource: "secrets"}, + adminSecret, + errors.New("denied"), + ) + }, + ) + options := runOptionsForWorkspace("1m") + options.Image = testImage + pod, err := c.k.buildPod(context.Background(), testPodID, options, false) + require.NoError(t, err) + assert.Equal(t, account.Name, pod.Spec.ServiceAccountName) + assert.Equal( + t, + []corev1.LocalObjectReference{{Name: "template-pull"}}, + pod.Spec.ImagePullSecrets, + ) + saved, err := c.cs.CoreV1().ServiceAccounts(testNamespace). + Get(context.Background(), account.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, account.ImagePullSecrets, saved.ImagePullSecrets) + assert.Zero(t, reads) + assert.NoFileExists(t, marker) +} diff --git a/pkg/driver/kubernetes/run.go b/pkg/driver/kubernetes/run.go index b25b7d620f..c1bbceb4af 100644 --- a/pkg/driver/kubernetes/run.go +++ b/pkg/driver/kubernetes/run.go @@ -764,9 +764,6 @@ func optionsEqual(a, b *provider2.ProviderKubernetesDriverConfig) bool { return aCopy == bCopy } -// withoutLocalOptions returns a copy of options without settings that only -// apply on the machine running Devsy. They must not be stored on the pod or -// decide whether it gets replaced. func withoutLocalOptions( options *provider2.ProviderKubernetesDriverConfig, ) provider2.ProviderKubernetesDriverConfig { diff --git a/pkg/driver/types.go b/pkg/driver/types.go index 3f0dcbb30f..afc94f4249 100644 --- a/pkg/driver/types.go +++ b/pkg/driver/types.go @@ -54,13 +54,13 @@ type WorkspaceChowner interface { RequiresWorkspaceChown() bool } -// ImageDetailsDriver is a capability interface for drivers that can supply the -// image config of a prebuilt image themselves, so the client does not need to -// read it from the registry. +type ImageDetailsRequest struct { + ImageName string + UnsupportedReason string +} + type ImageDetailsDriver interface { - // ImageDetails returns the configured details for imageName, or nil when - // none are configured. - ImageDetails(imageName string) (*config.ImageDetails, error) + ImageDetails(request ImageDetailsRequest) (*config.ImageDetails, error) } // ManagedLifecycleDriver identifies drivers that can run the container's main diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 7b29100079..e86217ea8d 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -452,9 +452,7 @@ type ProviderKubernetesDriverConfig struct { ClusterRole string `json:"clusterRole,omitempty"` ServiceAccount string `json:"serviceAccount,omitempty"` - Architecture string `json:"architecture,omitempty"` - // ImageConfigFile is a path to a JSON file with the image config of a prebuilt - // image, used instead of reading it from the registry on the client. + Architecture string `json:"architecture,omitempty"` ImageConfigFile string `json:"imageConfigFile,omitempty"` // InactivityTimeout is a deprecated alias of Agent.ContainerTimeout, used only // when the canonical field is empty. diff --git a/providers/kubernetes/provider.yaml b/providers/kubernetes/provider.yaml index 4abcd11d51..29d863ced5 100644 --- a/providers/kubernetes/provider.yaml +++ b/providers/kubernetes/provider.yaml @@ -119,7 +119,7 @@ options: description: The cpu architecture to use for the workspace pod, e.g. amd64, arm64, etc. type: string IMAGE_CONFIG_FILE: - description: Path to a JSON file with the image config of the devcontainer's prebuilt image (e.g. the output of `crane config `). If set, Devsy reads the user, env, entrypoint, cmd and devcontainer.metadata label from this file instead of fetching the image config from the registry, so the machine running Devsy needs no registry credentials. Devsy fails if the file is missing or invalid. Requires ARCHITECTURE to be set and to match the file's architecture. Pin the image by digest so the file matches the image the cluster pulls. Applies to the devcontainer.json image only; Dockerfile builds still read their base image from the registry. + description: Absolute local path to a version 1 image config file for a digest-pinned prebuilt image without Features. Requires matching ARCHITECTURE. type: string agent: containerInactivityTimeout: ${INACTIVITY_TIMEOUT} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx index 9cdf1036aa..0a4f5b9d35 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx @@ -53,7 +53,7 @@ The allowed options for the Kubernetes driver are: - **clusterRole**: if defined, Devsy will create a role binding for the given cluster role for the workspace container. This is useful if you need Kubernetes access within the workspace container - **serviceAccount**: if defined, Devsy will use the given service account for the dev container - **architecture**: the CPU architecture to use for the workspace pod, e.g. `amd64`, `arm64`. If empty, Devsy inspects the cluster's nodes to auto-detect it (and errors out if the cluster has mixed architectures). -- **imageConfigFile**: path to a JSON file with the image config of the devcontainer.json `image` (for example the output of `crane config `). When set, Devsy reads the user, env, entrypoint, cmd and `devcontainer.metadata` label from the file instead of fetching the image config from the registry on the client. See [Starting workspaces without registry access](#starting-workspaces-without-registry-access). +- **imageConfigFile**: path to a versioned JSON wrapper containing the digest-pinned image reference, platform and OCI config for the devcontainer.json `image`. When set, Devsy reads the user, env, entrypoint, cmd and `devcontainer.metadata` label from the file instead of fetching the image config from the registry on the client. See [Starting workspaces without registry access](#starting-workspaces-without-registry-access). - **inactivityTimeout**: after how much time to automatically stop the pod due to inactivity - **storageClass**: the storage class to use to create the persistent volume claim - **diskSize**: the default size for the persistent volume to use, e.g. `10Gi` @@ -84,20 +84,9 @@ Devsy builds images inside Kubernetes without Docker through the `dockerless` bu ### Starting workspaces without registry access -For a devcontainer.json with a prebuilt `image`, Devsy reads the image config (user, env, entrypoint, cmd and the `devcontainer.metadata` label) from the registry on the machine that runs Devsy. In clusters that already have their own registry credentials, the person starting the workspace would otherwise need registry read access and a local `docker login` just for this step. +The Kubernetes driver can use a local image configuration file for a prebuilt `image` in `devcontainer.json`. Devsy reads the user, environment, entrypoint, command and `devcontainer.metadata` label from this file instead of requesting image configuration from the registry on the client. See [Kubernetes registry access](../managing-providers/kubernetes-registry-access.mdx) for the file format, setup and limitations. -To skip it, produce the image config next to the image, for example in CI, and point the `IMAGE_CONFIG_FILE` option at the file: - -```bash -crane config registry.example.com/team/image@sha256: > image-config.json -devsy provider set kubernetes -o IMAGE_CONFIG_FILE=$PWD/image-config.json -o ARCHITECTURE=amd64 -``` - -The file can be an OCI image config (as printed by `crane config`), a single `docker image inspect` object, or the JSON of Devsy's image details. Devsy never contacts the registry for this step when the option is set, and it fails if the file is missing or invalid. Notes: - -- Pin the image by digest in devcontainer.json, so the file always matches the image the cluster pulls. -- This applies to the devcontainer.json `image`. Dockerfile builds still read their base image from the registry, and `features` are built from the image as usual. -- `ARCHITECTURE` (for example `amd64`) is required, so Devsy does not list the cluster's nodes to detect it, which needs cluster-scoped permissions. If the file has an `architecture`, it must match. On a cluster with mixed architectures, also schedule the pod onto matching nodes, for example with `NODE_SELECTOR`. +When a driver supports supplied image details, its `ImageDetails` capability method accepts an `ImageDetailsRequest` with `ImageName` and `UnsupportedReason`, then returns the validated `ImageDetails` or an error. The result is `nil, nil` only when the option is unset. Configured input must be validated for the requested image before use. Unsupported combinations and invalid files return errors. They must not fall back to registry inspection. ### Example Kubernetes Provider diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/kubernetes-registry-access.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/kubernetes-registry-access.mdx new file mode 100644 index 0000000000..bd94fb391b --- /dev/null +++ b/sites/docs-devsy-sh/content/docs/managing-providers/kubernetes-registry-access.mdx @@ -0,0 +1,67 @@ +--- +title: Kubernetes registry access without client credentials +sidebar_label: Kubernetes registry access +--- + +When a Kubernetes workspace uses a prebuilt `image` from a private registry, the cluster may already have permission to pull it through a node identity or a service account `imagePullSecret`. Devsy normally also reads the image configuration on the client to determine the container user, environment, entrypoint, command and `devcontainer.metadata`. The Kubernetes `IMAGE_CONFIG_FILE` option supplies those details locally, so the client does not need registry read credentials for this inspection. + +This option applies to a digest-pinned `image` with no Features. It does not support Dockerfile builds, Compose, recovery, prebuilds or workspace image builds. The file is trusted input. Its digest association prevents accidental mismatch when used correctly, but does not prove that the file contents came from the image. + +## Create the image details file + +Use a fully qualified digest reference and export the config for the selected Linux platform. Convert the OCI config to Devsy's versioned wrapper: + +```sh +IMAGE='registry.example.com/team/dev-image@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' +crane config --platform linux/amd64 "$IMAGE" | jq --arg image "$IMAGE" '{schemaVersion: 1, image: $image, os: "linux", architecture: "amd64", config: (.config | {User: (.User // ""), Env: (.Env // []), Labels: (.Labels // {}), Entrypoint: (.Entrypoint // []), Cmd: (.Cmd // [])})}' > image-details.json +``` + +The complete wrapper has this shape: + +```json +{ + "schemaVersion": 1, + "image": "registry.example.com/team/dev-image@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "os": "linux", + "architecture": "amd64", + "variant": "", + "config": { + "User": "1000:1000", + "Env": ["PATH=/usr/local/bin:/usr/bin"], + "Labels": {"devcontainer.metadata": "[]"}, + "Entrypoint": ["/usr/local/bin/start"], + "Cmd": ["serve"] + } +} +``` + +`variant` is optional. The file must be a regular UTF-8 file no larger than 1 MiB. It must contain one version 1 wrapper, a fully qualified digest reference matching the requested image, a Linux platform matching `ARCHITECTURE`, and a config object with valid fields. Missing or invalid input is a hard error, with no registry inspection fallback. + +Keep the image reference in `devcontainer.json` pinned to the same digest. Generate a separate file for each platform when publishing a multi-platform image. + +`ARCHITECTURE` selects the config platform and avoids listing cluster nodes. It does not schedule a pod onto a matching node. Use `NODE_SELECTOR` as well when the cluster has mixed architectures. + +Publish or distribute the image and its details file as separate artifacts. Importing the details file into Devsy does not upload it alongside the image or send it to the cluster. + +## Configure the Kubernetes provider + +The registry pull authorization must already be available to Kubernetes. Configure the workspace service account or pod template with the admin-managed `imagePullSecrets`, or use the node's registry identity. The Devsy client still needs its normal namespace permissions and access to the daemon-configuration Secrets it uses. It does not need permission to read the registry pull Secret. + +For cluster-managed pull credentials without client Docker credential reads, disable both client credential forwarding and Devsy pull-secret creation: + +```sh +devsy context set [context-name] --option SSH_INJECT_DOCKER_CREDENTIALS=false +devsy provider set kubernetes --option KUBERNETES_PULL_SECRETS_ENABLED=false +``` + +Set the file and architecture per workspace, using a stable absolute path: + +```sh +devsy workspace up ./project --provider kubernetes --ide none \ + --provider-option IMAGE_CONFIG_FILE="$PWD/image-details.json" \ + --provider-option ARCHITECTURE=amd64 +``` + +In Devsy Desktop, set `IMAGE_CONFIG_FILE` to an absolute path on the machine running Desktop. Keep that file at the same path while using the provider. Moving it does not update an existing workspace's saved image configuration. A normal `up` validates the configured file but keeps the workspace's saved launch recipe. Changing image metadata takes effect when you explicitly recreate the workspace with the updated file. A local path change alone does not replace a running pod. If you move to another machine or workspace host, copy the file there and configure its new absolute local path. Once a workspace exists, the file is not needed for status, logs, stop or delete operations. + +If cluster pull authorization is missing, Kubernetes reports an image pull failure such as `ImagePullBackOff`. Configure the service account, pod template or node identity that performs the pull. Adding credentials to the client's Docker config does not grant the cluster permission to pull the image. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx index a9f0702810..6b839494db 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx @@ -112,6 +112,10 @@ Change one later: devsy provider set --option KEY=VALUE ``` +## Kubernetes registry access + +For digest-pinned prebuilt images on Kubernetes, [configure cluster registry access without client credentials](./kubernetes-registry-access.mdx). + ## Docker provider Devsy follows the Docker CLI's rules for contexts and daemon endpoints. It keeps your existing Docker `config.json` and contexts, so contexts from Docker Desktop, OrbStack, or a remote endpoint keep working. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/meta.json b/sites/docs-devsy-sh/content/docs/managing-providers/meta.json index 775e246cff..e26c174546 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/meta.json +++ b/sites/docs-devsy-sh/content/docs/managing-providers/meta.json @@ -1,4 +1,4 @@ { "title": "Managing your Providers", - "pages": ["what-are-providers", "manage-providers"] + "pages": ["what-are-providers", "manage-providers", "kubernetes-registry-access"] } From c1b523dd68ee7cae10637ee6686182d1d644a3a0 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 31 Oct 2026 16:12:37 -0600 Subject: [PATCH 09/10] fix(kubernetes): keep image config paths out of container setup --- pkg/devcontainer/setup.go | 1 + pkg/devcontainer/setup_secrets_test.go | 32 +++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index 9e5e3c8721..2fd5904122 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -410,6 +410,7 @@ func (r *runner) compressWorkspaceConfig( Agent: r.workspaceConfig.Agent, ContentFolder: r.workspaceConfig.ContentFolder, } + workspaceConfig.Agent.Kubernetes.ImageConfigFile = "" lifecycle, err := r.lifecycleOptions(mergedConfig, dockerless, imageEntrypoint) if err != nil { return "", err diff --git a/pkg/devcontainer/setup_secrets_test.go b/pkg/devcontainer/setup_secrets_test.go index 7696922b59..2f258d60ff 100644 --- a/pkg/devcontainer/setup_secrets_test.go +++ b/pkg/devcontainer/setup_secrets_test.go @@ -4,12 +4,17 @@ import ( "encoding/json" "testing" + "github.com/devsy-org/devsy/pkg/agent" "github.com/devsy-org/devsy/pkg/compress" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/require" ) -const testTerminalSecretName = "API_KEY" +const ( + testTerminalSecretName = "API_KEY" + testTransportArchitecture = "arm64" +) // Secret values must never ride in the compressed CLIOptions embedded in the // setup command's arguments; the container pulls them over the tunnel instead. @@ -66,3 +71,28 @@ func TestCompressWorkspaceConfig_StripsSecretValues(t *testing.T) { } } } + +func TestCompressWorkspaceConfigKeepsImageConfigFileLocal(t *testing.T) { + const path = "/operator/image-config.json" + r := &runner{ + workspaceConfig: &provider.AgentWorkspaceInfo{ + Workspace: &provider.Workspace{}, + Agent: provider.ProviderAgentConfig{ + Kubernetes: provider.ProviderKubernetesDriverConfig{ + ImageConfigFile: path, + Architecture: testTransportArchitecture, + ServiceAccount: "cluster-account", + }, + }, + }, + } + compressed, err := r.compressWorkspaceConfig(&config.MergedDevContainerConfig{}, false, false) + require.NoError(t, err) + info, raw, err := agent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + require.Empty(t, info.Agent.Kubernetes.ImageConfigFile) + require.NotContains(t, raw, path) + require.Equal(t, path, r.workspaceConfig.Agent.Kubernetes.ImageConfigFile) + require.Equal(t, testTransportArchitecture, info.Agent.Kubernetes.Architecture) + require.Equal(t, "cluster-account", info.Agent.Kubernetes.ServiceAccount) +} From cf94f458b7e346495dd14583797e678b2e58eefe Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 31 Oct 2026 17:55:21 -0600 Subject: [PATCH 10/10] test(kubernetes): calibrate client credential lookup spy --- e2e/tests/up/provider_kubernetes_image_config_file.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/e2e/tests/up/provider_kubernetes_image_config_file.go b/e2e/tests/up/provider_kubernetes_image_config_file.go index 251a9c49fd..b0cc97f202 100644 --- a/e2e/tests/up/provider_kubernetes_image_config_file.go +++ b/e2e/tests/up/provider_kubernetes_image_config_file.go @@ -445,13 +445,18 @@ func (x *imageConfigFixture) configureCredentialSpy() { "docker-credential-desktop", "docker-credential-osxkeychain", "docker-credential-pass", "docker-credential-secretservice", } { - script := []byte("#!/bin/sh\necho called >> \"$IMAGE_CONFIG_HELPER_LOG\"\nexit 1\n") + script := []byte("#!/bin/sh\necho called >> \"$IMAGE_CONFIG_HELPER_LOG\"\n" + + "echo 'credentials not found in native keychain'\nexit 1\n") framework.ExpectNoError( (runenv.OsFileStore{}).WriteFile(filepath.Join(helperDir, helper), script, 0o700), ) } framework.SetEnvForSpec("PATH", helperDir+string(os.PathListSeparator)+os.Getenv("PATH")) framework.ExpectNoError(os.MkdirAll(filepath.Join(root, "empty-docker"), 0o700)) + imageConfigWrite( + filepath.Join(root, "empty-docker", "config.json"), + map[string]string{"credsStore": "pass"}, + ) framework.SetEnvForSpec("DEVSY_HOME", filepath.Join(root, "devsy")) framework.SetEnvForSpec("DEVSY_SECRETS_BACKEND", "file") framework.SetEnvForSpec("DEVSY_SECRETS_PASSPHRASE", "fixture") @@ -591,6 +596,8 @@ var _ = ginkgo.Describe( "logs", "--since", negativeStarted, x.registry) gomega.Expect(negativeLogs).To(gomega.ContainSubstring(" 401 ")) gomega.Expect(negativeLogs).To(gomega.ContainSubstring("/manifests/")) + _, err = os.Stat(filepath.Join(x.root, "helper-calls")) + framework.ExpectNoError(err) _, stderr, err = x.up( ctx, "--provider-option",