From 720d44326462833c26ae6a4d025626ae953f6b3b Mon Sep 17 00:00:00 2001 From: Samuel K Date: Wed, 14 Oct 2026 18:57:48 -0600 Subject: [PATCH] refactor(e2e): share kubectl fixture runner in up tests Replace per-file copies of the kubectl/ssh command runner and workspace pod discovery with one shared set in e2e/tests/up/fixture_command.go: runFixtureCommand, fixtureCommand, and fixtureWorkspacePod. Timeouts are kept per call site. Failure text for the shared wrapper is now 'fixture command ...' for both call paths. Pod discovery gains the selector and GinkgoWriter diagnostics on the dockerless path. --- e2e/tests/up/fixture_command.go | 84 ++++++++ .../up/provider_kubernetes_dockerless_path.go | 73 ++----- e2e/tests/up/provider_kubernetes_ssh_agent.go | 198 +++++++++--------- 3 files changed, 197 insertions(+), 158 deletions(-) create mode 100644 e2e/tests/up/fixture_command.go diff --git a/e2e/tests/up/fixture_command.go b/e2e/tests/up/fixture_command.go new file mode 100644 index 0000000000..880db873d9 --- /dev/null +++ b/e2e/tests/up/fixture_command.go @@ -0,0 +1,84 @@ +package up + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "time" + + "github.com/devsy-org/devsy/pkg/docker" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +// runFixtureCommand runs argv (argv[0] is the program), bounded by timeout. env entries +// are appended to the current environment. +func runFixtureCommand( + ctx context.Context, + timeout time.Duration, + env []string, + argv ...string, +) (string, error) { + commandCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + // #nosec G204 -- executable and arguments are controlled by this test fixture. + cmd := exec.CommandContext(commandCtx, argv[0], argv[1:]...) + docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = 5 * time.Second + cmd.Env = append(os.Environ(), env...) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// fixtureCommand is runFixtureCommand that fails the current spec on error. +func fixtureCommand( + ctx context.Context, + timeout time.Duration, + env []string, + argv ...string, +) string { + out, err := runFixtureCommand(ctx, timeout, env, argv...) + gomega.Expect(err). + NotTo(gomega.HaveOccurred(), "fixture command %v: %s", argv, out) + return out +} + +// fixtureWorkspacePod waits for exactly one ready workspace pod matching selector. +func fixtureWorkspacePod(ctx context.Context, namespace, selector string) corev1.Pod { + var pod corev1.Pod + gomega.Eventually(func(g gomega.Gomega) { + pollCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + out, err := runFixtureCommand( + pollCtx, + 10*time.Second, + nil, + "kubectl", + "get", + "pods", + "-n", + namespace, + "-l", + selector, + "-o", + "json", + ) + g.Expect(err).NotTo(gomega.HaveOccurred(), "workspace pod discovery: %s", out) + var pods corev1.PodList + g.Expect(json.Unmarshal([]byte(out), &pods)).To(gomega.Succeed()) + g.Expect(pods.Items).To(gomega.HaveLen(1), "workspace-specific selector %s", selector) + pod = pods.Items[0] + g.Expect(pod.UID).NotTo(gomega.BeEmpty()) + ready := false + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + ready = true + } + } + g.Expect(ready).To(gomega.BeTrue(), "workspace pod %s must be ready", pod.Name) + }).WithTimeout(30 * time.Second).WithPolling(time.Second).Should(gomega.Succeed()) + ginkgo.GinkgoWriter.Printf("workspace pod %s UID=%s\n", pod.Name, pod.UID) + return pod +} diff --git a/e2e/tests/up/provider_kubernetes_dockerless_path.go b/e2e/tests/up/provider_kubernetes_dockerless_path.go index 18ec366dfa..ff231bb9ef 100644 --- a/e2e/tests/up/provider_kubernetes_dockerless_path.go +++ b/e2e/tests/up/provider_kubernetes_dockerless_path.go @@ -5,14 +5,12 @@ import ( "encoding/json" "fmt" "os" - "os/exec" "path/filepath" "regexp" "strings" "time" "github.com/devsy-org/devsy/e2e/framework" - "github.com/devsy-org/devsy/pkg/docker" "github.com/devsy-org/devsy/pkg/driver/kubernetes" "github.com/devsy-org/devsy/pkg/flags/names" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -94,7 +92,7 @@ func newDockerlessPathFixture(ctx context.Context, appendRemotePath bool) *docke fixture.expectedPath += ":" + dockerlessPathRoot + "/remote-bin" addDockerlessRemotePath(fixtureDir) } - dockerlessPathKubectl(ctx, "create", "namespace", fixture.id) + fixtureCommand(ctx, time.Minute, nil, "kubectl", "create", "namespace", fixture.id) ginkgo.DeferCleanup(fixture.deleteNamespace) // Keep the built-in provider's Dockerless credentials enabled. // Debug output records the helper created by the real callback. @@ -111,7 +109,16 @@ func newDockerlessPathFixture(ctx context.Context, appendRemotePath bool) *docke func (fixture *dockerlessPathFixture) deleteNamespace(specCtx ginkgo.SpecContext) { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(specCtx), time.Minute) defer cancel() - dockerlessPathKubectl(cleanupCtx, "delete", "namespace", fixture.id, "--wait=false") + fixtureCommand( + cleanupCtx, + time.Minute, + nil, + "kubectl", + "delete", + "namespace", + fixture.id, + "--wait=false", + ) } func (fixture *dockerlessPathFixture) deleteProvider(specCtx ginkgo.SpecContext) { @@ -163,7 +170,7 @@ func (fixture *dockerlessPathFixture) up( func (fixture *dockerlessPathFixture) verify(ctx context.Context, phase string) corev1.Pod { selector := "devsy.sh/created=true,issue-1446=" + fixture.id - pod := dockerlessPathWorkspacePod(ctx, fixture.id, selector) + pod := fixtureWorkspacePod(ctx, fixture.id, selector) fixture.containerName = "" for _, container := range pod.Spec.Containers { if container.Name == kubernetes.DevContainerName { @@ -215,8 +222,10 @@ func (fixture *dockerlessPathFixture) verifyEnvironment( // Raw kubectl exec reads files without treating the pod's original builder // environment as the post-setup agent/SSH environment. execInPod := func(args ...string) string { - baseArgs := []string{"exec", "-n", fixture.id, podName, "-c", fixture.containerName, "--"} - return dockerlessPathKubectl(commandCtx, append(baseArgs, args...)...) + baseArgs := []string{ + "kubectl", "exec", "-n", fixture.id, podName, "-c", fixture.containerName, "--", + } + return fixtureCommand(commandCtx, time.Minute, nil, append(baseArgs, args...)...) } readFile := func(name string) string { return execInPod("/bin/cat", name) @@ -242,53 +251,3 @@ func (fixture *dockerlessPathFixture) verifyEnvironment( execInPod("/bin/sh", "-c", `for dir in /.dockerless/.docker/docker-credentials-*; do test ! -e "$dir" || exit 1; done`) } - -func dockerlessPathWorkspacePod(ctx context.Context, namespace, selector string) corev1.Pod { - var pod corev1.Pod - gomega.Eventually(func(g gomega.Gomega) { - pollCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - out, err := runDockerlessPathKubectl( - pollCtx, - "get", - "pods", - "-n", - namespace, - "-l", - selector, - "-o", - "json", - ) - g.Expect(err).NotTo(gomega.HaveOccurred(), "workspace pod discovery: %s", out) - var pods corev1.PodList - g.Expect(json.Unmarshal([]byte(out), &pods)).To(gomega.Succeed()) - g.Expect(pods.Items).To(gomega.HaveLen(1)) - pod = pods.Items[0] - g.Expect(pod.UID).NotTo(gomega.BeEmpty()) - ready := false - for _, condition := range pod.Status.Conditions { - if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { - ready = true - } - } - g.Expect(ready).To(gomega.BeTrue(), "workspace pod %s must be ready", pod.Name) - }).WithTimeout(30 * time.Second).WithPolling(time.Second).Should(gomega.Succeed()) - return pod -} - -func runDockerlessPathKubectl(ctx context.Context, args ...string) (string, error) { - commandCtx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - // #nosec G204 -- kubectl arguments are controlled by this test fixture. - cmd := exec.CommandContext(commandCtx, "kubectl", args...) - docker.PrepareForGroupCancellation(cmd) - cmd.WaitDelay = 5 * time.Second - out, err := cmd.CombinedOutput() - return string(out), err -} - -func dockerlessPathKubectl(ctx context.Context, args ...string) string { - out, err := runDockerlessPathKubectl(ctx, args...) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "kubectl %v: %s", args, out) - return out -} diff --git a/e2e/tests/up/provider_kubernetes_ssh_agent.go b/e2e/tests/up/provider_kubernetes_ssh_agent.go index 1dd018a554..4136049625 100644 --- a/e2e/tests/up/provider_kubernetes_ssh_agent.go +++ b/e2e/tests/up/provider_kubernetes_ssh_agent.go @@ -2,10 +2,8 @@ package up import ( "context" - "encoding/json" "fmt" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -13,10 +11,8 @@ import ( "al.essio.dev/pkg/shellescape" "github.com/devsy-org/devsy/e2e/framework" - "github.com/devsy-org/devsy/pkg/docker" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" ) var _ = ginkgo.Describe( @@ -39,15 +35,16 @@ var _ = ginkgo.Describe( selector := "devsy.sh/created=true,issue-1423=" + id source, sha := privateSSHGitFixture(ctx, initialDir, id, pubKey) - kindSSHCommand(ctx, nil, "kubectl", "create", "namespace", namespace) + fixtureCommand(ctx, 2*time.Minute, nil, "kubectl", "create", "namespace", namespace) ginkgo.DeferCleanup(func(specCtx ginkgo.SpecContext) { cleanupCtx, cancel := context.WithTimeout( context.WithoutCancel(specCtx), time.Minute, ) defer cancel() - kindSSHCommand( + fixtureCommand( cleanupCtx, + 2*time.Minute, nil, "kubectl", "delete", @@ -88,7 +85,11 @@ var _ = ginkgo.Describe( {"get", "events", "-n", namespace, "--sort-by=.lastTimestamp"}, {"logs", "-n", namespace, "-l", selector, "--all-containers", "--tail=200"}, } { - output, err := runKindSSHCommand(cleanupCtx, nil, "kubectl", args...) + output, err := runFixtureCommand( + cleanupCtx, + 2*time.Minute, + nil, + append([]string{"kubectl"}, args...)...) ginkgo.GinkgoWriter.Printf( "Kubernetes failure diagnostics: %s\n%s\nerror: %v\n", args, @@ -101,8 +102,9 @@ var _ = ginkgo.Describe( // Check the pod-to-service route before testing Devsy, so fixture failures // cannot be confused with a first-up failure. host, _, _ := strings.Cut(strings.TrimPrefix(source, "ssh://git@"), "/") - kindSSHCommand( + fixtureCommand( ctx, + 2*time.Minute, nil, "kubectl", "run", @@ -121,8 +123,9 @@ var _ = ginkgo.Describe( pollCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() return strings.TrimSpace( - kindSSHCommand( + fixtureCommand( pollCtx, + 2*time.Minute, nil, "kubectl", "get", @@ -189,15 +192,15 @@ var _ = ginkgo.Describe( "--id", id, ) - firstPod := kindSSHWorkspacePod(ctx, namespace, selector) + firstPod := fixtureWorkspacePod(ctx, namespace, selector) verifyCheckout() up("repeat", id) - samePod := kindSSHWorkspacePod(ctx, namespace, selector) + samePod := fixtureWorkspacePod(ctx, namespace, selector) gomega.Expect(samePod.UID). To(gomega.Equal(firstPod.UID), "second up must keep the pod") verifyCheckout() up("recreate", id, "--recreate") - recreated := kindSSHWorkspacePod(ctx, namespace, selector) + recreated := fixtureWorkspacePod(ctx, namespace, selector) gomega.Expect(recreated.UID). NotTo(gomega.Equal(firstPod.UID), "recreate must replace the pod") verifyCheckout() @@ -219,7 +222,15 @@ func privateSSHGitFixture( framework.ExpectNoError(err) ginkgo.DeferCleanup(os.RemoveAll, buildDir) sha := createKindSSHGitRepository(ctx, fixtureDir) - kindSSHCommand(ctx, nil, "cp", "-R", fixtureDir, filepath.Join(buildDir, "repository")) + fixtureCommand( + ctx, + 2*time.Minute, + nil, + "cp", + "-R", + fixtureDir, + filepath.Join(buildDir, "repository"), + ) authorizedKey := "restrict,command=\"git-upload-pack /srv/repository.git\" " + pubKey + "\n" framework.ExpectNoError( os.WriteFile(filepath.Join(buildDir, "authorized_keys"), []byte(authorizedKey), 0o600), @@ -239,7 +250,15 @@ func privateSSHGitFixture( gomega.Eventually(func() error { pollCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - out, err := runKindSSHCommand(pollCtx, env, "git", "ls-remote", source, "refs/heads/main") + out, err := runFixtureCommand( + pollCtx, + 2*time.Minute, + env, + "git", + "ls-remote", + source, + "refs/heads/main", + ) if err != nil { return fmt.Errorf("authenticated fixture readiness: %w: %s", err, out) } @@ -248,8 +267,9 @@ func privateSSHGitFixture( } return nil }).WithTimeout(time.Minute).WithPolling(time.Second).Should(gomega.Succeed()) - out, err := runKindSSHCommand( + out, err := runFixtureCommand( ctx, + 2*time.Minute, []string{"SSH_AUTH_SOCK=", "GIT_SSH_COMMAND=" + sshOptions + " -o IdentityAgent=none"}, "git", "ls-remote", @@ -263,10 +283,20 @@ func privateSSHGitFixture( } func createKindSSHGitRepository(ctx context.Context, fixtureDir string) string { - kindSSHCommand(ctx, nil, "git", "-C", fixtureDir, "init", "--initial-branch=main") - kindSSHCommand(ctx, nil, "git", "-C", fixtureDir, "add", ".") - kindSSHCommand( + fixtureCommand( + ctx, + 2*time.Minute, + nil, + "git", + "-C", + fixtureDir, + "init", + "--initial-branch=main", + ) + fixtureCommand(ctx, 2*time.Minute, nil, "git", "-C", fixtureDir, "add", ".") + fixtureCommand( ctx, + 2*time.Minute, nil, "git", "-C", @@ -281,33 +311,48 @@ func createKindSSHGitRepository(ctx context.Context, fixtureDir string) string { "-m", "Private SSH fixture", ) - sha := strings.TrimSpace(kindSSHCommand(ctx, nil, "git", "-C", fixtureDir, "rev-parse", "HEAD")) + sha := strings.TrimSpace( + fixtureCommand(ctx, 2*time.Minute, nil, "git", "-C", fixtureDir, "rev-parse", "HEAD"), + ) return sha } func startKindSSHGitServer(ctx context.Context, buildDir, id string) string { image := "devsy-private-ssh-git:" + id - kindSSHCommand(ctx, nil, "docker", "build", "-t", image, buildDir) + fixtureCommand(ctx, 2*time.Minute, nil, "docker", "build", "-t", image, buildDir) ginkgo.DeferCleanup( func(specCtx ginkgo.SpecContext) { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(specCtx), time.Minute) defer cancel() - kindSSHCommand(cleanupCtx, nil, "docker", "image", "rm", image) + fixtureCommand(cleanupCtx, 2*time.Minute, nil, "docker", "image", "rm", image) }, ) - kindSSHCommand(ctx, nil, "docker", "run", "-d", "--name", id, "--network", "kind", image) + fixtureCommand( + ctx, + 2*time.Minute, + nil, + "docker", + "run", + "-d", + "--name", + id, + "--network", + "kind", + image, + ) ginkgo.DeferCleanup(func(specCtx ginkgo.SpecContext) { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(specCtx), time.Minute) defer cancel() if ginkgo.CurrentSpecReport().Failed() { - out, err := runKindSSHCommand(cleanupCtx, nil, "docker", "logs", id) + out, err := runFixtureCommand(cleanupCtx, 2*time.Minute, nil, "docker", "logs", id) ginkgo.GinkgoWriter.Printf("private SSH Git service logs:\n%s\nerror: %v\n", out, err) } - kindSSHCommand(cleanupCtx, nil, "docker", "rm", "-f", id) + fixtureCommand(cleanupCtx, 2*time.Minute, nil, "docker", "rm", "-f", id) }) host := strings.TrimSpace( - kindSSHCommand( + fixtureCommand( ctx, + 2*time.Minute, nil, "docker", "inspect", @@ -321,42 +366,6 @@ func startKindSSHGitServer(ctx context.Context, buildDir, id string) string { return host } -func kindSSHWorkspacePod(ctx context.Context, namespace, selector string) corev1.Pod { - var pod corev1.Pod - gomega.Eventually(func(g gomega.Gomega) { - pollCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - output, err := runKindSSHCommand( - pollCtx, - nil, - "kubectl", - "get", - "pods", - "-n", - namespace, - "-l", - selector, - "-o", - "json", - ) - g.Expect(err).NotTo(gomega.HaveOccurred(), "workspace pod discovery: %s", output) - var pods corev1.PodList - g.Expect(json.Unmarshal([]byte(output), &pods)).To(gomega.Succeed()) - g.Expect(pods.Items).To(gomega.HaveLen(1), "workspace-specific selector %s", selector) - pod = pods.Items[0] - g.Expect(pod.UID).NotTo(gomega.BeEmpty()) - ready := false - for _, condition := range pod.Status.Conditions { - if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { - ready = true - } - } - g.Expect(ready).To(gomega.BeTrue(), "workspace pod %s must be ready", pod.Name) - }).WithTimeout(30 * time.Second).WithPolling(time.Second).Should(gomega.Succeed()) - ginkgo.GinkgoWriter.Printf("workspace pod %s UID=%s\n", pod.Name, pod.UID) - return pod -} - func verifyKindSSHAgentCleanup( ctx context.Context, f *framework.Framework, @@ -378,25 +387,31 @@ func verifyKindSSHAgentCleanup( "-o", "ConnectTimeout=10", } - kindSSHCommand( + fixtureCommand( ctx, + 2*time.Minute, env, - "ssh", - append( - options, - "-o", - "ForwardAgent=yes", - "-o", - "ControlMaster=yes", - "-o", - "ControlPersist=60", - "-N", - "-f", - host, - )...) + append([]string{cmdSSH}, + append( + options, + "-o", + "ForwardAgent=yes", + "-o", + "ControlMaster=yes", + "-o", + "ControlPersist=60", + "-N", + "-f", + host, + )...)...) closed := false closeMaster := func(cleanupCtx context.Context) { - kindSSHCommand(cleanupCtx, env, "ssh", append(options, "-O", "exit", host)...) + fixtureCommand( + cleanupCtx, + 2*time.Minute, + env, + append([]string{cmdSSH}, + append(options, "-O", "exit", host)...)...) } ginkgo.DeferCleanup(func(specCtx ginkgo.SpecContext) { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(specCtx), time.Minute) @@ -406,7 +421,12 @@ func verifyKindSSHAgentCleanup( } }) remote := func(command string) string { - return kindSSHCommand(ctx, env, "ssh", append(options, host, "--", command)...) + return fixtureCommand( + ctx, + 2*time.Minute, + env, + append([]string{cmdSSH}, + append(options, host, "--", command)...)...) } socket := strings.TrimSpace(remote("printf %s \"$SSH_AUTH_SOCK\"")) gomega.Expect(socket).NotTo(gomega.BeEmpty()) @@ -431,27 +451,3 @@ func waitForKindSSHSocketCleanup(ctx context.Context, f *framework.Framework, id filepath.Dir(socket), ) } - -func runKindSSHCommand( - ctx context.Context, - env []string, - name string, - args ...string, -) (string, error) { - commandCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) - defer cancel() - // #nosec G204 -- executable and arguments are controlled by this test fixture. - cmd := exec.CommandContext(commandCtx, name, args...) - docker.PrepareForGroupCancellation(cmd) - cmd.WaitDelay = 5 * time.Second - cmd.Env = append(os.Environ(), env...) - out, err := cmd.CombinedOutput() - return string(out), err -} - -func kindSSHCommand(ctx context.Context, env []string, name string, args ...string) string { - out, err := runKindSSHCommand(ctx, env, name, args...) - gomega.Expect(err). - NotTo(gomega.HaveOccurred(), "private SSH fixture command %s %v: %s", name, args, out) - return out -}