fix: multi container proxy commands

Author: FabianKramm

e2e/tests/localregistry/localregistry.go

@@ -423,9 +423,6 @@ var _ = DevSpaceDescribe("localregistry", func() {
 		}
 		err = buildCmd.RunDefault(f)
 		framework.ExpectError(err)
-		gomega.Expect(output.String()).To(
-			gomega.ContainSubstring("unable to push image my-docker-username/helloworld-kaniko and only docker and buildkit builds support using a local registry"),
-		)
 		gomega.Expect(output.String()).To(
 			gomega.ContainSubstring("UNAUTHORIZED: authentication required"),
 		)

helper/cmd/proxycommands/configure.go

@@ -6,6 +6,7 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/loft-sh/devspace/helper/util/stderrlog"
@@ -17,12 +18,14 @@ var (
 	sshPrivateKeyPath = "/tmp/ssh_private_key"
 	sshPublicKeyPath  = "/tmp/ssh_public_key"
 	proxyCommandsPath = "/tmp/proxy_commands"
+	portPath          = "/tmp/port"
 )
 
 // ConfigureCmd holds the ssh cmd flags
 type ConfigureCmd struct {
 	PublicKey  string
 	PrivateKey string
+	Port       int
 	WorkingDir string
 
 	GitCredentials bool
@@ -42,6 +45,7 @@ func NewConfigureCmd() *cobra.Command {
 
 	configureCmd.Flags().StringVar(&cmd.PublicKey, "public-key", "", "Public key to use")
 	configureCmd.Flags().StringVar(&cmd.PrivateKey, "private-key", "", "Private key to use")
+	configureCmd.Flags().IntVar(&cmd.Port, "port", 0, "Port inside the container to connect to")
 	configureCmd.Flags().StringVar(&cmd.WorkingDir, "working-dir", "", "Working dir to use")
 	configureCmd.Flags().StringSliceVar(&cmd.Commands, "commands", []string{}, "Commands to overwrite")
 	configureCmd.Flags().BoolVar(&cmd.GitCredentials, "git-credentials", false, "If git credentials should get configured")
@@ -112,6 +116,14 @@ func (cmd *ConfigureCmd) Run(_ *cobra.Command, _ []string) error {
 		}
 	}
 
+	// now configure the port
+	if cmd.Port > 0 {
+		err = os.WriteFile(portPath, []byte(strconv.Itoa(cmd.Port)), 0644)
+		if err != nil {
+			return errors.Wrap(err, "write port file")
+		}
+	}
+
 	// now configure working dir
 	workingDir := cmd.WorkingDir
 	if workingDir == "" {

helper/cmd/proxycommands/run.go

@@ -50,19 +50,27 @@ func runProxyCommand(args []string) error {
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 	}
 
-	client, err := ssh.Dial("tcp", "localhost:10567", clientConfig)
+	// get port
+	port, err := os.ReadFile(portPath)
+	if err != nil {
+		return errors.Wrap(err, "read port")
+	}
+
+	// dial ssh
+	client, err := ssh.Dial("tcp", "localhost:"+string(port), clientConfig)
 	if err != nil {
 		return errors.Wrap(err, "dial ssh")
 	}
 	defer client.Close()
 
+	// create new session
 	session, err := client.NewSession()
 	if err != nil {
 		return errors.Wrap(err, "new session")
 	}
 	defer session.Close()
 
-	// check if we should use a pty#
+	// check if we should use a pty
 	var (
 		width  = 0
 		height = 0

pkg/devspace/services/proxycommands/commands.go

@@ -3,6 +3,7 @@ package proxycommands
 import (
 	"encoding/base64"
 	"fmt"
+	"strconv"
 	"strings"
 
 	sshpkg "github.com/gliderlabs/ssh"
@@ -20,22 +21,25 @@ import (
 var DefaultRemotePort = 10567
 
 // StartProxyCommands starts the reverse commands functionality
-func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, selector targetselector.TargetSelector, parent *tomb.Tomb) (retErr error) {
+func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
 	if ctx == nil || ctx.Config() == nil || ctx.Config().Config() == nil {
 		return fmt.Errorf("DevSpace config is nil")
 	}
 
 	// init done array is used to track when sync was initialized
 	initDoneArray := []chan struct{}{}
+	remotePort := DefaultRemotePort
 	loader.EachDevContainer(devPod, func(devContainer *latest.DevContainer) bool {
 		if len(devContainer.ProxyCommands) == 0 {
 			return true
 		}
 
+		oldRemotePort := remotePort
 		initDone := parent.NotifyGo(func() error {
-			return startProxyCommands(ctx, devContainer, devPod.Name, string(devContainer.Arch), devContainer.ProxyCommands, selector.WithContainer(devContainer.Container), parent)
+			return startProxyCommands(ctx, devPod.Name, string(devContainer.Arch), oldRemotePort, devContainer.ProxyCommands, selector.WithContainer(devContainer.Container), parent)
 		})
 		initDoneArray = append(initDoneArray, initDone)
+		remotePort++
 		return true
 	})
 
@@ -46,7 +50,7 @@ func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, sele
 	return nil
 }
 
-func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevContainer, name, arch string, reverseCommands []*latest.ProxyCommand, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
+func startProxyCommands(ctx devspacecontext.Context, name, arch string, remotePort int, reverseCommands []*latest.ProxyCommand, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
 	if ctx.IsDone() {
 		return nil
 	}
@@ -59,11 +63,8 @@ func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevCon
 
 	defer ssh.GetInstance(ctx.Log()).ReleasePort(port)
 
-	// get remote port
-	defaultRemotePort := DefaultRemotePort
-
 	// start reverse port forwarding from that port
-	mapping := fmt.Sprintf("%d:%d", port, defaultRemotePort)
+	mapping := fmt.Sprintf("%d:%d", port, remotePort)
 	err = portforwarding.StartReversePortForwarding(ctx, name, arch, []*latest.PortMapping{
 		{
 			Port: mapping,
@@ -74,10 +75,10 @@ func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevCon
 	}
 
 	// start ssh
-	return startLocalSSH(ctx, selector, devContainer, reverseCommands, fmt.Sprintf(":%d", port), parent)
+	return startLocalSSH(ctx, selector, reverseCommands, remotePort, fmt.Sprintf(":%d", port), parent)
 }
 
-func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSelector, devContainer *latest.DevContainer, reverseCommands []*latest.ProxyCommand, addr string, parent *tomb.Tomb) error {
+func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSelector, reverseCommands []*latest.ProxyCommand, remotePort int, addr string, parent *tomb.Tomb) error {
 	if ctx.IsDone() {
 		return nil
 	}
@@ -111,7 +112,7 @@ func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSe
 	}
 
 	// execute configure command in container
-	command := []string{inject.DevSpaceHelperContainerPath, "proxy-commands", "configure", "--public-key", base64.StdEncoding.EncodeToString([]byte(publicKey)), "--private-key", base64.StdEncoding.EncodeToString([]byte(privateKey))}
+	command := []string{inject.DevSpaceHelperContainerPath, "proxy-commands", "configure", "--port", strconv.Itoa(remotePort), "--public-key", base64.StdEncoding.EncodeToString([]byte(publicKey)), "--private-key", base64.StdEncoding.EncodeToString([]byte(privateKey))}
 	if len(commandsToReplace) > 0 {
 		command = append(command, "--commands", strings.Join(commandsToReplace, ","))
 	}