Skip to content

Commit c385a26

Browse files
FabianKrammFabianKramm
authored
Merge pull request #2479 from FabianKramm/main
refactor: disable local registry in some cases
2 parents ef1a491 + 4cfd7c5 commit c385a26

7 files changed

Lines changed: 92 additions & 61 deletions

File tree

‎e2e/tests/localregistry/localregistry.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -423,9 +423,6 @@ var _ = DevSpaceDescribe("localregistry", func() {
423423
}
424424
err = buildCmd.RunDefault(f)
425425
framework.ExpectError(err)
426-
gomega.Expect(output.String()).To(
427-
gomega.ContainSubstring("unable to push image my-docker-username/helloworld-kaniko and only docker and buildkit builds support using a local registry"),
428-
)
429426
gomega.Expect(output.String()).To(
430427
gomega.ContainSubstring("UNAUTHORIZED: authentication required"),
431428
)

‎helper/cmd/proxycommands/configure.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"github.com/mitchellh/go-homedir"
77
"os"
88
"path/filepath"
9+
"strconv"
910
"strings"
1011

1112
"github.com/loft-sh/devspace/helper/util/stderrlog"
@@ -17,12 +18,14 @@ var (
1718
sshPrivateKeyPath = "/tmp/ssh_private_key"
1819
sshPublicKeyPath = "/tmp/ssh_public_key"
1920
proxyCommandsPath = "/tmp/proxy_commands"
21+
portPath = "/tmp/port"
2022
)
2123

2224
// ConfigureCmd holds the ssh cmd flags
2325
type ConfigureCmd struct {
2426
PublicKey string
2527
PrivateKey string
28+
Port int
2629
WorkingDir string
2730

2831
GitCredentials bool
@@ -42,6 +45,7 @@ func NewConfigureCmd() *cobra.Command {
4245

4346
configureCmd.Flags().StringVar(&cmd.PublicKey, "public-key", "", "Public key to use")
4447
configureCmd.Flags().StringVar(&cmd.PrivateKey, "private-key", "", "Private key to use")
48+
configureCmd.Flags().IntVar(&cmd.Port, "port", 0, "Port inside the container to connect to")
4549
configureCmd.Flags().StringVar(&cmd.WorkingDir, "working-dir", "", "Working dir to use")
4650
configureCmd.Flags().StringSliceVar(&cmd.Commands, "commands", []string{}, "Commands to overwrite")
4751
configureCmd.Flags().BoolVar(&cmd.GitCredentials, "git-credentials", false, "If git credentials should get configured")
@@ -112,6 +116,14 @@ func (cmd *ConfigureCmd) Run(_ *cobra.Command, _ []string) error {
112116
}
113117
}
114118

119+
// now configure the port
120+
if cmd.Port > 0 {
121+
err = os.WriteFile(portPath, []byte(strconv.Itoa(cmd.Port)), 0644)
122+
if err != nil {
123+
return errors.Wrap(err, "write port file")
124+
}
125+
}
126+
115127
// now configure working dir
116128
workingDir := cmd.WorkingDir
117129
if workingDir == "" {

‎helper/cmd/proxycommands/run.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,19 +50,27 @@ func runProxyCommand(args []string) error {
5050
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
5151
}
5252

53-
client, err := ssh.Dial("tcp", "localhost:10567", clientConfig)
53+
// get port
54+
port, err := os.ReadFile(portPath)
55+
if err != nil {
56+
return errors.Wrap(err, "read port")
57+
}
58+
59+
// dial ssh
60+
client, err := ssh.Dial("tcp", "localhost:"+string(port), clientConfig)
5461
if err != nil {
5562
return errors.Wrap(err, "dial ssh")
5663
}
5764
defer client.Close()
5865

66+
// create new session
5967
session, err := client.NewSession()
6068
if err != nil {
6169
return errors.Wrap(err, "new session")
6270
}
6371
defer session.Close()
6472

65-
// check if we should use a pty#
73+
// check if we should use a pty
6674
var (
6775
width = 0
6876
height = 0

‎pkg/devspace/build/build.go‎

Lines changed: 24 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ import (
77
"strings"
88

99
"github.com/loft-sh/devspace/pkg/devspace/build/builder"
10-
"github.com/loft-sh/devspace/pkg/devspace/build/builder/buildkit"
11-
"github.com/loft-sh/devspace/pkg/devspace/build/builder/docker"
1210
"github.com/loft-sh/devspace/pkg/devspace/build/registry"
1311
"github.com/loft-sh/devspace/pkg/devspace/build/types"
1412
"github.com/loft-sh/devspace/pkg/devspace/config/constants"
@@ -151,38 +149,34 @@ func (c *controller) Build(ctx devspacecontext.Context, images []string, options
151149
imageCache.ImageName = imageName
152150
imageCache.LocalRegistryImageName = ""
153151

154-
if registry.UseLocalRegistry(kubeClient, conf, options.SkipPush) && !registry.HasPushPermission(imageConf) {
155-
if SupportsLocalRegistry(builder) {
156-
// Not able to deploy a local registry without a valid kube context
157-
if kubeClient == nil {
158-
return fmt.Errorf("unable to push image %s and a valid kube context is not available", imageConf.Image)
159-
}
152+
if registry.UseLocalRegistry(kubeClient, conf, imageConf, builder, options.SkipPush) && !registry.HasPushPermission(imageConf) {
153+
// Not able to deploy a local registry without a valid kube context
154+
if kubeClient == nil {
155+
return fmt.Errorf("unable to push image %s and a valid kube context is not available", imageConf.Image)
156+
}
160157

161-
registryOptions := registry.NewDefaultOptions().
162-
WithNamespace(kubeClient.Namespace()).
163-
WithLocalRegistryConfig(conf.LocalRegistry)
158+
registryOptions := registry.NewDefaultOptions().
159+
WithNamespace(kubeClient.Namespace()).
160+
WithLocalRegistryConfig(conf.LocalRegistry)
164161

165-
// Create and start a local registry if one isn't already running
166-
localRegistry, err := registry.GetOrCreateLocalRegistry(ctx, registryOptions)
167-
if err != nil {
168-
return errors.Wrap(err, "get or create local registry")
169-
}
162+
// Create and start a local registry if one isn't already running
163+
localRegistry, err := registry.GetOrCreateLocalRegistry(ctx, registryOptions)
164+
if err != nil {
165+
return errors.Wrap(err, "get or create local registry")
166+
}
170167

171-
// Update cache for local registry use
172-
imageCache.LocalRegistryImageName, err = localRegistry.RewriteImage(imageName)
173-
if err != nil {
174-
return errors.Wrap(err, "rewrite image")
175-
}
176-
ctx.Config().LocalCache().SetImageCache(imageConfigName, imageCache)
168+
// Update cache for local registry use
169+
imageCache.LocalRegistryImageName, err = localRegistry.RewriteImage(imageName)
170+
if err != nil {
171+
return errors.Wrap(err, "rewrite image")
172+
}
173+
ctx.Config().LocalCache().SetImageCache(imageConfigName, imageCache)
177174

178-
// Reset the builder for local registry usage
179-
// TODO: refactor so this isn't necessary!
180-
builder, err = c.createBuilder(ctx, imageConfigName, imageConf, imageTags, options)
181-
if err != nil {
182-
return errors.Wrap(err, "create builder")
183-
}
184-
} else {
185-
ctx.Log().Warnf("unable to push image %s and only docker and buildkit builds support using a local registry", imageConf.Image)
175+
// Reset the builder for local registry usage
176+
// TODO: refactor so this isn't necessary!
177+
builder, err = c.createBuilder(ctx, imageConfigName, imageConf, imageTags, options)
178+
if err != nil {
179+
return errors.Wrap(err, "create builder")
186180
}
187181
}
188182

@@ -430,14 +424,3 @@ func (c *controller) waitForBuild(ctx devspacecontext.Context, errChan <-chan er
430424

431425
return nil
432426
}
433-
434-
func SupportsLocalRegistry(builder builder.Interface) bool {
435-
switch builder.(type) {
436-
case *buildkit.Builder:
437-
return true
438-
case *docker.Builder:
439-
return true
440-
default:
441-
return false
442-
}
443-
}

‎pkg/devspace/build/registry/util.go‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@ package registry
33
import (
44
"context"
55
"fmt"
6+
"github.com/loft-sh/devspace/pkg/devspace/build/builder"
7+
"github.com/loft-sh/devspace/pkg/devspace/build/builder/kaniko"
68
"io"
79
"net/http"
10+
"runtime"
811
"strings"
912

1013
"github.com/docker/docker/pkg/jsonmessage"
@@ -120,22 +123,49 @@ func CopyImageToRemote(ctx context.Context, client dockerclient.Client, imageNam
120123
return <-errChan
121124
}
122125

123-
func UseLocalRegistry(client kubectl.Client, config *latest.Config, skipPush bool) bool {
126+
func UseLocalRegistry(client kubectl.Client, config *latest.Config, imageConfig *latest.Image, imageBuilder builder.Interface, skipPush bool) bool {
124127
if skipPush {
125128
return false
129+
} else if client == nil {
130+
return false
126131
}
127132

128-
if client == nil {
133+
// TODO: better check for arm64 architectures
134+
if runtime.GOARCH != "amd64" {
129135
return false
130136
}
131137

138+
// check if builder is kaniko
139+
if imageBuilder != nil {
140+
_, ok := imageBuilder.(*kaniko.Builder)
141+
if ok {
142+
return false
143+
}
144+
}
145+
146+
// check if image looks weird like localhost / cluster.local
147+
if imageConfig != nil {
148+
if imageConfig.Kaniko != nil {
149+
return false
150+
} else if imageConfig.Custom != nil {
151+
return false
152+
} else if imageConfig.BuildKit != nil && imageConfig.BuildKit.InCluster != nil {
153+
return false
154+
}
155+
156+
imageWithoutPort := strings.Split(imageConfig.Image, ":")[0]
157+
if imageWithoutPort == "" || imageWithoutPort == "localhost" || imageWithoutPort == "127.0.0.1" || strings.HasSuffix(imageWithoutPort, ".cluster.local") {
158+
return false
159+
}
160+
}
161+
162+
// check if fallback
132163
if !IsLocalRegistryFallback(config) {
133164
return IsLocalRegistryEnabled(config)
134165
}
135166

136-
context := client.CurrentContext()
137-
138167
// Determine if this is a vcluster
168+
context := client.CurrentContext()
139169
isVClusterContext := strings.Contains(context, "vcluster_")
140170

141171
// Determine if this is a local kubernetes cluster

‎pkg/devspace/build/registry/util_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ func TestUseLocalRegistry(t *testing.T) {
307307
}
308308

309309
for _, testCase := range testCases {
310-
actual := UseLocalRegistry(testCase.client, testCase.config, testCase.skipPush)
310+
actual := UseLocalRegistry(testCase.client, testCase.config, nil, nil, testCase.skipPush)
311311
assert.Equal(t, actual, testCase.expected, "Unexpected result in test case %s", testCase.name)
312312
}
313313
}

‎pkg/devspace/services/proxycommands/commands.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package proxycommands
33
import (
44
"encoding/base64"
55
"fmt"
6+
"strconv"
67
"strings"
78

89
sshpkg "github.com/gliderlabs/ssh"
@@ -20,22 +21,25 @@ import (
2021
var DefaultRemotePort = 10567
2122

2223
// StartProxyCommands starts the reverse commands functionality
23-
func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, selector targetselector.TargetSelector, parent *tomb.Tomb) (retErr error) {
24+
func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
2425
if ctx == nil || ctx.Config() == nil || ctx.Config().Config() == nil {
2526
return fmt.Errorf("DevSpace config is nil")
2627
}
2728

2829
// init done array is used to track when sync was initialized
2930
initDoneArray := []chan struct{}{}
31+
remotePort := DefaultRemotePort
3032
loader.EachDevContainer(devPod, func(devContainer *latest.DevContainer) bool {
3133
if len(devContainer.ProxyCommands) == 0 {
3234
return true
3335
}
3436

37+
oldRemotePort := remotePort
3538
initDone := parent.NotifyGo(func() error {
36-
return startProxyCommands(ctx, devContainer, devPod.Name, string(devContainer.Arch), devContainer.ProxyCommands, selector.WithContainer(devContainer.Container), parent)
39+
return startProxyCommands(ctx, devPod.Name, string(devContainer.Arch), oldRemotePort, devContainer.ProxyCommands, selector.WithContainer(devContainer.Container), parent)
3740
})
3841
initDoneArray = append(initDoneArray, initDone)
42+
remotePort++
3943
return true
4044
})
4145

@@ -46,7 +50,7 @@ func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, sele
4650
return nil
4751
}
4852

49-
func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevContainer, name, arch string, reverseCommands []*latest.ProxyCommand, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
53+
func startProxyCommands(ctx devspacecontext.Context, name, arch string, remotePort int, reverseCommands []*latest.ProxyCommand, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
5054
if ctx.IsDone() {
5155
return nil
5256
}
@@ -59,11 +63,8 @@ func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevCon
5963

6064
defer ssh.GetInstance(ctx.Log()).ReleasePort(port)
6165

62-
// get remote port
63-
defaultRemotePort := DefaultRemotePort
64-
6566
// start reverse port forwarding from that port
66-
mapping := fmt.Sprintf("%d:%d", port, defaultRemotePort)
67+
mapping := fmt.Sprintf("%d:%d", port, remotePort)
6768
err = portforwarding.StartReversePortForwarding(ctx, name, arch, []*latest.PortMapping{
6869
{
6970
Port: mapping,
@@ -74,10 +75,10 @@ func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevCon
7475
}
7576

7677
// start ssh
77-
return startLocalSSH(ctx, selector, devContainer, reverseCommands, fmt.Sprintf(":%d", port), parent)
78+
return startLocalSSH(ctx, selector, reverseCommands, remotePort, fmt.Sprintf(":%d", port), parent)
7879
}
7980

80-
func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSelector, devContainer *latest.DevContainer, reverseCommands []*latest.ProxyCommand, addr string, parent *tomb.Tomb) error {
81+
func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSelector, reverseCommands []*latest.ProxyCommand, remotePort int, addr string, parent *tomb.Tomb) error {
8182
if ctx.IsDone() {
8283
return nil
8384
}
@@ -111,7 +112,7 @@ func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSe
111112
}
112113

113114
// execute configure command in container
114-
command := []string{inject.DevSpaceHelperContainerPath, "proxy-commands", "configure", "--public-key", base64.StdEncoding.EncodeToString([]byte(publicKey)), "--private-key", base64.StdEncoding.EncodeToString([]byte(privateKey))}
115+
command := []string{inject.DevSpaceHelperContainerPath, "proxy-commands", "configure", "--port", strconv.Itoa(remotePort), "--public-key", base64.StdEncoding.EncodeToString([]byte(publicKey)), "--private-key", base64.StdEncoding.EncodeToString([]byte(privateKey))}
115116
if len(commandsToReplace) > 0 {
116117
command = append(command, "--commands", strings.Join(commandsToReplace, ","))
117118
}

0 commit comments

Comments
 (0)