fix: allow single file syncing

FabianKramm · FabianKramm · commit dadc635861c6 · 2022-08-26T17:14:39.000+02:00
diff --git a/e2e/tests/sync/sync.go b/e2e/tests/sync/sync.go
@@ -695,4 +695,84 @@ var _ = DevSpaceDescribe("sync", func() {
 		// wait for the command to finish
 		waitGroup.Wait()
 	})
+
+	ginkgo.It("should sync single file to a container", func() {
+		tempDir, err := framework.CopyToTempDir("tests/sync/testdata/sync-single-file")
+		framework.ExpectNoError(err)
+		defer framework.CleanupTempDir(initialDir, tempDir)
+
+		ns, err := kubeClient.CreateNamespace("sync")
+		framework.ExpectNoError(err)
+		defer func() {
+			err := kubeClient.DeleteNamespace(ns)
+			framework.ExpectNoError(err)
+		}()
+
+		// deploy app to sync
+		deployCmd := &cmd.RunPipelineCmd{
+			GlobalFlags: &flags.GlobalFlags{
+				NoWarn:     true,
+				Namespace:  ns,
+				ConfigPath: "single.yaml",
+			},
+			Pipeline: "deploy",
+		}
+		err = deployCmd.RunDefault(f)
+		framework.ExpectNoError(err)
+
+		cancelCtx, stop := context.WithCancel(context.Background())
+		defer stop()
+
+		// sync command
+		syncCmd := &cmd.SyncCmd{
+			GlobalFlags: &flags.GlobalFlags{
+				NoWarn:     true,
+				Namespace:  ns,
+				ConfigPath: "single.yaml",
+			},
+			Wait: true,
+			Ctx:  cancelCtx,
+		}
+
+		// start the command
+		waitGroup := sync.WaitGroup{}
+		waitGroup.Add(1)
+		go func() {
+			defer ginkgo.GinkgoRecover()
+			defer waitGroup.Done()
+			err = syncCmd.Run(f)
+			framework.ExpectNoError(err)
+		}()
+
+		// check that uploadExcludePaths folder was not synced
+		framework.ExpectRemoteFileContents("alpine", ns, "/watch/test.txt", "Hello World")
+		framework.ExpectRemoteFileNotFound("alpine", ns, "/watch/single.yaml")
+
+		// write a file and check that it got synced
+		payload := randutil.GenerateRandomString(10000)
+		err = ioutil.WriteFile(filepath.Join(tempDir, "other-folder", "test.txt"), []byte(payload), 0666)
+		framework.ExpectNoError(err)
+		err = ioutil.WriteFile(filepath.Join(tempDir, "other-folder", "test123.txt"), []byte(payload), 0666)
+		framework.ExpectNoError(err)
+		framework.ExpectRemoteFileContents("alpine", ns, "/watch/test.txt", payload)
+		framework.ExpectRemoteFileNotFound("alpine", ns, "/watch/test123.txt")
+
+		// check that file is not created but updated
+		_, err = kubeClient.ExecByImageSelector("alpine", ns, []string{
+			"sh", "-c", "echo -n 'Hello World' > /watch/test.test",
+		})
+		framework.ExpectNoError(err)
+		_, err = kubeClient.ExecByImageSelector("alpine", ns, []string{
+			"sh", "-c", "echo -n 'Hello DevSpace' > /watch/test.txt",
+		})
+		framework.ExpectNoError(err)
+		framework.ExpectLocalFileContents(filepath.Join(tempDir, "other-folder", "test.txt"), "Hello DevSpace")
+		framework.ExpectLocalFileNotFound(filepath.Join(tempDir, "other-folder", "test.test"))
+
+		// stop command
+		stop()
+
+		// wait for the command to finish
+		waitGroup.Wait()
+	})
 })
diff --git a/e2e/tests/sync/testdata/sync-single-file/other-folder/test.txt b/e2e/tests/sync/testdata/sync-single-file/other-folder/test.txt
@@ -0,0 +1 @@
+Hello World
diff --git a/e2e/tests/sync/testdata/sync-single-file/single.yaml b/e2e/tests/sync/testdata/sync-single-file/single.yaml
@@ -0,0 +1,16 @@
+version: v2beta1
+vars:
+  IMAGE: alpine
+deployments:
+  test:
+    helm:
+      values:
+        containers:
+        - image: ${IMAGE}
+          command: ["sleep"]
+          args: ["999999999999"]
+dev:
+  sync:
+    imageSelector: ${IMAGE}
+    sync:
+    - path: other-folder/test.txt:/watch/test.txt
diff --git a/helper/cmd/sync/downstream.go b/helper/cmd/sync/downstream.go
@@ -13,7 +13,8 @@ type DownstreamCmd struct {
 
 	Throttle int64
 
-	Polling bool
+	Polling        bool
+	RecursiveWatch bool
 }
 
 // NewDownstreamCmd creates a new downstream command
@@ -29,6 +30,7 @@ func NewDownstreamCmd() *cobra.Command {
 	downstreamCmd.Flags().StringSliceVar(&cmd.Exclude, "exclude", []string{}, "The exclude paths for downstream watching")
 	downstreamCmd.Flags().Int64Var(&cmd.Throttle, "throttle", 5, "The amount of milliseconds to throttle change detection per 100 files")
 	downstreamCmd.Flags().BoolVar(&cmd.Polling, "polling", false, "If true, DevSpace will use polling instead of inotify")
+	downstreamCmd.Flags().BoolVar(&cmd.RecursiveWatch, "recursive-watch", true, "If false, DevSpace will not watch recursively")
 	return downstreamCmd
 }
 
@@ -43,9 +45,10 @@ func (cmd *DownstreamCmd) Run(cobraCmd *cobra.Command, args []string) error {
 		RemotePath:   absolutePath,
 		ExcludePaths: cmd.Exclude,
 
-		Throttle:    cmd.Throttle,
-		Polling:     cmd.Polling,
-		ExitOnClose: true,
-		Ping:        true,
+		Throttle:         cmd.Throttle,
+		Polling:          cmd.Polling,
+		ExitOnClose:      true,
+		NoRecursiveWatch: !cmd.RecursiveWatch,
+		Ping:             true,
 	})
 }
diff --git a/helper/server/downstream.go b/helper/server/downstream.go
@@ -31,10 +31,11 @@ const rescanPeriod = time.Minute * 15
 
 // DownstreamOptions holds the options for the downstream server
 type DownstreamOptions struct {
-	RemotePath   string
-	ExcludePaths []string
-	ExitOnClose  bool
-	Throttle     int64
+	RemotePath       string
+	ExcludePaths     []string
+	ExitOnClose      bool
+	NoRecursiveWatch bool
+	Throttle         int64
 
 	Polling bool
 	Ping    bool
@@ -78,7 +79,12 @@ func StartDownstreamServer(reader io.Reader, writer io.Writer, options *Downstre
 
 			go func() {
 				// set up a watchpoint listening for events within a directory tree rooted at specified directory
-				err := notify.WatchWithFilter(options.RemotePath+"/...", downStream.events, func(s string) bool {
+				watchPath := options.RemotePath + "/..."
+				if options.NoRecursiveWatch {
+					watchPath = options.RemotePath
+				}
+
+				err := notify.WatchWithFilter(watchPath, downStream.events, func(s string) bool {
 					if ignoreMatcher == nil || ignoreMatcher.RequireFullScan() {
 						return false
 					}
@@ -229,7 +235,7 @@ func (d *Downstream) ChangesCount(context.Context, *remote.Empty) (*remote.Chang
 
 	// Walk through the dir
 	if d.options.Polling {
-		walkDir(d.options.RemotePath, d.options.RemotePath, d.ignoreMatcher, newState, throttle)
+		walkDir(d.options.RemotePath, d.options.RemotePath, d.ignoreMatcher, newState, d.options.NoRecursiveWatch, throttle)
 	}
 
 	changeAmount := int64(0)
@@ -272,7 +278,7 @@ func (d *Downstream) getWatchState() map[string]*remote.Change {
 		d.lastRescan = &now
 
 		newState := make(map[string]*remote.Change)
-		walkDir(d.options.RemotePath, d.options.RemotePath, d.ignoreMatcher, newState, 0)
+		walkDir(d.options.RemotePath, d.options.RemotePath, d.ignoreMatcher, newState, d.options.NoRecursiveWatch, 0)
 		return newState
 	} else if changeAmount == 0 {
 		return nil
@@ -305,7 +311,7 @@ func (d *Downstream) Changes(empty *remote.Empty, stream remote.Downstream_Chang
 	if !d.options.Polling {
 		newState = d.getWatchState()
 	} else {
-		walkDir(d.options.RemotePath, d.options.RemotePath, d.ignoreMatcher, newState, throttle)
+		walkDir(d.options.RemotePath, d.options.RemotePath, d.ignoreMatcher, newState, d.options.NoRecursiveWatch, throttle)
 	}
 
 	if newState != nil {
@@ -384,7 +390,7 @@ func (d *Downstream) applyChange(newState map[string]*remote.Change, fullPath st
 			}
 		}
 
-		walkDir(d.options.RemotePath, fullPath, d.ignoreMatcher, newState, time.Duration(d.options.Throttle)*time.Millisecond)
+		walkDir(d.options.RemotePath, fullPath, d.ignoreMatcher, newState, d.options.NoRecursiveWatch, time.Duration(d.options.Throttle)*time.Millisecond)
 	} else {
 		if d.ignoreMatcher == nil || !d.ignoreMatcher.RequireFullScan() || !d.ignoreMatcher.Matches(relativePath, false) {
 			newState[fullPath] = &remote.Change{
@@ -525,7 +531,7 @@ func streamChanges(basePath string, oldState map[string]*remote.Change, newState
 	return changeAmount, nil
 }
 
-func walkDir(basePath string, path string, ignoreMatcher ignoreparser.IgnoreParser, state map[string]*remote.Change, throttle time.Duration) {
+func walkDir(basePath string, path string, ignoreMatcher ignoreparser.IgnoreParser, state map[string]*remote.Change, noRecursive bool, throttle time.Duration) {
 	files, err := ioutil.ReadDir(path)
 	if err != nil {
 		// We ignore errors here
@@ -566,7 +572,9 @@ func walkDir(basePath string, path string, ignoreMatcher ignoreparser.IgnorePars
 				}
 			}
 
-			walkDir(basePath, absolutePath, ignoreMatcher, state, throttle)
+			if !noRecursive {
+				walkDir(basePath, absolutePath, ignoreMatcher, state, noRecursive, throttle)
+			}
 		} else {
 			// Check if not ignored
 			if ignoreMatcher == nil || !ignoreMatcher.RequireFullScan() || !ignoreMatcher.Matches(absolutePath[len(basePath):], false) {
diff --git a/pkg/devspace/services/sync/controller.go b/pkg/devspace/services/sync/controller.go
@@ -9,6 +9,7 @@ import (
 	"github.com/mgutz/ansi"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -329,24 +330,6 @@ func (c *controller) initClient(ctx devspacecontext.Context, pod *v1.Pod, arch,
 	// make sure we resolve it correctly
 	localPath = ctx.ResolvePath(localPath)
 
-	// check if local path exists
-	_, err = os.Stat(localPath)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return nil, err
-		}
-
-		err = os.MkdirAll(localPath, os.ModePerm)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	err = inject.InjectDevSpaceHelper(ctx.Context(), ctx.KubeClient(), pod, container, arch, customLog)
-	if err != nil {
-		return nil, err
-	}
-
 	upstreamDisabled := syncConfig.DisableUpload
 	downstreamDisabled := syncConfig.DisableDownload
 	compareBy := latest.InitialSyncCompareByMTime
@@ -368,6 +351,36 @@ func (c *controller) initClient(ctx devspacecontext.Context, pod *v1.Pod, arch,
 		},
 	}
 
+	if len(syncConfig.ExcludePaths) > 0 {
+		options.ExcludePaths = syncConfig.ExcludePaths
+	}
+
+	// check if local path exists
+	stat, err := os.Stat(localPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+
+		err = os.MkdirAll(localPath, os.ModePerm)
+		if err != nil {
+			return nil, err
+		}
+	} else if !stat.IsDir() {
+		if path.Base(filepath.ToSlash(localPath)) != path.Base(containerPath) {
+			return nil, fmt.Errorf("if you want to sync a single file, make sure the filename matches on the local and container path. E.g.: local-path/my-file.txt:remote-path/my-file.txt")
+		}
+
+		fileName := path.Base(localPath)
+		localPath = path.Dir(localPath)
+		containerPath = path.Dir(containerPath)
+		options.ExcludePaths = []string{
+			"**",
+			"!/" + fileName,
+		}
+		options.NoRecursiveWatch = true
+	}
+
 	// Initialize log
 	if options.Log == nil {
 		options.Log = logpkg.GetFileLogger("sync")
@@ -378,8 +391,10 @@ func (c *controller) initClient(ctx devspacecontext.Context, pod *v1.Pod, arch,
 		options.Exec = syncConfig.OnUpload.Exec
 	}
 
-	if len(syncConfig.ExcludePaths) > 0 {
-		options.ExcludePaths = syncConfig.ExcludePaths
+	// inject devspace helper
+	err = inject.InjectDevSpaceHelper(ctx.Context(), ctx.KubeClient(), pod, container, arch, customLog)
+	if err != nil {
+		return nil, err
 	}
 
 	if syncConfig.ExcludeFile != "" {
@@ -494,6 +509,9 @@ func (c *controller) initClient(ctx devspacecontext.Context, pod *v1.Pod, arch,
 	for _, exclude := range options.ExcludePaths {
 		downstreamArgs = append(downstreamArgs, "--exclude", exclude)
 	}
+	if options.NoRecursiveWatch {
+		downstreamArgs = append(downstreamArgs, "--recursive-watch=false")
+	}
 	downstreamArgs = append(downstreamArgs, containerPath)
 
 	downStdinReader, downStdinWriter := io.Pipe()
diff --git a/pkg/devspace/sync/sync.go b/pkg/devspace/sync/sync.go
@@ -25,7 +25,8 @@ const waitForMoreChangesTimeout = time.Minute
 
 // Options holds the sync options
 type Options struct {
-	Polling bool
+	Polling          bool
+	NoRecursiveWatch bool
 
 	Exec []latest.SyncExec
 
@@ -85,7 +86,7 @@ type Sync struct {
 	readyChan chan bool
 }
 
-// NewSync creates a new sync for the given
+// NewSync creates a new sync for the given local path
 func NewSync(ctx context.Context, localPath string, options Options) (*Sync, error) {
 	cancelCtx, cancel := context.WithCancel(ctx)
 
@@ -233,7 +234,11 @@ func (s *Sync) startUpstream() {
 	s.tree = notify.NewTree()
 
 	// Set up a watchpoint listening for events within a directory tree rooted at specified directory
-	err := s.tree.Watch(s.LocalPath+"/...", s.upstream.events, func(path string) bool {
+	watchPath := s.LocalPath + "/..."
+	if s.Options.NoRecursiveWatch {
+		watchPath = s.LocalPath
+	}
+	err := s.tree.Watch(watchPath, s.upstream.events, func(path string) bool {
 		if s.ignoreMatcher == nil || s.ignoreMatcher.RequireFullScan() {
 			return false
 		}
