diff --git a/pkg/client/clientimplementation/locks.go b/pkg/client/clientimplementation/locks.go new file mode 100644 index 0000000000..97574f88d0 --- /dev/null +++ b/pkg/client/clientimplementation/locks.go @@ -0,0 +1,146 @@ +package clientimplementation + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/gofrs/flock" +) + +const ( + lockTimeout = 5 * time.Minute + lockRetry = time.Second +) + +// clientLocks lazily opens the file locks that serialize operations on a workspace. +type clientLocks struct { + // ids returns the context, workspace ID and machine ID used for the lock files. + // It is called once, on first use. An empty machine ID means no machine lock. + ids func() (contextName, workspaceID, machineID string) + + once sync.Once + initErr error + workspace *flock.Flock + machine *flock.Flock +} + +func newClientLocks(ids func() (string, string, string)) *clientLocks { + return &clientLocks{ids: ids} +} + +// Lock acquires the workspace lock, then the machine lock when one is configured. +func (l *clientLocks) Lock(ctx context.Context) error { + if err := l.ensureOpen(); err != nil { + return fmt.Errorf("init lock: %w", err) + } + + log.Debug("acquire workspace lock") + if err := l.lockWorkspace(ctx); err != nil { + return fmt.Errorf("lock workspace: %w", err) + } + log.Debug("acquired workspace lock") + + if l.machine != nil { + log.Debug("acquire machine lock") + if err := tryLock(ctx, l.machine, "machine"); err != nil { + if uerr := l.unlockWorkspace(); uerr != nil { + log.Warnf("unlock workspace: %v", uerr) + } + return fmt.Errorf("lock machine: %w", err) + } + log.Debug("acquired machine lock") + } + + return nil +} + +// Unlock releases the machine lock, then the workspace lock. +func (l *clientLocks) Unlock() { + if err := l.ensureOpen(); err != nil { + log.Warnf("init lock: %v", err) + return + } + + if l.machine != nil { + if err := l.machine.Unlock(); err != nil { + log.Warnf("unlock machine: %v", err) + } + } + if err := l.unlockWorkspace(); err != nil { + log.Warnf("unlock workspace: %v", err) + } +} + +func (l *clientLocks) ensureOpen() error { + l.once.Do(func() { + contextName, workspaceID, machineID := l.ids() + + dir, err := provider.GetLocksDir(contextName) + if err != nil { + l.initErr = fmt.Errorf("get workspaces dir: %w", err) + return + } + + // #nosec G301 -- TODO Consider using a more secure permission setting and ownership if needed. + if err := os.MkdirAll(dir, 0o755); err != nil { + l.initErr = fmt.Errorf("create workspace locks dir: %w", err) + return + } + + l.workspace = flock.New(filepath.Join(dir, workspaceID+".workspace.lock")) + if machineID != "" { + l.machine = flock.New(filepath.Join(dir, machineID+".machine.lock")) + } + }) + + return l.initErr +} + +// lockWorkspace blocks until the workspace lock is held. ensureOpen must have succeeded. +func (l *clientLocks) lockWorkspace(ctx context.Context) error { + return tryLock(ctx, l.workspace, "workspace") +} + +// unlockWorkspace releases the workspace lock. ensureOpen must have succeeded. +func (l *clientLocks) unlockWorkspace() error { + return l.workspace.Unlock() +} + +func tryLock(ctx context.Context, lock *flock.Flock, name string) error { + done := scheduleLogMessage( + fmt.Sprintf( + "Trying to lock %s, seems like another process is running that blocks this %s", + name, + name, + ), + ) + defer close(done) + + now := time.Now() + for time.Since(now) < lockTimeout { + locked, err := lock.TryLock() + if err != nil { + return err + } else if locked { + return nil + } + + select { + case <-time.After(lockRetry): + continue + case <-ctx.Done(): + return ctx.Err() + } + } + + return fmt.Errorf( + "timed out waiting to lock %s, seems like there is another process running on this machine that blocks it", + name, + ) +} diff --git a/pkg/client/clientimplementation/locks_test.go b/pkg/client/clientimplementation/locks_test.go new file mode 100644 index 0000000000..1a51f348a6 --- /dev/null +++ b/pkg/client/clientimplementation/locks_test.go @@ -0,0 +1,159 @@ +package clientimplementation + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/gofrs/flock" + "github.com/stretchr/testify/require" +) + +const ( + testLocksContext = "default" + lockTestWorkspaceID = "ws1" +) + +func useTempDevsyHome(t *testing.T, home string) { + t.Helper() + t.Setenv(config.EnvHome, home) + config.ResetPathManager() + t.Cleanup(config.ResetPathManager) +} + +func testLocksDir(t *testing.T) string { + t.Helper() + useTempDevsyHome(t, t.TempDir()) + + dir, err := provider.GetLocksDir(testLocksContext) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(dir, 0o750)) + + return dir +} + +func testLockIDs() (string, string, string) { + return testLocksContext, lockTestWorkspaceID, "" +} + +func TestClientLocksLockUnlockRoundTrip(t *testing.T) { + dir := testLocksDir(t) + workspaceLock := filepath.Join(dir, lockTestWorkspaceID+".workspace.lock") + + locks := newClientLocks(testLockIDs) + require.NoError(t, locks.Lock(context.Background())) + + probe := flock.New(workspaceLock) + locked, err := probe.TryLock() + require.NoError(t, err) + require.False(t, locked, "workspace lock should be held after Lock") + + locks.Unlock() + + locked, err = probe.TryLock() + require.NoError(t, err) + require.True(t, locked, "workspace lock should be released after Unlock") + require.NoError(t, probe.Unlock()) +} + +func TestClientLocksContendedWorkspaceFailsWhenContextCancelled(t *testing.T) { + dir := testLocksDir(t) + + holder := flock.New(filepath.Join(dir, lockTestWorkspaceID+".workspace.lock")) + locked, err := holder.TryLock() + require.NoError(t, err) + require.True(t, locked) + t.Cleanup(func() { require.NoError(t, holder.Unlock()) }) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + locks := newClientLocks(testLockIDs) + err = locks.Lock(ctx) + require.ErrorIs(t, err, context.Canceled) + require.ErrorContains(t, err, "lock workspace:") +} + +func TestClientLocksInitFailureIsCachedAndUnlockDoesNotPanic(t *testing.T) { + // A regular file as DEVSY_HOME makes creating the locks dir fail. + blocker := filepath.Join(t.TempDir(), "not-a-dir") + require.NoError(t, os.WriteFile(blocker, []byte("x"), 0o600)) + useTempDevsyHome(t, blocker) + + locks := newClientLocks(testLockIDs) + + err := locks.Lock(context.Background()) + require.ErrorContains(t, err, "init lock:") + require.NotPanics(t, locks.Unlock) + + // Once init has failed, a later valid home must not be picked up. + useTempDevsyHome(t, t.TempDir()) + err = locks.Lock(context.Background()) + require.ErrorContains(t, err, "init lock:") +} + +func TestClientLocksMachineLock(t *testing.T) { + t.Run("no machine id creates no machine lock", func(t *testing.T) { + dir := testLocksDir(t) + + locks := newClientLocks(testLockIDs) + require.NoError(t, locks.Lock(context.Background())) + locks.Unlock() + + _, err := os.Stat(filepath.Join(dir, ".machine.lock")) + require.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("machine id read lazily on first lock", func(t *testing.T) { + dir := testLocksDir(t) + machineID := "" + locks := newClientLocks(func() (string, string, string) { + return testLocksContext, lockTestWorkspaceID, machineID + }) + + machineID = "m1" + require.NoError(t, locks.Lock(context.Background())) + + machineLock := filepath.Join(dir, "m1.machine.lock") + probe := flock.New(machineLock) + locked, err := probe.TryLock() + require.NoError(t, err) + require.False(t, locked, "machine lock should be held after Lock") + + locks.Unlock() + + locked, err = probe.TryLock() + require.NoError(t, err) + require.True(t, locked, "machine lock should be released after Unlock") + require.NoError(t, probe.Unlock()) + }) + + t.Run("machine lock failure releases workspace lock", func(t *testing.T) { + dir := testLocksDir(t) + + holder := flock.New(filepath.Join(dir, "m1.machine.lock")) + locked, err := holder.TryLock() + require.NoError(t, err) + require.True(t, locked) + t.Cleanup(func() { require.NoError(t, holder.Unlock()) }) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + locks := newClientLocks(func() (string, string, string) { + return testLocksContext, lockTestWorkspaceID, "m1" + }) + err = locks.Lock(ctx) + require.ErrorContains(t, err, "lock machine:") + + probe := flock.New(filepath.Join(dir, lockTestWorkspaceID+".workspace.lock")) + locked, err = probe.TryLock() + require.NoError(t, err) + require.True(t, locked, "workspace lock must be released after machine lock failure") + require.NoError(t, probe.Unlock()) + }) +} diff --git a/pkg/client/clientimplementation/proxy_client.go b/pkg/client/clientimplementation/proxy_client.go index cc96d699a4..1c7ca79cb8 100644 --- a/pkg/client/clientimplementation/proxy_client.go +++ b/pkg/client/clientimplementation/proxy_client.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "path/filepath" "strings" "sync" "time" @@ -24,12 +23,6 @@ import ( "github.com/devsy-org/devsy/pkg/terminal" "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/types" - "github.com/gofrs/flock" -) - -const ( - lockTimeout = 5 * time.Minute - lockRetry = time.Second ) func NewProxyClient( @@ -43,14 +36,14 @@ func NewProxyClient( workspace: workspace, } pc.executor = &proxyExecutor{client: pc} + pc.locks = newClientLocks(pc.lockIDs) return pc, nil } type proxyClient struct { m sync.Mutex - workspaceLockOnce sync.Once - workspaceLock *flock.Flock + locks *clientLocks devsyConfig *config.Config config *provider.ProviderConfig @@ -109,12 +102,13 @@ func (e *proxyExecutor) executeWithJSONLog(ctx context.Context, params execParam } func (s *proxyClient) Lock(ctx context.Context) error { - s.initLock() + if err := s.locks.ensureOpen(); err != nil { + return fmt.Errorf("init lock: %w", err) + } // try to lock workspace log.Debugf("Acquire workspace lock") - err := tryLock(ctx, s.workspaceLock, "workspace") - if err != nil { + if err := s.locks.lockWorkspace(ctx); err != nil { return fmt.Errorf("error locking workspace: %w", err) } log.Debugf("Acquired workspace lock") @@ -123,48 +117,17 @@ func (s *proxyClient) Lock(ctx context.Context) error { } func (s *proxyClient) Unlock() { - s.initLock() + if err := s.locks.ensureOpen(); err != nil { + log.Warnf("init lock: %v", err) + return + } // try to unlock workspace - err := s.workspaceLock.Unlock() - if err != nil { + if err := s.locks.unlockWorkspace(); err != nil { log.Warnf("Error unlocking workspace: %v", err) } } -func tryLock(ctx context.Context, lock *flock.Flock, name string) error { - done := scheduleLogMessage( - fmt.Sprintf( - "Trying to lock %s, seems like another process is running that blocks this %s", - name, - name, - ), - ) - defer close(done) - - now := time.Now() - for time.Since(now) < lockTimeout { - locked, err := lock.TryLock() - if err != nil { - return err - } else if locked { - return nil - } - - select { - case <-time.After(lockRetry): - continue - case <-ctx.Done(): - return ctx.Err() - } - } - - return fmt.Errorf( - "timed out waiting to lock %s, seems like there is another process running on this machine that blocks it", - name, - ) -} - func (s *proxyClient) Provider() string { return s.config.Name } @@ -379,26 +342,11 @@ func (s *proxyClient) Status( return client.ParseStatus(status.State) } -func (s *proxyClient) initLock() { - s.workspaceLockOnce.Do(func() { - s.m.Lock() - defer s.m.Unlock() - - // get locks dir - workspaceLocksDir, err := provider.GetLocksDir(s.workspace.Context) - if err != nil { - panic(fmt.Errorf("get workspaces dir: %w", err)) - } - // #nosec G301 -- TODO Consider using a more secure permission setting and ownership if needed. - if err = os.MkdirAll(workspaceLocksDir, 0o755); err != nil { - panic(fmt.Errorf("create workspace locks dir: %w", err)) - } +func (s *proxyClient) lockIDs() (string, string, string) { + s.m.Lock() + defer s.m.Unlock() - // create workspace lock - s.workspaceLock = flock.New( - filepath.Join(workspaceLocksDir, s.workspace.ID+".workspace.lock"), - ) - }) + return s.workspace.Context, s.workspace.ID, "" } // checkPlatformVersion validates the platform provider version compatibility. diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 9e95d41e2a..16f96e43aa 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -9,7 +9,6 @@ import ( "io" "os" "os/exec" - "path/filepath" "strings" "sync" "time" @@ -31,7 +30,6 @@ import ( "github.com/devsy-org/devsy/pkg/task" "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/types" - "github.com/gofrs/flock" ) func NewWorkspaceClient( @@ -47,21 +45,20 @@ func NewWorkspaceClient( return nil, errors.New("machine id empty for machine provider") } - return &workspaceClient{ + c := &workspaceClient{ devsyConfig: devsyConfig, providerConfig: providerConfig, providerWorkspace: workspace, providerMachine: machine, - }, nil + } + c.locks = newClientLocks(c.lockIDs) + return c, nil } type workspaceClient struct { m sync.Mutex - workspaceLockOnce sync.Once - workspaceLockErr error - workspaceLock *flock.Flock - machineLock *flock.Flock + locks *clientLocks devsyConfig *config.Config providerConfig *provider.ProviderConfig @@ -155,44 +152,11 @@ func (s *workspaceClient) AgentInfo( } func (s *workspaceClient) Lock(ctx context.Context) error { - if err := s.initLock(); err != nil { - return fmt.Errorf("init lock: %w", err) - } - - log.Debug("acquire workspace lock") - if err := tryLock(ctx, s.workspaceLock, "workspace"); err != nil { - return fmt.Errorf("lock workspace: %w", err) - } - log.Debug("acquired workspace lock") - - if s.machineLock != nil { - log.Debug("acquire machine lock") - if err := tryLock(ctx, s.machineLock, "machine"); err != nil { - return fmt.Errorf("lock machine: %w", err) - } - log.Debug("acquired machine lock") - } - - return nil + return s.locks.Lock(ctx) } func (s *workspaceClient) Unlock() { - if err := s.initLock(); err != nil { - log.Warnf("init lock: %v", err) - return - } - - if s.machineLock != nil { - if err := s.machineLock.Unlock(); err != nil { - log.Warnf("unlock machine: %v", err) - } - } - - if s.workspaceLock != nil { - if err := s.workspaceLock.Unlock(); err != nil { - log.Warnf("unlock workspace: %v", err) - } - } + s.locks.Unlock() } func (s *workspaceClient) Create(ctx context.Context) error { @@ -796,32 +760,16 @@ func (s *workspaceClient) buildEnvironment(command string) ([]string, error) { }) } -func (s *workspaceClient) initLock() error { - s.workspaceLockOnce.Do(func() { - s.m.Lock() - defer s.m.Unlock() - - workspaceLocksDir, err := provider.GetLocksDir(s.providerWorkspace.Context) - if err != nil { - s.workspaceLockErr = fmt.Errorf("get workspaces dir: %w", err) - return - } - if err = os.MkdirAll(workspaceLocksDir, 0o755); err != nil { // #nosec G301 - s.workspaceLockErr = fmt.Errorf("create workspace locks dir: %w", err) - return - } +func (s *workspaceClient) lockIDs() (string, string, string) { + s.m.Lock() + defer s.m.Unlock() - s.workspaceLock = flock.New( - filepath.Join(workspaceLocksDir, s.providerWorkspace.ID+".workspace.lock"), - ) + machineID := "" + if s.providerMachine != nil { + machineID = s.providerMachine.ID + } - if s.providerMachine != nil { - s.machineLock = flock.New( - filepath.Join(workspaceLocksDir, s.providerMachine.ID+".machine.lock"), - ) - } - }) - return s.workspaceLockErr + return s.providerWorkspace.Context, s.providerWorkspace.ID, machineID } func stripProviderOptions(info *provider.AgentWorkspaceInfo) {