Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 146 additions & 0 deletions pkg/client/clientimplementation/locks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package clientimplementation

import (
"context"
"fmt"
"os"
"path/filepath"
"sync"
"time"

"github.com/devsy-org/devsy/pkg/log"
"github.com/devsy-org/devsy/pkg/provider"
"github.com/gofrs/flock"
)

const (
lockTimeout = 5 * time.Minute
lockRetry = time.Second
)

// clientLocks lazily opens the file locks that serialize operations on a workspace.
type clientLocks struct {
// ids returns the context, workspace ID and machine ID used for the lock files.
// It is called once, on first use. An empty machine ID means no machine lock.
ids func() (contextName, workspaceID, machineID string)

once sync.Once
initErr error
workspace *flock.Flock
machine *flock.Flock
}

func newClientLocks(ids func() (string, string, string)) *clientLocks {
return &clientLocks{ids: ids}
}

// Lock acquires the workspace lock, then the machine lock when one is configured.
func (l *clientLocks) Lock(ctx context.Context) error {
if err := l.ensureOpen(); err != nil {
return fmt.Errorf("init lock: %w", err)
}

log.Debug("acquire workspace lock")
if err := l.lockWorkspace(ctx); err != nil {
return fmt.Errorf("lock workspace: %w", err)
}
log.Debug("acquired workspace lock")

if l.machine != nil {
log.Debug("acquire machine lock")
if err := tryLock(ctx, l.machine, "machine"); err != nil {
if uerr := l.unlockWorkspace(); uerr != nil {
log.Warnf("unlock workspace: %v", uerr)
}
return fmt.Errorf("lock machine: %w", err)
}
log.Debug("acquired machine lock")
}

return nil
}

// Unlock releases the machine lock, then the workspace lock.
func (l *clientLocks) Unlock() {
if err := l.ensureOpen(); err != nil {
log.Warnf("init lock: %v", err)
return
}

if l.machine != nil {
if err := l.machine.Unlock(); err != nil {
log.Warnf("unlock machine: %v", err)
}
}
if err := l.unlockWorkspace(); err != nil {
log.Warnf("unlock workspace: %v", err)
}
}

func (l *clientLocks) ensureOpen() error {
l.once.Do(func() {
contextName, workspaceID, machineID := l.ids()

dir, err := provider.GetLocksDir(contextName)
if err != nil {
l.initErr = fmt.Errorf("get workspaces dir: %w", err)
return
}

// #nosec G301 -- TODO Consider using a more secure permission setting and ownership if needed.
if err := os.MkdirAll(dir, 0o755); err != nil {
l.initErr = fmt.Errorf("create workspace locks dir: %w", err)
return
}

l.workspace = flock.New(filepath.Join(dir, workspaceID+".workspace.lock"))
if machineID != "" {
l.machine = flock.New(filepath.Join(dir, machineID+".machine.lock"))
}
})

return l.initErr
}

// lockWorkspace blocks until the workspace lock is held. ensureOpen must have succeeded.
func (l *clientLocks) lockWorkspace(ctx context.Context) error {
return tryLock(ctx, l.workspace, "workspace")
}

// unlockWorkspace releases the workspace lock. ensureOpen must have succeeded.
func (l *clientLocks) unlockWorkspace() error {
return l.workspace.Unlock()
}

func tryLock(ctx context.Context, lock *flock.Flock, name string) error {
done := scheduleLogMessage(
fmt.Sprintf(
"Trying to lock %s, seems like another process is running that blocks this %s",
name,
name,
),
)
defer close(done)

now := time.Now()
for time.Since(now) < lockTimeout {
locked, err := lock.TryLock()
if err != nil {
return err
} else if locked {
return nil
}

select {
case <-time.After(lockRetry):
continue
case <-ctx.Done():
return ctx.Err()
}
}

return fmt.Errorf(
"timed out waiting to lock %s, seems like there is another process running on this machine that blocks it",
name,
)
}
159 changes: 159 additions & 0 deletions pkg/client/clientimplementation/locks_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package clientimplementation

import (
"context"
"os"
"path/filepath"
"testing"
"time"

"github.com/devsy-org/devsy/pkg/config"
"github.com/devsy-org/devsy/pkg/provider"
"github.com/gofrs/flock"
"github.com/stretchr/testify/require"
)

const (
testLocksContext = "default"
lockTestWorkspaceID = "ws1"
)

func useTempDevsyHome(t *testing.T, home string) {
t.Helper()
t.Setenv(config.EnvHome, home)
config.ResetPathManager()
t.Cleanup(config.ResetPathManager)
}

func testLocksDir(t *testing.T) string {
t.Helper()
useTempDevsyHome(t, t.TempDir())

dir, err := provider.GetLocksDir(testLocksContext)
require.NoError(t, err)
require.NoError(t, os.MkdirAll(dir, 0o750))

return dir
}

func testLockIDs() (string, string, string) {
return testLocksContext, lockTestWorkspaceID, ""
}

func TestClientLocksLockUnlockRoundTrip(t *testing.T) {
dir := testLocksDir(t)
workspaceLock := filepath.Join(dir, lockTestWorkspaceID+".workspace.lock")

locks := newClientLocks(testLockIDs)
require.NoError(t, locks.Lock(context.Background()))

probe := flock.New(workspaceLock)
locked, err := probe.TryLock()
require.NoError(t, err)
require.False(t, locked, "workspace lock should be held after Lock")

locks.Unlock()

locked, err = probe.TryLock()
require.NoError(t, err)
require.True(t, locked, "workspace lock should be released after Unlock")
require.NoError(t, probe.Unlock())
}

func TestClientLocksContendedWorkspaceFailsWhenContextCancelled(t *testing.T) {
dir := testLocksDir(t)

holder := flock.New(filepath.Join(dir, lockTestWorkspaceID+".workspace.lock"))
locked, err := holder.TryLock()
require.NoError(t, err)
require.True(t, locked)
t.Cleanup(func() { require.NoError(t, holder.Unlock()) })

ctx, cancel := context.WithCancel(context.Background())
cancel()

locks := newClientLocks(testLockIDs)
err = locks.Lock(ctx)
require.ErrorIs(t, err, context.Canceled)
require.ErrorContains(t, err, "lock workspace:")
}

func TestClientLocksInitFailureIsCachedAndUnlockDoesNotPanic(t *testing.T) {
// A regular file as DEVSY_HOME makes creating the locks dir fail.
blocker := filepath.Join(t.TempDir(), "not-a-dir")
require.NoError(t, os.WriteFile(blocker, []byte("x"), 0o600))
useTempDevsyHome(t, blocker)

locks := newClientLocks(testLockIDs)

err := locks.Lock(context.Background())
require.ErrorContains(t, err, "init lock:")
require.NotPanics(t, locks.Unlock)

// Once init has failed, a later valid home must not be picked up.
useTempDevsyHome(t, t.TempDir())
err = locks.Lock(context.Background())
require.ErrorContains(t, err, "init lock:")
}

func TestClientLocksMachineLock(t *testing.T) {
t.Run("no machine id creates no machine lock", func(t *testing.T) {
dir := testLocksDir(t)

locks := newClientLocks(testLockIDs)
require.NoError(t, locks.Lock(context.Background()))
locks.Unlock()

_, err := os.Stat(filepath.Join(dir, ".machine.lock"))
require.ErrorIs(t, err, os.ErrNotExist)
})

t.Run("machine id read lazily on first lock", func(t *testing.T) {
dir := testLocksDir(t)
machineID := ""
locks := newClientLocks(func() (string, string, string) {
return testLocksContext, lockTestWorkspaceID, machineID
})

machineID = "m1"
require.NoError(t, locks.Lock(context.Background()))

machineLock := filepath.Join(dir, "m1.machine.lock")
probe := flock.New(machineLock)
locked, err := probe.TryLock()
require.NoError(t, err)
require.False(t, locked, "machine lock should be held after Lock")

locks.Unlock()

locked, err = probe.TryLock()
require.NoError(t, err)
require.True(t, locked, "machine lock should be released after Unlock")
require.NoError(t, probe.Unlock())
})

t.Run("machine lock failure releases workspace lock", func(t *testing.T) {
dir := testLocksDir(t)

holder := flock.New(filepath.Join(dir, "m1.machine.lock"))
locked, err := holder.TryLock()
require.NoError(t, err)
require.True(t, locked)
t.Cleanup(func() { require.NoError(t, holder.Unlock()) })

ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()

locks := newClientLocks(func() (string, string, string) {
return testLocksContext, lockTestWorkspaceID, "m1"
})
err = locks.Lock(ctx)
require.ErrorContains(t, err, "lock machine:")

probe := flock.New(filepath.Join(dir, lockTestWorkspaceID+".workspace.lock"))
locked, err = probe.TryLock()
require.NoError(t, err)
require.True(t, locked, "workspace lock must be released after machine lock failure")
require.NoError(t, probe.Unlock())
})
}
Loading
Loading