Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 19 additions & 34 deletions pkg/task/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func (s *Store) ReconcileTask(ctx context.Context, id string) (*State, error) {
if err != nil {
return nil, err
}
if !needsReconciliation(current) {
if !current.NeedsReconciliation() {
return current, nil
}
lock, ok := s.claimDeadWorkerLock(id)
Expand All @@ -159,16 +159,11 @@ func (s *Store) ReconcileTask(ctx context.Context, id string) (*State, error) {
return s.reconcileClaimedTask(ctx, id)
}

func needsReconciliation(state *State) bool {
return state != nil && !state.Status.Terminal() &&
!state.LaunchPending
}

// ReconcileState marks a task after its worker exits without a result. A
// requested cancellation completes cleanup here; an abandoned task's tree is
// cleaned up by the next workspace stop or delete.
func (s *Store) ReconcileState(state *State) *State {
if !needsStateReconciliation(state) {
if !state.NeedsReconciliation() {
return state
}
lock, ok := s.claimDeadWorkerLock(state.ID)
Expand All @@ -177,18 +172,18 @@ func (s *Store) ReconcileState(state *State) *State {
}
defer func() { _ = lock.Unlock() }()
current, err := s.Get(state.ID)
if err != nil || !needsStateReconciliation(current) {
if err == nil {
return current
}
if err != nil {
return state
}
if !current.NeedsReconciliation() {
return current
}
if current.CancelRequested {
reconciled, err := s.reconcileClaimedCancellation(context.Background(), current)
if err == nil {
return reconciled
if err != nil {
return current
}
return current
return reconciled
}
return s.failAbandoned(current)
}
Expand Down Expand Up @@ -241,11 +236,6 @@ func (s *Store) CleanupCanceledWorkerTree(ctx context.Context, id string) error
return s.waitCanceledWorkerCleanup(ctx, id, ref)
}

func needsStateReconciliation(state *State) bool {
return state != nil && !state.Status.Terminal() &&
!state.LaunchPending
}

func hasProcessReference(state *State) bool {
_, ok := state.ProcessReference()
return ok
Expand Down Expand Up @@ -429,7 +419,7 @@ func (s *Store) workerObservation(id string) (WorkerObservation, bool, error) {

func (s *Store) reconcileClaimedTask(ctx context.Context, id string) (*State, error) {
current, err := s.Get(id)
if err != nil || !needsReconciliation(current) {
if err != nil || !current.NeedsReconciliation() {
return current, err
}
if current.CancelRequested {
Expand Down Expand Up @@ -499,23 +489,18 @@ func (s *Store) failAbandoned(state *State) *State {
// tryWorkerLock claims the task's worker lock when no worker owns it. Unlike
// claimDeadWorkerLock, it creates the lock file if the worker has not started.
func (s *Store) tryWorkerLock(id string) (*flock.Flock, bool, error) {
path, err := s.workerLockPath(id)
if err != nil {
return nil, false, err
}
lock := flock.New(path)
locked, err := lock.TryLock()
if err != nil {
return nil, false, err
}
if !locked {
return nil, false, nil
}
return lock, true, nil
return s.tryLock(id, s.workerLockPath)
}

func (s *Store) tryLauncherLock(id string) (*flock.Flock, bool, error) {
path, err := s.launcherLockPath(id)
return s.tryLock(id, s.launcherLockPath)
}

func (s *Store) tryLock(
id string,
lockPath func(string) (string, error),
) (*flock.Flock, bool, error) {
path, err := lockPath(id)
if err != nil {
return nil, false, err
}
Expand Down
19 changes: 19 additions & 0 deletions pkg/task/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -435,3 +435,22 @@ func TestActiveForWorkspaceTreatsLiveWorkerAsActive(t *testing.T) {
t.Errorf("live worker task not reported active: %+v", active)
}
}

func TestStateNeedsReconciliation(t *testing.T) {
for _, tc := range []struct {
name string
state *State
want bool
}{
{name: "nil", state: nil, want: false},
{name: "pending", state: &State{Status: StatusPending}, want: true},
{name: "running", state: &State{Status: StatusRunning}, want: true},
{name: "launch pending", state: &State{Status: StatusRunning, LaunchPending: true}, want: false},
{name: "succeeded", state: &State{Status: StatusSucceeded}, want: false},
{name: "failed", state: &State{Status: StatusFailed}, want: false},
} {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.want, tc.state.NeedsReconciliation())
})
}
}
6 changes: 6 additions & 0 deletions pkg/task/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,12 @@ func (s *State) NeedsCanceledWorkerCleanup() bool {
return ok
}

// NeedsReconciliation reports whether a task has no terminal result and is not
// mid-launch, so a worker that exited without recording one must be reconciled.
func (s *State) NeedsReconciliation() bool {
return s != nil && !s.Status.Terminal() && !s.LaunchPending
}

func (t *Task) BeginLaunch() error {
launcherLock, available, err := t.store.tryLauncherLock(t.id)
if err != nil {
Expand Down
Loading