Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion internal/app/sync_events.go
Original file line number Diff line number Diff line change
Expand Up @@ -502,10 +502,26 @@ func (a *App) handleAppStateSyncError(ctx context.Context, evt *events.AppStateS

a.emitWarning(
"app_state_lthash_mismatch",
fmt.Sprintf("warning: app state %s hit an LTHash mismatch; requesting recovery snapshot", name),
fmt.Sprintf("warning: app state %s hit an LTHash mismatch; attempting full sync", name),
map[string]any{"name": name},
)
go func() {
if err := a.wa.FetchAppState(ctx, name, true, false); err == nil {
return
} else if !errors.Is(err, appstate.ErrMismatchingLTHash) {
a.emitWarning(
"app_state_full_sync_failed",
fmt.Sprintf("warning: app state %s full sync failed: %v", name, err),
map[string]any{"name": name, "error": err.Error()},
)
return
}

a.emitWarning(
"app_state_full_sync_mismatch",
fmt.Sprintf("warning: app state %s full sync still has an LTHash mismatch; requesting recovery snapshot", name),
map[string]any{"name": name},
)
reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
reqID, err := a.wa.RequestAppStateRecovery(reqCtx, name)
Expand Down
71 changes: 69 additions & 2 deletions internal/app/sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -558,13 +558,14 @@ func TestAppStateCallLogDeleteRemovesStoredCallEvent(t *testing.T) {
}
}

func TestAppStateLTHashMismatchRequestsRecoveryOnce(t *testing.T) {
func TestAppStateLTHashMismatchAttemptsFullSyncBeforeRecovery(t *testing.T) {
a := newTestApp(t)
f := newFakeWA()
a.wa = f

var recoveries sync.Map
err := fmt.Errorf("failed to verify patch v5848: %w", appstate.ErrMismatchingLTHash)
f.appStateFetchErrs = []error{err}
a.handleAppStateSyncError(context.Background(), &events.AppStateSyncError{
Name: appstate.WAPatchRegularLow,
Error: err,
Expand All @@ -577,15 +578,81 @@ func TestAppStateLTHashMismatchRequestsRecoveryOnce(t *testing.T) {
waitForCondition(t, time.Second, func() bool {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.appStateRecoveries) == 1
return len(f.appStateFetches) == 1 && len(f.appStateRecoveries) == 1
})
f.mu.Lock()
defer f.mu.Unlock()
if len(f.appStateFetches) != 1 {
t.Fatalf("app state fetches = %+v, want one full sync", f.appStateFetches)
}
if fetch := f.appStateFetches[0]; fetch.name != string(appstate.WAPatchRegularLow) || !fetch.fullSync || fetch.onlyIfNotSynced {
t.Fatalf("app state fetch = %+v, want regular_low full sync", fetch)
}
if got := f.appStateRecoveries[0]; got != string(appstate.WAPatchRegularLow) {
t.Fatalf("recovery collection = %q", got)
}
}

func TestAppStateLTHashMismatchStopsAfterSuccessfulFullSync(t *testing.T) {
a := newTestApp(t)
f := newFakeWA()
a.wa = f

var recoveries sync.Map
err := fmt.Errorf("failed to verify patch v5848: %w", appstate.ErrMismatchingLTHash)
a.handleAppStateSyncError(context.Background(), &events.AppStateSyncError{
Name: appstate.WAPatchRegularHigh,
Error: err,
}, &recoveries)
a.handleAppStateSyncError(context.Background(), &events.AppStateSyncError{
Name: appstate.WAPatchRegularHigh,
Error: err,
}, &recoveries)

waitForCondition(t, time.Second, func() bool {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.appStateFetches) == 1
})
time.Sleep(20 * time.Millisecond)
f.mu.Lock()
defer f.mu.Unlock()
if len(f.appStateFetches) != 1 {
t.Fatalf("app state fetches = %+v, want one full sync", f.appStateFetches)
}
if fetch := f.appStateFetches[0]; fetch.name != string(appstate.WAPatchRegularHigh) || !fetch.fullSync || fetch.onlyIfNotSynced {
t.Fatalf("app state fetch = %+v, want regular_high full sync", fetch)
}
if len(f.appStateRecoveries) != 0 {
t.Fatalf("app state recoveries = %v, want none after successful full sync", f.appStateRecoveries)
}
}

func TestAppStateLTHashMismatchDoesNotEscalateOtherFullSyncFailures(t *testing.T) {
a := newTestApp(t)
f := newFakeWA()
a.wa = f
f.appStateFetchErr = errors.New("injected full sync transport failure")

var recoveries sync.Map
a.handleAppStateSyncError(context.Background(), &events.AppStateSyncError{
Name: appstate.WAPatchRegularLow,
Error: fmt.Errorf("failed to verify patch v5848: %w", appstate.ErrMismatchingLTHash),
}, &recoveries)

waitForCondition(t, time.Second, func() bool {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.appStateFetches) == 1
})
time.Sleep(20 * time.Millisecond)
f.mu.Lock()
defer f.mu.Unlock()
if len(f.appStateRecoveries) != 0 {
t.Fatalf("app state recoveries = %v, want none after non-LTHash full sync failure", f.appStateRecoveries)
}
}

func TestAppStateNonLTHashErrorDoesNotRequestRecovery(t *testing.T) {
a := newTestApp(t)
f := newFakeWA()
Expand Down
Loading