diff --git a/zetaclient/maintenance/tss_listener.go b/zetaclient/maintenance/tss_listener.go index afb0ca577b..b7ab75674b 100644 --- a/zetaclient/maintenance/tss_listener.go +++ b/zetaclient/maintenance/tss_listener.go @@ -13,7 +13,9 @@ import ( observertypes "github.com/zeta-chain/node/x/observer/types" ) -const tssListenerTicker = 5 * time.Second +// tssListenerTicker is how often the watchers re-query zetacore. A var rather than a const so +// tests can shrink it; nothing in production writes to it. +var tssListenerTicker = 5 * time.Second // TSSListener is a struct that listens for TSS updates, new keygen, and new TSS key generation. type TSSListener struct { @@ -32,6 +34,15 @@ func NewTSSListener(client ZetacoreClient, logger zerolog.Logger) *TSSListener { } // Listen listens for any maintenance regarding TSS and calls action specified. Works in the background. +// +// Both watchers key off the TSS itself: the address changing, or a new key landing in history. +// The keygen record is deliberately not watched. It is reset to "pending at block MaxInt64" on +// any observer set change, and restarting on that reset is what turns a routine validator +// unbonding into an outage — every signer shuts down at once and none of them can start again. +// +// Note this also removes the only trigger that restarted zetaclient for a scheduled keygen, so +// while a finalized key exists a rotation ceremony will not start. That is deliberate; see the +// step 5 comment in zetaclient/tss/setup.go. func (tl *TSSListener) Listen(ctx context.Context, action func()) { var ( withLogger = bg.WithLogger(tl.logger) @@ -40,7 +51,6 @@ func (tl *TSSListener) Listen(ctx context.Context, action func()) { bg.Work(ctx, tl.waitForUpdate, bg.WithName("tss.wait_for_update"), withLogger, onComplete) bg.Work(ctx, tl.waitForNewKeyGeneration, bg.WithName("tss.wait_for_generation"), withLogger, onComplete) - bg.Work(ctx, tl.waitForNewKeygen, bg.WithName("tss.wait_for_keygen"), withLogger, onComplete) } // waitForUpdate listens for TSS updates. Returns `nil` when the TSS address is updated @@ -126,48 +136,3 @@ func (tl *TSSListener) waitForNewKeyGeneration(ctx context.Context) error { } } } - -// waitForNewKeygen is a background thread that listens for new keygen; it returns when a new keygen is set -func (tl *TSSListener) waitForNewKeygen(ctx context.Context) error { - // Initial Keygen retrieval - keygen, err := tl.client.GetKeyGen(ctx) - if err != nil { - return errors.Wrap(err, "failed to get initial TSS history") - } - - ticker := time.NewTicker(tssListenerTicker) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - keygenUpdated, err := tl.client.GetKeyGen(ctx) - switch { - case err != nil: - tl.logger.Warn().Err(err).Msg("unable to get keygen") - continue - // Keygen is not pending it has already been successfully generated, continue loop - case keygenUpdated.Status == observertypes.KeygenStatus_KeyGenSuccess: - continue - // Keygen failed we to need to wait until a new keygen is set, continue loop - case keygenUpdated.Status == observertypes.KeygenStatus_KeyGenFailed: - continue - // Keygen is pending but block number is not updated, continue loop. - // Most likely the zetaclient is waiting for the keygen block to arrive. - case keygenUpdated.Status == observertypes.KeygenStatus_PendingKeygen && - keygenUpdated.BlockNumber <= keygen.BlockNumber: - continue - } - - // Trigger restart only when the following conditions are met: - // 1. Keygen is pending - // 2. Block number is updated - - tl.logger.Info().Int64("block_number", keygenUpdated.BlockNumber).Msg("got new keygen") - return nil - case <-ctx.Done(): - tl.logger.Info().Msg("stopped waiting for new keygen in the TSS listener") - return nil - } - } -} diff --git a/zetaclient/maintenance/tss_listener_test.go b/zetaclient/maintenance/tss_listener_test.go new file mode 100644 index 0000000000..cdc630f213 --- /dev/null +++ b/zetaclient/maintenance/tss_listener_test.go @@ -0,0 +1,124 @@ +package maintenance + +import ( + "context" + "io" + "testing" + "time" + + "github.com/rs/zerolog" + + observertypes "github.com/zeta-chain/node/x/observer/types" + "github.com/zeta-chain/node/zetaclient/testutils/mocks" +) + +const ( + tssPubkeyOld = "zetapub1addwnpepqtadxdyt037h86z60nl98t6zk56mw5zpnm79tsmvspln3hgt5phdc79kvfc" + tssPubkeyNew = "zetapub1addwnpepqglunjrgl3qg08duxq9pf28jmvrer3crwnnfzp6m0u0yh9jk9mnn5p76utc" +) + +// useFastTicker shrinks the watcher poll interval for the duration of one test. The assertions +// are about which events cause a shutdown, not about the real 5s cadence, so waiting on it only +// bought ~31s of sleep across this file. +func useFastTicker(t *testing.T) { + original := tssListenerTicker + tssListenerTicker = 10 * time.Millisecond + t.Cleanup(func() { tssListenerTicker = original }) +} + +// waitForListenerTick gives the listener room for at least one tick so a shutdown it was +// going to signal has actually had the chance to fire. +func waitForListenerTick() { + time.Sleep(20 * tssListenerTicker) +} + +func TestTSSListener(t *testing.T) { + // Deliberately not zerolog.NewTestWriter(t): Listen's workers keep running after the test + // returns, and logging into a finished t panics with "Log in goroutine after test completed". + logger := zerolog.New(io.Discard) + + oldTSS := observertypes.TSS{TssPubkey: tssPubkeyOld} + + // A blanked keygen record must not restart the client. + // + // zetacore writes exactly this on any observer set change (x/observer/abci.go + // BeginBlocker): status back to pending, grantees erased, block set to MaxInt64. On + // mainnet that reset restarted every signer at once, and none could start again because + // the erased grantee list left them with an empty p2p whitelist. + t.Run("blanked keygen record does not trigger a shutdown", func(t *testing.T) { + useFastTicker(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client := mocks.NewZetacoreClient(t) + client.Mock.On("GetTSS", ctx).Return(oldTSS, nil) + client.Mock.On("GetTSSHistory", ctx).Return([]observertypes.TSS{oldTSS}, nil) + + complete := make(chan interface{}) + NewTSSListener(client, logger).Listen(ctx, func() { close(complete) }) + + waitForListenerTick() + assertChannelNotClosed(t, complete) + }) + + t.Run("TSS address change still triggers a shutdown", func(t *testing.T) { + useFastTicker(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client := mocks.NewZetacoreClient(t) + client.Mock.On("GetTSSHistory", ctx).Return([]observertypes.TSS{oldTSS}, nil) + client.Mock.On("GetTSS", ctx).Return(oldTSS, nil).Once() + client.Mock.On("GetTSS", ctx).Return(observertypes.TSS{TssPubkey: tssPubkeyNew}, nil) + + complete := make(chan interface{}) + NewTSSListener(client, logger).Listen(ctx, func() { close(complete) }) + + <-complete + }) + + t.Run("new key in TSS history still triggers a shutdown", func(t *testing.T) { + useFastTicker(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + newTSS := observertypes.TSS{TssPubkey: tssPubkeyNew} + + client := mocks.NewZetacoreClient(t) + client.Mock.On("GetTSS", ctx).Return(oldTSS, nil) + client.Mock.On("GetTSSHistory", ctx).Return([]observertypes.TSS{oldTSS}, nil).Once() + client.Mock.On("GetTSSHistory", ctx).Return([]observertypes.TSS{oldTSS, newTSS}, nil) + + complete := make(chan interface{}) + NewTSSListener(client, logger).Listen(ctx, func() { close(complete) }) + + <-complete + }) +} + +// TestTSSListenerIgnoresKeygen pins that the listener never reads the keygen record. The mock +// fails the test on any unexpected call, so a reintroduced keygen watcher shows up here rather +// than as an outage. +func TestTSSListenerIgnoresKeygen(t *testing.T) { + useFastTicker(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Deliberately not zerolog.NewTestWriter(t): Listen's workers keep running after the test + // returns, and logging into a finished t panics with "Log in goroutine after test completed". + logger := zerolog.New(io.Discard) + oldTSS := observertypes.TSS{TssPubkey: tssPubkeyOld} + + client := mocks.NewZetacoreClient(t) + client.Mock.On("GetTSS", ctx).Return(oldTSS, nil) + client.Mock.On("GetTSSHistory", ctx).Return([]observertypes.TSS{oldTSS}, nil) + + NewTSSListener(client, logger).Listen(ctx, func() {}) + waitForListenerTick() + + client.Mock.AssertNotCalled(t, "GetKeyGen", ctx) +} diff --git a/zetaclient/tss/service.go b/zetaclient/tss/service.go index 215ac07264..b1bdf6e57e 100644 --- a/zetaclient/tss/service.go +++ b/zetaclient/tss/service.go @@ -18,6 +18,7 @@ import ( "github.com/zeta-chain/go-tss/keysign" "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/retry" observertypes "github.com/zeta-chain/node/x/observer/types" keyinterfaces "github.com/zeta-chain/node/zetaclient/keys/interfaces" "github.com/zeta-chain/node/zetaclient/logs" @@ -106,7 +107,12 @@ func WithPostBlame(postBlame bool) Opt { // Otherwise, no metrics will be collected. func WithMetrics(ctx context.Context, zetacore Zetacore, m *Metrics) Opt { return func(cfg *serviceConfig, _ zerolog.Logger) error { - keygen, err := zetacore.GetKeyGen(ctx) + // Retried like the other startup queries: this one is easy to miss because it sits in + // an option rather than in Setup, but it runs on the same path and is just as fatal. + keygen, err := retry.DoTypedWithBackoffAndRetry( + func() (observertypes.Keygen, error) { return zetacore.GetKeyGen(ctx) }, + retry.DefaultConstantBackoff(), + ) if err != nil { return errors.Wrap(err, "failed to get keygen (WithMetrics)") } diff --git a/zetaclient/tss/setup.go b/zetaclient/tss/setup.go index 9563c70be9..fd42468ed8 100644 --- a/zetaclient/tss/setup.go +++ b/zetaclient/tss/setup.go @@ -20,6 +20,7 @@ import ( "github.com/zeta-chain/go-tss/conversion" "github.com/zeta-chain/go-tss/tss" + "github.com/zeta-chain/node/pkg/retry" observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/config" "github.com/zeta-chain/node/zetaclient/logs" @@ -79,25 +80,11 @@ func Setup(ctx context.Context, p SetupProps, logger zerolog.Logger) (*Service, setupLogger.Info().Msg("resolved pre-params file") // 3. Prepare whitelist of peers - tssKeygen, err := p.Zetacore.GetKeyGen(ctx) + currentTSS, whitelistedPeers, keyAlreadyFinalized, err := resolveWhitelist(ctx, p.Zetacore, setupLogger) if err != nil { - return nil, errors.Wrap(err, "unable to get TSS keygen") + return nil, err } - setupLogger.Info().Msg("fetched TSS keygen info") - - whitelistedPeers := make([]peer.ID, len(tssKeygen.GranteePubkeys)) - for i, pk := range tssKeygen.GranteePubkeys { - whitelistedPeers[i], err = conversion.Bech32PubkeyToPeerID(pk) - if err != nil { - return nil, errors.Wrap(err, pk) - } - } - - setupLogger.Info(). - Any("whitelisted_peers", whitelistedPeers). - Msg("resolved whitelist peers") - // 4. Bootstrap go-tss TSS server tssServer, err := NewServer( bootstrapPeers, @@ -119,12 +106,32 @@ func Setup(ctx context.Context, p SetupProps, logger zerolog.Logger) (*Service, setupLogger.Info().Msg("started TSS server") // 5. Perform key generation (if needed) - tssInfo, err := KeygenCeremony(ctx, tssServer, p.Zetacore, logger) - if err != nil { - return nil, errors.Wrap(err, "unable to perform keygen ceremony") + // + // A finalized TSS means there is nothing left to generate, so the ceremony is skipped + // entirely rather than waiting on the keygen record to say so. Waiting is not safe: the + // record is reset to "pending at block MaxInt64" on any observer set change, and the + // ceremony would then block forever on a block that never arrives, taking the signer down + // with it. KeygenCeremony returns GetTSS on its own success path, so this is the same value + // it would have produced. + // + // Consequence worth being explicit about: while a finalized key exists, a keygen scheduled + // by MsgUpdateKeygen will NOT run, because nothing else calls KeygenCeremony. Rotating on + // purpose means removing the current TSS first, or reintroducing a deliberate trigger. + // Tracked in https://github.com/zeta-chain/node/issues/4623. + tssInfo := currentTSS + if !keyAlreadyFinalized { + tssInfo, err = KeygenCeremony(ctx, tssServer, p.Zetacore, logger) + if err != nil { + return nil, errors.Wrap(err, "unable to perform keygen ceremony") + } } - historicalTSSInfo, err := p.Zetacore.GetTSSHistory(ctx) + // Retried for the same reason as the two calls above: same startup path, same contended + // zetacore, and a one-shot failure here is equally fatal. + historicalTSSInfo, err := retry.DoTypedWithBackoffAndRetry( + func() ([]observertypes.TSS, error) { return p.Zetacore.GetTSSHistory(ctx) }, + retry.DefaultConstantBackoff(), + ) if err != nil { return nil, errors.Wrap(err, "unable to get TSS history") } @@ -185,6 +192,121 @@ func Setup(ctx context.Context, p SetupProps, logger zerolog.Logger) (*Service, return service, nil } +// tssFetcher is the slice of Zetacore that resolveWhitelist needs. Narrow on purpose: the +// whole point of splitting this out of Setup is that it can be exercised without a p2p server, +// key files on disk, or a full zetacore client. +type tssFetcher interface { + GetKeyGen(ctx context.Context) (observertypes.Keygen, error) + GetTSS(ctx context.Context) (observertypes.TSS, error) +} + +// resolveWhitelist decides which peers the TSS server may talk to, and reports whether a TSS +// key already exists so Setup can skip the keygen ceremony. +// +// A finalized TSS takes precedence over the keygen record; see resolveTSSPeers. +// +// "GetTSS failed" and "there is no TSS" are different answers and must not be merged. The +// record we would fall back to is the blanked one this function exists to survive, so +// treating a failed query as absence restores both symptoms: an empty whitelist, and a +// ceremony waiting on a block that never arrives. +// +// A failed query therefore falls back to the record, and the guard below is what keeps that +// safe: if the record has been blanked there is nothing to whitelist and startup stops +// there rather than continuing on a bad answer. When the record is intact the fallback is +// the right one anyway — its grantees are the same set, and a keygen that already succeeded +// makes the ceremony a noop, so a blip heals itself instead of taking the node down. +// Every error is retried, including the not-found a chain without a key answers with. The +// constant backoff is deliberate: sub-second retries give up inside an RPC restart, and the +// only node that pays the full wait is one on a chain with no key, which then waits for the +// keygen block anyway. +func resolveWhitelist( + ctx context.Context, + client tssFetcher, + setupLogger zerolog.Logger, +) (observertypes.TSS, []peer.ID, bool, error) { + // This query runs first, so leaving it a one-shot fatal would kill startup before the + // retry below could ever apply. + tssKeygen, err := retry.DoTypedWithBackoffAndRetry( + func() (observertypes.Keygen, error) { return client.GetKeyGen(ctx) }, + retry.DefaultConstantBackoff(), + ) + if err != nil { + return observertypes.TSS{}, nil, false, errors.Wrap(err, "unable to get TSS keygen") + } + + setupLogger.Info().Msg("fetched TSS keygen info") + + currentTSS, tssErr := retry.DoTypedWithBackoffAndRetry( + func() (observertypes.TSS, error) { return client.GetTSS(ctx) }, + retry.DefaultConstantBackoff(), + ) + + if tssErr != nil { + setupLogger.Warn().Err(tssErr).Msg("unable to get TSS; falling back to the keygen record") + currentTSS = observertypes.TSS{} + } + + peerSource, keyAlreadyFinalized := resolveTSSPeers(tssKeygen, currentTSS) + if keyAlreadyFinalized { + setupLogger.Info(). + Str("tss_pubkey", currentTSS.TssPubkey). + Int("grantees_in_keygen_record", len(tssKeygen.GranteePubkeys)). + Int("participants_in_tss", len(currentTSS.TssParticipantList)). + Msg("TSS key already finalized; whitelisting its participants") + } + + // Starting with nothing to whitelist is the failure this function guards against, so say + // so plainly instead of letting go-tss report it as missing bootstrap peers. + if len(peerSource) == 0 { + return observertypes.TSS{}, nil, false, errors.New( + "no TSS peers to whitelist: both the TSS participant list and the keygen grantees are empty", + ) + } + + whitelistedPeers := make([]peer.ID, len(peerSource)) + for i, pk := range peerSource { + peerID, err := conversion.Bech32PubkeyToPeerID(pk) + if err != nil { + return observertypes.TSS{}, nil, false, errors.Wrap(err, pk) + } + whitelistedPeers[i] = peerID + } + + setupLogger.Info(). + Any("whitelisted_peers", whitelistedPeers). + Msg("resolved whitelist peers") + + return currentTSS, whitelistedPeers, keyAlreadyFinalized, nil +} + +// resolveTSSPeers returns the pubkeys allowed into p2p, and whether the TSS key is already +// finalized (in which case no keygen ceremony is required). +// +// The keygen record describes a key that is yet to be generated, so it is only authoritative +// before the first ceremony. zetacore resets it to a blank value on any observer set change +// (x/observer/abci.go BeginBlocker): the grantee list is erased and the block set to MaxInt64. +// That says nothing about a key generated long ago and whose shares this node still holds, so +// a finalized TSS wins. Reading the record instead would leave a healthy signer with an empty +// whitelist and a ceremony scheduled for a block that never arrives. +// +// The participant list is the set that produced the key we are about to sign with, which makes +// it the correct whitelist: it must stay a superset of the participants recorded in the local +// key share, or peers are refused at the p2p layer during keysign. +func resolveTSSPeers(keygen observertypes.Keygen, currentTSS observertypes.TSS) (pubkeys []string, finalized bool) { + if currentTSS.TssPubkey == "" { + return keygen.GranteePubkeys, false + } + + // A TSS imported straight from genesis (x/observer/genesis.go) can carry a key with no + // participants recorded. The key is still real and must not be regenerated, so keep + // finalized set and take the peers from the keygen record instead of whitelisting nobody. + if len(currentTSS.TssParticipantList) == 0 { + return keygen.GranteePubkeys, true + } + + return currentTSS.TssParticipantList, true +} + // NewServer creates a new tss.TssServer (go-tss) instance for key signing. // - bootstrapPeers are used to discover other peers // - whitelistPeers are the only peers that are allowed in p2p key signing. diff --git a/zetaclient/tss/setup_test.go b/zetaclient/tss/setup_test.go new file mode 100644 index 0000000000..84b1780fc0 --- /dev/null +++ b/zetaclient/tss/setup_test.go @@ -0,0 +1,238 @@ +package tss + +import ( + "context" + "io" + "math" + "testing" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + observertypes "github.com/zeta-chain/node/x/observer/types" +) + +const ( + pubkeyA = "zetapub1addwnpepqglunjrgl3qg08duxq9pf28jmvrer3crwnnfzp6m0u0yh9jk9mnn5p76utc" + pubkeyB = "zetapub1addwnpepqwwpjwwnes7cywfkr0afme7ymk8rf5jzhn8pfr6qqvfm9v342486qsrh4f5" + tssKey = "zetapub1addwnpepqtadxdyt037h86z60nl98t6zk56mw5zpnm79tsmvspln3hgt5phdc79kvfc" + + // The libp2p peer IDs the three keys above convert to. Hardcoded rather than derived in + // the test, so a change in the conversion shows up as a failure instead of agreeing with + // itself. + peerIDA = "16Uiu2HAkyig859BKphpgkyiJAE3wmsFAJMf541DRbFosai13ECbK" + peerIDB = "16Uiu2HAmPALG7YS5PNAsbHpjeg5WwSrRYqhZFkKjyPmwwEWjTP35" +) + +func TestResolveTSSPeers(t *testing.T) { + + finalized := observertypes.TSS{ + TssPubkey: tssKey, + TssParticipantList: []string{pubkeyA, pubkeyB}, + } + + t.Run("no TSS yet, keygen record is authoritative", func(t *testing.T) { + keygen := observertypes.Keygen{ + Status: observertypes.KeygenStatus_PendingKeygen, + GranteePubkeys: []string{pubkeyA, pubkeyB}, + BlockNumber: 100, + } + + pubkeys, finalizedKey := resolveTSSPeers(keygen, observertypes.TSS{}) + + assert.False(t, finalizedKey) + assert.Equal(t, []string{pubkeyA, pubkeyB}, pubkeys) + }) + + t.Run("blanked keygen record does not empty the whitelist", func(t *testing.T) { + // The exact state zetacore writes on an observer set change: no grantees, pending, + // scheduled for a block that never arrives. + blanked := observertypes.Keygen{ + Status: observertypes.KeygenStatus_PendingKeygen, + BlockNumber: math.MaxInt64, + } + require.Empty(t, blanked.GranteePubkeys) + + pubkeys, finalizedKey := resolveTSSPeers(blanked, finalized) + + assert.True(t, finalizedKey, "a finalized key must skip the ceremony") + assert.Equal(t, []string{pubkeyA, pubkeyB}, pubkeys, "whitelist must come from the TSS participants") + }) + + t.Run("finalized key wins over a populated keygen record", func(t *testing.T) { + // A scheduled keygen must not strand the signer either: it would otherwise block on + // its own future block while the existing key is perfectly usable. + scheduled := observertypes.Keygen{ + Status: observertypes.KeygenStatus_PendingKeygen, + GranteePubkeys: []string{pubkeyA}, + BlockNumber: 26_400_000, + } + + pubkeys, finalizedKey := resolveTSSPeers(scheduled, finalized) + + assert.True(t, finalizedKey) + assert.Equal(t, []string{pubkeyA, pubkeyB}, pubkeys) + }) + + t.Run("genesis TSS without participants falls back to the grantees", func(t *testing.T) { + // x/observer/genesis.go imports a TSS verbatim, so it can carry a real key with no + // participant list. The key must still count as finalized, or startup would try to + // generate a replacement. + imported := observertypes.TSS{TssPubkey: tssKey} + + keygen := observertypes.Keygen{ + Status: observertypes.KeygenStatus_KeyGenSuccess, + GranteePubkeys: []string{pubkeyA, pubkeyB}, + } + + pubkeys, finalizedKey := resolveTSSPeers(keygen, imported) + + assert.True(t, finalizedKey, "an imported key is still a key; do not regenerate it") + assert.Equal(t, []string{pubkeyA, pubkeyB}, pubkeys, "must not whitelist nobody") + }) + + t.Run("successful keygen still resolves to the TSS participants", func(t *testing.T) { + succeeded := observertypes.Keygen{ + Status: observertypes.KeygenStatus_KeyGenSuccess, + GranteePubkeys: []string{pubkeyA, pubkeyB}, + BlockNumber: math.MaxInt64, + } + + pubkeys, finalizedKey := resolveTSSPeers(succeeded, finalized) + + assert.True(t, finalizedKey) + assert.Equal(t, finalized.TssParticipantList, pubkeys) + }) +} + +// stubTSSFetcher stands in for the zetacore client. The generated mock cannot be used here: +// zetaclient/testutils/mocks imports this package, so an in-package test importing it back +// would be an import cycle. +type stubTSSFetcher struct { + keygen observertypes.Keygen + keygenErr error + tss observertypes.TSS + err error + calls int +} + +func (s *stubTSSFetcher) GetKeyGen(context.Context) (observertypes.Keygen, error) { + return s.keygen, s.keygenErr +} + +func (s *stubTSSFetcher) GetTSS(context.Context) (observertypes.TSS, error) { + s.calls++ + return s.tss, s.err +} + +func peerIDStrings(t *testing.T, peers []peer.ID) []string { + t.Helper() + + out := make([]string, len(peers)) + for i, p := range peers { + out[i] = p.String() + } + + return out +} + +func TestResolveWhitelist(t *testing.T) { + logger := zerolog.New(io.Discard) + + // The exact record zetacore writes on an observer set change. + blanked := observertypes.Keygen{ + Status: observertypes.KeygenStatus_PendingKeygen, + BlockNumber: math.MaxInt64, + } + + populated := observertypes.Keygen{ + Status: observertypes.KeygenStatus_PendingKeygen, + GranteePubkeys: []string{pubkeyA, pubkeyB}, + BlockNumber: 100, + } + + finalized := observertypes.TSS{ + TssPubkey: tssKey, + TssParticipantList: []string{pubkeyA, pubkeyB}, + } + + // The mainnet case: the record has been erased, but the key it says nothing about is still + // there and still carries the participants to whitelist. + t.Run("finalized key survives a blanked record", func(t *testing.T) { + client := &stubTSSFetcher{keygen: blanked, tss: finalized} + + currentTSS, peers, keyFinalized, err := resolveWhitelist(context.Background(), client, logger) + + require.NoError(t, err) + assert.True(t, keyFinalized, "a finalized key must skip the ceremony") + assert.Equal(t, tssKey, currentTSS.TssPubkey) + assert.Equal(t, []string{peerIDA, peerIDB}, peerIDStrings(t, peers)) + }) + + t.Run("no TSS yet whitelists the keygen grantees", func(t *testing.T) { + client := &stubTSSFetcher{keygen: populated, tss: observertypes.TSS{}} + + currentTSS, peers, keyFinalized, err := resolveWhitelist(context.Background(), client, logger) + + require.NoError(t, err) + assert.False(t, keyFinalized, "a first-ever node still has to run the ceremony") + assert.Empty(t, currentTSS.TssPubkey) + assert.Equal(t, []string{peerIDA, peerIDB}, peerIDStrings(t, peers)) + }) + + // A failed query is not the same as "there is no key", but with an intact record the + // fallback lands on the same set anyway. + t.Run("query failure falls back to the keygen record", func(t *testing.T) { + client := &stubTSSFetcher{keygen: populated, err: context.Canceled} + + _, peers, keyFinalized, err := resolveWhitelist(context.Background(), client, logger) + + require.NoError(t, err) + assert.False(t, keyFinalized) + assert.Equal(t, []string{peerIDA, peerIDB}, peerIDStrings(t, peers)) + + // retry.Retry treats context errors as non-retryable, which is what keeps this test + // instant instead of sitting through the 5s x10 constant backoff. + assert.Equal(t, 1, client.calls) + }) + + // Both sources empty is the state that took mainnet down. Startup must stop here rather + // than hand go-tss an empty whitelist and let it fail as "missing bootstrap peers". + t.Run("query failure on a blanked record stops startup", func(t *testing.T) { + client := &stubTSSFetcher{keygen: blanked, err: context.Canceled} + + _, peers, keyFinalized, err := resolveWhitelist(context.Background(), client, logger) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no TSS peers to whitelist") + assert.Nil(t, peers) + assert.False(t, keyFinalized) + }) + + t.Run("a pubkey that cannot become a peer ID is reported", func(t *testing.T) { + keygen := observertypes.Keygen{GranteePubkeys: []string{pubkeyA, "not-a-pubkey"}} + client := &stubTSSFetcher{keygen: keygen, tss: observertypes.TSS{}} + + _, peers, _, err := resolveWhitelist(context.Background(), client, logger) + + require.Error(t, err) + assert.Contains(t, err.Error(), "not-a-pubkey") + assert.Nil(t, peers) + }) + + // The keygen fetch is the first query on the startup path, so its failure has to stop + // startup rather than fall through to an empty whitelist. + t.Run("keygen query failure stops startup", func(t *testing.T) { + client := &stubTSSFetcher{keygenErr: context.Canceled, tss: finalized} + + _, peers, keyFinalized, err := resolveWhitelist(context.Background(), client, logger) + + require.Error(t, err) + assert.Contains(t, err.Error(), "unable to get TSS keygen") + assert.Nil(t, peers) + assert.False(t, keyFinalized) + assert.Equal(t, 0, client.calls, "must not query the TSS after the keygen query failed") + }) +}