From 1eca2ecdb3214e726f2ff684196b26e1b653c150 Mon Sep 17 00:00:00 2001 From: smcio Date: Sat, 30 May 2026 02:20:27 +0200 Subject: [PATCH] implement & hook up turbine, gossip & repair --- cmd/mithril/configcmd/configcmd.go | 10 +- cmd/mithril/dashboardcmd/dashboard.go | 53 + cmd/mithril/dashboardcmd/data.go | 119 +- cmd/mithril/node/node.go | 119 +- cmd/mithril/setupcmd/migrate.go | 2 +- cmd/mithril/setupcmd/setup.go | 14 +- config.example.toml | 75 +- go.mod | 3 +- go.sum | 4 + nix/modules/shared/lib.nix | 1 + nix/modules/shared/options.nix | 6 + pkg/accountsdb/accountsdb.go | 30 +- pkg/blockstream/block_source.go | 626 +++- pkg/blockstream/block_source_test.go | 216 ++ pkg/config/config.go | 36 +- pkg/forkchoice/vote_parser.go | 9 +- pkg/gossip/bitvec.go | 64 - pkg/gossip/bloom.go | 105 - pkg/gossip/bloom_test.go | 91 - pkg/gossip/client.go | 668 +++- pkg/gossip/codec.go | 201 ++ pkg/gossip/contact_info.go | 619 ++++ pkg/gossip/crds.go | 74 - pkg/gossip/crds_test.go | 46 - pkg/gossip/gossip_test.go | 161 + pkg/gossip/ipecho.go | 70 + pkg/gossip/message.go | 249 ++ pkg/gossip/message_test.go | 210 -- pkg/gossip/ping.go | 245 -- pkg/gossip/ping_test.go | 65 - pkg/gossip/pull.go | 64 - pkg/gossip/schema.go | 3527 ---------------------- pkg/gossip/schema.yaml | 335 -- pkg/gossip/socketaddr.go | 66 - pkg/gossip/transaction.go | 115 - pkg/gossip/types.go | 15 - pkg/repair/protocol.go | 164 + pkg/repair/protocol_test.go | 133 + pkg/replay/block.go | 91 +- pkg/replay/consensus.go | 46 +- pkg/replay/consensus_fallback_test.go | 51 + pkg/replay/transaction.go | 3 +- pkg/rpcserver/rpcserver.go | 169 +- pkg/rpcserver/rpcserver_test.go | 81 + pkg/sbpf/program.go | 14 + pkg/sealevel/vote_program.go | 12 + pkg/sealevel/vote_program_decode_test.go | 32 + pkg/shred/entry.go | 42 - pkg/shred/shred.go | 174 -- pkg/shred/shredder.go | 21 - pkg/snapshot/build_db.go | 108 +- pkg/snapshot/build_db_with_incr.go | 10 +- pkg/snapshot/shard.go | 8 +- pkg/snapshotdl/snapshotdl.go | 214 +- pkg/snapshotdl/snapshotdl_test.go | 80 + pkg/turbine/assembler.go | 654 ++++ pkg/turbine/assembler_test.go | 584 ++++ pkg/turbine/entries.go | 157 + pkg/turbine/receiver.go | 241 ++ pkg/turbine/repair.go | 387 +++ pkg/turbine/shred.go | 381 +++ pkg/txverify/txverify.go | 48 + 62 files changed, 6629 insertions(+), 5589 deletions(-) delete mode 100644 pkg/gossip/bitvec.go delete mode 100644 pkg/gossip/bloom.go delete mode 100644 pkg/gossip/bloom_test.go create mode 100644 pkg/gossip/codec.go create mode 100644 pkg/gossip/contact_info.go delete mode 100644 pkg/gossip/crds.go delete mode 100644 pkg/gossip/crds_test.go create mode 100644 pkg/gossip/gossip_test.go create mode 100644 pkg/gossip/ipecho.go create mode 100644 pkg/gossip/message.go delete mode 100644 pkg/gossip/message_test.go delete mode 100644 pkg/gossip/ping.go delete mode 100644 pkg/gossip/ping_test.go delete mode 100644 pkg/gossip/pull.go delete mode 100644 pkg/gossip/schema.go delete mode 100644 pkg/gossip/schema.yaml delete mode 100644 pkg/gossip/socketaddr.go delete mode 100644 pkg/gossip/transaction.go delete mode 100644 pkg/gossip/types.go create mode 100644 pkg/repair/protocol.go create mode 100644 pkg/repair/protocol_test.go create mode 100644 pkg/rpcserver/rpcserver_test.go create mode 100644 pkg/sealevel/vote_program_decode_test.go delete mode 100644 pkg/shred/entry.go delete mode 100644 pkg/shred/shred.go delete mode 100644 pkg/shred/shredder.go create mode 100644 pkg/snapshotdl/snapshotdl_test.go create mode 100644 pkg/turbine/assembler.go create mode 100644 pkg/turbine/assembler_test.go create mode 100644 pkg/turbine/entries.go create mode 100644 pkg/turbine/receiver.go create mode 100644 pkg/turbine/repair.go create mode 100644 pkg/turbine/shred.go create mode 100644 pkg/txverify/txverify.go diff --git a/cmd/mithril/configcmd/configcmd.go b/cmd/mithril/configcmd/configcmd.go index 35517d59..0f1f7ed9 100644 --- a/cmd/mithril/configcmd/configcmd.go +++ b/cmd/mithril/configcmd/configcmd.go @@ -141,8 +141,16 @@ cluster = "mainnet-beta" # Required: "mainnet-beta" | "testnet" | "devnet" rpc = ["https://api.mainnet-beta.solana.com"] [block] -source = "rpc" # "rpc" | "lightbringer" +source = "rpc" # "rpc" | "lightbringer" | "turbine" # lightbringer_endpoint = "localhost:9000" +# turbine_bind_addr = "0.0.0.0:8001" + +# [turbine] +# bind_addr = "0.0.0.0:8001" +# gossip_entrypoint = "1.2.3.4:8000" +# gossip_bind_addr = "0.0.0.0:65401" +# advertised_ip = "203.0.113.10" +# shred_version = 0 # [lightbringer] # enabled = false diff --git a/cmd/mithril/dashboardcmd/dashboard.go b/cmd/mithril/dashboardcmd/dashboard.go index 1169533a..6ee4fc45 100644 --- a/cmd/mithril/dashboardcmd/dashboard.go +++ b/cmd/mithril/dashboardcmd/dashboard.go @@ -214,6 +214,11 @@ func newModel(cf string) model { {section: "storage", key: "logs", label: "Logs Path"}, {isSep: true}, {section: "block", key: "source", label: "Block Source"}, + {section: "block", key: "turbine_bind_addr", label: "Turbine UDP"}, + {section: "turbine", key: "gossip_entrypoint", label: "Turbine Gossip"}, + {section: "turbine", key: "gossip_bind_addr", label: "Gossip UDP"}, + {section: "turbine", key: "advertised_ip", label: "Advertised IP"}, + {section: "turbine", key: "shred_version", label: "Shred Version"}, {section: "block", key: "max_rps", label: "Block Max RPS"}, {section: "block", key: "max_inflight", label: "Block Max Inflight"}, {isSep: true}, @@ -714,6 +719,16 @@ func (m model) getFieldValue(f editFieldDef) string { return m.cfg.logsPath case "block.source": return m.cfg.blockSource + case "block.turbine_bind_addr": + return m.cfg.turbineBindAddr + case "turbine.gossip_entrypoint": + return m.cfg.turbineGossip + case "turbine.gossip_bind_addr": + return m.cfg.turbineGossipBind + case "turbine.advertised_ip": + return m.cfg.turbineAdvertisedIP + case "turbine.shred_version": + return m.cfg.turbineShredVersion case "block.max_rps": return m.cfg.blockMaxRPS case "block.max_inflight": @@ -759,6 +774,7 @@ func menuOptionsFor(section, key string) []editOption { return []editOption{ {label: "rpc", value: "rpc", desc: "Fetch blocks via RPC"}, {label: "lightbringer", value: "lightbringer", desc: "Sidecar streaming"}, + {label: "turbine", value: "turbine", desc: "Native shred receiver"}, } case "lightbringer.enabled": return []editOption{ @@ -877,6 +893,43 @@ func (m *model) applyEditField() { m.editErr = "Must be a number" return } + case key == "turbine.shred_version": + if value != "" { + n, err := strconv.Atoi(value) + if err != nil || n < 0 || n > 65535 { + m.editErr = "Must be 0-65535" + return + } + } + case key == "block.turbine_bind_addr" || key == "turbine.gossip_bind_addr": + if value != "" { + _, portStr, err := net.SplitHostPort(value) + if err != nil { + m.editErr = "Format: host:port or :port" + return + } + if p, perr := strconv.Atoi(portStr); perr != nil || p < 1 || p > 65535 { + m.editErr = "Port must be 1-65535" + return + } + } + case key == "turbine.gossip_entrypoint": + if value != "" { + host, portStr, err := net.SplitHostPort(value) + if err != nil || host == "" { + m.editErr = "Format: host:port" + return + } + if p, perr := strconv.Atoi(portStr); perr != nil || p < 1 || p > 65535 { + m.editErr = "Port must be 1-65535" + return + } + } + case key == "turbine.advertised_ip": + if value != "" && net.ParseIP(value) == nil { + m.editErr = "Must be an IP address" + return + } case key == "tuning.txpar": if value != "" { // empty = sequential (runtime default 0) if _, err := strconv.Atoi(value); err != nil { diff --git a/cmd/mithril/dashboardcmd/data.go b/cmd/mithril/dashboardcmd/data.go index 604b22a1..b8e549fc 100644 --- a/cmd/mithril/dashboardcmd/data.go +++ b/cmd/mithril/dashboardcmd/data.go @@ -61,26 +61,31 @@ func readState(accountsPath string) *nodeState { // ── Config reading ────────────────────────────────────────────────────── type configData struct { - cluster string - rpcEndpoints []string - blockSource string - lbEnabled bool - lbGossip string - lbGrpcAddr string - lbRpcAddr string - lbExternalEndpoint string // block.lightbringer_endpoint for external LB mode - lbBinaryPath string - lbQuiet bool - accountsPath string - snapshotsPath string - shredstorePath string - logsPath string - txpar string - blockMaxRPS string - blockInflight string - rpcPort string - logLevel string - bootstrapMode string + cluster string + rpcEndpoints []string + blockSource string + lbEnabled bool + lbGossip string + lbGrpcAddr string + lbRpcAddr string + lbExternalEndpoint string // block.lightbringer_endpoint for external LB mode + lbBinaryPath string + lbQuiet bool + turbineBindAddr string + turbineGossip string + turbineGossipBind string + turbineAdvertisedIP string + turbineShredVersion string + accountsPath string + snapshotsPath string + shredstorePath string + logsPath string + txpar string + blockMaxRPS string + blockInflight string + rpcPort string + logLevel string + bootstrapMode string } func readConfig(configFile string) *configData { @@ -106,28 +111,37 @@ func readConfig(configFile string) *configData { if logsPath == "" { logsPath = v.GetString("log.dir") } + turbineBindAddr := v.GetString("block.turbine_bind_addr") + if turbineBindAddr == "" { + turbineBindAddr = v.GetString("turbine.bind_addr") + } return &configData{ - cluster: cluster, - rpcEndpoints: v.GetStringSlice("network.rpc"), - blockSource: v.GetString("block.source"), - lbEnabled: v.GetBool("lightbringer.enabled"), - lbGossip: v.GetString("lightbringer.gossip_entrypoint"), - lbGrpcAddr: v.GetString("lightbringer.grpc_addr"), - lbRpcAddr: v.GetString("lightbringer.rpc_addr"), - lbQuiet: v.GetBool("lightbringer.quiet"), - lbExternalEndpoint: v.GetString("block.lightbringer_endpoint"), - lbBinaryPath: v.GetString("lightbringer.binary_path"), - accountsPath: v.GetString("storage.accounts"), - snapshotsPath: v.GetString("storage.snapshots"), - shredstorePath: v.GetString("storage.shredstore"), - logsPath: logsPath, - txpar: txpar, - blockMaxRPS: v.GetString("block.max_rps"), - blockInflight: v.GetString("block.max_inflight"), - rpcPort: v.GetString("rpc.port"), - logLevel: v.GetString("log.level"), - bootstrapMode: v.GetString("bootstrap.mode"), + cluster: cluster, + rpcEndpoints: v.GetStringSlice("network.rpc"), + blockSource: v.GetString("block.source"), + lbEnabled: v.GetBool("lightbringer.enabled"), + lbGossip: v.GetString("lightbringer.gossip_entrypoint"), + lbGrpcAddr: v.GetString("lightbringer.grpc_addr"), + lbRpcAddr: v.GetString("lightbringer.rpc_addr"), + lbQuiet: v.GetBool("lightbringer.quiet"), + lbExternalEndpoint: v.GetString("block.lightbringer_endpoint"), + lbBinaryPath: v.GetString("lightbringer.binary_path"), + turbineBindAddr: turbineBindAddr, + turbineGossip: v.GetString("turbine.gossip_entrypoint"), + turbineGossipBind: v.GetString("turbine.gossip_bind_addr"), + turbineAdvertisedIP: v.GetString("turbine.advertised_ip"), + turbineShredVersion: v.GetString("turbine.shred_version"), + accountsPath: v.GetString("storage.accounts"), + snapshotsPath: v.GetString("storage.snapshots"), + shredstorePath: v.GetString("storage.shredstore"), + logsPath: logsPath, + txpar: txpar, + blockMaxRPS: v.GetString("block.max_rps"), + blockInflight: v.GetString("block.max_inflight"), + rpcPort: v.GetString("rpc.port"), + logLevel: v.GetString("log.level"), + bootstrapMode: v.GetString("bootstrap.mode"), } } @@ -392,6 +406,28 @@ func runDoctorChecks(configFile string, cfg *configData) []checkResult { } else if cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint == "" { // Invalid: source=lightbringer but no sidecar and no endpoint results = append(results, checkResult{"Lightbringer", "fail", "block.source=lightbringer requires enabled sidecar or endpoint"}) + } else if cfg.blockSource == "turbine" { + if cfg.turbineBindAddr == "" { + results = append(results, checkResult{"Turbine UDP", "fail", "block.source=turbine requires block.turbine_bind_addr or turbine.bind_addr"}) + } else if _, _, err := net.SplitHostPort(cfg.turbineBindAddr); err != nil { + results = append(results, checkResult{"Turbine UDP", "fail", "invalid format: " + cfg.turbineBindAddr}) + } else { + results = append(results, checkResult{"Turbine UDP", "pass", cfg.turbineBindAddr}) + } + if cfg.turbineGossip == "" { + results = append(results, checkResult{"Turbine gossip", "warn", "empty; UDP-only receiver mode"}) + } else if _, _, err := net.SplitHostPort(cfg.turbineGossip); err != nil { + results = append(results, checkResult{"Turbine gossip", "fail", "invalid format: " + cfg.turbineGossip}) + } else { + results = append(results, checkResult{"Turbine gossip", "pass", cfg.turbineGossip}) + } + if cfg.turbineGossipBind != "" { + if _, _, err := net.SplitHostPort(cfg.turbineGossipBind); err != nil { + results = append(results, checkResult{"Turbine gossip UDP", "fail", "invalid format: " + cfg.turbineGossipBind}) + } else { + results = append(results, checkResult{"Turbine gossip UDP", "pass", cfg.turbineGossipBind}) + } + } } else { results = append(results, checkResult{"Lightbringer", "pass", "disabled"}) } @@ -485,7 +521,8 @@ func saveConfigValue(configFile, section, key, value string) error { var tomlValue string switch { case fullKey == "block.max_rps" || fullKey == "block.max_inflight" || - fullKey == "tuning.txpar" || fullKey == "rpc.port": + fullKey == "tuning.txpar" || fullKey == "rpc.port" || + fullKey == "turbine.shred_version": tomlValue = value // numeric — no quoting case fullKey == "lightbringer.enabled" || fullKey == "lightbringer.quiet": tomlValue = value // boolean — no quoting diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index cfa011f2..845d6238 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -63,7 +63,7 @@ var ( scratchDirectory string rpcEndpoints []string cluster string // "mainnet-beta", "testnet", "devnet" - blockSource string // "rpc" or "lightbringer" + blockSource string // "rpc", "lightbringer", or "turbine" lightbringerEndpoint string blockMaxRPS int // Rate limit for block fetching blockMaxInflight int // Max concurrent block fetch workers @@ -111,6 +111,13 @@ var ( lightbringerBlockConfirmHTTP string lightbringerBlockConfirmWS string lightbringerQuiet bool + + // Native turbine receiver config + turbineBindAddr string + turbineGossipEntrypoint string + turbineGossipBindAddr string + turbineAdvertisedIP string + turbineShredVersion int ) func snapshotEpochForState(manifest *snapshot.SnapshotManifest) uint64 { @@ -220,9 +227,15 @@ func init() { Run.Flags().Uint64Var(¶mArenaSizeMB, "param-arena-size-mb", 512, "Size in MB for serialized parameter arena (0 to disable)") Run.Flags().Uint64Var(&borrowedAccountArenaSize, "borrowed-account-arena-size", 1024, "Number of borrowed accounts to preallocate in arena (0 to disable)") Run.Flags().IntVar(&snapshot.ZstdDecoderConcurrency, "zstd-decoder-concurrency", runtime.NumCPU(), "Zstd decoder concurrency") - Run.Flags().IntVar(&snapshot.MaxConcurrentFlushers, "max-concurrent-flushers", 16, "Bound for number of log shards to flush to Accounts DB Index at once.") + Run.Flags().IntVar(&snapshot.MaxConcurrentFlushers, "max-concurrent-flushers", snapshot.DefaultSnapshotMaxConcurrentFlushers, "Bound for number of log shards to flush to Accounts DB Index at once") + Run.Flags().IntVar(&snapshot.SnapshotAppendVecCopyingWorkers, "snapshot-append-vec-workers", snapshot.DefaultSnapshotAppendVecCopyingWorkers, "Snapshot bootstrap appendvec write workers") + Run.Flags().IntVar(&snapshot.SnapshotIndexEntryBuilderWorkers, "snapshot-index-builder-workers", snapshot.DefaultSnapshotIndexEntryBuilderWorkers, "Snapshot bootstrap account-index parser workers") + Run.Flags().IntVar(&snapshot.SnapshotIndexEntryCommitterWorkers, "snapshot-index-committer-workers", snapshot.DefaultSnapshotIndexEntryCommitterWorkers, "Snapshot bootstrap account-index shard enqueue workers") + Run.Flags().IntVar(&snapshot.SnapshotIndexShards, "snapshot-index-shards", snapshot.DefaultSnapshotIndexShards, "Snapshot bootstrap account-index shard count") + Run.Flags().StringVar(&snapshot.SnapshotIndexTempDir, "snapshot-index-temp-dir", "", "Optional directory for snapshot index shard logs/SST staging") Run.Flags().BoolVar(&sbpf.UsePool, "use-pool", true, "Disable to allocate fresh slices") Run.Flags().IntVar(&accountsdb.StoreAccountsWorkers, "store-accounts-workers", 128, "Number of workers to write account updates") + Run.Flags().IntVar(&accountsdb.ProgramCacheMaxMB, "program-cache-max-mb", accountsdb.DefaultProgramCacheMaxMB, "Maximum approximate SBPF program cache size in MiB") // [tuning.pprof] section flags Run.Flags().Int64Var(&pprofPort, "pprof-port", -1, "Port to serve HTTP pprof endpoint") @@ -237,8 +250,13 @@ func init() { Run.Flags().StringVar(&scratchDirectory, "scratch-directory", "/tmp", "Path for downloads (e.g. snapshots) and other temp state") // [block] section flags - Run.Flags().StringVar(&blockSource, "block-source", "rpc", "Block source: 'rpc' or 'lightbringer'") + Run.Flags().StringVar(&blockSource, "block-source", "rpc", "Block source: 'rpc', 'lightbringer', or 'turbine'") Run.Flags().StringVar(&lightbringerEndpoint, "lightbringer-endpoint", "", "Address for Lightbringer endpoint (only used when block-source=lightbringer)") + Run.Flags().StringVar(&turbineBindAddr, "turbine-bind-addr", "", "UDP address for native turbine shred receiver (only used when block-source=turbine)") + Run.Flags().StringVar(&turbineGossipEntrypoint, "turbine-gossip-entrypoint", "", "Solana gossip entrypoint for native turbine tree joining") + Run.Flags().StringVar(&turbineGossipBindAddr, "turbine-gossip-bind-addr", "", "UDP address for native turbine gossip traffic (only used when block-source=turbine)") + Run.Flags().StringVar(&turbineAdvertisedIP, "turbine-advertised-ip", "", "Public IP advertised by native turbine gossip (optional)") + Run.Flags().IntVar(&turbineShredVersion, "turbine-shred-version", 0, "Shred version for native turbine gossip (0 = discover from entrypoint)") Run.Flags().IntVar(&blockMaxRPS, "block-max-rps", 0, "Max RPC requests per second for block fetching (0 = use default)") Run.Flags().IntVar(&blockMaxInflight, "block-max-inflight", 0, "Max concurrent block fetch workers (0 = use default)") Run.Flags().IntVar(&blockTipPollIntervalMs, "block-tip-poll-ms", 0, "Tip poll interval in milliseconds (0 = use default)") @@ -475,6 +493,14 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { blockSource = "rpc" // default } lightbringerEndpoint = getString("lightbringer-endpoint", "block.lightbringer_endpoint") + turbineBindAddr = getString("turbine-bind-addr", "block.turbine_bind_addr") + if turbineBindAddr == "" { + turbineBindAddr = config.GetString("turbine.bind_addr") + } + turbineGossipEntrypoint = getString("turbine-gossip-entrypoint", "turbine.gossip_entrypoint") + turbineGossipBindAddr = getString("turbine-gossip-bind-addr", "turbine.gossip_bind_addr") + turbineAdvertisedIP = getString("turbine-advertised-ip", "turbine.advertised_ip") + turbineShredVersion = getInt("turbine-shred-version", "turbine.shred_version") // [lightbringer] section — sidecar management lightbringerEnabled = config.GetBool("lightbringer.enabled") @@ -543,8 +569,18 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { if len(rpcEndpoints) == 0 { return fmt.Errorf("block.source=lightbringer requires RPC endpoints for catchup (set network.rpc)") } + case "turbine": + if turbineBindAddr == "" { + return fmt.Errorf("block.source=turbine requires block.turbine_bind_addr or turbine.bind_addr") + } + if turbineShredVersion < 0 || turbineShredVersion > 0xffff { + return fmt.Errorf("turbine.shred_version must be between 0 and 65535") + } + if len(rpcEndpoints) == 0 { + return fmt.Errorf("block.source=turbine requires RPC endpoints for catchup and tip polling (set network.rpc)") + } default: - return fmt.Errorf("invalid block.source %q - must be 'rpc' or 'lightbringer'", blockSource) + return fmt.Errorf("invalid block.source %q - must be 'rpc', 'lightbringer', or 'turbine'", blockSource) } blockMaxRPS = getInt("block-max-rps", "block.max_rps") @@ -636,8 +672,32 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { snapshot.ZstdDecoderConcurrency = getInt("zstd-decoder-concurrency", "tuning.zstd_decoder_concurrency") snapshot.MaxConcurrentFlushers = getInt("max-concurrent-flushers", "tuning.max_concurrent_flushers") + snapshot.SnapshotAppendVecCopyingWorkers = getInt("snapshot-append-vec-workers", "tuning.snapshot_append_vec_workers") + snapshot.SnapshotIndexEntryBuilderWorkers = getInt("snapshot-index-builder-workers", "tuning.snapshot_index_builder_workers") + snapshot.SnapshotIndexEntryCommitterWorkers = getInt("snapshot-index-committer-workers", "tuning.snapshot_index_committer_workers") + snapshot.SnapshotIndexShards = getInt("snapshot-index-shards", "tuning.snapshot_index_shards") + snapshot.SnapshotIndexTempDir = getString("snapshot-index-temp-dir", "tuning.snapshot_index_temp_dir") + if snapshot.MaxConcurrentFlushers <= 0 { + return fmt.Errorf("tuning.max_concurrent_flushers must be > 0") + } + if snapshot.SnapshotAppendVecCopyingWorkers <= 0 { + return fmt.Errorf("tuning.snapshot_append_vec_workers must be > 0") + } + if snapshot.SnapshotIndexEntryBuilderWorkers <= 0 { + return fmt.Errorf("tuning.snapshot_index_builder_workers must be > 0") + } + if snapshot.SnapshotIndexEntryCommitterWorkers <= 0 { + return fmt.Errorf("tuning.snapshot_index_committer_workers must be > 0") + } + if snapshot.SnapshotIndexShards <= 0 || snapshot.SnapshotIndexShards > 1000 { + return fmt.Errorf("tuning.snapshot_index_shards must be between 1 and 1000") + } sbpf.UsePool = getBool("use-pool", "tuning.use_pool") accountsdb.StoreAccountsWorkers = getInt("store-accounts-workers", "tuning.store_accounts_workers") + accountsdb.ProgramCacheMaxMB = getInt("program-cache-max-mb", "tuning.program_cache_max_mb") + if accountsdb.ProgramCacheMaxMB <= 0 { + return fmt.Errorf("tuning.program_cache_max_mb must be > 0") + } return nil } @@ -697,6 +757,9 @@ func buildSnapshotConfig(rpcEndpoints []string) snapshotdl.SnapshotConfig { if config.IsSet("snapshot.allowed_node_versions") { cfg.AllowedNodeVersions = config.GetStringSlice("snapshot.allowed_node_versions") } + if config.IsSet("snapshot.node_blacklist") { + cfg.NodeBlacklist = config.GetStringSlice("snapshot.node_blacklist") + } if config.IsSet("snapshot.full_threshold") { cfg.FullThreshold = config.GetInt("snapshot.full_threshold") } @@ -812,6 +875,7 @@ func runLive(c *cobra.Command, args []string) { // Lightbringer sidecar management var lbManager *lightbringer.Manager useLightbringer := blockSource == "lightbringer" + useTurbine := blockSource == "turbine" if lightbringerEnabled { lbLogWriter := mlog.Log.CreateSubprocessWriter("lightbringer") @@ -878,6 +942,13 @@ func runLive(c *cobra.Command, args []string) { } else if useLightbringer { // block.source=lightbringer but lightbringer.enabled=false — standalone Lightbringer mode mlog.Log.Infof("block.source=lightbringer with external Lightbringer at %s", lightbringerEndpoint) + } else if useTurbine { + mlog.Log.Infof("block.source=turbine with native turbine receiver on %s", turbineBindAddr) + if turbineGossipEntrypoint != "" { + mlog.Log.Infof("native turbine gossip enabled with entrypoint %s", turbineGossipEntrypoint) + } else { + mlog.Log.Warnf("native turbine gossip entrypoint is empty; Mithril will receive turbine packets only if shreds are sent directly to %s", turbineBindAddr) + } } dbgOpts, err := replay.NewDebugOptions(debugTxs, debugAcctWrites, debugDumpEpochVotingRewardDiff) @@ -1467,14 +1538,14 @@ postBootstrap: } consensusEnforceSource := config.GetString("consensus.enforce_on_source") if consensusEnforceSource == "" { - consensusEnforceSource = "lightbringer" + consensusEnforceSource = "stream" } switch consensusEnforceSource { - case "lightbringer", "all": + case "lightbringer", "turbine", "stream", "all": // valid default: - mlog.Log.Errorf("invalid consensus.enforce_on_source %q (must be \"lightbringer\" or \"all\"), defaulting to \"lightbringer\"", consensusEnforceSource) - consensusEnforceSource = "lightbringer" + mlog.Log.Errorf("invalid consensus.enforce_on_source %q (must be \"lightbringer\", \"turbine\", \"stream\", or \"all\"), defaulting to \"stream\"", consensusEnforceSource) + consensusEnforceSource = "stream" } consensusOpts := &replay.ConsensusOpts{ SkipPathMaxDepth: consensusMaxDepth, @@ -1486,10 +1557,14 @@ postBootstrap: if rpcServer != nil { slotCtxSetter = rpcServer } - result := runReplayWithRecovery(ctx, accountsDb, accountsPath, manifest, resumeState, uint64(startSlot), liveEndSlot, rpcEndpoints, lightbringerEndpoint, blockstorePath, int(txParallelism), true, useLightbringer, dbgOpts, metricsWriter, slotCtxSetter, mithrilState, blockFetchOpts, consensusOpts, replayStartTime) + result := runReplayWithRecovery(ctx, accountsDb, accountsPath, manifest, resumeState, uint64(startSlot), liveEndSlot, rpcEndpoints, lightbringerEndpoint, turbineBindAddr, turbineGossipEntrypoint, turbineGossipBindAddr, turbineAdvertisedIP, uint16(turbineShredVersion), blockstorePath, int(txParallelism), true, useLightbringer, useTurbine, dbgOpts, metricsWriter, slotCtxSetter, mithrilState, blockFetchOpts, consensusOpts, replayStartTime) - if result.Error != nil && result.LastPersistedSlot == 0 { - mlog.Log.Errorf("Replay stopped before persisting the first post-start slot: %v", result.Error) + if result.Error != nil { + if result.LastPersistedSlot == 0 { + mlog.Log.Errorf("Replay stopped before persisting the first post-start slot: %v", result.Error) + } else { + mlog.Log.Errorf("Replay stopped with error after persisting slot %d: %v", result.LastPersistedSlot, result.Error) + } } // Update state file with last persisted slot and shutdown context @@ -1879,6 +1954,8 @@ func printStartupInfo(commandName string) { fmt.Printf(" %s(managed sidecar)%s\n", dim, reset) case blockSource == "lightbringer": fmt.Printf(" %s(external)%s\n", dim, reset) + case blockSource == "turbine": + fmt.Printf(" %s(native)%s\n", dim, reset) default: fmt.Println() } @@ -1893,6 +1970,18 @@ func printStartupInfo(commandName string) { if blockSource == "lightbringer" && lightbringerEndpoint != "" { fmt.Printf(" Lightbringer: %s%s%s\n", gold, lightbringerEndpoint, reset) } + if blockSource == "turbine" && turbineBindAddr != "" { + fmt.Printf(" Turbine UDP: %s%s%s\n", gold, turbineBindAddr, reset) + } + if blockSource == "turbine" && turbineGossipEntrypoint != "" { + fmt.Printf(" Gossip: %s%s%s\n", gold, turbineGossipEntrypoint, reset) + } + if blockSource == "turbine" && turbineGossipBindAddr != "" { + fmt.Printf(" Gossip UDP: %s%s%s\n", gold, turbineGossipBindAddr, reset) + } + if blockSource == "turbine" && turbineAdvertisedIP != "" { + fmt.Printf(" Advertised: %s%s%s\n", gold, turbineAdvertisedIP, reset) + } fmt.Println() } @@ -2346,10 +2435,16 @@ func runReplayWithRecovery( startSlot, endSlot uint64, rpcEndpoints []string, // RPC endpoints in priority order (first = primary) lightbringerEndpoint string, + turbineBindAddr string, + turbineGossipEntrypoint string, + turbineGossipBindAddr string, + turbineAdvertisedIP string, + turbineShredVersion uint16, blockDir string, txParallelism int, isLive bool, useLightbringer bool, + useTurbine bool, dbgOpts *replay.DebugOptions, metricsWriter io.Writer, rpcServer replay.SlotCtxSetter, @@ -2465,6 +2560,6 @@ func runReplayWithRecovery( } }() - result = replay.ReplayBlocks(ctx, accountsDb, accountsDbPath, mithrilState, resumeState, startSlot, endSlot, rpcEndpoints, lightbringerEndpoint, blockDir, txParallelism, isLive, useLightbringer, dbgOpts, metricsWriter, rpcServer, blockFetchOpts, consensusOpts, onCancelWriteState) + result = replay.ReplayBlocks(ctx, accountsDb, accountsDbPath, mithrilState, resumeState, startSlot, endSlot, rpcEndpoints, lightbringerEndpoint, turbineBindAddr, turbineGossipEntrypoint, turbineGossipBindAddr, turbineAdvertisedIP, turbineShredVersion, blockDir, txParallelism, isLive, useLightbringer, useTurbine, dbgOpts, metricsWriter, rpcServer, blockFetchOpts, consensusOpts, onCancelWriteState) return result } diff --git a/cmd/mithril/setupcmd/migrate.go b/cmd/mithril/setupcmd/migrate.go index 6ac65362..a0684dc1 100644 --- a/cmd/mithril/setupcmd/migrate.go +++ b/cmd/mithril/setupcmd/migrate.go @@ -34,7 +34,7 @@ func MigrateConfig(configPath string) bool { # [consensus] # skip_path_max_depth = 64 # unresolved_policy = "halt" -# enforce_on_source = "lightbringer" +# enforce_on_source = "stream" ` } diff --git a/cmd/mithril/setupcmd/setup.go b/cmd/mithril/setupcmd/setup.go index 69217667..cae51a33 100644 --- a/cmd/mithril/setupcmd/setup.go +++ b/cmd/mithril/setupcmd/setup.go @@ -866,7 +866,8 @@ func (m setupModel) generateConfig() (tea.Model, tea.Cmd) { cfg.WriteString("[consensus]\n") fmt.Fprintf(&cfg, "unresolved_policy = %q\n", m.consensusPolicy) - cfg.WriteString("skip_path_max_depth = 64\n\n") + cfg.WriteString("skip_path_max_depth = 64\n") + cfg.WriteString("enforce_on_source = \"stream\"\n\n") cfg.WriteString("[snapshot]\n") fmt.Fprintf(&cfg, "max_full_snapshots = %s\n\n", m.snapshotKeep) @@ -915,11 +916,19 @@ cluster = "mainnet-beta" # Required: "mainnet-beta" | "testnet" | "devnet" rpc = ["https://api.mainnet-beta.solana.com"] [block] -source = "rpc" # "rpc" | "lightbringer" +source = "rpc" # "rpc" | "lightbringer" | "turbine" +# turbine_bind_addr = "0.0.0.0:8001" # lightbringer_endpoint = "localhost:9000" max_rps = 8 max_inflight = 8 +# [turbine] +# bind_addr = "0.0.0.0:8001" +# gossip_entrypoint = "1.2.3.4:8000" +# gossip_bind_addr = "0.0.0.0:65401" +# advertised_ip = "203.0.113.10" +# shred_version = 0 + # [lightbringer] # enabled = false # binary_path = "./lightbringer" @@ -935,6 +944,7 @@ txpar = %d # Recommended: 2x your CPU core count [consensus] unresolved_policy = "halt" # "halt" | "warn" skip_path_max_depth = 64 +enforce_on_source = "stream" [snapshot] max_full_snapshots = 1 # 0 = stream only, saves disk diff --git a/config.example.toml b/config.example.toml index 8dad5bb0..baab2e89 100644 --- a/config.example.toml +++ b/config.example.toml @@ -120,11 +120,17 @@ name = "mithril" # "rpc" - Fetch blocks via getBlock RPC calls # "lightbringer" - Stream via Lightbringer sidecar (see [lightbringer] section) # Uses RPC for catchup, hands off to live stream near tip. + # "turbine" - Native UDP turbine shred receiver. + # Uses RPC for catchup, hands off to reconstructed shreds near tip. source = "rpc" # Lightbringer endpoint address (only used when source = "lightbringer") # lightbringer_endpoint = "localhost:9000" + # Native turbine UDP bind address (only used when source = "turbine"). + # turbine_bind_addr = "0.0.0.0:8001" + # Gossip settings for native turbine live under [turbine]. + # ========================================================================= # Global Fetch Tuning # ========================================================================= @@ -209,8 +215,38 @@ name = "mithril" # Which block source to enforce consensus on: # "lightbringer" - Only enforce on Lightbringer blocks (RPC blocks are trusted) + # "turbine" - Only enforce on native turbine blocks + # "stream" - Enforce on any live shred-stream path # "all" - Enforce on all block sources (not yet implemented) - enforce_on_source = "lightbringer" + enforce_on_source = "stream" + + +# ============================================================================ +# [turbine] - Native Turbine Receiver (block.source = "turbine") +# ============================================================================ +# +# Native turbine mode reconstructs blocks from data shreds inside Mithril. +# RPC remains configured for catchup, tip polling, and repair/fallback paths. + +[turbine] + # UDP address where Mithril listens for turbine shreds. + # bind_addr = "0.0.0.0:8001" + + # Solana gossip entrypoint used to discover the cluster shred version and + # advertise Mithril's turbine/TVU socket into the turbine tree. + # gossip_entrypoint = "1.2.3.4:8000" + + # UDP address where Mithril listens for gossip traffic. Leave empty to let + # the OS choose an available port. Set a fixed port if your firewall/NAT + # needs an explicit rule. + # gossip_bind_addr = "0.0.0.0:65401" + + # Optional public IP override. Leave empty to ask the gossip entrypoint's + # IP echo service. + # advertised_ip = "203.0.113.10" + + # Optional override. Leave at zero/empty to discover from gossip_entrypoint. + # shred_version = 0 # ============================================================================ @@ -301,8 +337,30 @@ name = "mithril" # Zstd decoder concurrency (defaults to NumCPU) # zstd_decoder_concurrency = 16 - # Bound for number of log shards to flush to Accounts DB Index at once - max_concurrent_flushers = 16 + # Snapshot bootstrap I/O tuning. + # These defaults deliberately avoid flooding a single NVMe with hundreds of + # concurrent writes. Increase cautiously on very fast multi-disk systems. + + # AppendVec files copied from the snapshot archive concurrently. + snapshot_append_vec_workers = 32 + + # Workers parsing AppendVecs into account-index entries. + snapshot_index_builder_workers = 64 + + # Workers enqueueing account-index entries into shard logs. + snapshot_index_committer_workers = 64 + + # Number of temporary account-index shard logs/SSTs. + snapshot_index_shards = 64 + + # Optional temp directory for snapshot index shard logs/SST staging. + # This can move tens of GB of temporary write/read churn off the AccountsDB + # disk. On RAM-rich machines, /dev/shm/mithril-snapshot-index is useful. + # Leave empty to stage under storage.accounts. + snapshot_index_temp_dir = "" + + # Bound for number of index shards to convert to SSTs at once. + max_concurrent_flushers = 8 # Size in MB for serialized parameter arena (0 to disable) param_arena_size_mb = 512 @@ -316,6 +374,9 @@ name = "mithril" # Number of goroutines to store modified accounts at the end of each block store_accounts_workers = 128 + # Approximate maximum retained SBPF program cache size in MiB. + program_cache_max_mb = 1024 + # [tuning.pprof] - CPU/Memory Profiling # # Usage (assuming port = 6060): @@ -474,6 +535,14 @@ name = "mithril" # Example: allowed_node_versions = ["2.2.0", "3.0.0"] # allowed_node_versions = [] + # Snapshot source blacklist. Entries may be full RPC URLs, host:port pairs, + # or bare host/IP values. Matching nodes are skipped before snapshot probing. + node_blacklist = [ + # "http://203.0.113.10:8899", + # "198.51.100.24:8899", + # "bad-snapshot-node.example.com", + ] + # ------------------------------------------------------------------------- # Performance # ------------------------------------------------------------------------- diff --git a/go.mod b/go.mod index 2483faa8..9b0bdf38 100644 --- a/go.mod +++ b/go.mod @@ -140,7 +140,8 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/reedsolomon v1.14.0 github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect diff --git a/go.sum b/go.sum index 7ea49725..3cdae32a 100644 --- a/go.sum +++ b/go.sum @@ -201,6 +201,10 @@ github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYW github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0= +github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/nix/modules/shared/lib.nix b/nix/modules/shared/lib.nix index 5481b942..6423b95e 100644 --- a/nix/modules/shared/lib.nix +++ b/nix/modules/shared/lib.nix @@ -66,6 +66,7 @@ borrowed_account_arena_size = cfg.configSchema.tuningBorrowedAccountArenaSize; use_pool = cfg.configSchema.tuningUsePool; store_accounts_workers = cfg.configSchema.tuningStoreAccountsWorkers; + program_cache_max_mb = cfg.configSchema.tuningProgramCacheMaxMb; pprof = { port = cfg.configSchema.tuningPprofPort; cpu_profile_path = cfg.configSchema.tuningPprofCpuProfilePath; diff --git a/nix/modules/shared/options.nix b/nix/modules/shared/options.nix index c716b701..98695177 100644 --- a/nix/modules/shared/options.nix +++ b/nix/modules/shared/options.nix @@ -356,6 +356,12 @@ description = "Store accounts workers."; }; + tuningProgramCacheMaxMb = lib.mkOption { + type = lib.types.int; + default = 1024; + description = "Approximate maximum retained SBPF program cache size in MiB."; + }; + tuningPprofPort = lib.mkOption { type = lib.types.nullOr lib.types.int; default = null; diff --git a/pkg/accountsdb/accountsdb.go b/pkg/accountsdb/accountsdb.go index a057c83d..acca39e0 100644 --- a/pkg/accountsdb/accountsdb.go +++ b/pkg/accountsdb/accountsdb.go @@ -69,11 +69,14 @@ var ( ErrNoAccount = errors.New("ErrNoAccount") StoreAccountsWorkers = 128 + ProgramCacheMaxMB = DefaultProgramCacheMaxMB ) const ( indexPebbleMemTableSize = 64 << 20 indexPebbleMemTableStopWritesThreshold = 4 + DefaultProgramCacheMaxMB = 1024 + programCacheCostUnitBytes = 1 << 20 ) func NewAccountsIndexPebbleOptions(logger pebble.Logger) *pebble.Options { @@ -175,9 +178,9 @@ func (accountsDb *AccountsDb) InitCaches() { panic(err) } - accountsDb.ProgramCache, err = otter.MustBuilder[solana.PublicKey, *ProgramCacheEntry](2000). + accountsDb.ProgramCache, err = otter.MustBuilder[solana.PublicKey, *ProgramCacheEntry](programCacheCapacityUnits()). Cost(func(key solana.PublicKey, progEntry *ProgramCacheEntry) uint32 { - return 1 + return progEntry.CostUnits() }). Build() if err != nil { @@ -199,6 +202,29 @@ type ProgramCacheEntry struct { DeploymentSlot uint64 } +func programCacheCapacityUnits() int { + if ProgramCacheMaxMB <= 0 { + return DefaultProgramCacheMaxMB + } + return ProgramCacheMaxMB +} + +func (entry *ProgramCacheEntry) CostUnits() uint32 { + if entry == nil || entry.Program == nil { + return 1 + } + bytes := entry.Program.MemoryBytes() + units := (bytes + programCacheCostUnitBytes - 1) / programCacheCostUnitBytes + if units == 0 { + return 1 + } + max := uint64(^uint32(0)) + if units > max { + return ^uint32(0) + } + return uint32(units) +} + func (accountsDb *AccountsDb) MaybeGetProgramFromCache(pubkey solana.PublicKey) (*ProgramCacheEntry, bool) { if accountsDb == nil { return nil, false diff --git a/pkg/blockstream/block_source.go b/pkg/blockstream/block_source.go index 256b708c..a0ea80d2 100644 --- a/pkg/blockstream/block_source.go +++ b/pkg/blockstream/block_source.go @@ -16,9 +16,12 @@ import ( "github.com/Overclock-Validator/mithril/pkg/block" b "github.com/Overclock-Validator/mithril/pkg/block" + gossipclient "github.com/Overclock-Validator/mithril/pkg/gossip" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/overcast" "github.com/Overclock-Validator/mithril/pkg/rpcclient" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" "golang.org/x/time/rate" "google.golang.org/grpc" @@ -32,15 +35,22 @@ const ( BlockSourceRpc = iota BlockSourceFile BlockSourceLightbringer + BlockSourceTurbine ) type BlockSourceOpts struct { - RpcClient *rpcclient.RpcClient // Primary RPC for block fetching (getBlock) - SourceType BlockSourceType - LightbringerEndpoint string - StartSlot uint64 - EndSlot uint64 - BlockDir string + RpcClient *rpcclient.RpcClient // Primary RPC for block fetching (getBlock) + SourceType BlockSourceType + LightbringerEndpoint string + TurbineBindAddr string + TurbineGossipEntrypoint string + TurbineGossipBindAddr string + TurbineAdvertisedIP string + TurbineShredVersion uint16 + LeaderForSlot func(slot uint64) (solana.PublicKey, bool) + StartSlot uint64 + EndSlot uint64 + BlockDir string // When enabled, an active near-tip Lightbringer stream is delivered to replay // as an observation feed for consensus buffering instead of requiring the // block source to resolve every local gap before delivery. @@ -81,13 +91,14 @@ const ( // fetchResult is sent from workers to the emitter type fetchResult struct { - slot uint64 - block *b.Block - err error - skipped bool // true if SlotSkipped error - absentOK bool // true if a secondary confirmed-RPC probe verified the slot is absent - rpcIdx int32 // which RPC endpoint produced this result (for error attribution) - latencyMs int64 // fetch latency in milliseconds (for stall diagnostics) + slot uint64 + block *b.Block + err error + skipped bool // true if SlotSkipped error + absentOK bool // true if a secondary confirmed-RPC probe verified the slot is absent + rpcIdx int32 // which RPC endpoint produced this result (for error attribution) + latencyMs int64 // fetch latency in milliseconds (for stall diagnostics) + liveStreamGeneration uint64 // live-stream result generation, used to discard stale handoff blocks } var errBeyondTip = errors.New("slot beyond confirmed tip") @@ -265,6 +276,12 @@ type BlockSource struct { // Lightbringer live-stream handoff lightbringerEndpoint string + turbineBindAddr string + turbineGossipEntrypoint string + turbineGossipBindAddr string + turbineAdvertisedIP string + turbineShredVersion uint16 + leaderForSlot func(slot uint64) (solana.PublicKey, bool) lightbringerStarted atomic.Bool lightbringerConnected atomic.Bool lightbringerLastStreamSlot atomic.Uint64 @@ -273,6 +290,7 @@ type BlockSource struct { lightbringerCancelMu sync.Mutex lightbringerCancel context.CancelFunc lightbringerHandoffSlot atomic.Uint64 // First slot from the active stream connection, 0 = no active handoff + lightbringerResultGeneration atomic.Uint64 // Incremented whenever a live-stream handoff/runway is invalidated lightbringerForceRPCUntil atomic.Uint64 // While set, ignore Lightbringer and use RPC until this slot is executed lightbringerCooldownUntil atomic.Uint64 // After a missing-slot recovery, keep RPC active until this slot executes lightbringerNeedRPCResume atomic.Bool // Set when a live handoff disconnects and RPC must fill the gap again @@ -446,6 +464,12 @@ func NewBlockSource(opts *BlockSourceOpts) *BlockSource { stallTimeout: defaultStallTimeout, catchupTipSafety: tipSafetyMargin, // Store original for switching back to catchup lightbringerEndpoint: opts.LightbringerEndpoint, + turbineBindAddr: opts.TurbineBindAddr, + turbineGossipEntrypoint: opts.TurbineGossipEntrypoint, + turbineGossipBindAddr: opts.TurbineGossipBindAddr, + turbineAdvertisedIP: opts.TurbineAdvertisedIP, + turbineShredVersion: opts.TurbineShredVersion, + leaderForSlot: opts.LeaderForSlot, lightbringerBuffer: make(map[uint64]*b.Block), consensusManagedLightbringer: opts.ConsensusManagedLightbringer, @@ -474,11 +498,34 @@ func NewBlockSource(opts *BlockSourceOpts) *BlockSource { } if opts.SourceType == BlockSourceLightbringer && opts.LightbringerEndpoint != "" { mlog.Log.Infof("Lightbringer live handoff configured for %s (RPC catchup remains enabled)", opts.LightbringerEndpoint) + } else if opts.SourceType == BlockSourceTurbine && opts.TurbineBindAddr != "" { + mlog.Log.Infof("Native turbine live handoff configured on %s (RPC catchup remains enabled)", opts.TurbineBindAddr) + if opts.TurbineGossipEntrypoint != "" { + mlog.Log.Infof("Native turbine gossip configured with entrypoint %s", opts.TurbineGossipEntrypoint) + } } return bs } +func (bs *BlockSource) usesLiveShredStream() bool { + switch bs.sourceType { + case BlockSourceLightbringer: + return bs.lightbringerEndpoint != "" + case BlockSourceTurbine: + return bs.turbineBindAddr != "" + default: + return false + } +} + +func (bs *BlockSource) liveShredStreamName() string { + if bs.sourceType == BlockSourceTurbine { + return "TURBINE" + } + return "LIGHTBRINGER" +} + func (bs *BlockSource) setStopReason(reason blockSourceStopReason, slot uint64) { if reason == blockSourceStopReasonNone { return @@ -535,7 +582,7 @@ func (bs *BlockSource) updateMode() { } } - if bs.sourceType == BlockSourceLightbringer && bs.lightbringerEndpoint != "" { + if bs.usesLiveShredStream() { bs.maybeStartLightbringerStream() if bs.isNearTip.Load() { bs.maybePrepareLightbringerHandoff() @@ -562,7 +609,7 @@ func (bs *BlockSource) consensusBufferedLightbringerMaxSourceGap() uint64 { } func (bs *BlockSource) shouldDeferCatchupForConsensusBufferedLightbringer(gap uint64, lastExecuted uint64, tip uint64) bool { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return false } if !bs.consensusManagedLightbringer || !bs.lightbringerActive.Load() || !bs.lightbringerConnected.Load() { @@ -630,15 +677,36 @@ func (bs *BlockSource) rewindConsensusManagedFrontierForRPCFallbackLocked() (wai return replayNextSlot, previousWaitingSlot } +func (bs *BlockSource) ForceRPCFallback(reason string) { + if reason == "" { + reason = "requested" + } + tip := bs.confirmedTip.Load() + lastExecuted := bs.lastExecutedSlot.Load() + var gap uint64 + if tip > lastExecuted { + gap = tip - lastExecuted + } + bs.forceRPCForCatchupWithReason(gap, reason) +} + func (bs *BlockSource) forceRPCForCatchup(gap uint64) { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + bs.forceRPCForCatchupWithReason(gap, "lost_tip") +} + +func (bs *BlockSource) forceRPCForCatchupWithReason(gap uint64, reason string) { + if !bs.usesLiveShredStream() { return } + if reason == "" { + reason = "lost_tip" + } bs.lightbringerForceRPCUntil.Store(0) bs.lightbringerCooldownUntil.Store(0) oldHandoff := bs.lightbringerHandoffSlot.Swap(0) wasActive := bs.lightbringerActive.Swap(false) + resultGeneration := bs.invalidateLightbringerResults() bs.lightbringerNeedRPCResume.Store(true) bs.clearLightbringerGapWatch() bs.resetLightbringerRepairSlot() @@ -692,26 +760,26 @@ func (bs *BlockSource) forceRPCForCatchup(gap uint64) { if wasActive { if previousWaitingSlot != waitingSlot { - mlog.Log.Warnf("BLOCK SOURCE SWITCH: LIGHTBRINGER -> RPC at slot %d | reason=lost_tip | gap=%d | rewound_emission_frontier_from=%d | cleared_buffered_lightbringer=%d | dropped_prefetched_lightbringer=%d", - waitingSlot, gap, previousWaitingSlot, len(removedSlots), clearedPrefetched) + mlog.Log.Warnf("BLOCK SOURCE SWITCH: %s -> RPC at slot %d | reason=%s | gap=%d | rewound_emission_frontier_from=%d | cleared_buffered_live_stream=%d | dropped_prefetched_live_stream=%d | live_generation=%d", + bs.liveShredStreamName(), waitingSlot, reason, gap, previousWaitingSlot, len(removedSlots), clearedPrefetched, resultGeneration) return } - mlog.Log.Warnf("BLOCK SOURCE SWITCH: LIGHTBRINGER -> RPC at slot %d | reason=lost_tip | gap=%d | cleared_buffered_lightbringer=%d | dropped_prefetched_lightbringer=%d", - waitingSlot, gap, len(removedSlots), clearedPrefetched) + mlog.Log.Warnf("BLOCK SOURCE SWITCH: %s -> RPC at slot %d | reason=%s | gap=%d | cleared_buffered_live_stream=%d | dropped_prefetched_live_stream=%d | live_generation=%d", + bs.liveShredStreamName(), waitingSlot, reason, gap, len(removedSlots), clearedPrefetched, resultGeneration) return } if oldHandoff != 0 || len(removedSlots) > 0 || clearedPrefetched > 0 { if previousWaitingSlot != waitingSlot { - mlog.Log.Warnf("BLOCK SOURCE STATUS: abandoning pending Lightbringer handoff and forcing RPC catchup | waiting_slot=%d | gap=%d | rewound_emission_frontier_from=%d | cleared_buffered_lightbringer=%d | dropped_prefetched_lightbringer=%d", - waitingSlot, gap, previousWaitingSlot, len(removedSlots), clearedPrefetched) + mlog.Log.Warnf("BLOCK SOURCE STATUS: abandoning pending %s handoff and forcing RPC catchup | reason=%s | waiting_slot=%d | gap=%d | rewound_emission_frontier_from=%d | cleared_buffered_live_stream=%d | dropped_prefetched_live_stream=%d | live_generation=%d", + bs.liveShredStreamName(), reason, waitingSlot, gap, previousWaitingSlot, len(removedSlots), clearedPrefetched, resultGeneration) return } - mlog.Log.Warnf("BLOCK SOURCE STATUS: abandoning pending Lightbringer handoff and forcing RPC catchup | waiting_slot=%d | gap=%d | cleared_buffered_lightbringer=%d | dropped_prefetched_lightbringer=%d", - waitingSlot, gap, len(removedSlots), clearedPrefetched) + mlog.Log.Warnf("BLOCK SOURCE STATUS: abandoning pending %s handoff and forcing RPC catchup | reason=%s | waiting_slot=%d | gap=%d | cleared_buffered_live_stream=%d | dropped_prefetched_live_stream=%d | live_generation=%d", + bs.liveShredStreamName(), reason, waitingSlot, gap, len(removedSlots), clearedPrefetched, resultGeneration) return } - mlog.Log.Infof("BLOCK SOURCE STATUS: catchup mode is using RPC | waiting_slot=%d | gap=%d", - waitingSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: catchup mode is using RPC | reason=%s | waiting_slot=%d | gap=%d", + reason, waitingSlot, gap) } func (bs *BlockSource) clearLightbringerGapWatch() { @@ -721,6 +789,10 @@ func (bs *BlockSource) clearLightbringerGapWatch() { bs.lightbringerGapReconnectSlot.Store(0) } +func (bs *BlockSource) invalidateLightbringerResults() uint64 { + return bs.lightbringerResultGeneration.Add(1) +} + func (bs *BlockSource) setLightbringerCancel(cancel context.CancelFunc) { bs.lightbringerCancelMu.Lock() bs.lightbringerCancel = cancel @@ -737,7 +809,6 @@ func (bs *BlockSource) requestLightbringerReconnect(reason string) bool { if !bs.lightbringerConnected.Load() { return false } - mlog.Log.Warnf("in requestLightbringerReconnect") if !bs.lightbringerReconnectRequested.CompareAndSwap(false, true) { return false } @@ -748,8 +819,8 @@ func (bs *BlockSource) requestLightbringerReconnect(reason string) bool { bs.reorderMu.Unlock() latestStreamed := bs.lightbringerLastStreamSlot.Load() - mlog.Log.Warnf("Lightbringer stream reconnect requested: %s | waiting_slot=%d | last_emitted=%d | latest_streamed=%d", - reason, waitingSlot, lastEmitted, latestStreamed) + mlog.Log.Warnf("%s stream reconnect requested: %s | waiting_slot=%d | last_emitted=%d | latest_streamed=%d", + bs.liveShredStreamName(), reason, waitingSlot, lastEmitted, latestStreamed) bs.lightbringerCancelMu.Lock() cancel := bs.lightbringerCancel @@ -773,7 +844,7 @@ func isLightbringerReconnectCancel(err error) bool { } func (bs *BlockSource) maybeReconnectActiveLightbringerForNoProgress(stallDuration time.Duration) { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return } if !bs.lightbringerActive.Load() || !bs.isNearTip.Load() { @@ -806,10 +877,10 @@ func (bs *BlockSource) maybeReconnectActiveLightbringerForNoProgress(stallDurati return } - reason := fmt.Sprintf("no block emitted for %s while Lightbringer is active and replay is waiting on slot %d", - stallDuration.Round(time.Second), waitingSlot) + reason := fmt.Sprintf("no block emitted for %s while %s is active and replay is waiting on slot %d", + stallDuration.Round(time.Second), bs.liveShredStreamName(), waitingSlot) if foundBuffered { - reason = fmt.Sprintf("no block emitted for %s while waiting on slot %d (first_buffered=%d buffered_lightbringer=%d last_emitted=%d)", + reason = fmt.Sprintf("no block emitted for %s while waiting on slot %d (first_buffered=%d buffered_live_stream=%d last_emitted=%d)", stallDuration.Round(time.Second), waitingSlot, firstBufferedSlot, bufferedLightbringer, lastEmitted) } @@ -872,7 +943,7 @@ func (bs *BlockSource) waitingLightbringerParentMismatchLocked() (waitingSlot ui } func (bs *BlockSource) repairLightbringerGap(waitingSlot, firstBufferedSlot, firstBufferedParentSlot uint64, bufferedCount int) { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return } if !bs.lightbringerActive.Load() { @@ -886,13 +957,14 @@ func (bs *BlockSource) repairLightbringerGap(waitingSlot, firstBufferedSlot, fir gapAge = time.Since(time.Unix(0, gapSinceUnix)) } - waitReason := fmt.Sprintf("first buffered Lightbringer block %d still depends on parent slot %d", firstBufferedSlot, firstBufferedParentSlot) + streamName := bs.liveShredStreamName() + waitReason := fmt.Sprintf("first buffered %s block %d still depends on parent slot %d", streamName, firstBufferedSlot, firstBufferedParentSlot) switch { case firstBufferedParentSlot == waitingSlot: - waitReason = fmt.Sprintf("waiting on live Lightbringer slot %d; later buffered block %d still depends on it", waitingSlot, firstBufferedSlot) + waitReason = fmt.Sprintf("waiting on live %s slot %d; later buffered block %d still depends on it", streamName, waitingSlot, firstBufferedSlot) case firstBufferedParentSlot > waitingSlot: - waitReason = fmt.Sprintf("waiting on missing Lightbringer ancestry range %d-%d; later buffered block %d still depends on slot %d", - waitingSlot, firstBufferedParentSlot, firstBufferedSlot, firstBufferedParentSlot) + waitReason = fmt.Sprintf("waiting on missing %s ancestry range %d-%d; later buffered block %d still depends on slot %d", + streamName, waitingSlot, firstBufferedParentSlot, firstBufferedSlot, firstBufferedParentSlot) case firstBufferedParentSlot == bs.lastEmittedBlockSlot: waitReason = fmt.Sprintf("later buffered block %d points back to anchor %d, but that only proves an observed alternate branch and is not treated as a canonical skipped run", firstBufferedSlot, bs.lastEmittedBlockSlot) } @@ -901,8 +973,8 @@ func (bs *BlockSource) repairLightbringerGap(waitingSlot, firstBufferedSlot, fir lastLog := time.Unix(0, bs.lightbringerGapLastLogUnix.Load()) if lastLog.IsZero() || now.Sub(lastLog) >= reorderGapWarnInterval { bs.lightbringerGapLastLogUnix.Store(now.UnixNano()) - mlog.Log.Warnf("BLOCK SOURCE STATUS: waiting for missing Lightbringer slot %d from live stream while keeping Lightbringer active | first_buffered=%d | first_parent_slot=%d | buffered_lightbringer=%d | reason=%s | mode=%s", - waitingSlot, firstBufferedSlot, firstBufferedParentSlot, bufferedCount, waitReason, bs.currentModeString()) + mlog.Log.Warnf("BLOCK SOURCE STATUS: waiting for missing %s slot %d from live stream while keeping %s active | first_buffered=%d | first_parent_slot=%d | buffered_live_stream=%d | reason=%s | mode=%s", + streamName, waitingSlot, streamName, firstBufferedSlot, firstBufferedParentSlot, bufferedCount, waitReason, bs.currentModeString()) } shouldReconnect := false @@ -910,28 +982,28 @@ func (bs *BlockSource) repairLightbringerGap(waitingSlot, firstBufferedSlot, fir switch { case firstBufferedParentSlot != waitingSlot: if firstBufferedParentSlot <= waitingSlot { - // A reconnect only helps when later buffered Lightbringer blocks still + // A reconnect only helps when later buffered live-stream blocks still // depend on an unseen ancestor beyond the current anchor. break } switch { case bufferedCount >= reorderGapWarnThreshold && gapAge >= lightbringerDeepGapReconnect: shouldReconnect = true - reconnectReason = fmt.Sprintf("waiting %s for Lightbringer ancestry range %d-%d while later buffered block %d still depends on slot %d", - gapAge.Round(time.Second), waitingSlot, firstBufferedParentSlot, firstBufferedSlot, firstBufferedParentSlot) + reconnectReason = fmt.Sprintf("waiting %s for %s ancestry range %d-%d while later buffered block %d still depends on slot %d", + gapAge.Round(time.Second), streamName, waitingSlot, firstBufferedParentSlot, firstBufferedSlot, firstBufferedParentSlot) case gapAge >= lightbringerGapReconnectAfter: shouldReconnect = true - reconnectReason = fmt.Sprintf("waiting %s for Lightbringer ancestry range %d-%d while later buffered block %d still depends on slot %d", - gapAge.Round(time.Second), waitingSlot, firstBufferedParentSlot, firstBufferedSlot, firstBufferedParentSlot) + reconnectReason = fmt.Sprintf("waiting %s for %s ancestry range %d-%d while later buffered block %d still depends on slot %d", + gapAge.Round(time.Second), streamName, waitingSlot, firstBufferedParentSlot, firstBufferedSlot, firstBufferedParentSlot) } case bufferedCount >= reorderGapWarnThreshold && gapAge >= lightbringerDeepGapReconnect: shouldReconnect = true - reconnectReason = fmt.Sprintf("waiting %s for live Lightbringer slot %d while %d later buffered blocks still depend on it", - gapAge.Round(time.Second), waitingSlot, bufferedCount) + reconnectReason = fmt.Sprintf("waiting %s for live %s slot %d while %d later buffered blocks still depend on it", + gapAge.Round(time.Second), streamName, waitingSlot, bufferedCount) case gapAge >= lightbringerGapReconnectAfter: shouldReconnect = true - reconnectReason = fmt.Sprintf("waiting %s for live Lightbringer slot %d while later buffered blocks still depend on it", - gapAge.Round(time.Second), waitingSlot) + reconnectReason = fmt.Sprintf("waiting %s for live %s slot %d while later buffered blocks still depend on it", + gapAge.Round(time.Second), streamName, waitingSlot) } if shouldReconnect && bs.lightbringerGapReconnectSlot.CompareAndSwap(0, waitingSlot) { @@ -940,7 +1012,7 @@ func (bs *BlockSource) repairLightbringerGap(waitingSlot, firstBufferedSlot, fir } func (bs *BlockSource) forceRPCForLightbringerGap(waitingSlot, firstBufferedSlot, firstBufferedParentSlot uint64, bufferedCount int) { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return } @@ -953,6 +1025,7 @@ func (bs *BlockSource) forceRPCForLightbringerGap(waitingSlot, firstBufferedSlot bs.lightbringerCooldownUntil.Store(recoveryUntil) oldHandoff := bs.lightbringerHandoffSlot.Swap(0) wasActive := bs.lightbringerActive.Swap(false) + resultGeneration := bs.invalidateLightbringerResults() bs.lightbringerNeedRPCResume.Store(true) bs.clearLightbringerGapWatch() bs.resetLightbringerRepairSlot() @@ -978,13 +1051,13 @@ func (bs *BlockSource) forceRPCForLightbringerGap(waitingSlot, firstBufferedSlot } if wasActive || oldHandoff != 0 { - mlog.Log.Warnf("BLOCK SOURCE SWITCH: LIGHTBRINGER -> RPC at slot %d | reason=missing_lightbringer_slot | first_buffered=%d | first_parent_slot=%d | buffered_lightbringer=%d | cleared_prefetched_lightbringer=%d | rpc_forced_until=%d | rpc_cooldown_until=%d | mode=%s", - waitingSlot, firstBufferedSlot, firstBufferedParentSlot, bufferedCount, clearedPrefetched, waitingSlot, recoveryUntil, bs.currentModeString()) + mlog.Log.Warnf("BLOCK SOURCE SWITCH: %s -> RPC at slot %d | reason=missing_streamed_slot | first_buffered=%d | first_parent_slot=%d | buffered_live_stream=%d | cleared_prefetched_live_stream=%d | rpc_forced_until=%d | rpc_cooldown_until=%d | live_generation=%d | mode=%s", + bs.liveShredStreamName(), waitingSlot, firstBufferedSlot, firstBufferedParentSlot, bufferedCount, clearedPrefetched, waitingSlot, recoveryUntil, resultGeneration, bs.currentModeString()) return } - mlog.Log.Warnf("BLOCK SOURCE STATUS: forcing RPC because Lightbringer skipped waiting slot %d | first_buffered=%d | first_parent_slot=%d | buffered_lightbringer=%d | cleared_prefetched_lightbringer=%d | rpc_forced_until=%d | rpc_cooldown_until=%d | mode=%s", - waitingSlot, firstBufferedSlot, firstBufferedParentSlot, bufferedCount, clearedPrefetched, waitingSlot, recoveryUntil, bs.currentModeString()) + mlog.Log.Warnf("BLOCK SOURCE STATUS: forcing RPC because %s skipped waiting slot %d | first_buffered=%d | first_parent_slot=%d | buffered_live_stream=%d | cleared_prefetched_live_stream=%d | rpc_forced_until=%d | rpc_cooldown_until=%d | live_generation=%d | mode=%s", + bs.liveShredStreamName(), waitingSlot, firstBufferedSlot, firstBufferedParentSlot, bufferedCount, clearedPrefetched, waitingSlot, recoveryUntil, resultGeneration, bs.currentModeString()) } func (bs *BlockSource) handleDetectedLightbringerGap(waitingSlot, firstBufferedSlot, firstBufferedParentSlot uint64, bufferedCount int) { @@ -996,7 +1069,7 @@ func (bs *BlockSource) handleDetectedLightbringerGap(waitingSlot, firstBufferedS } func (bs *BlockSource) forceRPCForLightbringerParentMismatch(waitingSlot, observedParentSlot, expectedParentSlot uint64) { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return } @@ -1009,6 +1082,7 @@ func (bs *BlockSource) forceRPCForLightbringerParentMismatch(waitingSlot, observ bs.lightbringerCooldownUntil.Store(recoveryUntil) oldHandoff := bs.lightbringerHandoffSlot.Swap(0) wasActive := bs.lightbringerActive.Swap(false) + resultGeneration := bs.invalidateLightbringerResults() bs.lightbringerNeedRPCResume.Store(true) bs.clearLightbringerGapWatch() bs.resetLightbringerRepairSlot() @@ -1039,17 +1113,17 @@ func (bs *BlockSource) forceRPCForLightbringerParentMismatch(waitingSlot, observ } if wasActive || oldHandoff != 0 { - mlog.Log.Warnf("BLOCK SOURCE SWITCH: LIGHTBRINGER -> RPC at slot %d | reason=%s | observed_parent_slot=%d | expected_parent_slot=%d | cleared_prefetched_lightbringer=%d | rpc_forced_until=%d | rpc_cooldown_until=%d | mode=%s", - waitingSlot, reason, observedParentSlot, expectedParentSlot, clearedPrefetched, waitingSlot, recoveryUntil, bs.currentModeString()) + mlog.Log.Warnf("BLOCK SOURCE SWITCH: %s -> RPC at slot %d | reason=%s | observed_parent_slot=%d | expected_parent_slot=%d | cleared_prefetched_live_stream=%d | rpc_forced_until=%d | rpc_cooldown_until=%d | live_generation=%d | mode=%s", + bs.liveShredStreamName(), waitingSlot, reason, observedParentSlot, expectedParentSlot, clearedPrefetched, waitingSlot, recoveryUntil, resultGeneration, bs.currentModeString()) return } - mlog.Log.Warnf("BLOCK SOURCE STATUS: rejecting Lightbringer handoff at slot %d | reason=%s | observed_parent_slot=%d | expected_parent_slot=%d | cleared_prefetched_lightbringer=%d | rpc_forced_until=%d | rpc_cooldown_until=%d | mode=%s", - waitingSlot, reason, observedParentSlot, expectedParentSlot, clearedPrefetched, waitingSlot, recoveryUntil, bs.currentModeString()) + mlog.Log.Warnf("BLOCK SOURCE STATUS: rejecting %s handoff at slot %d | reason=%s | observed_parent_slot=%d | expected_parent_slot=%d | cleared_prefetched_live_stream=%d | rpc_forced_until=%d | rpc_cooldown_until=%d | live_generation=%d | mode=%s", + bs.liveShredStreamName(), waitingSlot, reason, observedParentSlot, expectedParentSlot, clearedPrefetched, waitingSlot, recoveryUntil, resultGeneration, bs.currentModeString()) } func (bs *BlockSource) logLightbringerModeState(mode string, gap uint64) { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return } @@ -1067,52 +1141,56 @@ func (bs *BlockSource) logLightbringerModeState(mode string, gap uint64) { switch mode { case "near-tip": if active { - mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained and blocks are already arriving from LIGHTBRINGER | waiting_slot=%d | handoff_slot=%d | gap=%d", - waitingSlot, handoffSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained and blocks are already arriving from %s | waiting_slot=%d | handoff_slot=%d | gap=%d", + bs.liveShredStreamName(), waitingSlot, handoffSlot, gap) return } if cooldownUntil != 0 { if connected && lastSlot != 0 { - mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; staying on RPC until slot %d after a Lightbringer gap (latest streamed slot %d) | waiting_slot=%d | gap=%d", - cooldownUntil, lastSlot, waitingSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; staying on RPC until slot %d after a %s gap (latest streamed slot %d) | waiting_slot=%d | gap=%d", + cooldownUntil, bs.liveShredStreamName(), lastSlot, waitingSlot, gap) return } - mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; staying on RPC until slot %d after a Lightbringer gap | waiting_slot=%d | gap=%d", - cooldownUntil, waitingSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; staying on RPC until slot %d after a %s gap | waiting_slot=%d | gap=%d", + cooldownUntil, bs.liveShredStreamName(), waitingSlot, gap) return } if handoffSlot != 0 { - mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; waiting to switch block receipt from RPC to LIGHTBRINGER at handoff slot %d | waiting_slot=%d | gap=%d", - handoffSlot, waitingSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; waiting to switch block receipt from RPC to %s at handoff slot %d | waiting_slot=%d | gap=%d", + bs.liveShredStreamName(), handoffSlot, waitingSlot, gap) return } if connected { if lastSlot != 0 { - mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; Lightbringer stream is connected (latest streamed slot %d) and waiting to arm handoff | waiting_slot=%d | gap=%d", - lastSlot, waitingSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; %s stream is connected (latest streamed slot %d) and waiting to arm handoff | waiting_slot=%d | gap=%d", + bs.liveShredStreamName(), lastSlot, waitingSlot, gap) return } - mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; Lightbringer stream is connected and waiting for its first streamed slot | waiting_slot=%d | gap=%d", - waitingSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; %s stream is connected and waiting for its first streamed slot | waiting_slot=%d | gap=%d", + bs.liveShredStreamName(), waitingSlot, gap) return } if started { - mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; waiting for Lightbringer stream connection before handoff | waiting_slot=%d | gap=%d", - waitingSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; waiting for %s stream connection before handoff | waiting_slot=%d | gap=%d", + bs.liveShredStreamName(), waitingSlot, gap) return } - mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; preparing to switch block receipt from RPC to LIGHTBRINGER | waiting_slot=%d | gap=%d", - waitingSlot, gap) + mlog.Log.Infof("BLOCK SOURCE STATUS: near-tip regained; preparing to switch block receipt from RPC to %s | waiting_slot=%d | gap=%d", + bs.liveShredStreamName(), waitingSlot, gap) } } func (bs *BlockSource) maybeStartLightbringerStream() { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return } if bs.lightbringerStarted.CompareAndSwap(false, true) { bs.lightbringerWg.Add(1) - go bs.runLightbringerStream() + if bs.sourceType == BlockSourceTurbine { + go bs.runTurbineStream() + } else { + go bs.runLightbringerStream() + } } } @@ -1251,9 +1329,13 @@ func (bs *BlockSource) lightbringerLiveEdgeHandoffMaxLag() uint64 { return maxLag } +func (bs *BlockSource) allowsLiveEdgeHandoff() bool { + return bs.consensusManagedLightbringer || bs.sourceType == BlockSourceTurbine +} + func (bs *BlockSource) lightbringerHandoffRequiredLastSlot(waitingSlot uint64) uint64 { requiredLastSlot := lightbringerDefaultHandoffLastSlot(waitingSlot) - if !bs.consensusManagedLightbringer || !bs.lightbringerConnected.Load() { + if !bs.allowsLiveEdgeHandoff() || !bs.lightbringerConnected.Load() { return requiredLastSlot } @@ -1384,9 +1466,9 @@ func (bs *BlockSource) lightbringerHandoffWaitReason(waitingSlot uint64, anchorS runway, coveredUntil, firstBufferedSlot, maxBufferedSlot, ok := bs.connectedLightbringerRunwayLocked(waitingSlot, anchorSlot) if !ok { if len(bs.lightbringerBuffer) == 0 { - return "waiting for first buffered Lightbringer slot" + return fmt.Sprintf("waiting for first buffered %s slot", bs.liveShredStreamName()) } - return fmt.Sprintf("no buffered Lightbringer slot at or beyond waiting slot %d", waitingSlot) + return fmt.Sprintf("no buffered %s slot at or beyond waiting slot %d", bs.liveShredStreamName(), waitingSlot) } requiredLastSlot := bs.lightbringerHandoffRequiredLastSlot(waitingSlot) @@ -1400,8 +1482,8 @@ func (bs *BlockSource) lightbringerHandoffWaitReason(waitingSlot uint64, anchorS firstRunwaySlot := runway[0].Slot if firstRunwaySlot > waitingSlot { - return fmt.Sprintf("waiting for slot %d or skipped-slot inference; first buffered Lightbringer block is slot %d (parent %d), connected runway covers through slot %d, latest buffered slot %d", - waitingSlot, firstRunwaySlot, runway[0].SourceParentSlot, coveredUntil, maxBufferedSlot) + return fmt.Sprintf("waiting for slot %d or skipped-slot inference; first buffered %s block is slot %d (parent %d), connected runway covers through slot %d, latest buffered slot %d", + waitingSlot, bs.liveShredStreamName(), firstRunwaySlot, runway[0].SourceParentSlot, coveredUntil, maxBufferedSlot) } if firstBufferedSlot != 0 && firstRunwaySlot != firstBufferedSlot { @@ -1416,6 +1498,7 @@ func (bs *BlockSource) enqueueLightbringerBlocks(blocks []*b.Block) { if len(blocks) == 0 { return } + generation := bs.lightbringerResultGeneration.Load() sort.Slice(blocks, func(i, j int) bool { return blocks[i].Slot < blocks[j].Slot @@ -1427,7 +1510,7 @@ func (bs *BlockSource) enqueueLightbringerBlocks(blocks []*b.Block) { } select { - case bs.resultQueue <- fetchResult{slot: blk.Slot, block: blk, rpcIdx: -1}: + case bs.resultQueue <- fetchResult{slot: blk.Slot, block: blk, rpcIdx: -1, liveStreamGeneration: generation}: case <-bs.stopChan: return } @@ -1435,7 +1518,7 @@ func (bs *BlockSource) enqueueLightbringerBlocks(blocks []*b.Block) { } func (bs *BlockSource) maybePrepareLightbringerHandoff() { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" || !bs.isNearTip.Load() { + if !bs.usesLiveShredStream() || !bs.isNearTip.Load() { return } if bs.lightbringerForceRPCUntil.Load() != 0 { @@ -1463,8 +1546,8 @@ func (bs *BlockSource) maybePrepareLightbringerHandoff() { if len(blocks) > 0 { runwayCoveredUntil = blocks[len(blocks)-1].Slot } - mlog.Log.Infof("Lightbringer handoff ready at slot %d (connected runway buffered through slot %d; RPC catchup continues until then)", - handoffSlot, runwayCoveredUntil) + mlog.Log.Infof("%s handoff ready at slot %d (connected runway buffered through slot %d; RPC catchup continues until then)", + bs.liveShredStreamName(), handoffSlot, runwayCoveredUntil) bs.enqueueLightbringerBlocks(blocks) } @@ -1522,7 +1605,7 @@ func (bs *BlockSource) shouldDecodeLightbringerSlot(slot uint64) bool { } func (bs *BlockSource) shouldUseRPCForSlot(slot uint64) bool { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return true } if bs.lightbringerForceRPCUntil.Load() != 0 { @@ -1547,7 +1630,7 @@ func (bs *BlockSource) shouldUseRPCForSlot(slot uint64) bool { } func (bs *BlockSource) shouldDiscardRPCResultAfterHandoff(slot uint64, blk *b.Block) bool { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return false } handoffSlot := bs.lightbringerHandoffSlot.Load() @@ -1567,6 +1650,30 @@ func (bs *BlockSource) shouldDiscardSkippedSlotAfterHandoff(slot uint64) bool { return !bs.lightbringerSynthesizedSkips[slot] } +func (bs *BlockSource) shouldDiscardLiveStreamResult(slot uint64, generation uint64) bool { + if generation != bs.lightbringerResultGeneration.Load() { + return true + } + if bs.lightbringerForceRPCUntil.Load() != 0 { + return true + } + if bs.lightbringerCooldownUntil.Load() != 0 { + return true + } + if bs.isLightbringerRepairSlot(slot) { + return true + } + if !bs.isNearTip.Load() { + return true + } + if bs.consensusManagedLightbringer && bs.lightbringerActive.Load() { + return false + } + + handoffSlot := bs.lightbringerHandoffSlot.Load() + return handoffSlot == 0 || slot < handoffSlot +} + // inspectLaterLightbringerBlocksLocked summarizes later buffered Lightbringer // traffic for the currently waiting slot so diagnostics can distinguish // "we have later stream traffic" from "we have a descendant that reconnects @@ -1625,6 +1732,267 @@ func (bs *BlockSource) waitForStopOrTimeout(delay time.Duration) bool { } } +func (bs *BlockSource) ingestLiveShredBlock(blk *b.Block) bool { + if blk == nil { + return true + } + + bs.lightbringerLastStreamSlot.Store(blk.Slot) + bs.lightbringerLastRecvUnix.Store(time.Now().Unix()) + + if !bs.shouldDecodeLightbringerSlot(blk.Slot) { + return true + } + + if bs.lightbringerHandoffSlot.Load() == 0 { + // Stage a bounded runway before near-tip so handoff does not have + // to build its whole connected run while replay is already at tip. + bs.bufferLightbringerBlock(blk) + bs.maybePrepareLightbringerHandoff() + return true + } + + if !bs.isNearTip.Load() || blk.Slot < bs.lightbringerHandoffSlot.Load() { + return true + } + + select { + case bs.resultQueue <- fetchResult{slot: blk.Slot, block: blk, rpcIdx: -1, liveStreamGeneration: bs.lightbringerResultGeneration.Load()}: + return true + case <-bs.stopChan: + return false + } +} + +func (bs *BlockSource) handleLiveShredStreamClosed(reason string) int { + bs.lightbringerConnected.Store(false) + bs.clearLightbringerCancel() + interrupted := bs.lightbringerHandoffSlot.Load() != 0 || bs.lightbringerActive.Load() + if interrupted { + bs.reorderMu.Lock() + waitingSlot := bs.nextSlotToSend + bs.reorderMu.Unlock() + if waitingSlot == 0 { + if lastExecuted := bs.lastExecutedSlot.Load(); lastExecuted != 0 { + waitingSlot = lastExecuted + 1 + } else { + waitingSlot = bs.startSlot + } + } + bs.forceRPCForLightbringerGap(waitingSlot, 0, 0, 0) + mlog.Log.Warnf("%s handoff interrupted; replay will resume RPC fallback from slot %d until a fresh stream runway is armed", + bs.liveShredStreamName(), waitingSlot) + if reason != "" { + mlog.Log.Warnf("%s stream closed: %s", bs.liveShredStreamName(), reason) + } + return 0 + } + + bs.invalidateLightbringerResults() + clearedPrefetched := bs.clearBufferedLightbringerBlocks() + bs.lightbringerHandoffSlot.Store(0) + if reason != "" { + mlog.Log.Warnf("%s stream closed: %s", bs.liveShredStreamName(), reason) + } + return clearedPrefetched +} + +func (bs *BlockSource) runTurbineStream() { + defer bs.lightbringerWg.Done() + + backoff := lightbringerRetryBackoff + for { + if bs.stopped.Load() { + return + } + + bs.lightbringerConnected.Store(false) + streamCtx, cancelStream := context.WithCancel(context.Background()) + bs.setLightbringerCancel(cancelStream) + + var gossipClient *gossipclient.Client + var gossipDone <-chan error + if bs.turbineGossipEntrypoint != "" { + gossipCfg := gossipclient.Config{ + Entrypoint: bs.turbineGossipEntrypoint, + BindAddr: bs.turbineGossipBindAddr, + TVUAddr: bs.turbineBindAddr, + AdvertisedIP: bs.turbineAdvertisedIP, + ShredVersion: bs.turbineShredVersion, + Name: gossipclient.ClientName, + } + client, err := gossipclient.NewClient(gossipCfg) + if err != nil { + cancelStream() + bs.handleLiveShredStreamClosed(fmt.Sprintf("native turbine gossip client setup failed: %v", err)) + if bs.waitForStopOrTimeout(backoff) { + return + } + backoff *= 2 + if backoff > lightbringerMaxRetryBackoff { + backoff = lightbringerMaxRetryBackoff + } + continue + } + gossipClient = client + } + + receiver := turbine.NewUDPReceiver(bs.turbineBindAddr) + receiver.SetLeaderForSlot(bs.leaderForSlot) + if gossipClient != nil { + if err := receiver.SetRepairPeerSource(gossipClient.Identity(), gossipClient.RepairPeers); err != nil { + cancelStream() + bs.handleLiveShredStreamClosed(fmt.Sprintf("native turbine repair setup failed: %v", err)) + if bs.waitForStopOrTimeout(backoff) { + return + } + backoff *= 2 + if backoff > lightbringerMaxRetryBackoff { + backoff = lightbringerMaxRetryBackoff + } + continue + } + } + streamDone := make(chan error, 1) + go func() { + streamDone <- receiver.Run(streamCtx) + }() + go func() { + select { + case <-bs.stopChan: + cancelStream() + case <-streamCtx.Done(): + } + }() + + select { + case err := <-receiver.Ready(): + if err != nil { + cancelStream() + <-streamDone + bs.handleLiveShredStreamClosed(err.Error()) + if bs.waitForStopOrTimeout(backoff) { + return + } + backoff *= 2 + if backoff > lightbringerMaxRetryBackoff { + backoff = lightbringerMaxRetryBackoff + } + continue + } + case <-bs.stopChan: + cancelStream() + <-streamDone + return + } + + mlog.Log.Infof("Native turbine receiver listening on %s", bs.turbineBindAddr) + bs.lightbringerConnected.Store(true) + bs.lightbringerLastRecvUnix.Store(time.Now().Unix()) + backoff = lightbringerRetryBackoff + + if gossipClient != nil { + done := make(chan error, 1) + go func() { + done <- gossipClient.Run(streamCtx) + }() + gossipDone = done + bindAddr := bs.turbineGossipBindAddr + if bindAddr == "" { + bindAddr = gossipclient.DefaultBindAddr + } + mlog.Log.Infof("Native turbine gossip client starting: entrypoint=%s bind=%s client=%s repair=enabled", bs.turbineGossipEntrypoint, bindAddr, gossipclient.ClientName) + } else { + mlog.Log.Warnf("Native turbine gossip entrypoint is not configured; receiver is running UDP-only on %s with repair disabled", bs.turbineBindAddr) + } + + var streamErr error + streamDoneConsumed := false + var ignoredPacketCount uint64 + var ignoredPacketLastLog time.Time + statsTicker := time.NewTicker(10 * time.Second) + streamLoop: + for { + select { + case blk, ok := <-receiver.Blocks(): + if !ok { + streamErr = <-streamDone + streamDoneConsumed = true + break streamLoop + } + if !bs.ingestLiveShredBlock(blk) { + statsTicker.Stop() + cancelStream() + <-streamDone + return + } + case err, ok := <-receiver.Errors(): + if ok && err != nil { + ignoredPacketCount++ + now := time.Now() + if ignoredPacketLastLog.IsZero() || now.Sub(ignoredPacketLastLog) >= 10*time.Second { + mlog.Log.FileOnlyf("native turbine packets ignored: count=%d latest=%v", ignoredPacketCount, err) + ignoredPacketCount = 0 + ignoredPacketLastLog = now + } else { + mlog.Log.Debugf("native turbine packet ignored: %v", err) + } + } + case <-statsTicker.C: + stats := receiver.Stats() + lastPacketAge := "never" + if stats.LastPacketUnix != 0 { + lastPacketAge = time.Since(time.Unix(stats.LastPacketUnix, 0)).Round(time.Second).String() + } + mlog.Log.FileOnlyf("native turbine receiver stats: packets=%d data=%d coding=%d recovered=%d blocks=%d active_slots=%d evicted_slots=%d ignored_old_shreds=%d repair_requests=%d repair_responses=%d repair_timeouts=%d repair_outstanding=%d repair_peers=%d repair_pings=%d/%d repair_errors=%d parse_errors=%d sig_errors=%d missing_leaders=%d assembly_errors=%d last_packet=%s last_data_slot=%d last_block_slot=%d", + stats.Packets, stats.DataShreds, stats.CodingShreds, stats.RecoveredData, stats.BlocksEmitted, stats.ActiveSlots, + stats.EvictedSlots, stats.IgnoredOldShreds, stats.Repair.Requests, stats.Repair.Responses, stats.Repair.Timeouts, stats.Repair.Outstanding, stats.Repair.Peers, + stats.Repair.Pings, stats.Repair.Pongs, stats.Repair.Errors, stats.ParseErrors, stats.SignatureErrors, stats.MissingLeaders, stats.AssemblyErrors, + lastPacketAge, stats.LastDataSlot, stats.LastBlockSlot) + case err := <-streamDone: + streamErr = err + streamDoneConsumed = true + break streamLoop + case err := <-gossipDone: + if err != nil { + streamErr = fmt.Errorf("native turbine gossip client stopped: %w", err) + } else { + streamErr = fmt.Errorf("native turbine gossip client stopped") + } + break streamLoop + case <-bs.stopChan: + statsTicker.Stop() + cancelStream() + <-streamDone + return + } + } + + statsTicker.Stop() + cancelStream() + if !streamDoneConsumed { + if err := <-streamDone; streamErr == nil { + streamErr = err + } + } + if streamErr == nil && bs.stopped.Load() { + return + } + reason := "" + if streamErr != nil { + reason = streamErr.Error() + } + bs.handleLiveShredStreamClosed(reason) + if bs.waitForStopOrTimeout(backoff) { + return + } + backoff *= 2 + if backoff > lightbringerMaxRetryBackoff { + backoff = lightbringerMaxRetryBackoff + } + } +} + func (bs *BlockSource) runLightbringerStream() { defer bs.lightbringerWg.Done() @@ -1767,18 +2135,7 @@ func (bs *BlockSource) runLightbringerStream() { connectionClosedOnce.Do(func() { close(connectionClosed) }) - bs.lightbringerConnected.Store(false) - bs.clearLightbringerCancel() - clearedPrefetched := bs.clearBufferedLightbringerBlocks() - if bs.lightbringerHandoffSlot.Load() != 0 { - bs.reorderMu.Lock() - waitingSlot := bs.nextSlotToSend - bs.reorderMu.Unlock() - mlog.Log.Warnf("Lightbringer handoff interrupted; replay will resume RPC fallback from slot %d until the stream recovers | dropped_prefetched_lightbringer=%d", - waitingSlot, clearedPrefetched) - bs.lightbringerNeedRPCResume.Store(true) - } - bs.lightbringerHandoffSlot.Store(0) + bs.handleLiveShredStreamClosed("") cancelStream() <-streamDone _ = conn.Close() @@ -1840,22 +2197,7 @@ func (bs *BlockSource) runLightbringerStream() { } blk := block.FromLightbringerStreamMsg(resp) - - if bs.lightbringerHandoffSlot.Load() == 0 { - // Stage a bounded runway before near-tip so handoff does not have - // to build its whole connected run while replay is already at tip. - bs.bufferLightbringerBlock(blk) - bs.maybePrepareLightbringerHandoff() - continue - } - - if !bs.isNearTip.Load() || blk.Slot < bs.lightbringerHandoffSlot.Load() { - continue - } - - select { - case bs.resultQueue <- fetchResult{slot: blk.Slot, block: blk, rpcIdx: -1}: - case <-bs.stopChan: + if !bs.ingestLiveShredBlock(blk) { cancelStream() <-streamDone _ = conn.Close() @@ -1937,7 +2279,7 @@ func (bs *BlockSource) maybeLogReorderGapLocked() { } func (bs *BlockSource) detectLightbringerGapLocked() (waitingSlot uint64, firstBufferedSlot uint64, firstBufferedParentSlot uint64, bufferedCount int, shouldFallback bool) { - if bs.sourceType != BlockSourceLightbringer || bs.lightbringerEndpoint == "" { + if !bs.usesLiveShredStream() { return 0, 0, 0, 0, false } if bs.lightbringerForceRPCUntil.Load() != 0 { @@ -2012,7 +2354,7 @@ func (bs *BlockSource) detectLightbringerGapLocked() (waitingSlot uint64, firstB } func (bs *BlockSource) shouldDeliverLightbringerObservationDirectLocked(slot uint64, blk *b.Block) bool { - if !bs.consensusManagedLightbringer || bs.sourceType != BlockSourceLightbringer { + if !bs.consensusManagedLightbringer || !bs.usesLiveShredStream() { return false } if blk == nil || !blk.FromLightbringer { @@ -2910,6 +3252,10 @@ func (bs *BlockSource) emitOrderedBlocks() { } if result.block != nil && result.block.FromLightbringer { + if bs.shouldDiscardLiveStreamResult(result.slot, result.liveStreamGeneration) { + bs.reorderMu.Unlock() + continue + } handoffSlot := bs.lightbringerHandoffSlot.Load() if handoffSlot != 0 && result.slot >= handoffSlot { if existing := bs.reorderBuffer[result.slot]; existing != nil && !existing.FromLightbringer { @@ -3096,19 +3442,19 @@ func (bs *BlockSource) emitOrderedBlocks() { bs.reorderMu.Unlock() repairingSlot := bs.isLightbringerRepairSlot(blk.Slot) - if bs.sourceType == BlockSourceLightbringer { + if bs.usesLiveShredStream() { if blk.FromLightbringer { if repairingSlot { bs.clearLightbringerRepairSlot(blk.Slot) } if !bs.lightbringerActive.Swap(true) { - mlog.Log.Infof("BLOCK SOURCE SWITCH: RPC -> LIGHTBRINGER at slot %d | mode=%s", blk.Slot, bs.currentModeString()) + mlog.Log.Infof("BLOCK SOURCE SWITCH: RPC -> %s at slot %d | mode=%s", bs.liveShredStreamName(), blk.Slot, bs.currentModeString()) } } else if repairingSlot { bs.clearLightbringerRepairSlot(blk.Slot) - mlog.Log.Infof("BLOCK SOURCE STATUS: missing Lightbringer slot recovered via RPC at slot %d; staying on Lightbringer stream", blk.Slot) + mlog.Log.Infof("BLOCK SOURCE STATUS: missing streamed slot recovered via RPC at slot %d; staying on %s stream", blk.Slot, bs.liveShredStreamName()) } else if bs.lightbringerActive.Swap(false) { - mlog.Log.Warnf("BLOCK SOURCE SWITCH: LIGHTBRINGER -> RPC at slot %d | mode=%s", blk.Slot, bs.currentModeString()) + mlog.Log.Warnf("BLOCK SOURCE SWITCH: %s -> RPC at slot %d | mode=%s", bs.liveShredStreamName(), blk.Slot, bs.currentModeString()) } } @@ -3148,10 +3494,10 @@ func (bs *BlockSource) emitOrderedBlocks() { IsSkipped: true, } - if bs.sourceType == BlockSourceLightbringer { + if bs.usesLiveShredStream() { if bs.isLightbringerRepairSlot(skippedSlot) { bs.clearLightbringerRepairSlot(skippedSlot) - mlog.Log.Infof("BLOCK SOURCE STATUS: missing Lightbringer slot %d was confirmed skipped via RPC; staying on Lightbringer stream", skippedSlot) + mlog.Log.Infof("BLOCK SOURCE STATUS: missing streamed slot %d was confirmed skipped via RPC; staying on %s stream", skippedSlot, bs.liveShredStreamName()) } } bs.streamChan <- skipBlock @@ -3589,7 +3935,7 @@ func (bs *BlockSource) Start() { return } - if bs.sourceType == BlockSourceLightbringer && bs.lightbringerEndpoint != "" { + if bs.usesLiveShredStream() { bs.maybeStartLightbringerStream() } @@ -3674,7 +4020,7 @@ func (bs *BlockSource) fetchAndParseBlockSequential(slot uint64) (*b.Block, erro } blk = block.FromBlockResult(blockResult, slot, rpc) } - } else if bs.sourceType == BlockSourceLightbringer { + } else if bs.sourceType == BlockSourceLightbringer || bs.sourceType == BlockSourceTurbine { // Legacy sequential mode does not support the live stream handoff. // If this path is ever used, fall back to the RPC catchup fetcher. rpc := bs.getActiveRpc() @@ -3779,7 +4125,7 @@ type FetchStatsSnapshot struct { WaitingSlotRetries int // How many times the waiting slot has been retried InflightCount int // Number of slots currently being fetched RetryQueueLen int // Number of slots waiting to be retried - CurrentSource string // "rpc", "lightbringer", or "file" + CurrentSource string // "rpc", "lightbringer", "turbine", or "file" SourceStatus string // Human-readable description of current source state HandoffSlot uint64 // First slot at which Lightbringer can take over (0 = none pending) } @@ -3790,23 +4136,27 @@ func (bs *BlockSource) currentSourceSnapshot() (string, string, uint64) { return "file", "file playback", 0 case BlockSourceRpc: return "rpc", "rpc", 0 - case BlockSourceLightbringer: + case BlockSourceLightbringer, BlockSourceTurbine: + source := "lightbringer" + if bs.sourceType == BlockSourceTurbine { + source = "turbine" + } handoffSlot := bs.lightbringerHandoffSlot.Load() cooldownUntil := bs.lightbringerCooldownUntil.Load() connected := bs.lightbringerConnected.Load() lastStreamSlot := bs.lightbringerLastStreamSlot.Load() if bs.lightbringerActive.Load() { - return "lightbringer", "lightbringer live stream", handoffSlot + return source, source + " live stream", handoffSlot } if cooldownUntil != 0 { if connected && lastStreamSlot != 0 { - return "rpc", fmt.Sprintf("rpc, stabilising after lightbringer gap until slot %d (latest streamed slot %d)", cooldownUntil, lastStreamSlot), 0 + return "rpc", fmt.Sprintf("rpc, stabilising after %s gap until slot %d (latest streamed slot %d)", source, cooldownUntil, lastStreamSlot), 0 } - return "rpc", fmt.Sprintf("rpc, stabilising after lightbringer gap until slot %d", cooldownUntil), 0 + return "rpc", fmt.Sprintf("rpc, stabilising after %s gap until slot %d", source, cooldownUntil), 0 } if bs.isNearTip.Load() { if handoffSlot != 0 { - return "rpc", fmt.Sprintf("rpc, waiting for lightbringer handoff at slot %d", handoffSlot), handoffSlot + return "rpc", fmt.Sprintf("rpc, waiting for %s handoff at slot %d", source, handoffSlot), handoffSlot } if connected { bs.reorderMu.Lock() @@ -3815,17 +4165,17 @@ func (bs *BlockSource) currentSourceSnapshot() (string, string, uint64) { bs.reorderMu.Unlock() waitReason := bs.lightbringerHandoffWaitReason(waitingSlot, anchorSlot) if lastStreamSlot != 0 { - return "rpc", fmt.Sprintf("rpc, lightbringer connected (latest streamed slot %d); %s", lastStreamSlot, waitReason), 0 + return "rpc", fmt.Sprintf("rpc, %s connected (latest streamed slot %d); %s", source, lastStreamSlot, waitReason), 0 } - return "rpc", fmt.Sprintf("rpc, lightbringer connected; %s", waitReason), 0 + return "rpc", fmt.Sprintf("rpc, %s connected; %s", source, waitReason), 0 } if bs.lightbringerStarted.Load() { - return "rpc", "rpc, waiting for lightbringer stream connection", 0 + return "rpc", fmt.Sprintf("rpc, waiting for %s stream connection", source), 0 } - return "rpc", "rpc, starting lightbringer stream", 0 + return "rpc", fmt.Sprintf("rpc, starting %s stream", source), 0 } if connected && lastStreamSlot != 0 { - return "rpc", fmt.Sprintf("rpc catchup (lightbringer connected, latest streamed slot %d)", lastStreamSlot), 0 + return "rpc", fmt.Sprintf("rpc catchup (%s connected, latest streamed slot %d)", source, lastStreamSlot), 0 } return "rpc", "rpc catchup", 0 default: diff --git a/pkg/blockstream/block_source_test.go b/pkg/blockstream/block_source_test.go index 9b218013..ade6f572 100644 --- a/pkg/blockstream/block_source_test.go +++ b/pkg/blockstream/block_source_test.go @@ -34,6 +34,32 @@ func TestLightbringerBlockConnectsLocked(t *testing.T) { } } +func TestCurrentSourceSnapshotUsesTurbineSourceName(t *testing.T) { + bs := NewBlockSource(&BlockSourceOpts{ + SourceType: BlockSourceTurbine, + TurbineBindAddr: "127.0.0.1:0", + StartSlot: 100, + EndSlot: 200, + }) + + bs.isNearTip.Store(true) + bs.lightbringerConnected.Store(true) + bs.lightbringerLastStreamSlot.Store(150) + + source, status, handoff := bs.currentSourceSnapshot() + if source != "rpc" || handoff != 0 || !strings.Contains(status, "turbine connected") { + t.Fatalf("expected pre-handoff turbine status, got source=%q status=%q handoff=%d", source, status, handoff) + } + + bs.lightbringerActive.Store(true) + bs.lightbringerHandoffSlot.Store(151) + + source, status, handoff = bs.currentSourceSnapshot() + if source != "turbine" || status != "turbine live stream" || handoff != 151 { + t.Fatalf("expected active turbine status, got source=%q status=%q handoff=%d", source, status, handoff) + } +} + func TestForceRPCForLightbringerParentMismatchClearsBufferedState(t *testing.T) { bs := NewBlockSource(&BlockSourceOpts{ SourceType: BlockSourceLightbringer, @@ -94,6 +120,70 @@ func TestForceRPCForLightbringerParentMismatchClearsBufferedState(t *testing.T) } } +func TestHandleLiveShredStreamClosedForcesRPCAndInvalidatesBufferedRunway(t *testing.T) { + bs := NewBlockSource(&BlockSourceOpts{ + SourceType: BlockSourceTurbine, + TurbineBindAddr: "127.0.0.1:0", + StartSlot: 100, + EndSlot: 200, + }) + + bs.isNearTip.Store(true) + bs.lightbringerConnected.Store(true) + bs.lightbringerHandoffSlot.Store(121) + bs.lightbringerActive.Store(true) + bs.nextSlotToSend = 122 + bs.lastEmittedBlockSlot = 121 + bs.reorderBuffer[123] = &b.Block{Slot: 123, FromLightbringer: true, SourceParentSlot: 122} + bs.reorderBuffer[124] = &b.Block{Slot: 124, FromLightbringer: true, SourceParentSlot: 123} + bs.reorderBuffer[125] = &b.Block{Slot: 125, FromLightbringer: false} + bs.slotState[123] = slotDone + bs.slotState[124] = slotDone + bs.slotState[125] = slotDone + bs.lightbringerBuffer[126] = &b.Block{Slot: 126, FromLightbringer: true, SourceParentSlot: 125} + bs.lightbringerBufferOrder = []uint64{126} + oldGeneration := bs.lightbringerResultGeneration.Load() + + bs.handleLiveShredStreamClosed("test reconnect") + + if got := bs.lightbringerResultGeneration.Load(); got != oldGeneration+1 { + t.Fatalf("expected live stream generation to advance, got %d want %d", got, oldGeneration+1) + } + if got := bs.lightbringerForceRPCUntil.Load(); got != 122 { + t.Fatalf("expected RPC to be forced from waiting slot 122, got %d", got) + } + if got := bs.lightbringerHandoffSlot.Load(); got != 0 { + t.Fatalf("expected handoff slot to be cleared, got %d", got) + } + if bs.lightbringerActive.Load() { + t.Fatalf("expected turbine to be marked inactive after stream close") + } + if !bs.lightbringerNeedRPCResume.Load() { + t.Fatalf("expected RPC resume flag after stream close") + } + if _, exists := bs.reorderBuffer[123]; exists { + t.Fatalf("expected stale turbine slot 123 to be removed from reorder buffer") + } + if _, exists := bs.reorderBuffer[124]; exists { + t.Fatalf("expected stale turbine slot 124 to be removed from reorder buffer") + } + if _, exists := bs.reorderBuffer[125]; !exists { + t.Fatalf("expected buffered RPC slot 125 to remain") + } + if _, exists := bs.slotState[123]; exists { + t.Fatalf("expected stale turbine slot state 123 to be cleared") + } + if _, exists := bs.slotState[124]; exists { + t.Fatalf("expected stale turbine slot state 124 to be cleared") + } + if _, exists := bs.slotState[125]; !exists { + t.Fatalf("expected RPC slot state 125 to remain") + } + if len(bs.lightbringerBuffer) != 0 || len(bs.lightbringerBufferOrder) != 0 { + t.Fatalf("expected prefetched turbine buffer to be cleared") + } +} + func TestPrepareLightbringerHandoffAllowsSkippedGapFromParentSlot(t *testing.T) { bs := NewBlockSource(&BlockSourceOpts{ SourceType: BlockSourceLightbringer, @@ -200,6 +290,40 @@ func TestPrepareLightbringerHandoffAllowsLiveEdgeRunwayAtTip(t *testing.T) { } } +func TestPrepareTurbineHandoffAllowsLiveEdgeRunwayAtTipWithoutConsensusBuffering(t *testing.T) { + bs := NewBlockSource(&BlockSourceOpts{ + SourceType: BlockSourceTurbine, + TurbineBindAddr: "127.0.0.1:8001", + StartSlot: 100, + EndSlot: 200, + }) + + bs.isNearTip.Store(true) + bs.lightbringerConnected.Store(true) + bs.lastExecutedSlot.Store(150) + bs.confirmedTip.Store(151) + bs.lightbringerLastStreamSlot.Store(151) + bs.lastEmittedBlockSlot = 150 + bs.lightbringerBuffer[151] = &b.Block{Slot: 151, FromLightbringer: true, SourceParentSlot: 150} + bs.lightbringerBufferOrder = append(bs.lightbringerBufferOrder, 151) + + reason := bs.lightbringerHandoffWaitReason(151, 150) + if !strings.Contains(reason, "handoff-ready runway buffered through slot 151") { + t.Fatalf("expected live-edge turbine runway to be handoff-ready, got %q", reason) + } + + blocks, handoffSlot, prepared := bs.prepareLightbringerHandoff(151, 150) + if !prepared { + t.Fatalf("expected turbine handoff to prepare at the live edge") + } + if handoffSlot != 151 { + t.Fatalf("expected handoff slot 151, got %d", handoffSlot) + } + if len(blocks) != 1 || blocks[0].Slot != 151 { + t.Fatalf("expected single live-edge turbine block to be enqueued, got %+v", blocks) + } +} + func TestPrepareLightbringerHandoffKeepsMinimumRunwayWhenLightbringerLagsTip(t *testing.T) { bs := NewBlockSource(&BlockSourceOpts{ SourceType: BlockSourceLightbringer, @@ -930,6 +1054,59 @@ func TestForceRPCForCatchupRewindsConsensusManagedFrontier(t *testing.T) { } } +func TestForceRPCFallbackRewindsConsensusManagedTurbineFrontier(t *testing.T) { + bs := NewBlockSource(&BlockSourceOpts{ + SourceType: BlockSourceTurbine, + TurbineBindAddr: "127.0.0.1:8001", + StartSlot: 100, + EndSlot: 200, + ConsensusManagedLightbringer: true, + }) + + bs.lightbringerActive.Store(true) + bs.lightbringerHandoffSlot.Store(121) + bs.lastExecutedSlot.Store(120) + bs.confirmedTip.Store(180) + bs.nextSlotToSend = 150 + bs.reorderBuffer[121] = &b.Block{Slot: 121, FromLightbringer: true} + bs.reorderBuffer[149] = &b.Block{Slot: 149, FromLightbringer: true} + bs.reorderBuffer[151] = &b.Block{Slot: 151, FromLightbringer: false} + bs.slotState[121] = slotDone + bs.slotState[149] = slotDone + bs.slotState[151] = slotInflight + bs.retrySlots = []uint64{119, 121, 149, 151} + + bs.ForceRPCFallback("consensus_depth_exceeded") + + if got := bs.nextSlotToSend; got != 121 { + t.Fatalf("expected RPC fallback frontier to rewind to replay's next slot 121, got %d", got) + } + if bs.lightbringerActive.Load() { + t.Fatalf("expected turbine to be marked inactive") + } + if got := bs.lightbringerHandoffSlot.Load(); got != 0 { + t.Fatalf("expected turbine handoff to be cleared, got %d", got) + } + if !bs.lightbringerNeedRPCResume.Load() { + t.Fatalf("expected scheduler to resume RPC from the rewound frontier") + } + if _, exists := bs.reorderBuffer[121]; exists { + t.Fatalf("expected turbine slot 121 to be dropped for RPC refetch") + } + if _, exists := bs.reorderBuffer[149]; exists { + t.Fatalf("expected turbine slot 149 to be dropped for RPC refetch") + } + if _, exists := bs.reorderBuffer[151]; !exists { + t.Fatalf("expected buffered RPC slot 151 to remain") + } + if _, exists := bs.slotState[151]; exists { + t.Fatalf("expected future slot state to be cleared for RPC rescheduling") + } + if len(bs.retrySlots) != 1 || bs.retrySlots[0] != 119 { + t.Fatalf("expected only retries before the replay frontier to remain, got %+v", bs.retrySlots) + } +} + func TestForceRPCForCatchupKeepsPendingHandoffEmissionFrontier(t *testing.T) { bs := NewBlockSource(&BlockSourceOpts{ SourceType: BlockSourceLightbringer, @@ -1005,6 +1182,45 @@ func TestEmitOrderedBlocksDirectlyStreamsConsensusManagedLightbringerObservation } } +func TestEmitOrderedBlocksDropsStaleLiveStreamGeneration(t *testing.T) { + bs := NewBlockSource(&BlockSourceOpts{ + SourceType: BlockSourceTurbine, + TurbineBindAddr: "127.0.0.1:0", + StartSlot: 100, + EndSlot: 200, + }) + + bs.isNearTip.Store(true) + bs.lightbringerActive.Store(true) + bs.lightbringerHandoffSlot.Store(100) + staleGeneration := bs.lightbringerResultGeneration.Load() + bs.invalidateLightbringerResults() + + done := make(chan struct{}) + go func() { + bs.emitOrderedBlocks() + close(done) + }() + + bs.resultQueue <- fetchResult{ + slot: 100, + block: &b.Block{Slot: 100, FromLightbringer: true, SourceParentSlot: 99}, + liveStreamGeneration: staleGeneration, + } + close(bs.resultQueue) + <-done + + if len(bs.streamChan) != 0 { + t.Fatalf("expected stale turbine result to be dropped without emission") + } + if _, exists := bs.reorderBuffer[100]; exists { + t.Fatalf("expected stale turbine result not to enter reorder buffer") + } + if got := bs.nextSlotToSend; got != 100 { + t.Fatalf("expected stale turbine result to leave emission frontier at 100, got %d", got) + } +} + func TestEmitOrderedBlocksDropsResultsBehindEmissionFrontier(t *testing.T) { bs := NewBlockSource(&BlockSourceOpts{ SourceType: BlockSourceRpc, diff --git a/pkg/config/config.go b/pkg/config/config.go index 8b9dedc7..b4fe0bda 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -50,13 +50,19 @@ type DebugConfig struct { // DevelopmentConfig holds development/tuning configuration (matches Firedancer [development] section) type DevelopmentConfig struct { - ZstdDecoderConcurrency int `toml:"zstd_decoder_concurrency" mapstructure:"zstd_decoder_concurrency"` // was: zstd-decoder-concurrency - MaxConcurrentFlushers int `toml:"max_concurrent_flushers" mapstructure:"max_concurrent_flushers"` // was: max-concurrent-flushers - ParamArenaSizeMB uint64 `toml:"param_arena_size_mb" mapstructure:"param_arena_size_mb"` // was: param-arena-size-mb - BorrowedAccountArenaSize uint64 `toml:"borrowed_account_arena_size" mapstructure:"borrowed_account_arena_size"` // was: borrowed-account-arena-size - UsePool bool `toml:"use_pool" mapstructure:"use_pool"` // was: use-pool - Pprof PprofConfig `toml:"pprof" mapstructure:"pprof"` - Debug DebugConfig `toml:"debug" mapstructure:"debug"` + ZstdDecoderConcurrency int `toml:"zstd_decoder_concurrency" mapstructure:"zstd_decoder_concurrency"` // was: zstd-decoder-concurrency + MaxConcurrentFlushers int `toml:"max_concurrent_flushers" mapstructure:"max_concurrent_flushers"` // was: max-concurrent-flushers + SnapshotAppendVecWorkers int `toml:"snapshot_append_vec_workers" mapstructure:"snapshot_append_vec_workers"` // Snapshot appendvec write workers + SnapshotIndexBuilderWorkers int `toml:"snapshot_index_builder_workers" mapstructure:"snapshot_index_builder_workers"` // Snapshot index parsing workers + SnapshotIndexCommitterWorkers int `toml:"snapshot_index_committer_workers" mapstructure:"snapshot_index_committer_workers"` // Snapshot index shard enqueue workers + SnapshotIndexShards int `toml:"snapshot_index_shards" mapstructure:"snapshot_index_shards"` // Snapshot account-index shard count + SnapshotIndexTempDir string `toml:"snapshot_index_temp_dir" mapstructure:"snapshot_index_temp_dir"` // Optional temp dir for snapshot index shard logs/SST staging + ParamArenaSizeMB uint64 `toml:"param_arena_size_mb" mapstructure:"param_arena_size_mb"` // was: param-arena-size-mb + BorrowedAccountArenaSize uint64 `toml:"borrowed_account_arena_size" mapstructure:"borrowed_account_arena_size"` // was: borrowed-account-arena-size + UsePool bool `toml:"use_pool" mapstructure:"use_pool"` // was: use-pool + Pprof PprofConfig `toml:"pprof" mapstructure:"pprof"` + ProgramCacheMaxMB int `toml:"program_cache_max_mb" mapstructure:"program_cache_max_mb"` // Approximate SBPF program cache size in MiB + Debug DebugConfig `toml:"debug" mapstructure:"debug"` } // ReportingConfig holds metrics/reporting configuration (matches Firedancer [reporting] section) @@ -66,8 +72,9 @@ type ReportingConfig struct { // BlockConfig holds block source configuration type BlockConfig struct { - Source string `toml:"source" mapstructure:"source"` // "rpc" or "lightbringer" + Source string `toml:"source" mapstructure:"source"` // "rpc", "lightbringer", or "turbine" LightbringerEndpoint string `toml:"lightbringer_endpoint" mapstructure:"lightbringer_endpoint"` // Lightbringer endpoint (optional) + TurbineBindAddr string `toml:"turbine_bind_addr" mapstructure:"turbine_bind_addr"` // Native turbine UDP receiver bind address (optional) // Global fetch tuning MaxRPS int `toml:"max_rps" mapstructure:"max_rps"` // Rate limit (requests per second) @@ -87,6 +94,15 @@ type BlockConfig struct { NearTipLookahead int `toml:"near_tip_lookahead" mapstructure:"near_tip_lookahead"` // Slots ahead to schedule } +// TurbineConfig holds native gossip/turbine receiver configuration. +type TurbineConfig struct { + BindAddr string `toml:"bind_addr" mapstructure:"bind_addr"` // UDP address for incoming turbine shreds + GossipEntrypoint string `toml:"gossip_entrypoint" mapstructure:"gossip_entrypoint"` // Solana gossip entrypoint for turbine tree joining + GossipBindAddr string `toml:"gossip_bind_addr" mapstructure:"gossip_bind_addr"` // UDP address for Mithril gossip traffic + AdvertisedIP string `toml:"advertised_ip" mapstructure:"advertised_ip"` // Public IP to advertise in gossip + ShredVersion uint16 `toml:"shred_version" mapstructure:"shred_version"` // Optional override; normally discovered from entrypoint +} + // SnapshotConfig holds snapshot download configuration type SnapshotConfig struct { // MaxFullSnapshots controls both saving and retention: @@ -122,6 +138,7 @@ type SnapshotConfig struct { TCPTimeoutMs int `toml:"tcp_timeout_ms" mapstructure:"tcp_timeout_ms"` MinNodeVersion string `toml:"min_node_version" mapstructure:"min_node_version"` AllowedNodeVersions []string `toml:"allowed_node_versions" mapstructure:"allowed_node_versions"` + NodeBlacklist []string `toml:"node_blacklist" mapstructure:"node_blacklist"` // Snapshot age thresholds FullThreshold int `toml:"full_threshold" mapstructure:"full_threshold"` @@ -178,7 +195,7 @@ type LogConfig struct { type ConsensusConfig struct { SkipPathMaxDepth int `toml:"skip_path_max_depth" mapstructure:"skip_path_max_depth"` // Max slots the skip-path solver explores (default: 64) UnresolvedPolicy string `toml:"unresolved_policy" mapstructure:"unresolved_policy"` // "halt" or "warn" (default: "halt") - EnforceOnSource string `toml:"enforce_on_source" mapstructure:"enforce_on_source"` // "lightbringer" or "all" (default: "lightbringer") + EnforceOnSource string `toml:"enforce_on_source" mapstructure:"enforce_on_source"` // "lightbringer", "turbine", "stream", or "all" (default: "stream") } // Config holds all configuration options for Mithril (Firedancer-style hierarchy) @@ -194,6 +211,7 @@ type Config struct { Block BlockConfig `toml:"block" mapstructure:"block"` Consensus ConsensusConfig `toml:"consensus" mapstructure:"consensus"` Lightbringer LightbringerConfig `toml:"lightbringer" mapstructure:"lightbringer"` + Turbine TurbineConfig `toml:"turbine" mapstructure:"turbine"` Snapshot SnapshotConfig `toml:"snapshot" mapstructure:"snapshot"` Development DevelopmentConfig `toml:"development" mapstructure:"development"` Reporting ReportingConfig `toml:"reporting" mapstructure:"reporting"` diff --git a/pkg/forkchoice/vote_parser.go b/pkg/forkchoice/vote_parser.go index 7f6a0ed0..557abba6 100644 --- a/pkg/forkchoice/vote_parser.go +++ b/pkg/forkchoice/vote_parser.go @@ -32,7 +32,14 @@ func parseAndValidateVoteTx(tx *solana.Transaction, authorizedVoters *epochstake return nil, false } -func parseAndValidateVoteInstruction(tx *solana.Transaction, instr solana.CompiledInstruction, authorizedVoters *epochstakes.EpochAuthorizedVotersCache) (*voteInfo, bool) { +func parseAndValidateVoteInstruction(tx *solana.Transaction, instr solana.CompiledInstruction, authorizedVoters *epochstakes.EpochAuthorizedVotersCache) (info *voteInfo, ok bool) { + defer func() { + if recover() != nil { + info = nil + ok = false + } + }() + if len(instr.Accounts) < 1 { return nil, false } diff --git a/pkg/gossip/bitvec.go b/pkg/gossip/bitvec.go deleted file mode 100644 index a990256e..00000000 --- a/pkg/gossip/bitvec.go +++ /dev/null @@ -1,64 +0,0 @@ -package gossip - -func MakeBitVecU8(bits []byte, len int) BitVecU8 { - if len <= 0 { - return BitVecU8{ - Bits: BitVecU8Inner{Value: nil}, - Len: 0, - } - } - return BitVecU8{ - Bits: BitVecU8Inner{Value: &bits}, - Len: uint64(len), - } -} - -func (bv *BitVecU8) Get(pos uint64) bool { - if pos >= bv.Len { - panic("get bit out of bounds") - } - return (*bv.Bits.Value)[pos/8]&(1<<(pos%8)) != 0 -} - -func (bv *BitVecU8) Set(pos uint64, b bool) { - if pos >= bv.Len { - panic("get bit out of bounds") - } - if b { - (*bv.Bits.Value)[pos/8] |= 1 << (pos % 8) - } else { - (*bv.Bits.Value)[pos/8] &= ^uint8(1 << (pos % 8)) - } -} - -func MakeBitVecU64(bits []uint64, len int) BitVecU64 { - if len <= 0 { - return BitVecU64{ - Bits: BitVecU64Inner{Value: nil}, - Len: 0, - } - } - return BitVecU64{ - Bits: BitVecU64Inner{Value: &bits}, - Len: uint64(len), - } -} - -func (bv *BitVecU64) Get(pos uint64) bool { - if pos >= bv.Len { - panic("get bit out of bounds") - } - return (*bv.Bits.Value)[pos/64]&(1<<(pos%64)) != 0 -} - -func (bv *BitVecU64) Set(pos uint64, b bool) { - if pos >= bv.Len { - panic("get bit out of bounds") - } - bits := *bv.Bits.Value - if b { - bits[pos/64] |= 1 << (pos % 64) - } else { - bits[pos/64] &= ^uint64(1 << (pos % 64)) - } -} diff --git a/pkg/gossip/bloom.go b/pkg/gossip/bloom.go deleted file mode 100644 index f7529af1..00000000 --- a/pkg/gossip/bloom.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2022 Solana Foundation. -// Go port by Richard Patel -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package gossip - -// Original Rust source: https://crates.io/crates/solana-bloom - -import ( - "math" - "math/rand" -) - -const MaxBloomSize = 928 - -func NewBloom(numBits uint64, keys []uint64) *Bloom { - bits := make([]uint64, (numBits+63)/64) - ret := &Bloom{ - Keys: keys, - Bits: BitVecU64{ - Bits: BitVecU64Inner{Value: &bits}, - Len: numBits, - }, - NumBitsSet: 0, - } - return ret -} - -func NewBloomRandom(numItems uint64, falseRate float64, maxBits uint64) *Bloom { - m := BloomNumBits(float64(numItems), falseRate) - numBits := uint64(m) - if maxBits < numBits { - numBits = maxBits - } - if numBits == 0 { - numBits = 1 - } - numKeys := uint64(BloomNumKeys(float64(numBits), float64(numItems))) - keys := make([]uint64, numKeys) - for i := range keys { - keys[i] = rand.Uint64() - } - return NewBloom(numBits, keys) -} - -func BloomNumBits(n, p float64) float64 { - return math.Ceil((n * math.Log(p)) / math.Log(1/math.Pow(2, math.Log(2)))) -} - -func BloomNumKeys(m, n float64) float64 { - if n == 0 { - return 0 - } - return math.Max(1, math.Round((m/n)*math.Log(2))) -} - -func BloomMaxItems(m, p, k float64) float64 { - return math.Ceil(m / (-k / math.Log(1-math.Exp(math.Log(p)/k)))) -} - -func BloomMaskBits(numItems, maxItems float64) uint32 { - return uint32(math.Max(math.Ceil(math.Log2(numItems/maxItems)), 0)) -} - -func (b *Bloom) Pos(key *Hash, k uint64) uint64 { - return FNV1a(key[:], k) % b.Bits.Len -} - -func (b *Bloom) Clear() { - bits := *b.Bits.Bits.Value - for i := range bits { - bits[i] = 0 - } - b.NumBitsSet = 0 -} - -func (b *Bloom) Add(key *Hash) { - for _, k := range b.Keys { - pos := b.Pos(key, k) - if !b.Bits.Get(pos) { - b.NumBitsSet += 1 - b.Bits.Set(pos, true) - } - } -} - -func (b *Bloom) Contains(key *Hash) bool { - for _, k := range b.Keys { - if !b.Bits.Get(b.Pos(key, k)) { - return false - } - } - return true -} - -func FNV1a(slice []byte, hash uint64) uint64 { - for _, c := range slice { - hash ^= uint64(c) - hash *= 1099511628211 - } - return hash -} diff --git a/pkg/gossip/bloom_test.go b/pkg/gossip/bloom_test.go deleted file mode 100644 index ea21b997..00000000 --- a/pkg/gossip/bloom_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package gossip - -import ( - "crypto/sha256" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewBloomRandom(t *testing.T) { - cases := []struct { - name string - numItems uint64 - falseRate float64 - maxBits uint64 - - wantKeys int - wantBits uint64 - }{ - { - name: "Empty", - numItems: 0, falseRate: 0.1, maxBits: 100, - wantKeys: 0, wantBits: 1, - }, - { - name: "Random", - numItems: 10, falseRate: 0.1, maxBits: 100, - wantKeys: 3, wantBits: 48, - }, - { - name: "Random", - numItems: 100, falseRate: 0.1, maxBits: 100, - wantKeys: 1, wantBits: 100, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - bloom := NewBloomRandom(tc.numItems, tc.falseRate, tc.maxBits) - require.NotNil(t, bloom) - assert.Equal(t, tc.wantKeys, len(bloom.Keys)) - assert.Equal(t, tc.wantBits, bloom.Bits.Len) - }) - } -} - -func TestBloom_FilterMath(t *testing.T) { - assert.Equal(t, uint64(480), uint64(BloomNumBits(100, 0.1))) - assert.Equal(t, uint64(959), uint64(BloomNumBits(100, 0.01))) - assert.Equal(t, uint64(14), uint64(BloomNumKeys(1000, 50))) - assert.Equal(t, uint64(28), uint64(BloomNumKeys(2000, 50))) - assert.Equal(t, uint64(55), uint64(BloomNumKeys(2000, 25))) - assert.Equal(t, uint64(1), uint64(BloomNumKeys(20, 1000))) -} - -func TestBloom_AddContains(t *testing.T) { - bloom := NewBloomRandom(100, 0.1, 100) - require.NotNil(t, bloom) - // known keys to avoid false positives in the test - bloom.Keys = []uint64{0, 1, 2, 3} - - var key Hash - - key = sha256.Sum256([]byte("hello")) - assert.False(t, bloom.Contains(&key)) - bloom.Add(&key) - assert.True(t, bloom.Contains(&key)) - - key = sha256.Sum256([]byte("world")) - assert.False(t, bloom.Contains(&key)) - bloom.Add(&key) - assert.True(t, bloom.Contains(&key)) -} - -func TestBloom_Randomness(t *testing.T) { - b1 := NewBloomRandom(10, 0.1, 100) - b2 := NewBloomRandom(10, 0.1, 100) - require.NotNil(t, b1) - require.NotNil(t, b2) - - sort.Slice(b1.Keys, func(i, j int) bool { return b1.Keys[i] < b1.Keys[j] }) - sort.Slice(b2.Keys, func(i, j int) bool { return b2.Keys[i] < b2.Keys[j] }) - - assert.NotEqual(t, b1.Keys, b2.Keys) -} - -func TestBloom_MaxItems(t *testing.T) { - assert.Equal(t, float64(9), BloomMaxItems(80, 0.01, 8)) -} diff --git a/pkg/gossip/client.go b/pkg/gossip/client.go index 77d9f6c0..efb9de2c 100644 --- a/pkg/gossip/client.go +++ b/pkg/gossip/client.go @@ -2,102 +2,650 @@ package gossip import ( "context" + "crypto/ed25519" + "crypto/rand" + "errors" + "fmt" "net" - "net/netip" + "sync" "sync/atomic" + "time" + + "github.com/Overclock-Validator/mithril/pkg/mlog" +) + +const ( + DefaultBindAddr = "0.0.0.0:0" + defaultPushInterval = 2 * time.Second + defaultPingInterval = 10 * time.Second + defaultEchoTimeout = 5 * time.Second + maxKnownGossipPeers = 128 + peerExpirationWindow = 15 * time.Minute ) -// Driver implements the network main loop. -// -// Note: This uses Go 1.19 standard library networking, which processes packets one-by-one. (slow!) -// Looks like Go 1.20 will add batch packet receive: https://github.com/golang/go/issues/45886 -type Driver struct { - handler *Handler - so *net.UDPConn +type Config struct { + Entrypoint string + BindAddr string + TVUAddr string + AdvertisedIP string + ShredVersion uint16 + Identity ed25519.PrivateKey + PushInterval time.Duration + PingInterval time.Duration + EntrypointTimeout time.Duration + Name string +} + +type Client struct { + cfg Config + + entrypoint *net.UDPAddr + bindAddr *net.UDPAddr + tvuAddr *net.UDPAddr + identity ed25519.PrivateKey + pubkey Pubkey + + contactMu sync.RWMutex + contact *ContactInfo + + peerMu sync.Mutex + peers map[udpAddrKey]knownPeer + + repairPeerMu sync.Mutex + repairPeers map[Pubkey]RepairPeer + + rxPackets atomic.Uint64 + txPackets atomic.Uint64 + rxDecodeErrors atomic.Uint64 + txErrors atomic.Uint64 + rxPings atomic.Uint64 + rxPongs atomic.Uint64 + rxContacts atomic.Uint64 + acceptedContacts atomic.Uint64 + rxPullRequests atomic.Uint64 + txPushMessages atomic.Uint64 + txPingMessages atomic.Uint64 + txPongMessages atomic.Uint64 + txPullResponses atomic.Uint64 + lastRxUnix atomic.Int64 + lastTxUnix atomic.Int64 +} + +type knownPeer struct { + addr *net.UDPAddr + lastSeen time.Time +} + +type udpAddrKey struct { + ip [16]byte + port int +} + +type RepairPeer struct { + Pubkey Pubkey + Addr *net.UDPAddr + LastSeen time.Time +} + +type Stats struct { + RxPackets uint64 + TxPackets uint64 + RxDecodeErrors uint64 + TxErrors uint64 + RxPings uint64 + RxPongs uint64 + RxContacts uint64 + AcceptedContacts uint64 + RxPullRequests uint64 + TxPushMessages uint64 + TxPingMessages uint64 + TxPongMessages uint64 + TxPullResponses uint64 + LastRxUnix int64 + LastTxUnix int64 + Peers int + RepairPeers int +} + +func NewClient(cfg Config) (*Client, error) { + if cfg.Name == "" { + cfg.Name = ClientName + } + if cfg.Entrypoint == "" { + return nil, fmt.Errorf("gossip entrypoint is required") + } + if cfg.BindAddr == "" { + cfg.BindAddr = DefaultBindAddr + } + if cfg.PushInterval == 0 { + cfg.PushInterval = defaultPushInterval + } + if cfg.PingInterval == 0 { + cfg.PingInterval = defaultPingInterval + } + if cfg.EntrypointTimeout == 0 { + cfg.EntrypointTimeout = defaultEchoTimeout + } + entrypoint, err := net.ResolveUDPAddr("udp", cfg.Entrypoint) + if err != nil { + return nil, fmt.Errorf("resolve gossip entrypoint: %w", err) + } + bindAddr, err := net.ResolveUDPAddr("udp", cfg.BindAddr) + if err != nil { + return nil, fmt.Errorf("resolve gossip bind address: %w", err) + } + tvuAddr, err := net.ResolveUDPAddr("udp", cfg.TVUAddr) + if err != nil { + return nil, fmt.Errorf("resolve TVU address: %w", err) + } + identity := cfg.Identity + if len(identity) == 0 { + _, identity, err = ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate gossip identity: %w", err) + } + } + if len(identity) != ed25519.PrivateKeySize { + return nil, fmt.Errorf("invalid gossip identity size %d", len(identity)) + } + pubkey, err := pubkeyFromPrivateKey(identity) + if err != nil { + return nil, err + } + + return &Client{ + cfg: cfg, + entrypoint: entrypoint, + bindAddr: bindAddr, + tvuAddr: tvuAddr, + identity: identity, + pubkey: pubkey, + peers: make(map[udpAddrKey]knownPeer), + repairPeers: make(map[Pubkey]RepairPeer), + }, nil +} + +func (c *Client) Pubkey() Pubkey { + return c.pubkey } -func NewDriver(handler *Handler, so *net.UDPConn) *Driver { - return &Driver{ - handler: handler, - so: so, +func (c *Client) Identity() ed25519.PrivateKey { + return append(ed25519.PrivateKey(nil), c.identity...) +} + +func (c *Client) ShredVersion() uint16 { + c.contactMu.RLock() + defer c.contactMu.RUnlock() + if c.contact == nil { + return c.cfg.ShredVersion } + return c.contact.ShredVer } -// Run processes packets until the context is cancelled. -// -// Destroys all handlers and closes the socket after returning. -// Returns any network error or nil if the context closed. -func (c *Driver) Run(ctx context.Context) error { - defer c.handler.Close() +func (c *Client) Run(ctx context.Context) error { + conn, err := net.ListenUDP("udp", c.bindAddr) + if err != nil { + return fmt.Errorf("bind gossip socket %s: %w", c.bindAddr, err) + } + defer conn.Close() + + if err := c.initializeContact(conn.LocalAddr().(*net.UDPAddr)); err != nil { + return err + } + c.contactMu.RLock() + contact := c.contact + c.contactMu.RUnlock() + if contact != nil { + mlog.Log.Infof("gossip client listening: local=%s advertised_gossip=%s advertised_tvu=%s shred_version=%d client=%s", + conn.LocalAddr().String(), contact.GossipAddr.String(), contact.TVUAddr.String(), contact.ShredVer, c.cfg.Name) + } + c.recordPeer(c.entrypoint) ctx, cancel := context.WithCancel(ctx) defer cancel() - - var graceful atomic.Bool - // Monitor close signal go func() { - defer c.so.Close() - defer graceful.Store(true) <-ctx.Done() + _ = conn.Close() + }() + + errCh := make(chan error, 1) + go func() { + errCh <- c.recvLoop(ctx, conn) }() - var buf [PacketSize]byte + if err := c.pushContact(conn); err != nil { + return err + } + _ = c.sendPing(conn, c.entrypoint) + + pushTicker := time.NewTicker(c.cfg.PushInterval) + defer pushTicker.Stop() + pingTicker := time.NewTicker(c.cfg.PingInterval) + defer pingTicker.Stop() + statsTicker := time.NewTicker(10 * time.Second) + defer statsTicker.Stop() + for { - n, _, _, addr, err := c.so.ReadMsgUDPAddrPort(buf[:], nil) - if n > 0 { - c.handler.HandlePacket(buf[:n], addr) + select { + case <-ctx.Done(): + return nil + case err := <-errCh: + if err == nil || ctx.Err() != nil { + return nil + } + return err + case <-pushTicker.C: + if err := c.pushContact(conn); err != nil { + return err + } + case <-pingTicker.C: + _ = c.sendPing(conn, c.entrypoint) + case <-statsTicker.C: + stats := c.Stats() + mlog.Log.FileOnlyf("gossip client stats: rx=%d tx=%d peers=%d repair_peers=%d decode_errors=%d tx_errors=%d pings=%d/%d pongs=%d/%d contacts_rx=%d contacts_accepted=%d pull_requests=%d pull_responses=%d last_rx=%s last_tx=%s", + stats.RxPackets, stats.TxPackets, stats.Peers, stats.RepairPeers, stats.RxDecodeErrors, stats.TxErrors, + stats.RxPings, stats.TxPingMessages, stats.RxPongs, stats.TxPongMessages, + stats.RxContacts, stats.AcceptedContacts, stats.RxPullRequests, stats.TxPullResponses, + formatAge(stats.LastRxUnix), formatAge(stats.LastTxUnix)) + } + } +} + +func (c *Client) Stats() Stats { + c.peerMu.Lock() + peers := len(c.peers) + c.peerMu.Unlock() + c.repairPeerMu.Lock() + repairPeers := len(c.repairPeers) + c.repairPeerMu.Unlock() + return Stats{ + RxPackets: c.rxPackets.Load(), + TxPackets: c.txPackets.Load(), + RxDecodeErrors: c.rxDecodeErrors.Load(), + TxErrors: c.txErrors.Load(), + RxPings: c.rxPings.Load(), + RxPongs: c.rxPongs.Load(), + RxContacts: c.rxContacts.Load(), + AcceptedContacts: c.acceptedContacts.Load(), + RxPullRequests: c.rxPullRequests.Load(), + TxPushMessages: c.txPushMessages.Load(), + TxPingMessages: c.txPingMessages.Load(), + TxPongMessages: c.txPongMessages.Load(), + TxPullResponses: c.txPullResponses.Load(), + LastRxUnix: c.lastRxUnix.Load(), + LastTxUnix: c.lastTxUnix.Load(), + Peers: peers, + RepairPeers: repairPeers, + } +} + +func (c *Client) RepairPeers() []RepairPeer { + now := time.Now() + c.repairPeerMu.Lock() + defer c.repairPeerMu.Unlock() + for key, peer := range c.repairPeers { + if now.Sub(peer.LastSeen) > peerExpirationWindow { + delete(c.repairPeers, key) + } + } + peers := make([]RepairPeer, 0, len(c.repairPeers)) + for _, peer := range c.repairPeers { + peers = append(peers, peer) + } + return peers +} + +func (c *Client) initializeContact(localGossipAddr *net.UDPAddr) error { + shredVersion := c.cfg.ShredVersion + var advertisedIP net.IP + if c.cfg.AdvertisedIP != "" { + advertisedIP = net.ParseIP(c.cfg.AdvertisedIP) + if advertisedIP == nil { + return fmt.Errorf("invalid advertised IP %q", c.cfg.AdvertisedIP) } + } + + if shredVersion == 0 || advertisedIP == nil { + echo, err := QueryEntrypoint(c.entrypoint, c.cfg.EntrypointTimeout) if err != nil { - if graceful.Load() { - return nil + if shredVersion == 0 { + return fmt.Errorf("query gossip entrypoint %s: %w", c.entrypoint.String(), err) + } + } else { + if shredVersion == 0 { + shredVersion = echo.ShredVersion } + if advertisedIP == nil { + advertisedIP = echo.Address + } + } + } + if advertisedIP == nil { + advertisedIP = localGossipAddr.IP + } + advertisedIP = normalizedIP(advertisedIP) + if advertisedIP == nil || advertisedIP.IsUnspecified() { + return fmt.Errorf("could not determine advertised gossip IP") + } + if shredVersion == 0 { + return fmt.Errorf("could not determine shred version") + } + + gossipAddr := &net.UDPAddr{IP: advertisedIP, Port: localGossipAddr.Port} + tvuAddr := &net.UDPAddr{IP: advertisedIP, Port: c.tvuAddr.Port} + if tvuIP := normalizedIP(c.tvuAddr.IP); tvuIP != nil && !tvuIP.IsUnspecified() { + tvuAddr.IP = tvuIP + } + + contact, err := NewContactInfo(c.pubkey, shredVersion, gossipAddr, tvuAddr) + if err != nil { + return err + } + c.contactMu.Lock() + c.contact = contact + c.contactMu.Unlock() + return nil +} + +func (c *Client) currentContactValue() (CrdsValue, error) { + c.contactMu.RLock() + contact := c.contact + c.contactMu.RUnlock() + if contact == nil { + return CrdsValue{}, fmt.Errorf("gossip contact info is not initialized") + } + return signCrdsContactInfo(contact.CloneWithWallclock(wallclockMillis()), c.identity) +} + +func (c *Client) pushContact(conn *net.UDPConn) error { + value, err := c.currentContactValue() + if err != nil { + return err + } + packet, err := encodePushMessage(c.pubkey, []CrdsValue{value}) + if err != nil { + return err + } + for _, peer := range c.currentPeers() { + if err := sendUDP(conn, packet, peer); err != nil { + c.txErrors.Add(1) return err } + c.recordTx() + c.txPushMessages.Add(1) } + return nil } -// Handler is a network-agnostic multiplexer for incoming gossip messages. -type Handler struct { - *PullClient - *PingClient - *PingServer +func (c *Client) sendPullResponse(conn *net.UDPConn, addr *net.UDPAddr) error { + value, err := c.currentContactValue() + if err != nil { + return err + } + packet, err := encodePullResponse(c.pubkey, []CrdsValue{value}) + if err != nil { + return err + } + if err := sendUDP(conn, packet, addr); err != nil { + c.txErrors.Add(1) + return err + } + c.recordTx() + c.txPullResponses.Add(1) + return nil +} - numInvalidMsgs uint64 - numIgnoredMsgs uint64 +func (c *Client) sendPing(conn *net.UDPConn, addr *net.UDPAddr) error { + ping, err := newPing(c.identity) + if err != nil { + return err + } + if err := sendUDP(conn, encodePingMessage(ping), addr); err != nil { + c.txErrors.Add(1) + return err + } + c.recordTx() + c.txPingMessages.Add(1) + return nil } -// HandlePacket is the entrypoint of the RX side. -func (h *Handler) HandlePacket(packet []byte, from netip.AddrPort) { - msg, err := BincodeDeserializeMessage(packet) +func (c *Client) sendPong(conn *net.UDPConn, ping Ping, addr *net.UDPAddr) error { + if !ping.Verify() { + return nil + } + pong, err := newPong(ping, c.identity) if err != nil { - atomic.AddUint64(&h.numInvalidMsgs, 1) - return + return err + } + if err := sendUDP(conn, encodePongMessage(pong), addr); err != nil { + c.txErrors.Add(1) + return err } - switch x := msg.(type) { - case *Message__PullResponse: - if h.PullClient != nil { - h.PullClient.HandlePullResponse(x, from) - return + c.recordTx() + c.txPongMessages.Add(1) + return nil +} + +func (c *Client) recvLoop(ctx context.Context, conn *net.UDPConn) error { + buf := make([]byte, packetDataSize) + for { + n, addr, err := conn.ReadFromUDP(buf) + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return err } - case *Message__Ping: - if h.PingServer != nil { - h.PingServer.HandlePing(x, from) - return + c.rxPackets.Add(1) + c.lastRxUnix.Store(time.Now().Unix()) + if err := c.handlePacket(conn, buf[:n], addr); err != nil { + c.rxDecodeErrors.Add(1) + continue } - case *Message__Pong: - if h.PingClient != nil { - h.PingClient.HandlePong(x, from) - return + } +} + +func (c *Client) handlePacket(conn *net.UDPConn, packet []byte, from *net.UDPAddr) error { + shredVersion := c.ShredVersion() + decoded, err := decodePacketWithContactHandler(packet, func(record contactRecord) { + c.handleContactRecord(record, shredVersion) + }) + if err != nil { + return err + } + switch decoded.Kind { + case packetPing: + c.rxPings.Add(1) + return c.sendPong(conn, decoded.Ping, from) + case packetPong: + c.rxPongs.Add(1) + if decoded.Pong.Verify() { + c.recordPeer(from) } + case packetContacts: + c.rxContacts.Add(uint64(decoded.ContactCount)) + case packetPullRequest: + c.rxPullRequests.Add(1) + return c.sendPullResponse(conn, from) } - atomic.AddUint64(&h.numIgnoredMsgs, 1) + return nil } -// Close destroys all handlers. -func (h *Handler) Close() { - h.PingClient.Close() +func (c *Client) handleContactRecord(record contactRecord, shredVersion uint16) { + if record.ShredVer != shredVersion || !record.GossipAddr.ok { + return + } + if sameEndpointUDPAddr(record.GossipAddr, c.entrypoint) { + return + } + c.recordPeerEndpoint(record.GossipAddr) + c.recordRepairPeerRecord(record) + c.acceptedContacts.Add(1) } -type udpSender interface { - WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (int, error) +func (c *Client) recordRepairPeer(contact *ContactInfo) { + if contact == nil || contact.ServeRepairAddr == nil || contact.ServeRepairAddr.Port == 0 { + return + } + now := time.Now() + key := contact.Pubkey + c.repairPeerMu.Lock() + if existing, ok := c.repairPeers[key]; ok && sameAddr(existing.Addr, contact.ServeRepairAddr) { + existing.LastSeen = now + c.repairPeers[key] = existing + c.repairPeerMu.Unlock() + return + } + ip := normalizedIP(contact.ServeRepairAddr.IP) + if ip == nil || ip.IsUnspecified() { + c.repairPeerMu.Unlock() + return + } + addr := &net.UDPAddr{IP: ip, Port: contact.ServeRepairAddr.Port} + c.repairPeers[key] = RepairPeer{ + Pubkey: contact.Pubkey, + Addr: addr, + LastSeen: now, + } + c.repairPeerMu.Unlock() +} + +func (c *Client) recordRepairPeerRecord(record contactRecord) { + if !record.ServeRepairAddr.ok || record.ServeRepairAddr.port == 0 { + return + } + now := time.Now() + key := record.Pubkey + c.repairPeerMu.Lock() + if existing, ok := c.repairPeers[key]; ok && sameEndpointUDPAddr(record.ServeRepairAddr, existing.Addr) { + existing.LastSeen = now + c.repairPeers[key] = existing + c.repairPeerMu.Unlock() + return + } + c.repairPeers[key] = RepairPeer{ + Pubkey: record.Pubkey, + Addr: record.ServeRepairAddr.UDPAddr(), + LastSeen: now, + } + c.repairPeerMu.Unlock() +} + +func (c *Client) recordTx() { + c.txPackets.Add(1) + c.lastTxUnix.Store(time.Now().Unix()) +} + +func (c *Client) recordPeer(addr *net.UDPAddr) { + if addr == nil || addr.Port == 0 { + return + } + ip := normalizedIP(addr.IP) + if ip == nil || ip.IsUnspecified() { + return + } + key, ok := makeUDPAddrKey(addr) + if !ok { + return + } + now := time.Now() + c.peerMu.Lock() + defer c.peerMu.Unlock() + if existing, exists := c.peers[key]; exists { + existing.lastSeen = now + c.peers[key] = existing + return + } + if len(c.peers) >= maxKnownGossipPeers { + c.prunePeersLocked(now) + } + if len(c.peers) >= maxKnownGossipPeers { + return + } + c.peers[key] = knownPeer{addr: cloneUDPAddr(addr), lastSeen: now} +} + +func (c *Client) recordPeerEndpoint(endpoint contactEndpoint) { + if !endpoint.ok || endpoint.port == 0 { + return + } + key, ok := makeUDPAddrKeyFromEndpoint(endpoint) + if !ok { + return + } + now := time.Now() + c.peerMu.Lock() + defer c.peerMu.Unlock() + if existing, exists := c.peers[key]; exists { + existing.lastSeen = now + c.peers[key] = existing + return + } + if len(c.peers) >= maxKnownGossipPeers { + c.prunePeersLocked(now) + } + if len(c.peers) >= maxKnownGossipPeers { + return + } + c.peers[key] = knownPeer{addr: endpoint.UDPAddr(), lastSeen: now} +} + +func (c *Client) currentPeers() []*net.UDPAddr { + now := time.Now() + c.peerMu.Lock() + defer c.peerMu.Unlock() + c.prunePeersLocked(now) + peers := make([]*net.UDPAddr, 0, len(c.peers)) + for _, peer := range c.peers { + peers = append(peers, cloneUDPAddr(peer.addr)) + } + return peers +} + +func (c *Client) prunePeersLocked(now time.Time) { + for key, peer := range c.peers { + if now.Sub(peer.lastSeen) > peerExpirationWindow { + delete(c.peers, key) + } + } +} + +func formatAge(unix int64) string { + if unix == 0 { + return "never" + } + return time.Since(time.Unix(unix, 0)).Round(time.Second).String() +} + +func makeUDPAddrKey(addr *net.UDPAddr) (udpAddrKey, bool) { + var key udpAddrKey + if addr == nil || addr.Port == 0 { + return key, false + } + ip := addr.IP.To16() + if ip == nil { + return key, false + } + copy(key.ip[:], ip) + key.port = addr.Port + return key, true +} + +func makeUDPAddrKeyFromEndpoint(endpoint contactEndpoint) (udpAddrKey, bool) { + var key udpAddrKey + if !endpoint.ok || endpoint.port == 0 { + return key, false + } + copy(key.ip[:], endpoint.ip[:]) + key.port = endpoint.port + return key, true +} + +func sameEndpointUDPAddr(endpoint contactEndpoint, addr *net.UDPAddr) bool { + key, ok := makeUDPAddrKeyFromEndpoint(endpoint) + if !ok { + return addr == nil + } + addrKey, ok := makeUDPAddrKey(addr) + return ok && key == addrKey } diff --git a/pkg/gossip/codec.go b/pkg/gossip/codec.go new file mode 100644 index 00000000..b2a535c9 --- /dev/null +++ b/pkg/gossip/codec.go @@ -0,0 +1,201 @@ +package gossip + +import ( + "encoding/binary" + "fmt" + "io" + "net" +) + +type encoder struct { + buf []byte +} + +func (e *encoder) bytes() []byte { + return e.buf +} + +func (e *encoder) u8(v uint8) { + e.buf = append(e.buf, v) +} + +func (e *encoder) bool(v bool) { + if v { + e.u8(1) + return + } + e.u8(0) +} + +func (e *encoder) u16(v uint16) { + e.buf = binary.LittleEndian.AppendUint16(e.buf, v) +} + +func (e *encoder) u32(v uint32) { + e.buf = binary.LittleEndian.AppendUint32(e.buf, v) +} + +func (e *encoder) u64(v uint64) { + e.buf = binary.LittleEndian.AppendUint64(e.buf, v) +} + +func (e *encoder) variant(index uint32) { + e.u32(index) +} + +func (e *encoder) fixed(b []byte) { + e.buf = append(e.buf, b...) +} + +func (e *encoder) varint(v uint64) { + for v >= 0x80 { + e.u8(byte(v) | 0x80) + v >>= 7 + } + e.u8(byte(v)) +} + +func (e *encoder) shortLen(n int) error { + if n > 0xffff { + return fmt.Errorf("short_vec length %d overflows u16", n) + } + e.varint(uint64(n)) + return nil +} + +type decoder struct { + data []byte + off int +} + +func newDecoder(data []byte) *decoder { + return &decoder{data: data} +} + +func (d *decoder) remaining() int { + return len(d.data) - d.off +} + +func (d *decoder) read(n int) ([]byte, error) { + if n < 0 || d.remaining() < n { + return nil, io.ErrUnexpectedEOF + } + out := d.data[d.off : d.off+n] + d.off += n + return out, nil +} + +func (d *decoder) u8() (uint8, error) { + b, err := d.read(1) + if err != nil { + return 0, err + } + return b[0], nil +} + +func (d *decoder) bool() (bool, error) { + v, err := d.u8() + if err != nil { + return false, err + } + switch v { + case 0: + return false, nil + case 1: + return true, nil + default: + return false, fmt.Errorf("invalid bool tag %d", v) + } +} + +func (d *decoder) u16() (uint16, error) { + b, err := d.read(2) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint16(b), nil +} + +func (d *decoder) u32() (uint32, error) { + b, err := d.read(4) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint32(b), nil +} + +func (d *decoder) u64() (uint64, error) { + b, err := d.read(8) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint64(b), nil +} + +func (d *decoder) variant() (uint32, error) { + return d.u32() +} + +func (d *decoder) varint(maxBytes int) (uint64, error) { + var out uint64 + for i := 0; i < maxBytes; i++ { + b, err := d.u8() + if err != nil { + return 0, err + } + out |= uint64(b&0x7f) << uint(7*i) + if b&0x80 == 0 { + return out, nil + } + } + return 0, fmt.Errorf("varint overflows %d bytes", maxBytes) +} + +func (d *decoder) shortLen() (int, error) { + v, err := d.varint(3) + if err != nil { + return 0, err + } + if v > 0xffff { + return 0, fmt.Errorf("short_vec length %d overflows u16", v) + } + return int(v), nil +} + +func encodeIP(e *encoder, ip net.IP) error { + if v4 := ip.To4(); v4 != nil { + e.variant(0) + e.fixed(v4) + return nil + } + v6 := ip.To16() + if v6 == nil { + return fmt.Errorf("invalid IP address %q", ip.String()) + } + e.variant(1) + e.fixed(v6) + return nil +} + +func decodeIP(d *decoder) (net.IP, error) { + variant, err := d.variant() + if err != nil { + return nil, err + } + switch variant { + case 0: + b, err := d.read(4) + if err != nil { + return nil, err + } + return net.IPv4(b[0], b[1], b[2], b[3]), nil + case 1: + b, err := d.read(16) + if err != nil { + return nil, err + } + return net.IP(append([]byte(nil), b...)), nil + default: + return nil, fmt.Errorf("unknown IP variant %d", variant) + } +} diff --git a/pkg/gossip/contact_info.go b/pkg/gossip/contact_info.go new file mode 100644 index 00000000..6986cc40 --- /dev/null +++ b/pkg/gossip/contact_info.go @@ -0,0 +1,619 @@ +package gossip + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "fmt" + "net" + "sort" + "time" +) + +const ( + ClientName = "mithril" + + packetDataSize = 1232 + + crdsDataLegacyContactInfo = 0 + crdsDataContactInfo = 11 + + protocolPullRequest = 0 + protocolPullResponse = 1 + protocolPushMessage = 2 + protocolPruneMessage = 3 + protocolPingMessage = 4 + protocolPongMessage = 5 + + socketTagGossip = 0 + socketTagServeRepair = 4 + socketTagTVU = 10 + + // Agave's Version has a numeric ClientId, not a string. Unknown IDs are + // accepted, so use a stable "MI" marker while keeping ClientName in logs. + versionClientMithril = 0x4d49 +) + +type Pubkey [32]byte +type Signature [64]byte +type Hash [32]byte + +type Version struct { + Major uint16 + Minor uint16 + Patch uint16 + Commit uint32 + FeatureSet uint32 + Client uint16 +} + +type SocketEntry struct { + Key uint8 + Index uint8 + Port uint16 +} + +type ContactInfo struct { + Pubkey Pubkey + Wallclock uint64 + Outset uint64 + ShredVer uint16 + Version Version + Addrs []net.IP + Sockets []SocketEntry + GossipAddr *net.UDPAddr + ServeRepairAddr *net.UDPAddr + TVUAddr *net.UDPAddr + Extensions [][]byte +} + +type contactEndpoint struct { + ip [16]byte + port int + ok bool +} + +type contactRecord struct { + Pubkey Pubkey + ShredVer uint16 + GossipAddr contactEndpoint + ServeRepairAddr contactEndpoint + signature Signature + data []byte +} + +func NewContactInfo(pubkey Pubkey, shredVersion uint16, gossipAddr, tvuAddr *net.UDPAddr) (*ContactInfo, error) { + info := &ContactInfo{ + Pubkey: pubkey, + Wallclock: wallclockMillis(), + Outset: uint64(time.Now().UnixMicro()), + ShredVer: shredVersion, + Version: Version{ + Client: versionClientMithril, + }, + } + if gossipAddr != nil { + if err := info.SetSocket(socketTagGossip, gossipAddr); err != nil { + return nil, err + } + info.GossipAddr = cloneUDPAddr(gossipAddr) + } + if tvuAddr != nil { + if err := info.SetSocket(socketTagTVU, tvuAddr); err != nil { + return nil, err + } + info.TVUAddr = cloneUDPAddr(tvuAddr) + } + return info, nil +} + +func (r contactRecord) Verify() bool { + return ed25519.Verify(ed25519.PublicKey(r.Pubkey[:]), r.data, r.signature[:]) +} + +func (r contactRecord) ContactInfo() *ContactInfo { + return &ContactInfo{ + Pubkey: r.Pubkey, + ShredVer: r.ShredVer, + GossipAddr: r.GossipAddr.UDPAddr(), + ServeRepairAddr: r.ServeRepairAddr.UDPAddr(), + } +} + +func (e contactEndpoint) UDPAddr() *net.UDPAddr { + if !e.ok || e.port == 0 { + return nil + } + ip := make(net.IP, 16) + copy(ip, e.ip[:]) + return &net.UDPAddr{IP: ip, Port: e.port} +} + +func (c *ContactInfo) CloneWithWallclock(wallclock uint64) *ContactInfo { + cp := *c + cp.Wallclock = wallclock + cp.Addrs = append([]net.IP(nil), c.Addrs...) + cp.Sockets = append([]SocketEntry(nil), c.Sockets...) + cp.Extensions = append([][]byte(nil), c.Extensions...) + cp.GossipAddr = cloneUDPAddr(c.GossipAddr) + cp.ServeRepairAddr = cloneUDPAddr(c.ServeRepairAddr) + cp.TVUAddr = cloneUDPAddr(c.TVUAddr) + return &cp +} + +func (c *ContactInfo) SetSocket(key uint8, addr *net.UDPAddr) error { + if addr == nil { + return fmt.Errorf("nil socket for key %d", key) + } + if addr.Port <= 0 || addr.Port > 0xffff { + return fmt.Errorf("invalid port %d for key %d", addr.Port, key) + } + ip := normalizedIP(addr.IP) + if ip == nil || ip.IsUnspecified() { + return fmt.Errorf("invalid IP %q for key %d", addr.IP.String(), key) + } + addrIndex := -1 + for idx, existing := range c.Addrs { + if existing.Equal(ip) { + addrIndex = idx + break + } + } + if addrIndex < 0 { + if len(c.Addrs) >= 256 { + return fmt.Errorf("too many contact IP addresses") + } + addrIndex = len(c.Addrs) + c.Addrs = append(c.Addrs, ip) + } + + filtered := c.Sockets[:0] + for _, entry := range c.Sockets { + if entry.Key != key { + filtered = append(filtered, entry) + } + } + c.Sockets = append(filtered, SocketEntry{Key: key, Index: uint8(addrIndex), Port: uint16(addr.Port)}) + sort.Slice(c.Sockets, func(i, j int) bool { + if c.Sockets[i].Port == c.Sockets[j].Port { + return c.Sockets[i].Key < c.Sockets[j].Key + } + return c.Sockets[i].Port < c.Sockets[j].Port + }) + return nil +} + +func (c *ContactInfo) encode(e *encoder) error { + e.fixed(c.Pubkey[:]) + e.varint(c.Wallclock) + e.u64(c.Outset) + e.u16(c.ShredVer) + encodeVersion(e, c.Version) + + if err := e.shortLen(len(c.Addrs)); err != nil { + return err + } + for _, ip := range c.Addrs { + if err := encodeIP(e, ip); err != nil { + return err + } + } + + if err := e.shortLen(len(c.Sockets)); err != nil { + return err + } + var previousPort uint16 + for _, socket := range c.Sockets { + if socket.Port < previousPort { + return fmt.Errorf("socket ports are not sorted") + } + e.u8(socket.Key) + e.u8(socket.Index) + e.varint(uint64(socket.Port - previousPort)) + previousPort = socket.Port + } + + if err := e.shortLen(len(c.Extensions)); err != nil { + return err + } + for _, ext := range c.Extensions { + e.fixed(ext) + } + return nil +} + +func decodeContactInfo(d *decoder) (*ContactInfo, error) { + info := &ContactInfo{} + pubkey, err := d.read(32) + if err != nil { + return nil, err + } + copy(info.Pubkey[:], pubkey) + if info.Wallclock, err = d.varint(10); err != nil { + return nil, err + } + if info.Outset, err = d.u64(); err != nil { + return nil, err + } + if info.ShredVer, err = d.u16(); err != nil { + return nil, err + } + if info.Version, err = decodeVersion(d); err != nil { + return nil, err + } + + numAddrs, err := d.shortLen() + if err != nil { + return nil, err + } + info.Addrs = make([]net.IP, numAddrs) + for i := range info.Addrs { + ip, err := decodeIP(d) + if err != nil { + return nil, err + } + info.Addrs[i] = normalizedIP(ip) + } + + numSockets, err := d.shortLen() + if err != nil { + return nil, err + } + info.Sockets = make([]SocketEntry, 0, numSockets) + var port uint16 + for i := 0; i < numSockets; i++ { + key, err := d.u8() + if err != nil { + return nil, err + } + index, err := d.u8() + if err != nil { + return nil, err + } + offset, err := d.varint(3) + if err != nil { + return nil, err + } + if offset > 0xffff || uint64(port)+offset > 0xffff { + return nil, fmt.Errorf("socket port offset overflow") + } + port += uint16(offset) + if int(index) >= len(info.Addrs) { + return nil, fmt.Errorf("socket IP index %d out of range", index) + } + entry := SocketEntry{Key: key, Index: index, Port: port} + info.Sockets = append(info.Sockets, entry) + addr := &net.UDPAddr{IP: info.Addrs[index], Port: int(port)} + switch key { + case socketTagGossip: + info.GossipAddr = cloneUDPAddr(addr) + case socketTagServeRepair: + info.ServeRepairAddr = cloneUDPAddr(addr) + case socketTagTVU: + info.TVUAddr = cloneUDPAddr(addr) + } + } + + numExt, err := d.shortLen() + if err != nil { + return nil, err + } + for i := 0; i < numExt; i++ { + return nil, errUnsupportedCRDSValue + } + return info, nil +} + +func decodeContactRecord(d *decoder) (contactRecord, error) { + var record contactRecord + pubkey, err := d.read(32) + if err != nil { + return contactRecord{}, err + } + copy(record.Pubkey[:], pubkey) + if _, err := d.varint(10); err != nil { + return contactRecord{}, err + } + if _, err := d.u64(); err != nil { + return contactRecord{}, err + } + if record.ShredVer, err = d.u16(); err != nil { + return contactRecord{}, err + } + if _, err = decodeVersion(d); err != nil { + return contactRecord{}, err + } + + numAddrs, err := d.shortLen() + if err != nil { + return contactRecord{}, err + } + const inlineAddrCount = 8 + var inlineAddrs [inlineAddrCount]contactEndpoint + var addrs []contactEndpoint + if numAddrs > inlineAddrCount { + addrs = make([]contactEndpoint, numAddrs) + } + for i := 0; i < numAddrs; i++ { + addr, err := decodeContactIP(d) + if err != nil { + return contactRecord{}, err + } + if addrs != nil { + addrs[i] = addr + } else { + inlineAddrs[i] = addr + } + } + addrAt := func(index uint8) (contactEndpoint, bool) { + if int(index) >= numAddrs { + return contactEndpoint{}, false + } + if addrs != nil { + return addrs[index], true + } + return inlineAddrs[index], true + } + + numSockets, err := d.shortLen() + if err != nil { + return contactRecord{}, err + } + var port uint16 + for i := 0; i < numSockets; i++ { + key, err := d.u8() + if err != nil { + return contactRecord{}, err + } + index, err := d.u8() + if err != nil { + return contactRecord{}, err + } + offset, err := d.varint(3) + if err != nil { + return contactRecord{}, err + } + if offset > 0xffff || uint64(port)+offset > 0xffff { + return contactRecord{}, fmt.Errorf("socket port offset overflow") + } + port += uint16(offset) + addr, ok := addrAt(index) + if !ok { + return contactRecord{}, fmt.Errorf("socket IP index %d out of range", index) + } + addr.port = int(port) + addr.ok = true + switch key { + case socketTagGossip: + record.GossipAddr = addr + case socketTagServeRepair: + record.ServeRepairAddr = addr + } + } + + numExt, err := d.shortLen() + if err != nil { + return contactRecord{}, err + } + if numExt > 0 { + return contactRecord{}, errUnsupportedCRDSValue + } + return record, nil +} + +func decodeContactIP(d *decoder) (contactEndpoint, error) { + var out contactEndpoint + variant, err := d.variant() + if err != nil { + return contactEndpoint{}, err + } + switch variant { + case 0: + b, err := d.read(4) + if err != nil { + return contactEndpoint{}, err + } + out.ip[10] = 0xff + out.ip[11] = 0xff + copy(out.ip[12:], b) + case 1: + b, err := d.read(16) + if err != nil { + return contactEndpoint{}, err + } + copy(out.ip[:], b) + default: + return contactEndpoint{}, fmt.Errorf("unknown IP variant %d", variant) + } + out.ok = true + return out, nil +} + +func encodeVersion(e *encoder, version Version) { + e.varint(uint64(version.Major)) + e.varint(uint64(version.Minor)) + e.varint(uint64(version.Patch)) + e.u32(version.Commit) + e.u32(version.FeatureSet) + e.varint(uint64(version.Client)) +} + +func decodeVersion(d *decoder) (Version, error) { + major, err := d.varint(3) + if err != nil { + return Version{}, err + } + minor, err := d.varint(3) + if err != nil { + return Version{}, err + } + patch, err := d.varint(3) + if err != nil { + return Version{}, err + } + commit, err := d.u32() + if err != nil { + return Version{}, err + } + featureSet, err := d.u32() + if err != nil { + return Version{}, err + } + client, err := d.varint(3) + if err != nil { + return Version{}, err + } + return Version{ + Major: uint16(major), + Minor: uint16(minor), + Patch: uint16(patch), + Commit: commit, + FeatureSet: featureSet, + Client: uint16(client), + }, nil +} + +func encodeCrdsDataContactInfo(info *ContactInfo) ([]byte, error) { + var e encoder + e.variant(crdsDataContactInfo) + if err := info.encode(&e); err != nil { + return nil, err + } + return e.bytes(), nil +} + +func decodeCrdsDataContactInfo(d *decoder) (*ContactInfo, error) { + variant, err := d.variant() + if err != nil { + return nil, err + } + switch variant { + case crdsDataContactInfo: + return decodeContactInfo(d) + default: + return nil, errUnsupportedCRDSValue + } +} + +func decodeCrdsDataContactRecord(d *decoder) (contactRecord, error) { + variant, err := d.variant() + if err != nil { + return contactRecord{}, err + } + switch variant { + case crdsDataContactInfo: + return decodeContactRecord(d) + default: + return contactRecord{}, errUnsupportedCRDSValue + } +} + +func signCrdsContactInfo(info *ContactInfo, identity ed25519.PrivateKey) (CrdsValue, error) { + data, err := encodeCrdsDataContactInfo(info) + if err != nil { + return CrdsValue{}, err + } + sig := ed25519.Sign(identity, data) + value := CrdsValue{ContactInfo: info, Data: data} + copy(value.Signature[:], sig) + return value, nil +} + +func decodeCrdsContactRecord(d *decoder) (contactRecord, error) { + sig, err := d.read(64) + if err != nil { + return contactRecord{}, err + } + dataStart := d.off + record, err := decodeCrdsDataContactRecord(d) + if err != nil { + return contactRecord{}, err + } + record.data = d.data[dataStart:d.off] + copy(record.signature[:], sig) + return record, nil +} + +type CrdsValue struct { + Signature Signature + ContactInfo *ContactInfo + Data []byte +} + +func (v CrdsValue) encode(e *encoder) { + e.fixed(v.Signature[:]) + e.fixed(v.Data) +} + +func decodeCrdsValue(d *decoder) (CrdsValue, error) { + sig, err := d.read(64) + if err != nil { + return CrdsValue{}, err + } + dataStart := d.off + info, err := decodeCrdsDataContactInfo(d) + if err != nil { + return CrdsValue{}, err + } + data := d.data[dataStart:d.off] + value := CrdsValue{ContactInfo: info, Data: data} + copy(value.Signature[:], sig) + return value, nil +} + +func (v CrdsValue) Verify() bool { + if v.ContactInfo == nil { + return false + } + return ed25519.Verify(ed25519.PublicKey(v.ContactInfo.Pubkey[:]), v.Data, v.Signature[:]) +} + +func hashPingToken(token [32]byte) Hash { + h := sha256.New() + h.Write([]byte("SOLANA_PING_PONG")) + h.Write(token[:]) + var out Hash + copy(out[:], h.Sum(nil)) + return out +} + +func normalizedIP(ip net.IP) net.IP { + if ip == nil { + return nil + } + if v4 := ip.To4(); v4 != nil { + return net.IPv4(v4[0], v4[1], v4[2], v4[3]) + } + if v6 := ip.To16(); v6 != nil { + return append(net.IP(nil), v6...) + } + return nil +} + +func cloneUDPAddr(addr *net.UDPAddr) *net.UDPAddr { + if addr == nil { + return nil + } + return &net.UDPAddr{IP: append(net.IP(nil), addr.IP...), Port: addr.Port, Zone: addr.Zone} +} + +func wallclockMillis() uint64 { + return uint64(time.Now().UnixMilli()) +} + +func pubkeyFromPrivateKey(key ed25519.PrivateKey) (Pubkey, error) { + pub, ok := key.Public().(ed25519.PublicKey) + if !ok || len(pub) != ed25519.PublicKeySize { + return Pubkey{}, fmt.Errorf("invalid ed25519 private key") + } + var out Pubkey + copy(out[:], pub) + return out, nil +} + +func sameAddr(a, b *net.UDPAddr) bool { + if a == nil || b == nil { + return a == b + } + return a.Port == b.Port && bytes.Equal(normalizedIP(a.IP), normalizedIP(b.IP)) +} diff --git a/pkg/gossip/crds.go b/pkg/gossip/crds.go deleted file mode 100644 index 00d23226..00000000 --- a/pkg/gossip/crds.go +++ /dev/null @@ -1,74 +0,0 @@ -package gossip - -import ( - "crypto/ed25519" - "encoding/binary" - "math" -) - -// CrdsBloomP is bloom filter 'p' parameter (probability) -const CrdsBloomP = 0.1 - -func (f *CrdsFilter) Contains(item *Hash) bool { - if !f.TestMask(item) { - return false - } - return f.Filter.Contains(item) -} - -func (f *CrdsFilter) TestMask(item *Hash) bool { - ones := uint64(math.MaxUint64) >> f.MaskBits - bits := binary.LittleEndian.Uint64(item[:8]) | ones - return bits == f.Mask -} - -type CrdsFilterSet []CrdsFilter - -func NewCrdsFilterSet(numItems, maxBytes uint64) CrdsFilterSet { - maxBits := maxBytes * 8 - maxItems := BloomMaxItems(float64(maxBits), CrdsBloomP, 8) - maskBits := BloomMaskBits(float64(numItems), maxItems) - filters := make([]CrdsFilter, 1<> maskBits), - MaskBits: maskBits, - } - } - return filters -} - -func (c CrdsFilterSet) Add(h Hash) { - index := binary.LittleEndian.Uint64(h[:8]) - index >>= 64 - c[0].MaskBits - c[index].Filter.Add(&h) -} - -func (c *CrdsValue) Sign(identity ed25519.PrivateKey) error { - // Write pubkey into data field - pubkey := ed25519.PublicKey(c.Data.Pubkey()[:]) - copy( - pubkey[:ed25519.PublicKeySize], - identity.Public().(ed25519.PublicKey)[:ed25519.PublicKeySize], - ) - - msg, err := c.Data.BincodeSerialize() - if err != nil { - return err - } - sig := ed25519.Sign(identity, msg) - copy(c.Signature[:ed25519.SignatureSize], sig[:ed25519.SignatureSize]) - return nil -} - -func (c *CrdsValue) VerifySignature() bool { - msg, err := c.Data.BincodeSerialize() - if err != nil { - return false - } - pubkey := ed25519.PublicKey(c.Data.Pubkey()[:]) - return ed25519.Verify(pubkey, msg, c.Signature[:]) -} diff --git a/pkg/gossip/crds_test.go b/pkg/gossip/crds_test.go deleted file mode 100644 index 33672a30..00000000 --- a/pkg/gossip/crds_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package gossip - -import ( - "math" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNewCrdsFilterSet(t *testing.T) { - filters := NewCrdsFilterSet(55345017, 4098) - assert.Equal(t, 16384, len(filters)) - maskBits := filters[0].MaskBits - rightShift := 64 - maskBits - ones := uint64(math.MaxUint64) >> maskBits - for i, filter := range filters { - assert.Equal(t, maskBits, filter.MaskBits) - assert.Equal(t, uint64(i), filter.Mask>>rightShift) - assert.Equal(t, ones, ones&filter.Mask) - } -} - -func TestCrdsFilterSet_Add(t *testing.T) { - filters := NewCrdsFilterSet(9672788, 8196) - hashValues := make([]Hash, 1024) - for i := range hashValues { - rand.Read(hashValues[i][:]) - filters.Add(hashValues[i]) - } - for _, hashValue := range hashValues { - numHits := uint(0) - falsePositives := uint(0) - for _, filter := range filters { - if filter.TestMask(&hashValue) { - numHits++ - assert.True(t, filter.Contains(&hashValue)) - assert.True(t, filter.Filter.Contains(&hashValue)) - } else if filter.Filter.Contains(&hashValue) { - falsePositives++ - } - } - assert.Equal(t, numHits, uint(1)) - assert.Less(t, falsePositives, uint(5)) - } -} diff --git a/pkg/gossip/gossip_test.go b/pkg/gossip/gossip_test.go new file mode 100644 index 00000000..19181604 --- /dev/null +++ b/pkg/gossip/gossip_test.go @@ -0,0 +1,161 @@ +package gossip + +import ( + "crypto/ed25519" + "crypto/rand" + "net" + "testing" + "time" +) + +func TestVarintAndShortLenEncoding(t *testing.T) { + var e encoder + if err := e.shortLen(0xffff); err != nil { + t.Fatalf("shortLen returned error: %v", err) + } + if got, want := e.bytes(), []byte{0xff, 0xff, 0x03}; string(got) != string(want) { + t.Fatalf("shortLen bytes = %x, want %x", got, want) + } + + d := newDecoder(e.bytes()) + got, err := d.shortLen() + if err != nil { + t.Fatalf("shortLen decode returned error: %v", err) + } + if got != 0xffff { + t.Fatalf("shortLen decode = %d, want %d", got, 0xffff) + } +} + +func TestContactInfoRoundTripAndSignature(t *testing.T) { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("GenerateKey returned error: %v", err) + } + pubkey, err := pubkeyFromPrivateKey(priv) + if err != nil { + t.Fatalf("pubkeyFromPrivateKey returned error: %v", err) + } + contact, err := NewContactInfo( + pubkey, + 1234, + &net.UDPAddr{IP: net.ParseIP("203.0.113.10"), Port: 65400}, + &net.UDPAddr{IP: net.ParseIP("203.0.113.10"), Port: 8001}, + ) + if err != nil { + t.Fatalf("NewContactInfo returned error: %v", err) + } + if err := contact.SetSocket(socketTagServeRepair, &net.UDPAddr{IP: net.ParseIP("203.0.113.10"), Port: 8008}); err != nil { + t.Fatalf("SetSocket serve repair returned error: %v", err) + } + + value, err := signCrdsContactInfo(contact, priv) + if err != nil { + t.Fatalf("signCrdsContactInfo returned error: %v", err) + } + if !value.Verify() { + t.Fatalf("signed CRDS value did not verify") + } + + var e encoder + value.encode(&e) + decoded, err := decodeCrdsValue(newDecoder(e.bytes())) + if err != nil { + t.Fatalf("decodeCrdsValue returned error: %v", err) + } + if !decoded.Verify() { + t.Fatalf("decoded CRDS value did not verify") + } + if decoded.ContactInfo.ShredVer != 1234 { + t.Fatalf("shred version = %d, want 1234", decoded.ContactInfo.ShredVer) + } + if decoded.ContactInfo.GossipAddr.String() != "203.0.113.10:65400" { + t.Fatalf("gossip addr = %s", decoded.ContactInfo.GossipAddr.String()) + } + if decoded.ContactInfo.TVUAddr.String() != "203.0.113.10:8001" { + t.Fatalf("tvu addr = %s", decoded.ContactInfo.TVUAddr.String()) + } + if decoded.ContactInfo.ServeRepairAddr.String() != "203.0.113.10:8008" { + t.Fatalf("serve repair addr = %s", decoded.ContactInfo.ServeRepairAddr.String()) + } +} + +func TestPingPongRoundTrip(t *testing.T) { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("GenerateKey returned error: %v", err) + } + ping, err := newPing(priv) + if err != nil { + t.Fatalf("newPing returned error: %v", err) + } + if !ping.Verify() { + t.Fatalf("ping did not verify") + } + pong, err := newPong(ping, priv) + if err != nil { + t.Fatalf("newPong returned error: %v", err) + } + if !pong.Verify() { + t.Fatalf("pong did not verify") + } + + decoded, err := decodePacket(encodePingMessage(ping)) + if err != nil { + t.Fatalf("decodePacket ping returned error: %v", err) + } + if decoded.Kind != packetPing || !decoded.Ping.Verify() { + t.Fatalf("decoded ping did not verify") + } + + decoded, err = decodePacket(encodePongMessage(pong)) + if err != nil { + t.Fatalf("decodePacket pong returned error: %v", err) + } + if decoded.Kind != packetPong || !decoded.Pong.Verify() { + t.Fatalf("decoded pong did not verify") + } +} + +func TestQueryEntrypointParsesEchoResponse(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen returned error: %v", err) + } + defer listener.Close() + + go func() { + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(time.Second)) + buf := make([]byte, ipEchoHeaderLength+16+1) + _, _ = conn.Read(buf) + var e encoder + e.fixed([]byte{0, 0, 0, 0}) + if err := encodeIP(&e, net.ParseIP("198.51.100.7")); err != nil { + t.Errorf("encodeIP returned error: %v", err) + return + } + e.bool(true) + e.u16(4321) + _, _ = conn.Write(e.bytes()) + }() + + addr, err := net.ResolveUDPAddr("udp", listener.Addr().String()) + if err != nil { + t.Fatalf("ResolveUDPAddr returned error: %v", err) + } + resp, err := QueryEntrypoint(addr, time.Second) + if err != nil { + t.Fatalf("QueryEntrypoint returned error: %v", err) + } + if resp.ShredVersion != 4321 { + t.Fatalf("shred version = %d, want 4321", resp.ShredVersion) + } + if !resp.Address.Equal(net.ParseIP("198.51.100.7")) { + t.Fatalf("address = %s", resp.Address.String()) + } +} diff --git a/pkg/gossip/ipecho.go b/pkg/gossip/ipecho.go new file mode 100644 index 00000000..8498041a --- /dev/null +++ b/pkg/gossip/ipecho.go @@ -0,0 +1,70 @@ +package gossip + +import ( + "fmt" + "net" + "time" +) + +const ( + ipEchoHeaderLength = 4 + ipEchoResponseLength = ipEchoHeaderLength + 23 +) + +type EchoResponse struct { + Address net.IP + ShredVersion uint16 +} + +func QueryEntrypoint(entrypoint *net.UDPAddr, timeout time.Duration) (EchoResponse, error) { + if entrypoint == nil { + return EchoResponse{}, fmt.Errorf("nil gossip entrypoint") + } + tcpAddr := &net.TCPAddr{IP: entrypoint.IP, Port: entrypoint.Port, Zone: entrypoint.Zone} + conn, err := net.DialTimeout("tcp", tcpAddr.String(), timeout) + if err != nil { + return EchoResponse{}, err + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil { + return EchoResponse{}, err + } + + req := make([]byte, 0, ipEchoHeaderLength+16+1) + req = append(req, 0, 0, 0, 0) + req = append(req, make([]byte, 16)...) + req = append(req, '\n') + if _, err := conn.Write(req); err != nil { + return EchoResponse{}, err + } + + resp := make([]byte, ipEchoResponseLength) + n, err := conn.Read(resp) + if err != nil { + return EchoResponse{}, err + } + if n < ipEchoHeaderLength { + return EchoResponse{}, fmt.Errorf("short IP echo response: %d bytes", n) + } + if resp[0] != 0 || resp[1] != 0 || resp[2] != 0 || resp[3] != 0 { + return EchoResponse{}, fmt.Errorf("invalid IP echo response header %v", resp[:ipEchoHeaderLength]) + } + + d := newDecoder(resp[ipEchoHeaderLength:n]) + ip, err := decodeIP(d) + if err != nil { + return EchoResponse{}, err + } + hasShredVersion, err := d.bool() + if err != nil { + return EchoResponse{}, err + } + if !hasShredVersion { + return EchoResponse{}, fmt.Errorf("entrypoint did not return a shred version") + } + shredVersion, err := d.u16() + if err != nil { + return EchoResponse{}, err + } + return EchoResponse{Address: normalizedIP(ip), ShredVersion: shredVersion}, nil +} diff --git a/pkg/gossip/message.go b/pkg/gossip/message.go new file mode 100644 index 00000000..4b1d5689 --- /dev/null +++ b/pkg/gossip/message.go @@ -0,0 +1,249 @@ +package gossip + +import ( + "crypto/ed25519" + "crypto/rand" + "errors" + "fmt" + "net" +) + +var errUnsupportedCRDSValue = errors.New("unsupported CRDS value") + +type Ping struct { + From Pubkey + Token [32]byte + Signature Signature +} + +type Pong struct { + From Pubkey + Hash Hash + Signature Signature +} + +func newPing(identity ed25519.PrivateKey) (Ping, error) { + var token [32]byte + if _, err := rand.Read(token[:]); err != nil { + return Ping{}, err + } + pubkey, err := pubkeyFromPrivateKey(identity) + if err != nil { + return Ping{}, err + } + sig := ed25519.Sign(identity, token[:]) + ping := Ping{From: pubkey, Token: token} + copy(ping.Signature[:], sig) + return ping, nil +} + +func newPong(ping Ping, identity ed25519.PrivateKey) (Pong, error) { + pubkey, err := pubkeyFromPrivateKey(identity) + if err != nil { + return Pong{}, err + } + hash := hashPingToken(ping.Token) + sig := ed25519.Sign(identity, hash[:]) + pong := Pong{From: pubkey, Hash: hash} + copy(pong.Signature[:], sig) + return pong, nil +} + +func (p Ping) Verify() bool { + return ed25519.Verify(ed25519.PublicKey(p.From[:]), p.Token[:], p.Signature[:]) +} + +func (p Pong) Verify() bool { + return ed25519.Verify(ed25519.PublicKey(p.From[:]), p.Hash[:], p.Signature[:]) +} + +func encodePingMessage(ping Ping) []byte { + var e encoder + e.variant(protocolPingMessage) + e.fixed(ping.From[:]) + e.fixed(ping.Token[:]) + e.fixed(ping.Signature[:]) + return e.bytes() +} + +func encodePongMessage(pong Pong) []byte { + var e encoder + e.variant(protocolPongMessage) + e.fixed(pong.From[:]) + e.fixed(pong.Hash[:]) + e.fixed(pong.Signature[:]) + return e.bytes() +} + +func encodePushMessage(from Pubkey, values []CrdsValue) ([]byte, error) { + var e encoder + e.variant(protocolPushMessage) + e.fixed(from[:]) + e.u64(uint64(len(values))) + for _, value := range values { + value.encode(&e) + } + if len(e.bytes()) > packetDataSize { + return nil, fmt.Errorf("push message size %d exceeds packet size %d", len(e.bytes()), packetDataSize) + } + return e.bytes(), nil +} + +func encodePullResponse(from Pubkey, values []CrdsValue) ([]byte, error) { + var e encoder + e.variant(protocolPullResponse) + e.fixed(from[:]) + e.u64(uint64(len(values))) + for _, value := range values { + value.encode(&e) + } + if len(e.bytes()) > packetDataSize { + return nil, fmt.Errorf("pull response size %d exceeds packet size %d", len(e.bytes()), packetDataSize) + } + return e.bytes(), nil +} + +func decodePing(d *decoder) (Ping, error) { + var ping Ping + from, err := d.read(32) + if err != nil { + return Ping{}, err + } + token, err := d.read(32) + if err != nil { + return Ping{}, err + } + sig, err := d.read(64) + if err != nil { + return Ping{}, err + } + copy(ping.From[:], from) + copy(ping.Token[:], token) + copy(ping.Signature[:], sig) + return ping, nil +} + +func decodePong(d *decoder) (Pong, error) { + var pong Pong + from, err := d.read(32) + if err != nil { + return Pong{}, err + } + hash, err := d.read(32) + if err != nil { + return Pong{}, err + } + sig, err := d.read(64) + if err != nil { + return Pong{}, err + } + copy(pong.From[:], from) + copy(pong.Hash[:], hash) + copy(pong.Signature[:], sig) + return pong, nil +} + +func decodeContactRecords(d *decoder, handleContact func(contactRecord)) (int, []*ContactInfo, error) { + n, err := d.u64() + if err != nil { + return 0, nil, err + } + if n > 128 { + return 0, nil, fmt.Errorf("too many CRDS values in packet: %d", n) + } + var contacts []*ContactInfo + if handleContact == nil { + contacts = make([]*ContactInfo, 0, n) + } + contactCount := 0 + for i := uint64(0); i < n; i++ { + record, err := decodeCrdsContactRecord(d) + if err != nil { + if errors.Is(err, errUnsupportedCRDSValue) { + return contactCount, contacts, errUnsupportedCRDSValue + } + return contactCount, contacts, err + } + if !record.Verify() { + continue + } + contactCount++ + if handleContact != nil { + handleContact(record) + } else { + contacts = append(contacts, record.ContactInfo()) + } + } + return contactCount, contacts, nil +} + +type packetKind int + +const ( + packetIgnored packetKind = iota + packetPing + packetPong + packetContacts + packetPullRequest +) + +type decodedPacket struct { + Kind packetKind + Ping Ping + Pong Pong + Contacts []*ContactInfo + ContactCount int +} + +func decodePacket(packet []byte) (decodedPacket, error) { + return decodePacketWithContactHandler(packet, nil) +} + +func decodePacketWithContactHandler(packet []byte, handleContact func(contactRecord)) (decodedPacket, error) { + d := newDecoder(packet) + variant, err := d.variant() + if err != nil { + return decodedPacket{}, err + } + + switch variant { + case protocolPullRequest: + return decodedPacket{Kind: packetPullRequest}, nil + case protocolPullResponse, protocolPushMessage: + if _, err := d.read(32); err != nil { + return decodedPacket{}, err + } + contactCount, contacts, err := decodeContactRecords(d, handleContact) + if err != nil { + if errors.Is(err, errUnsupportedCRDSValue) { + return decodedPacket{Kind: packetIgnored}, nil + } + return decodedPacket{}, err + } + return decodedPacket{Kind: packetContacts, Contacts: contacts, ContactCount: contactCount}, nil + case protocolPingMessage: + ping, err := decodePing(d) + if err != nil { + return decodedPacket{}, err + } + return decodedPacket{Kind: packetPing, Ping: ping}, nil + case protocolPongMessage: + pong, err := decodePong(d) + if err != nil { + return decodedPacket{}, err + } + return decodedPacket{Kind: packetPong, Pong: pong}, nil + case protocolPruneMessage: + return decodedPacket{Kind: packetIgnored}, nil + default: + return decodedPacket{Kind: packetIgnored}, nil + } +} + +func sendUDP(conn *net.UDPConn, packet []byte, addr *net.UDPAddr) error { + if len(packet) > packetDataSize { + return fmt.Errorf("gossip packet size %d exceeds packet size %d", len(packet), packetDataSize) + } + _, err := conn.WriteToUDP(packet, addr) + return err +} diff --git a/pkg/gossip/message_test.go b/pkg/gossip/message_test.go deleted file mode 100644 index dcfb024c..00000000 --- a/pkg/gossip/message_test.go +++ /dev/null @@ -1,210 +0,0 @@ -package gossip - -import ( - "net/netip" - "testing" - - "github.com/Overclock-Validator/mithril/fixtures" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMessage(t *testing.T) { - cases := []struct { - name string - fixture string - message Message - err string - }{ - { - name: "PullRequest", - fixture: "gossip/pull_request.bin", - message: &Message__PullRequest{ - Filter: CrdsFilter{ - Filter: Bloom{ - Keys: []uint64{ - 0x0e1f75695561c8f4, - 0x7edda0f717d55580, - 0x221a275bb8650ed4, - }, - Bits: MakeBitVecU64(make([]uint64, 97), 6168), - }, - Mask: 0x3ffffffffffffff, - MaskBits: 6, - }, - Value: CrdsValue{ - Signature: Signature{ - 0x50, 0xd2, 0xbd, 0xa4, 0x5b, 0x66, 0xd9, 0xf5, 0xc7, 0x7e, 0xef, 0x3b, 0x25, 0x65, 0xcd, 0xf6, - 0x5a, 0xcb, 0xd6, 0x0f, 0x56, 0xce, 0x77, 0xb7, 0xf4, 0xd8, 0xb0, 0x28, 0x4f, 0x03, 0x12, 0xeb, - 0xf9, 0x78, 0xf8, 0x2f, 0xa6, 0xbe, 0x7e, 0xeb, 0x9f, 0xdd, 0x55, 0x5c, 0x9c, 0x9f, 0x61, 0xc5, - 0xdc, 0x60, 0x57, 0x26, 0xd5, 0xc0, 0xe2, 0x93, 0x4e, 0x16, 0x31, 0xb6, 0x68, 0xa5, 0xde, 0x09, - }, - Data: &CrdsData__ContactInfo{ - Value: ContactInfo{ - Id: Pubkey{ - 0xdd, 0x52, 0xbd, 0x9b, 0x7d, 0xb5, 0xcd, 0x06, - 0x1a, 0xe6, 0xbe, 0x46, 0x98, 0xd9, 0x32, 0x96, - 0xe9, 0x09, 0xa1, 0xb9, 0xc4, 0xec, 0x10, 0xf7, - 0xbc, 0xb1, 0x43, 0x0d, 0xed, 0xf7, 0xb6, 0x30, - }, - Wallclock: 1660627129489, - }, - }, - }, - }, - }, - { - name: "PullResponse_ContactInfo", - fixture: "gossip/pull_response_contact_info.bin", - message: &Message__PullResponse{ - Pubkey: Pubkey{ - 0x7a, 0x1f, 0xe3, 0x85, 0x3f, 0x19, 0xa1, 0xcc, 0x52, 0x82, 0x2c, 0x0d, 0x03, 0x2d, 0x19, 0x16, - 0xaf, 0x37, 0x50, 0xbb, 0xd8, 0x25, 0x16, 0x3e, 0x46, 0xe2, 0x87, 0x16, 0xde, 0x8e, 0x35, 0x6f, - }, - Values: []CrdsValue{ - { - Signature: Signature{ - 0x6b, 0x43, 0x11, 0xd0, 0x90, 0x85, 0x22, 0xe1, 0x28, 0xf9, 0xcf, 0xf9, 0x72, 0xaa, 0xfb, 0xa5, - 0x05, 0x33, 0xdb, 0x3c, 0x85, 0xfa, 0x83, 0x97, 0x22, 0x70, 0xb3, 0xaf, 0x02, 0x7f, 0x1f, 0x3f, - 0x32, 0xb3, 0xb9, 0x7b, 0x42, 0x81, 0xc7, 0x55, 0x61, 0x43, 0x35, 0x48, 0x08, 0x65, 0x26, 0x1a, - 0x7b, 0x66, 0xfa, 0x4a, 0x60, 0xc4, 0xc4, 0xb8, 0x8c, 0x5f, 0xae, 0xcb, 0x40, 0x76, 0x7b, 0x03, - }, - Data: &CrdsData__ContactInfo{ - Value: ContactInfo{ - Id: Pubkey{ - 0x7a, 0x1f, 0xe3, 0x85, 0x3f, 0x19, 0xa1, 0xcc, 0x52, 0x82, 0x2c, 0x0d, 0x03, 0x2d, 0x19, 0x16, - 0xaf, 0x37, 0x50, 0xbb, 0xd8, 0x25, 0x16, 0x3e, 0x46, 0xe2, 0x87, 0x16, 0xde, 0x8e, 0x35, 0x6f, - }, - Gossip: SocketAddr{netip.MustParseAddrPort("127.0.0.1:1024")}, - Tvu: SocketAddr{netip.MustParseAddrPort("127.0.0.1:1025")}, - TvuForwards: SocketAddr{netip.MustParseAddrPort("127.0.0.1:1026")}, - Repair: SocketAddr{netip.MustParseAddrPort("127.0.0.1:1031")}, - Tpu: SocketAddr{netip.MustParseAddrPort("127.0.0.1:1027")}, - TpuForwards: SocketAddr{netip.MustParseAddrPort("127.0.0.1:1028")}, - TpuVote: SocketAddr{netip.MustParseAddrPort("127.0.0.1:1029")}, - Rpc: SocketAddr{netip.MustParseAddrPort("127.0.0.1:8899")}, - RpcPubsub: SocketAddr{netip.MustParseAddrPort("127.0.0.1:8900")}, - ServeRepair: SocketAddr{netip.MustParseAddrPort("127.0.0.1:1032")}, - Wallclock: 1660658416429, - ShredVersion: 25514, - }, - }, - }, - }, - }, - }, - { - name: "PullResponse_SnapshotHashes", - fixture: "gossip/pull_response_snapshot_hashes.bin", - message: &Message__PullResponse{ - Pubkey: Pubkey{ - 0x7a, 0x1f, 0xe3, 0x85, 0x3f, 0x19, 0xa1, 0xcc, 0x52, 0x82, 0x2c, 0x0d, 0x03, 0x2d, 0x19, 0x16, - 0xaf, 0x37, 0x50, 0xbb, 0xd8, 0x25, 0x16, 0x3e, 0x46, 0xe2, 0x87, 0x16, 0xde, 0x8e, 0x35, 0x6f, - }, - Values: []CrdsValue{ - { - Signature: Signature{ - 0xbe, 0x85, 0x8b, 0xf3, 0xb2, 0x65, 0x47, 0x7d, 0xac, 0x2b, 0x86, 0xfc, 0x80, 0x8b, 0x9a, 0x78, - 0x99, 0xbc, 0xac, 0x25, 0xbd, 0xa2, 0xd0, 0x66, 0x6f, 0x3b, 0x80, 0x48, 0xea, 0x1f, 0x1f, 0xc2, - 0xfe, 0x11, 0xe1, 0xe3, 0x4d, 0x27, 0xd4, 0xfc, 0xd8, 0xd0, 0x89, 0xcf, 0x04, 0xef, 0x78, 0x45, - 0xb5, 0x12, 0x70, 0x57, 0x1d, 0xff, 0x82, 0xc0, 0x23, 0x9a, 0x6d, 0xf6, 0x75, 0xcd, 0x9a, 0x06, - }, - Data: &CrdsData__SnapshotHashes{ - Value: SnapshotHashes{ - From: Pubkey{ - 0x7a, 0x1f, 0xe3, 0x85, 0x3f, 0x19, 0xa1, 0xcc, 0x52, 0x82, 0x2c, 0x0d, 0x03, 0x2d, 0x19, 0x16, - 0xaf, 0x37, 0x50, 0xbb, 0xd8, 0x25, 0x16, 0x3e, 0x46, 0xe2, 0x87, 0x16, 0xde, 0x8e, 0x35, 0x6f, - }, - Hashes: []SlotHash{ - { - Slot: 47411, - Hash: Hash{ - 0xa6, 0xb1, 0x85, 0x23, 0xe7, 0xaa, 0xca, 0x36, 0xe4, 0xda, 0x16, 0xc8, 0x8f, 0x5b, 0xa9, 0xad, - 0xd8, 0x77, 0xf7, 0x62, 0x0b, 0x8f, 0xf2, 0xcc, 0xe4, 0x35, 0x7c, 0x8e, 0xb8, 0xed, 0x3c, 0x8a, - }, - }, - }, - Wallclock: 1660658416429, - }, - }, - }, - }, - }, - }, - { - name: "PullResponse_Version", - fixture: "gossip/pull_response_version.bin", - message: &Message__PullResponse{ - Pubkey: Pubkey{ - 0x7a, 0x1f, 0xe3, 0x85, 0x3f, 0x19, 0xa1, 0xcc, 0x52, 0x82, 0x2c, 0x0d, 0x03, 0x2d, 0x19, 0x16, - 0xaf, 0x37, 0x50, 0xbb, 0xd8, 0x25, 0x16, 0x3e, 0x46, 0xe2, 0x87, 0x16, 0xde, 0x8e, 0x35, 0x6f, - }, - Values: []CrdsValue{ - { - Signature: Signature{ - 0x96, 0x9f, 0x40, 0x41, 0xdd, 0x80, 0x5e, 0x6f, 0x89, 0x58, 0x21, 0xf7, 0x28, 0xe2, 0x95, 0xb2, - 0x91, 0xfc, 0x85, 0xaa, 0xc2, 0x2e, 0x88, 0x51, 0xea, 0x95, 0x02, 0xab, 0x38, 0x0b, 0x5d, 0x9f, - 0xe7, 0x9e, 0xb1, 0x54, 0x36, 0x78, 0x5c, 0x49, 0xd6, 0x74, 0x1b, 0xf0, 0xb0, 0x59, 0x5f, 0x77, - 0xe0, 0xb7, 0x1b, 0x39, 0xb0, 0x39, 0x68, 0x76, 0x5b, 0x71, 0x81, 0x7f, 0x07, 0x79, 0x15, 0x0f, - }, - Data: &CrdsData__Version{ - From: Pubkey{ - 0x7a, 0x1f, 0xe3, 0x85, 0x3f, 0x19, 0xa1, 0xcc, 0x52, 0x82, 0x2c, 0x0d, 0x03, 0x2d, 0x19, 0x16, - 0xaf, 0x37, 0x50, 0xbb, 0xd8, 0x25, 0x16, 0x3e, 0x46, 0xe2, 0x87, 0x16, 0xde, 0x8e, 0x35, 0x6f, - }, - Wallclock: 1660658416907, - Major: 1, - Minor: 12, - Patch: 0, - Commit: nil, - FeatureSet: 0x1800dbd1, - }, - }, - }, - }, - }, - { - name: "PullResponse_NodeInstance", - fixture: "gossip/pull_response_node_instance.bin", - message: &Message__PullResponse{ - Pubkey: Pubkey{ - 0x7a, 0x1f, 0xe3, 0x85, 0x3f, 0x19, 0xa1, 0xcc, 0x52, 0x82, 0x2c, 0x0d, 0x03, 0x2d, 0x19, 0x16, - 0xaf, 0x37, 0x50, 0xbb, 0xd8, 0x25, 0x16, 0x3e, 0x46, 0xe2, 0x87, 0x16, 0xde, 0x8e, 0x35, 0x6f, - }, - Values: []CrdsValue{ - { - Signature: Signature{ - 0x61, 0x74, 0xf0, 0x84, 0x41, 0xcb, 0xd8, 0x9f, 0xcd, 0xb6, 0xbf, 0xee, 0x92, 0x47, 0x17, 0xec, - 0x28, 0xd8, 0xeb, 0xa3, 0xe4, 0x17, 0x4d, 0x75, 0x45, 0x9f, 0x34, 0x02, 0xa4, 0x91, 0x1f, 0xc0, - 0x15, 0x34, 0xb6, 0x0c, 0xdb, 0x84, 0x4d, 0xe3, 0xaa, 0xf7, 0xcb, 0x3d, 0xf0, 0x4e, 0x71, 0xaa, - 0x24, 0xc2, 0x16, 0xe5, 0x8a, 0x17, 0x37, 0x90, 0xe9, 0x50, 0xbc, 0xd2, 0x4c, 0x5a, 0xc7, 0x02, - }, - Data: &CrdsData__NodeInstance{ - From: Pubkey{ - 0x7a, 0x1f, 0xe3, 0x85, 0x3f, 0x19, 0xa1, 0xcc, 0x52, 0x82, 0x2c, 0x0d, 0x03, 0x2d, 0x19, 0x16, - 0xaf, 0x37, 0x50, 0xbb, 0xd8, 0x25, 0x16, 0x3e, 0x46, 0xe2, 0x87, 0x16, 0xde, 0x8e, 0x35, 0x6f, - }, - Wallclock: 1660658416907, - Timestamp: 1660658416429, - Token: 0x5d229535ca896c95, - }, - }, - }, - }, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - frame := fixtures.Load(t, tc.fixture) - msg, err := BincodeDeserializeMessage(frame) - if tc.err == "" { - require.NoError(t, err) - assert.Equal(t, tc.message, msg) - } else { - assert.Nil(t, tc.message) - assert.EqualError(t, err, tc.err) - } - }) - } -} diff --git a/pkg/gossip/ping.go b/pkg/gossip/ping.go deleted file mode 100644 index cb737ffc..00000000 --- a/pkg/gossip/ping.go +++ /dev/null @@ -1,245 +0,0 @@ -package gossip - -import ( - "context" - "crypto/ed25519" - "crypto/sha256" - "math/rand" - "net/netip" - "sync" - "sync/atomic" - - "k8s.io/klog/v2" -) - -// PingSize is the size of a serialized ping message. -const PingSize = 128 - -// NewPing creates and signs a new ping message. -// -// Panics if the provided private key is invalid. -func NewPing(token [32]byte, key ed25519.PrivateKey) (p Ping) { - sig := ed25519.Sign(key, token[:]) - copy(p.From[:], key.Public().(ed25519.PublicKey)) - copy(p.Token[:], token[:]) - copy(p.Signature[:], sig[:]) - return p -} - -func NewPingRandom(key ed25519.PrivateKey) Ping { - var token [32]byte - rand.Read(token[:]) - return NewPing(token, key) -} - -// Verify checks the Ping's signature. -func (p *Ping) Verify() bool { - return ed25519.Verify(p.From[:], p.Token[:], p.Signature[:]) -} - -// HashPingToken returns the pong token given a ping token. -func HashPingToken(token [32]byte) [32]byte { - h := sha256.New() - h.Write([]byte("SOLANA_PING_PONG")) - h.Write(token[:]) - return *(*[32]byte)(h.Sum(nil)) -} - -// PingClient implements the stateful client (initiator) side of the gossip ping protocol. -// -// It tracks every pending request to match it with solicited pong frames. -type PingClient struct { - identity ed25519.PrivateKey - so udpSender - - lock sync.RWMutex - reqs map[Hash]*pingSession // pong token => session - - NumSent atomic.Uint64 // ping messages sent - NumOK atomic.Uint64 // successful ping transaction - NumInvalid atomic.Uint64 // invalid sig in pong - NumTimeout atomic.Uint64 // context errored before pong arrived - NumSendFail atomic.Uint64 // socket refused to send (tx buffer full) - NumMartian atomic.Uint64 // unsolicited pong -} - -func NewPingClient(identity ed25519.PrivateKey, so udpSender) *PingClient { - return &PingClient{ - identity: identity, - so: so, - reqs: make(map[Hash]*pingSession), - } -} - -type pingSession struct { - out chan<- pongResponse - done atomic.Bool -} - -type pongResponse struct { - from netip.AddrPort - pong Ping -} - -// Ping sends a gossip ping packet. -// Blocks until a valid matching pong packet arrives or the context is cancelled. -// -// Note that this mechanism is unrelated to ICMP pings. -func (p *PingClient) Ping(ctx context.Context, target netip.AddrPort) (pong Ping, responder netip.AddrPort, err error) { - // Allocate session for lifetime of scope - ping, pongToken, resp := p.createSession() - defer p.destroySession(pongToken) - - // Send ping to "server" - pingMsg := &Message__Ping{ping} - packet, err := pingMsg.BincodeSerialize() - if err != nil { - klog.Errorf("Failed to serialize ping: %s", err) - return - } - if _, err = p.so.WriteToUDPAddrPort(packet, target); err != nil { - p.NumSendFail.Add(1) - return - } - p.NumSent.Add(1) - - // Block until something happens - select { - case <-ctx.Done(): - err = ctx.Err() - p.NumTimeout.Add(1) - return - case resp, ok := <-resp: - if !ok { - // sanity check: cancellation can only happen before or after select - panic("race condition") - } - pong = resp.pong - responder = resp.from - p.NumOK.Add(1) - return - } -} - -// HandlePong processes incoming gossip pong messages. -func (p *PingClient) HandlePong(msg *Message__Pong, from netip.AddrPort) { - pong := msg.Value - - // map lookup is cheaper than Ed25519 verify, so do that first - sess := p.getSession(pong.Token) - if sess == nil { - p.NumMartian.Add(1) - return - } - - // We might receive two valid pongs before the initiating goroutine cleans up. - // Bail here because we are only allowed to send one pong back to the channel. - if sess.done.Swap(true) { - p.NumMartian.Add(1) - return - } - - if !pong.Verify() { - p.NumInvalid.Add(1) - return - } - - // Upgrade to write lock to prevent initiating goroutine - // from closing the channel we're about to send on. - // TODO: this is probably very slow - p.lock.Lock() - defer p.lock.Unlock() - sess = p.reqs[pong.Token] - if sess == nil { - // session was cancelled while we were verifying the pong - p.NumMartian.Add(1) - return - } - sess.out <- pongResponse{ - from: from, - pong: pong, - } -} - -func (p *PingClient) createSession() (ping Ping, pongToken Hash, resp <-chan pongResponse) { - ping = NewPingRandom(p.identity) - pongToken = HashPingToken(ping.Token) - - respBi := make(chan pongResponse, 1) - resp = respBi // recv-only - - p.lock.Lock() - defer p.lock.Unlock() - p.reqs[pongToken] = &pingSession{ - out: respBi, // send-only - } - return -} - -func (p *PingClient) getSession(pongToken Hash) *pingSession { - p.lock.RLock() - defer p.lock.RUnlock() - return p.reqs[pongToken] -} - -func (p *PingClient) destroySession(pongToken Hash) { - p.lock.Lock() - defer p.lock.Unlock() - - session, ok := p.reqs[pongToken] - if !ok { - return - } - close(session.out) - delete(p.reqs, pongToken) -} - -func (p *PingClient) Close() { - p.lock.Lock() - defer p.lock.Unlock() -} - -// PingServer implements the stateless server (reactor) side of the gossip ping protocol. -// -// It implements no rate-limits and is thus vulnerable to packet floods. -type PingServer struct { - identity ed25519.PrivateKey - so udpSender - - NumOK atomic.Uint64 // handled pings, though uncertain whether pong arrived - NumInvalid atomic.Uint64 // invalid sig in ping - NumSendFail atomic.Uint64 // socket refused to send (tx buffer full) -} - -func NewPingServer(identity ed25519.PrivateKey, so udpSender) *PingServer { - return &PingServer{ - identity: identity, - so: so, - } -} - -// HandlePing processes incoming gossip ping messages. -func (p *PingServer) HandlePing(ping *Message__Ping, from netip.AddrPort) { - // Verify signature of ping. - if !ping.Value.Verify() { - p.NumInvalid.Add(1) - return - } - - // SHA-256 hash token and sign with identity. - // Note: Possible signature forging attack vector. - pong := NewPing(HashPingToken(ping.Value.Token), p.identity) - pongMsg := &Message__Pong{pong} - - // Respond to sender. - packet, err := pongMsg.BincodeSerialize() - if err != nil { - klog.Errorf("Failed to serialize pong: %s", err) - return - } - if _, err = p.so.WriteToUDPAddrPort(packet, from); err != nil { - p.NumSendFail.Add(1) - return - } - p.NumOK.Add(1) -} diff --git a/pkg/gossip/ping_test.go b/pkg/gossip/ping_test.go deleted file mode 100644 index 6aec6f85..00000000 --- a/pkg/gossip/ping_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package gossip - -import ( - "context" - "crypto/ed25519" - "crypto/rand" - "net" - "net/netip" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" -) - -func TestPingServer(t *testing.T) { - conn, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.MustParseAddrPort("[::1]:0"))) - require.NoError(t, err) - defer conn.Close() - - lo := conn.LocalAddr().(*net.UDPAddr).AddrPort() - - _, identity, err := ed25519.GenerateKey(rand.Reader) - require.NoError(t, err) - - handler := &Handler{ - PingClient: NewPingClient(identity, conn), - PingServer: NewPingServer(identity, conn), - } - client := NewDriver(handler, conn) - - ctx := context.Background() - ctx, cancel := context.WithCancel(ctx) - - group, ctx := errgroup.WithContext(ctx) - group.Go(func() error { - return client.Run(ctx) - }) - group.Go(func() error { - defer cancel() - pinger := handler.PingClient - - for i := 0; i < 100; i++ { - pong, responder, err := pinger.Ping(ctx, lo) - require.NoError(t, err) - assert.Equal(t, lo, responder) - assert.True(t, pong.Verify()) - } - return nil - }) - err = group.Wait() - assert.NoError(t, err) -} - -func BenchmarkPing_SignHashVerify(b *testing.B) { - _, identity, err := ed25519.GenerateKey(rand.Reader) - require.NoError(b, err) - - for i := 0; i < b.N; i++ { - ping := NewPingRandom(identity) - assert.True(b, ping.Verify()) - pong := NewPing(HashPingToken(ping.Token), identity) - assert.True(b, pong.Verify()) - } -} diff --git a/pkg/gossip/pull.go b/pkg/gossip/pull.go deleted file mode 100644 index e78f4cb7..00000000 --- a/pkg/gossip/pull.go +++ /dev/null @@ -1,64 +0,0 @@ -package gossip - -import ( - "crypto/ed25519" - "encoding/json" - "fmt" - "net/netip" - "time" -) - -const PacketSize = 1232 - -// PullClient implements the stateful client (initiator) side of the gossip pull protocol. -type PullClient struct { - identity ed25519.PrivateKey - so udpSender -} - -func NewPullClient(identity ed25519.PrivateKey, so udpSender) *PullClient { - return &PullClient{ - identity: identity, - so: so, - } -} - -func (p *PullClient) Pull(target netip.AddrPort) error { - filters := NewCrdsFilterSet(65536, MaxBloomSize) - for _, filter := range filters { - if err := p.sendPullRequest(target, filter); err != nil { - return err - } - } - return nil -} - -func (p *PullClient) sendPullRequest(target netip.AddrPort, filter CrdsFilter) error { - msg := &Message__PullRequest{ - Filter: filter, - Value: CrdsValue{ - Data: &CrdsData__ContactInfo{ - Value: ContactInfo{ - Wallclock: uint64(time.Now().UnixMilli()), - }, - }, - }, - } - err := msg.Value.Sign(p.identity) - if err != nil { - panic("failed to sign pull request: " + err.Error()) - } - - packet, err := msg.BincodeSerialize() - if err != nil { - panic("failed to serialize packet: " + err.Error()) - } - - _, err = p.so.WriteToUDPAddrPort(packet, target) - return err -} - -func (p *PullClient) HandlePullResponse(msg *Message__PullResponse, _ netip.AddrPort) { - jsonBuf, _ := json.MarshalIndent(msg, "", "\t") - fmt.Println(string(jsonBuf)) -} diff --git a/pkg/gossip/schema.go b/pkg/gossip/schema.go deleted file mode 100644 index f7ff2890..00000000 --- a/pkg/gossip/schema.go +++ /dev/null @@ -1,3527 +0,0 @@ -// Code generated by "serde-generate"; DO NOT EDIT. - -package gossip - -import ( - "fmt" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/bincode" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/serde" -) - -type BitVecU64 struct { - Bits BitVecU64Inner - Len uint64 -} - -func (obj *BitVecU64) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.Bits.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Len); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *BitVecU64) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeBitVecU64(deserializer serde.Deserializer) (BitVecU64, error) { - var obj BitVecU64 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeBitVecU64Inner(deserializer); err == nil { - obj.Bits = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Len = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeBitVecU64(input []byte) (BitVecU64, error) { - if input == nil { - var obj BitVecU64 - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeBitVecU64(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type BitVecU64Inner struct { - Value *[]uint64 -} - -func (obj *BitVecU64Inner) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serialize_option_vector_u64(obj.Value, serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *BitVecU64Inner) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeBitVecU64Inner(deserializer serde.Deserializer) (BitVecU64Inner, error) { - var obj BitVecU64Inner - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserialize_option_vector_u64(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeBitVecU64Inner(input []byte) (BitVecU64Inner, error) { - if input == nil { - var obj BitVecU64Inner - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeBitVecU64Inner(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type BitVecU8 struct { - Bits BitVecU8Inner - Len uint64 -} - -func (obj *BitVecU8) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.Bits.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Len); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *BitVecU8) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeBitVecU8(deserializer serde.Deserializer) (BitVecU8, error) { - var obj BitVecU8 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeBitVecU8Inner(deserializer); err == nil { - obj.Bits = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Len = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeBitVecU8(input []byte) (BitVecU8, error) { - if input == nil { - var obj BitVecU8 - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeBitVecU8(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type BitVecU8Inner struct { - Value *[]uint8 -} - -func (obj *BitVecU8Inner) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serialize_option_vector_u8(obj.Value, serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *BitVecU8Inner) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeBitVecU8Inner(deserializer serde.Deserializer) (BitVecU8Inner, error) { - var obj BitVecU8Inner - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserialize_option_vector_u8(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeBitVecU8Inner(input []byte) (BitVecU8Inner, error) { - if input == nil { - var obj BitVecU8Inner - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeBitVecU8Inner(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type Bloom struct { - Keys []uint64 - Bits BitVecU64 - NumBitsSet uint64 -} - -func (obj *Bloom) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serialize_vector_u64(obj.Keys, serializer); err != nil { - return err - } - if err := obj.Bits.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.NumBitsSet); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Bloom) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeBloom(deserializer serde.Deserializer) (Bloom, error) { - var obj Bloom - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserialize_vector_u64(deserializer); err == nil { - obj.Keys = val - } else { - return obj, err - } - if val, err := DeserializeBitVecU64(deserializer); err == nil { - obj.Bits = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.NumBitsSet = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeBloom(input []byte) (Bloom, error) { - if input == nil { - var obj Bloom - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeBloom(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type CompressedSlots interface { - isCompressedSlots() - Serialize(serializer serde.Serializer) error - BincodeSerialize() ([]byte, error) -} - -func DeserializeCompressedSlots(deserializer serde.Deserializer) (CompressedSlots, error) { - index, err := deserializer.DeserializeVariantIndex() - if err != nil { - return nil, err - } - - switch index { - case 0: - if val, err := load_CompressedSlots__Flate2(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 1: - if val, err := load_CompressedSlots__Uncompressed(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - default: - return nil, fmt.Errorf("Unknown variant index for CompressedSlots: %d", index) - } -} - -func BincodeDeserializeCompressedSlots(input []byte) (CompressedSlots, error) { - if input == nil { - var obj CompressedSlots - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeCompressedSlots(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type CompressedSlots__Flate2 struct { - Value SlotsFlate2 -} - -func (*CompressedSlots__Flate2) isCompressedSlots() {} - -func (obj *CompressedSlots__Flate2) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(0) - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CompressedSlots__Flate2) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_CompressedSlots__Flate2(deserializer serde.Deserializer) (CompressedSlots__Flate2, error) { - var obj CompressedSlots__Flate2 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeSlotsFlate2(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CompressedSlots__Uncompressed struct { - Value SlotsUncompressed -} - -func (*CompressedSlots__Uncompressed) isCompressedSlots() {} - -func (obj *CompressedSlots__Uncompressed) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(1) - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CompressedSlots__Uncompressed) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_CompressedSlots__Uncompressed(deserializer serde.Deserializer) (CompressedSlots__Uncompressed, error) { - var obj CompressedSlots__Uncompressed - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeSlotsUncompressed(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type ContactInfo struct { - Id Pubkey - Gossip SocketAddr - Tvu SocketAddr - TvuForwards SocketAddr - Repair SocketAddr - Tpu SocketAddr - TpuForwards SocketAddr - TpuVote SocketAddr - Rpc SocketAddr - RpcPubsub SocketAddr - ServeRepair SocketAddr - Wallclock uint64 - ShredVersion uint16 -} - -func (obj *ContactInfo) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.Id.Serialize(serializer); err != nil { - return err - } - if err := obj.Gossip.Serialize(serializer); err != nil { - return err - } - if err := obj.Tvu.Serialize(serializer); err != nil { - return err - } - if err := obj.TvuForwards.Serialize(serializer); err != nil { - return err - } - if err := obj.Repair.Serialize(serializer); err != nil { - return err - } - if err := obj.Tpu.Serialize(serializer); err != nil { - return err - } - if err := obj.TpuForwards.Serialize(serializer); err != nil { - return err - } - if err := obj.TpuVote.Serialize(serializer); err != nil { - return err - } - if err := obj.Rpc.Serialize(serializer); err != nil { - return err - } - if err := obj.RpcPubsub.Serialize(serializer); err != nil { - return err - } - if err := obj.ServeRepair.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - if err := serializer.SerializeU16(obj.ShredVersion); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *ContactInfo) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeContactInfo(deserializer serde.Deserializer) (ContactInfo, error) { - var obj ContactInfo - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.Id = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.Gossip = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.Tvu = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.TvuForwards = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.Repair = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.Tpu = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.TpuForwards = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.TpuVote = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.Rpc = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.RpcPubsub = val - } else { - return obj, err - } - if val, err := DeserializeSocketAddr(deserializer); err == nil { - obj.ServeRepair = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.ShredVersion = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeContactInfo(input []byte) (ContactInfo, error) { - if input == nil { - var obj ContactInfo - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeContactInfo(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type CrdsData interface { - isCrdsData() - Serialize(serializer serde.Serializer) error - BincodeSerialize() ([]byte, error) - Pubkey() *Pubkey -} - -func DeserializeCrdsData(deserializer serde.Deserializer) (CrdsData, error) { - index, err := deserializer.DeserializeVariantIndex() - if err != nil { - return nil, err - } - - switch index { - case 0: - if val, err := load_CrdsData__ContactInfo(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 1: - if val, err := load_CrdsData__Vote(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 2: - if val, err := load_CrdsData__LowestSlot(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 3: - if val, err := load_CrdsData__SnapshotHashes(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 4: - if val, err := load_CrdsData__AccountsHashes(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 5: - if val, err := load_CrdsData__EpochSlots(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 6: - if val, err := load_CrdsData__LegacyVersion(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 7: - if val, err := load_CrdsData__Version(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 8: - if val, err := load_CrdsData__NodeInstance(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 9: - if val, err := load_CrdsData__DuplicateShred(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 10: - if val, err := load_CrdsData__IncrementalSnapshotHashes(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - default: - return nil, fmt.Errorf("Unknown variant index for CrdsData: %d", index) - } -} - -func BincodeDeserializeCrdsData(input []byte) (CrdsData, error) { - if input == nil { - var obj CrdsData - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeCrdsData(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type CrdsData__ContactInfo struct { - Value ContactInfo -} - -func (*CrdsData__ContactInfo) isCrdsData() {} - -func (obj *CrdsData__ContactInfo) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(0) - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__ContactInfo) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__ContactInfo) Pubkey() *Pubkey { - return &obj.Value.Id -} - -func load_CrdsData__ContactInfo(deserializer serde.Deserializer) (CrdsData__ContactInfo, error) { - var obj CrdsData__ContactInfo - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeContactInfo(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__Vote struct { - Field0 uint8 - Field1 Vote -} - -func (*CrdsData__Vote) isCrdsData() {} - -func (obj *CrdsData__Vote) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(1) - if err := serializer.SerializeU8(obj.Field0); err != nil { - return err - } - if err := obj.Field1.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__Vote) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__Vote) Pubkey() *Pubkey { - return &obj.Field1.From -} - -func load_CrdsData__Vote(deserializer serde.Deserializer) (CrdsData__Vote, error) { - var obj CrdsData__Vote - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserializer.DeserializeU8(); err == nil { - obj.Field0 = val - } else { - return obj, err - } - if val, err := DeserializeVote(deserializer); err == nil { - obj.Field1 = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__LowestSlot struct { - Field0 uint8 - Field1 LowestSlot -} - -func (*CrdsData__LowestSlot) isCrdsData() {} - -func (obj *CrdsData__LowestSlot) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(2) - if err := serializer.SerializeU8(obj.Field0); err != nil { - return err - } - if err := obj.Field1.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__LowestSlot) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__LowestSlot) Pubkey() *Pubkey { - return &obj.Field1.From -} - -func load_CrdsData__LowestSlot(deserializer serde.Deserializer) (CrdsData__LowestSlot, error) { - var obj CrdsData__LowestSlot - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserializer.DeserializeU8(); err == nil { - obj.Field0 = val - } else { - return obj, err - } - if val, err := DeserializeLowestSlot(deserializer); err == nil { - obj.Field1 = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__SnapshotHashes struct { - Value SnapshotHashes -} - -func (*CrdsData__SnapshotHashes) isCrdsData() {} - -func (obj *CrdsData__SnapshotHashes) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(3) - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__SnapshotHashes) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__SnapshotHashes) Pubkey() *Pubkey { - return &obj.Value.From -} - -func load_CrdsData__SnapshotHashes(deserializer serde.Deserializer) (CrdsData__SnapshotHashes, error) { - var obj CrdsData__SnapshotHashes - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeSnapshotHashes(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__AccountsHashes struct { - Value SnapshotHashes -} - -func (*CrdsData__AccountsHashes) isCrdsData() {} - -func (obj *CrdsData__AccountsHashes) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(4) - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__AccountsHashes) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__AccountsHashes) Pubkey() *Pubkey { - return &obj.Value.From -} - -func load_CrdsData__AccountsHashes(deserializer serde.Deserializer) (CrdsData__AccountsHashes, error) { - var obj CrdsData__AccountsHashes - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeSnapshotHashes(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__EpochSlots struct { - Field0 uint8 - Field1 EpochSlots -} - -func (*CrdsData__EpochSlots) isCrdsData() {} - -func (obj *CrdsData__EpochSlots) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(5) - if err := serializer.SerializeU8(obj.Field0); err != nil { - return err - } - if err := obj.Field1.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__EpochSlots) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__EpochSlots) Pubkey() *Pubkey { - return &obj.Field1.From -} - -func load_CrdsData__EpochSlots(deserializer serde.Deserializer) (CrdsData__EpochSlots, error) { - var obj CrdsData__EpochSlots - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserializer.DeserializeU8(); err == nil { - obj.Field0 = val - } else { - return obj, err - } - if val, err := DeserializeEpochSlots(deserializer); err == nil { - obj.Field1 = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__LegacyVersion struct { - From Pubkey - Wallclock uint64 - Major uint16 - Minor uint16 - Patch uint16 - Commit *uint32 -} - -func (*CrdsData__LegacyVersion) isCrdsData() {} - -func (obj *CrdsData__LegacyVersion) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(6) - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - if err := serializer.SerializeU16(obj.Major); err != nil { - return err - } - if err := serializer.SerializeU16(obj.Minor); err != nil { - return err - } - if err := serializer.SerializeU16(obj.Patch); err != nil { - return err - } - if err := serialize_option_u32(obj.Commit, serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__LegacyVersion) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__LegacyVersion) Pubkey() *Pubkey { - return &obj.From -} - -func load_CrdsData__LegacyVersion(deserializer serde.Deserializer) (CrdsData__LegacyVersion, error) { - var obj CrdsData__LegacyVersion - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.Major = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.Minor = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.Patch = val - } else { - return obj, err - } - if val, err := deserialize_option_u32(deserializer); err == nil { - obj.Commit = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__Version struct { - From Pubkey - Wallclock uint64 - Major uint16 - Minor uint16 - Patch uint16 - Commit *uint32 - FeatureSet uint32 -} - -func (*CrdsData__Version) isCrdsData() {} - -func (obj *CrdsData__Version) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(7) - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - if err := serializer.SerializeU16(obj.Major); err != nil { - return err - } - if err := serializer.SerializeU16(obj.Minor); err != nil { - return err - } - if err := serializer.SerializeU16(obj.Patch); err != nil { - return err - } - if err := serialize_option_u32(obj.Commit, serializer); err != nil { - return err - } - if err := serializer.SerializeU32(obj.FeatureSet); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__Version) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__Version) Pubkey() *Pubkey { - return &obj.From -} - -func load_CrdsData__Version(deserializer serde.Deserializer) (CrdsData__Version, error) { - var obj CrdsData__Version - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.Major = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.Minor = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.Patch = val - } else { - return obj, err - } - if val, err := deserialize_option_u32(deserializer); err == nil { - obj.Commit = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU32(); err == nil { - obj.FeatureSet = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__NodeInstance struct { - From Pubkey - Wallclock uint64 - Timestamp uint64 - Token uint64 -} - -func (*CrdsData__NodeInstance) isCrdsData() {} - -func (obj *CrdsData__NodeInstance) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(8) - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Timestamp); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Token); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__NodeInstance) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__NodeInstance) Pubkey() *Pubkey { - return &obj.From -} - -func load_CrdsData__NodeInstance(deserializer serde.Deserializer) (CrdsData__NodeInstance, error) { - var obj CrdsData__NodeInstance - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Timestamp = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Token = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__DuplicateShred struct { - Field0 uint16 - Field1 DuplicateShred -} - -func (*CrdsData__DuplicateShred) isCrdsData() {} - -func (obj *CrdsData__DuplicateShred) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(9) - if err := serializer.SerializeU16(obj.Field0); err != nil { - return err - } - if err := obj.Field1.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__DuplicateShred) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__DuplicateShred) Pubkey() *Pubkey { - return &obj.Field1.From -} - -func load_CrdsData__DuplicateShred(deserializer serde.Deserializer) (CrdsData__DuplicateShred, error) { - var obj CrdsData__DuplicateShred - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.Field0 = val - } else { - return obj, err - } - if val, err := DeserializeDuplicateShred(deserializer); err == nil { - obj.Field1 = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsData__IncrementalSnapshotHashes struct { - Value IncrementalSnapshotHashes -} - -func (*CrdsData__IncrementalSnapshotHashes) isCrdsData() {} - -func (obj *CrdsData__IncrementalSnapshotHashes) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(10) - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsData__IncrementalSnapshotHashes) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func (obj *CrdsData__IncrementalSnapshotHashes) Pubkey() *Pubkey { - return &obj.Value.From -} - -func load_CrdsData__IncrementalSnapshotHashes(deserializer serde.Deserializer) (CrdsData__IncrementalSnapshotHashes, error) { - var obj CrdsData__IncrementalSnapshotHashes - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeIncrementalSnapshotHashes(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type CrdsFilter struct { - Filter Bloom - Mask uint64 - MaskBits uint32 -} - -func (obj *CrdsFilter) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.Filter.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Mask); err != nil { - return err - } - if err := serializer.SerializeU32(obj.MaskBits); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsFilter) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeCrdsFilter(deserializer serde.Deserializer) (CrdsFilter, error) { - var obj CrdsFilter - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeBloom(deserializer); err == nil { - obj.Filter = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Mask = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU32(); err == nil { - obj.MaskBits = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeCrdsFilter(input []byte) (CrdsFilter, error) { - if input == nil { - var obj CrdsFilter - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeCrdsFilter(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type CrdsValue struct { - Signature Signature - Data CrdsData -} - -func (obj *CrdsValue) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.Signature.Serialize(serializer); err != nil { - return err - } - if err := obj.Data.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *CrdsValue) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeCrdsValue(deserializer serde.Deserializer) (CrdsValue, error) { - var obj CrdsValue - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeSignature(deserializer); err == nil { - obj.Signature = val - } else { - return obj, err - } - if val, err := DeserializeCrdsData(deserializer); err == nil { - obj.Data = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeCrdsValue(input []byte) (CrdsValue, error) { - if input == nil { - var obj CrdsValue - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeCrdsValue(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type DuplicateShred struct { - From Pubkey - Wallclock uint64 - Slot uint64 - ShredIndex uint32 - ShredType uint8 - NumChunks uint8 - ChunkIndex uint8 - Chunk []uint8 -} - -func (obj *DuplicateShred) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Slot); err != nil { - return err - } - if err := serializer.SerializeU32(obj.ShredIndex); err != nil { - return err - } - if err := serializer.SerializeU8(obj.ShredType); err != nil { - return err - } - if err := serializer.SerializeU8(obj.NumChunks); err != nil { - return err - } - if err := serializer.SerializeU8(obj.ChunkIndex); err != nil { - return err - } - if err := serialize_vector_u8(obj.Chunk, serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *DuplicateShred) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeDuplicateShred(deserializer serde.Deserializer) (DuplicateShred, error) { - var obj DuplicateShred - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Slot = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU32(); err == nil { - obj.ShredIndex = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU8(); err == nil { - obj.ShredType = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU8(); err == nil { - obj.NumChunks = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU8(); err == nil { - obj.ChunkIndex = val - } else { - return obj, err - } - if val, err := deserialize_vector_u8(deserializer); err == nil { - obj.Chunk = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeDuplicateShred(input []byte) (DuplicateShred, error) { - if input == nil { - var obj DuplicateShred - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeDuplicateShred(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type EpochSlots struct { - From Pubkey - Slots []CompressedSlots - Wallclock uint64 -} - -func (obj *EpochSlots) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := serialize_vector_CompressedSlots(obj.Slots, serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *EpochSlots) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeEpochSlots(deserializer serde.Deserializer) (EpochSlots, error) { - var obj EpochSlots - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := deserialize_vector_CompressedSlots(deserializer); err == nil { - obj.Slots = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeEpochSlots(input []byte) (EpochSlots, error) { - if input == nil { - var obj EpochSlots - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeEpochSlots(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type Hash [32]uint8 - -func (obj *Hash) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serialize_array32_u8_array((([32]uint8)(*obj)), serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Hash) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeHash(deserializer serde.Deserializer) (Hash, error) { - var obj [32]uint8 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return (Hash)(obj), err - } - if val, err := deserialize_array32_u8_array(deserializer); err == nil { - obj = val - } else { - return ((Hash)(obj)), err - } - deserializer.DecreaseContainerDepth() - return (Hash)(obj), nil -} - -func BincodeDeserializeHash(input []byte) (Hash, error) { - if input == nil { - var obj Hash - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeHash(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type IncrementalSnapshotHashes struct { - From Pubkey - Base SlotHash - Hashes []SlotHash - Wallclock uint64 -} - -func (obj *IncrementalSnapshotHashes) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := obj.Base.Serialize(serializer); err != nil { - return err - } - if err := serialize_vector_SlotHash(obj.Hashes, serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *IncrementalSnapshotHashes) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeIncrementalSnapshotHashes(deserializer serde.Deserializer) (IncrementalSnapshotHashes, error) { - var obj IncrementalSnapshotHashes - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := DeserializeSlotHash(deserializer); err == nil { - obj.Base = val - } else { - return obj, err - } - if val, err := deserialize_vector_SlotHash(deserializer); err == nil { - obj.Hashes = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeIncrementalSnapshotHashes(input []byte) (IncrementalSnapshotHashes, error) { - if input == nil { - var obj IncrementalSnapshotHashes - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeIncrementalSnapshotHashes(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type LowestSlot struct { - From Pubkey - Root uint64 - Lowest uint64 - Slots []uint64 - Stash []struct{} - Wallclock uint64 -} - -func (obj *LowestSlot) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Root); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Lowest); err != nil { - return err - } - if err := serialize_vector_u64(obj.Slots, serializer); err != nil { - return err - } - if err := serialize_vector_unit(obj.Stash, serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *LowestSlot) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeLowestSlot(deserializer serde.Deserializer) (LowestSlot, error) { - var obj LowestSlot - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Root = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Lowest = val - } else { - return obj, err - } - if val, err := deserialize_vector_u64(deserializer); err == nil { - obj.Slots = val - } else { - return obj, err - } - if val, err := deserialize_vector_unit(deserializer); err == nil { - obj.Stash = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeLowestSlot(input []byte) (LowestSlot, error) { - if input == nil { - var obj LowestSlot - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeLowestSlot(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type Message interface { - isMessage() - Serialize(serializer serde.Serializer) error - BincodeSerialize() ([]byte, error) -} - -func DeserializeMessage(deserializer serde.Deserializer) (Message, error) { - index, err := deserializer.DeserializeVariantIndex() - if err != nil { - return nil, err - } - - switch index { - case 0: - if val, err := load_Message__PullRequest(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 1: - if val, err := load_Message__PullResponse(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 2: - if val, err := load_Message__PushMessage(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 3: - if val, err := load_Message__PruneMessage(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 4: - if val, err := load_Message__Ping(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 5: - if val, err := load_Message__Pong(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - default: - return nil, fmt.Errorf("Unknown variant index for Message: %d", index) - } -} - -func BincodeDeserializeMessage(input []byte) (Message, error) { - if input == nil { - var obj Message - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeMessage(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type Message__PullRequest struct { - Filter CrdsFilter - Value CrdsValue -} - -func (*Message__PullRequest) isMessage() {} - -func (obj *Message__PullRequest) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(0) - if err := obj.Filter.Serialize(serializer); err != nil { - return err - } - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Message__PullRequest) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_Message__PullRequest(deserializer serde.Deserializer) (Message__PullRequest, error) { - var obj Message__PullRequest - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeCrdsFilter(deserializer); err == nil { - obj.Filter = val - } else { - return obj, err - } - if val, err := DeserializeCrdsValue(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type Message__PullResponse struct { - Pubkey Pubkey - Values []CrdsValue -} - -func (*Message__PullResponse) isMessage() {} - -func (obj *Message__PullResponse) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(1) - if err := obj.Pubkey.Serialize(serializer); err != nil { - return err - } - if err := serialize_vector_CrdsValue(obj.Values, serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Message__PullResponse) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_Message__PullResponse(deserializer serde.Deserializer) (Message__PullResponse, error) { - var obj Message__PullResponse - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.Pubkey = val - } else { - return obj, err - } - if val, err := deserialize_vector_CrdsValue(deserializer); err == nil { - obj.Values = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type Message__PushMessage struct { - Pubkey Pubkey - Values []CrdsValue -} - -func (*Message__PushMessage) isMessage() {} - -func (obj *Message__PushMessage) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(2) - if err := obj.Pubkey.Serialize(serializer); err != nil { - return err - } - if err := serialize_vector_CrdsValue(obj.Values, serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Message__PushMessage) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_Message__PushMessage(deserializer serde.Deserializer) (Message__PushMessage, error) { - var obj Message__PushMessage - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.Pubkey = val - } else { - return obj, err - } - if val, err := deserialize_vector_CrdsValue(deserializer); err == nil { - obj.Values = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type Message__PruneMessage struct { - Pubkey Pubkey - Data PruneData -} - -func (*Message__PruneMessage) isMessage() {} - -func (obj *Message__PruneMessage) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(3) - if err := obj.Pubkey.Serialize(serializer); err != nil { - return err - } - if err := obj.Data.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Message__PruneMessage) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_Message__PruneMessage(deserializer serde.Deserializer) (Message__PruneMessage, error) { - var obj Message__PruneMessage - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.Pubkey = val - } else { - return obj, err - } - if val, err := DeserializePruneData(deserializer); err == nil { - obj.Data = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type Message__Ping struct { - Value Ping -} - -func (*Message__Ping) isMessage() {} - -func (obj *Message__Ping) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(4) - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Message__Ping) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_Message__Ping(deserializer serde.Deserializer) (Message__Ping, error) { - var obj Message__Ping - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePing(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type Message__Pong struct { - Value Ping -} - -func (*Message__Pong) isMessage() {} - -func (obj *Message__Pong) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(5) - if err := obj.Value.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Message__Pong) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_Message__Pong(deserializer serde.Deserializer) (Message__Pong, error) { - var obj Message__Pong - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePing(deserializer); err == nil { - obj.Value = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -type Ping struct { - From Pubkey - Token Hash - Signature Signature -} - -func (obj *Ping) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := obj.Token.Serialize(serializer); err != nil { - return err - } - if err := obj.Signature.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Ping) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializePing(deserializer serde.Deserializer) (Ping, error) { - var obj Ping - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := DeserializeHash(deserializer); err == nil { - obj.Token = val - } else { - return obj, err - } - if val, err := DeserializeSignature(deserializer); err == nil { - obj.Signature = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializePing(input []byte) (Ping, error) { - if input == nil { - var obj Ping - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializePing(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type PruneData struct { - Pubkey Pubkey - Prunes []Pubkey - Signature Signature - Destination Pubkey - Wallclock uint64 -} - -func (obj *PruneData) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.Pubkey.Serialize(serializer); err != nil { - return err - } - if err := serialize_vector_Pubkey(obj.Prunes, serializer); err != nil { - return err - } - if err := obj.Signature.Serialize(serializer); err != nil { - return err - } - if err := obj.Destination.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *PruneData) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializePruneData(deserializer serde.Deserializer) (PruneData, error) { - var obj PruneData - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.Pubkey = val - } else { - return obj, err - } - if val, err := deserialize_vector_Pubkey(deserializer); err == nil { - obj.Prunes = val - } else { - return obj, err - } - if val, err := DeserializeSignature(deserializer); err == nil { - obj.Signature = val - } else { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.Destination = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializePruneData(input []byte) (PruneData, error) { - if input == nil { - var obj PruneData - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializePruneData(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type Pubkey [32]uint8 - -func (obj *Pubkey) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serialize_array32_u8_array((([32]uint8)(*obj)), serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Pubkey) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializePubkey(deserializer serde.Deserializer) (Pubkey, error) { - var obj [32]uint8 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return (Pubkey)(obj), err - } - if val, err := deserialize_array32_u8_array(deserializer); err == nil { - obj = val - } else { - return ((Pubkey)(obj)), err - } - deserializer.DecreaseContainerDepth() - return (Pubkey)(obj), nil -} - -func BincodeDeserializePubkey(input []byte) (Pubkey, error) { - if input == nil { - var obj Pubkey - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializePubkey(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type RawAddr interface { - isRawAddr() - Serialize(serializer serde.Serializer) error - BincodeSerialize() ([]byte, error) -} - -func DeserializeRawAddr(deserializer serde.Deserializer) (RawAddr, error) { - index, err := deserializer.DeserializeVariantIndex() - if err != nil { - return nil, err - } - - switch index { - case 0: - if val, err := load_RawAddr__V4(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - case 1: - if val, err := load_RawAddr__V6(deserializer); err == nil { - return &val, nil - } else { - return nil, err - } - - default: - return nil, fmt.Errorf("Unknown variant index for RawAddr: %d", index) - } -} - -func BincodeDeserializeRawAddr(input []byte) (RawAddr, error) { - if input == nil { - var obj RawAddr - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeRawAddr(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type RawAddr__V4 [4]uint8 - -func (*RawAddr__V4) isRawAddr() {} - -func (obj *RawAddr__V4) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(0) - if err := serialize_array4_u8_array((([4]uint8)(*obj)), serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *RawAddr__V4) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_RawAddr__V4(deserializer serde.Deserializer) (RawAddr__V4, error) { - var obj [4]uint8 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return (RawAddr__V4)(obj), err - } - if val, err := deserialize_array4_u8_array(deserializer); err == nil { - obj = val - } else { - return ((RawAddr__V4)(obj)), err - } - deserializer.DecreaseContainerDepth() - return (RawAddr__V4)(obj), nil -} - -type RawAddr__V6 [16]uint8 - -func (*RawAddr__V6) isRawAddr() {} - -func (obj *RawAddr__V6) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - serializer.SerializeVariantIndex(1) - if err := serialize_array16_u8_array((([16]uint8)(*obj)), serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *RawAddr__V6) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func load_RawAddr__V6(deserializer serde.Deserializer) (RawAddr__V6, error) { - var obj [16]uint8 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return (RawAddr__V6)(obj), err - } - if val, err := deserialize_array16_u8_array(deserializer); err == nil { - obj = val - } else { - return ((RawAddr__V6)(obj)), err - } - deserializer.DecreaseContainerDepth() - return (RawAddr__V6)(obj), nil -} - -type RawSocketAddr struct { - Addr Addr - Port uint16 -} - -func (obj *RawSocketAddr) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.Addr.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU16(obj.Port); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *RawSocketAddr) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeRawSocketAddr(deserializer serde.Deserializer) (RawSocketAddr, error) { - var obj RawSocketAddr - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializeAddr(deserializer); err == nil { - obj.Addr = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU16(); err == nil { - obj.Port = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeRawSocketAddr(input []byte) (RawSocketAddr, error) { - if input == nil { - var obj RawSocketAddr - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeRawSocketAddr(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type Signature [64]uint8 - -func (obj *Signature) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serialize_array64_u8_array((([64]uint8)(*obj)), serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Signature) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeSignature(deserializer serde.Deserializer) (Signature, error) { - var obj [64]uint8 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return (Signature)(obj), err - } - if val, err := deserialize_array64_u8_array(deserializer); err == nil { - obj = val - } else { - return ((Signature)(obj)), err - } - deserializer.DecreaseContainerDepth() - return (Signature)(obj), nil -} - -func BincodeDeserializeSignature(input []byte) (Signature, error) { - if input == nil { - var obj Signature - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeSignature(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type SlotHash struct { - Slot uint64 - Hash Hash -} - -func (obj *SlotHash) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Slot); err != nil { - return err - } - if err := obj.Hash.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *SlotHash) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeSlotHash(deserializer serde.Deserializer) (SlotHash, error) { - var obj SlotHash - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Slot = val - } else { - return obj, err - } - if val, err := DeserializeHash(deserializer); err == nil { - obj.Hash = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeSlotHash(input []byte) (SlotHash, error) { - if input == nil { - var obj SlotHash - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeSlotHash(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type SlotsFlate2 struct { - FirstSlot uint64 - Num uint64 - Compressed []uint8 -} - -func (obj *SlotsFlate2) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serializer.SerializeU64(obj.FirstSlot); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Num); err != nil { - return err - } - if err := serialize_vector_u8(obj.Compressed, serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *SlotsFlate2) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeSlotsFlate2(deserializer serde.Deserializer) (SlotsFlate2, error) { - var obj SlotsFlate2 - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.FirstSlot = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Num = val - } else { - return obj, err - } - if val, err := deserialize_vector_u8(deserializer); err == nil { - obj.Compressed = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeSlotsFlate2(input []byte) (SlotsFlate2, error) { - if input == nil { - var obj SlotsFlate2 - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeSlotsFlate2(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type SlotsUncompressed struct { - FirstSlot uint64 - Num uint64 - Slots BitVecU8 -} - -func (obj *SlotsUncompressed) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := serializer.SerializeU64(obj.FirstSlot); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Num); err != nil { - return err - } - if err := obj.Slots.Serialize(serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *SlotsUncompressed) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeSlotsUncompressed(deserializer serde.Deserializer) (SlotsUncompressed, error) { - var obj SlotsUncompressed - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.FirstSlot = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Num = val - } else { - return obj, err - } - if val, err := DeserializeBitVecU8(deserializer); err == nil { - obj.Slots = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeSlotsUncompressed(input []byte) (SlotsUncompressed, error) { - if input == nil { - var obj SlotsUncompressed - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeSlotsUncompressed(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type SnapshotHashes struct { - From Pubkey - Hashes []SlotHash - Wallclock uint64 -} - -func (obj *SnapshotHashes) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := serialize_vector_SlotHash(obj.Hashes, serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *SnapshotHashes) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeSnapshotHashes(deserializer serde.Deserializer) (SnapshotHashes, error) { - var obj SnapshotHashes - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := deserialize_vector_SlotHash(deserializer); err == nil { - obj.Hashes = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeSnapshotHashes(input []byte) (SnapshotHashes, error) { - if input == nil { - var obj SnapshotHashes - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeSnapshotHashes(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} - -type Vote struct { - From Pubkey - Transaction Transaction - Wallclock uint64 - Slot *uint64 -} - -func (obj *Vote) Serialize(serializer serde.Serializer) error { - if err := serializer.IncreaseContainerDepth(); err != nil { - return err - } - if err := obj.From.Serialize(serializer); err != nil { - return err - } - if err := obj.Transaction.Serialize(serializer); err != nil { - return err - } - if err := serializer.SerializeU64(obj.Wallclock); err != nil { - return err - } - if err := serialize_option_u64(obj.Slot, serializer); err != nil { - return err - } - serializer.DecreaseContainerDepth() - return nil -} - -func (obj *Vote) BincodeSerialize() ([]byte, error) { - if obj == nil { - return nil, fmt.Errorf("Cannot serialize null object") - } - serializer := bincode.NewSerializer() - if err := obj.Serialize(serializer); err != nil { - return nil, err - } - return serializer.GetBytes(), nil -} - -func DeserializeVote(deserializer serde.Deserializer) (Vote, error) { - var obj Vote - if err := deserializer.IncreaseContainerDepth(); err != nil { - return obj, err - } - if val, err := DeserializePubkey(deserializer); err == nil { - obj.From = val - } else { - return obj, err - } - if val, err := DeserializeTransaction(deserializer); err == nil { - obj.Transaction = val - } else { - return obj, err - } - if val, err := deserializer.DeserializeU64(); err == nil { - obj.Wallclock = val - } else { - return obj, err - } - if val, err := deserialize_option_u64(deserializer); err == nil { - obj.Slot = val - } else { - return obj, err - } - deserializer.DecreaseContainerDepth() - return obj, nil -} - -func BincodeDeserializeVote(input []byte) (Vote, error) { - if input == nil { - var obj Vote - return obj, fmt.Errorf("Cannot deserialize null array") - } - deserializer := bincode.NewDeserializer(input) - obj, err := DeserializeVote(deserializer) - if err == nil && deserializer.GetBufferOffset() < uint64(len(input)) { - return obj, fmt.Errorf("Some input bytes were not read") - } - return obj, err -} -func serialize_array16_u8_array(value [16]uint8, serializer serde.Serializer) error { - for _, item := range value { - if err := serializer.SerializeU8(item); err != nil { - return err - } - } - return nil -} - -func deserialize_array16_u8_array(deserializer serde.Deserializer) ([16]uint8, error) { - var obj [16]uint8 - for i := range obj { - if val, err := deserializer.DeserializeU8(); err == nil { - obj[i] = val - } else { - return obj, err - } - } - return obj, nil -} - -func serialize_array32_u8_array(value [32]uint8, serializer serde.Serializer) error { - for _, item := range value { - if err := serializer.SerializeU8(item); err != nil { - return err - } - } - return nil -} - -func deserialize_array32_u8_array(deserializer serde.Deserializer) ([32]uint8, error) { - var obj [32]uint8 - for i := range obj { - if val, err := deserializer.DeserializeU8(); err == nil { - obj[i] = val - } else { - return obj, err - } - } - return obj, nil -} - -func serialize_array4_u8_array(value [4]uint8, serializer serde.Serializer) error { - for _, item := range value { - if err := serializer.SerializeU8(item); err != nil { - return err - } - } - return nil -} - -func deserialize_array4_u8_array(deserializer serde.Deserializer) ([4]uint8, error) { - var obj [4]uint8 - for i := range obj { - if val, err := deserializer.DeserializeU8(); err == nil { - obj[i] = val - } else { - return obj, err - } - } - return obj, nil -} - -func serialize_array64_u8_array(value [64]uint8, serializer serde.Serializer) error { - for _, item := range value { - if err := serializer.SerializeU8(item); err != nil { - return err - } - } - return nil -} - -func deserialize_array64_u8_array(deserializer serde.Deserializer) ([64]uint8, error) { - var obj [64]uint8 - for i := range obj { - if val, err := deserializer.DeserializeU8(); err == nil { - obj[i] = val - } else { - return obj, err - } - } - return obj, nil -} - -func serialize_option_u32(value *uint32, serializer serde.Serializer) error { - if value != nil { - if err := serializer.SerializeOptionTag(true); err != nil { - return err - } - if err := serializer.SerializeU32((*value)); err != nil { - return err - } - } else { - if err := serializer.SerializeOptionTag(false); err != nil { - return err - } - } - return nil -} - -func deserialize_option_u32(deserializer serde.Deserializer) (*uint32, error) { - tag, err := deserializer.DeserializeOptionTag() - if err != nil { - return nil, err - } - if tag { - value := new(uint32) - if val, err := deserializer.DeserializeU32(); err == nil { - *value = val - } else { - return nil, err - } - return value, nil - } else { - return nil, nil - } -} - -func serialize_option_u64(value *uint64, serializer serde.Serializer) error { - if value != nil { - if err := serializer.SerializeOptionTag(true); err != nil { - return err - } - if err := serializer.SerializeU64((*value)); err != nil { - return err - } - } else { - if err := serializer.SerializeOptionTag(false); err != nil { - return err - } - } - return nil -} - -func deserialize_option_u64(deserializer serde.Deserializer) (*uint64, error) { - tag, err := deserializer.DeserializeOptionTag() - if err != nil { - return nil, err - } - if tag { - value := new(uint64) - if val, err := deserializer.DeserializeU64(); err == nil { - *value = val - } else { - return nil, err - } - return value, nil - } else { - return nil, nil - } -} - -func serialize_option_vector_u64(value *[]uint64, serializer serde.Serializer) error { - if value != nil { - if err := serializer.SerializeOptionTag(true); err != nil { - return err - } - if err := serialize_vector_u64((*value), serializer); err != nil { - return err - } - } else { - if err := serializer.SerializeOptionTag(false); err != nil { - return err - } - } - return nil -} - -func deserialize_option_vector_u64(deserializer serde.Deserializer) (*[]uint64, error) { - tag, err := deserializer.DeserializeOptionTag() - if err != nil { - return nil, err - } - if tag { - value := new([]uint64) - if val, err := deserialize_vector_u64(deserializer); err == nil { - *value = val - } else { - return nil, err - } - return value, nil - } else { - return nil, nil - } -} - -func serialize_option_vector_u8(value *[]uint8, serializer serde.Serializer) error { - if value != nil { - if err := serializer.SerializeOptionTag(true); err != nil { - return err - } - if err := serialize_vector_u8((*value), serializer); err != nil { - return err - } - } else { - if err := serializer.SerializeOptionTag(false); err != nil { - return err - } - } - return nil -} - -func deserialize_option_vector_u8(deserializer serde.Deserializer) (*[]uint8, error) { - tag, err := deserializer.DeserializeOptionTag() - if err != nil { - return nil, err - } - if tag { - value := new([]uint8) - if val, err := deserialize_vector_u8(deserializer); err == nil { - *value = val - } else { - return nil, err - } - return value, nil - } else { - return nil, nil - } -} - -func serialize_vector_CompressedSlots(value []CompressedSlots, serializer serde.Serializer) error { - if err := serializer.SerializeLen(uint64(len(value))); err != nil { - return err - } - for _, item := range value { - if err := item.Serialize(serializer); err != nil { - return err - } - } - return nil -} - -func deserialize_vector_CompressedSlots(deserializer serde.Deserializer) ([]CompressedSlots, error) { - length, err := deserializer.DeserializeLen() - if err != nil { - return nil, err - } - obj := make([]CompressedSlots, length) - for i := range obj { - if val, err := DeserializeCompressedSlots(deserializer); err == nil { - obj[i] = val - } else { - return nil, err - } - } - return obj, nil -} - -func serialize_vector_CrdsValue(value []CrdsValue, serializer serde.Serializer) error { - if err := serializer.SerializeLen(uint64(len(value))); err != nil { - return err - } - for _, item := range value { - if err := item.Serialize(serializer); err != nil { - return err - } - } - return nil -} - -func deserialize_vector_CrdsValue(deserializer serde.Deserializer) ([]CrdsValue, error) { - length, err := deserializer.DeserializeLen() - if err != nil { - return nil, err - } - obj := make([]CrdsValue, length) - for i := range obj { - if val, err := DeserializeCrdsValue(deserializer); err == nil { - obj[i] = val - } else { - return nil, err - } - } - return obj, nil -} - -func serialize_vector_Pubkey(value []Pubkey, serializer serde.Serializer) error { - if err := serializer.SerializeLen(uint64(len(value))); err != nil { - return err - } - for _, item := range value { - if err := item.Serialize(serializer); err != nil { - return err - } - } - return nil -} - -func deserialize_vector_Pubkey(deserializer serde.Deserializer) ([]Pubkey, error) { - length, err := deserializer.DeserializeLen() - if err != nil { - return nil, err - } - obj := make([]Pubkey, length) - for i := range obj { - if val, err := DeserializePubkey(deserializer); err == nil { - obj[i] = val - } else { - return nil, err - } - } - return obj, nil -} - -func serialize_vector_SlotHash(value []SlotHash, serializer serde.Serializer) error { - if err := serializer.SerializeLen(uint64(len(value))); err != nil { - return err - } - for _, item := range value { - if err := item.Serialize(serializer); err != nil { - return err - } - } - return nil -} - -func deserialize_vector_SlotHash(deserializer serde.Deserializer) ([]SlotHash, error) { - length, err := deserializer.DeserializeLen() - if err != nil { - return nil, err - } - obj := make([]SlotHash, length) - for i := range obj { - if val, err := DeserializeSlotHash(deserializer); err == nil { - obj[i] = val - } else { - return nil, err - } - } - return obj, nil -} - -func serialize_vector_u64(value []uint64, serializer serde.Serializer) error { - if err := serializer.SerializeLen(uint64(len(value))); err != nil { - return err - } - for _, item := range value { - if err := serializer.SerializeU64(item); err != nil { - return err - } - } - return nil -} - -func deserialize_vector_u64(deserializer serde.Deserializer) ([]uint64, error) { - length, err := deserializer.DeserializeLen() - if err != nil { - return nil, err - } - obj := make([]uint64, length) - for i := range obj { - if val, err := deserializer.DeserializeU64(); err == nil { - obj[i] = val - } else { - return nil, err - } - } - return obj, nil -} - -func serialize_vector_u8(value []uint8, serializer serde.Serializer) error { - if err := serializer.SerializeLen(uint64(len(value))); err != nil { - return err - } - for _, item := range value { - if err := serializer.SerializeU8(item); err != nil { - return err - } - } - return nil -} - -func deserialize_vector_u8(deserializer serde.Deserializer) ([]uint8, error) { - length, err := deserializer.DeserializeLen() - if err != nil { - return nil, err - } - obj := make([]uint8, length) - for i := range obj { - if val, err := deserializer.DeserializeU8(); err == nil { - obj[i] = val - } else { - return nil, err - } - } - return obj, nil -} - -func serialize_vector_unit(value []struct{}, serializer serde.Serializer) error { - if err := serializer.SerializeLen(uint64(len(value))); err != nil { - return err - } - for _, item := range value { - if err := serializer.SerializeUnit(item); err != nil { - return err - } - } - return nil -} - -func deserialize_vector_unit(deserializer serde.Deserializer) ([]struct{}, error) { - length, err := deserializer.DeserializeLen() - if err != nil { - return nil, err - } - obj := make([]struct{}, length) - for i := range obj { - if val, err := deserializer.DeserializeUnit(); err == nil { - obj[i] = val - } else { - return nil, err - } - } - return obj, nil -} diff --git a/pkg/gossip/schema.yaml b/pkg/gossip/schema.yaml deleted file mode 100644 index 7a17f008..00000000 --- a/pkg/gossip/schema.yaml +++ /dev/null @@ -1,335 +0,0 @@ ---- -# ------------------------ -# Basic types -# ------------------------ -Pubkey: - NEWTYPESTRUCT: - TUPLEARRAY: - CONTENT: U8 - SIZE: 32 -Hash: - NEWTYPESTRUCT: - TUPLEARRAY: - CONTENT: U8 - SIZE: 32 -Signature: - NEWTYPESTRUCT: - TUPLEARRAY: - CONTENT: U8 - SIZE: 64 -RawSocketAddr: - STRUCT: - - addr: - TYPENAME: Addr - - port: U16 -RawAddr: - ENUM: - 0: - V4: - NEWTYPE: - TUPLEARRAY: - CONTENT: U8 - SIZE: 4 - 1: - V6: - NEWTYPE: - TUPLEARRAY: - CONTENT: U8 - SIZE: 16 - -# ------------------------ -# Message Enum -# ------------------------ -Message: - ENUM: - 0: - PullRequest: - STRUCT: - - filter: - TYPENAME: CrdsFilter - - value: - TYPENAME: CrdsValue - 1: - PullResponse: - STRUCT: - - pubkey: - TYPENAME: Pubkey - - values: - SEQ: - TYPENAME: CrdsValue - 2: - PushMessage: - STRUCT: - - pubkey: - TYPENAME: Pubkey - - values: - SEQ: - TYPENAME: CrdsValue - 3: - PruneMessage: - STRUCT: - - pubkey: - TYPENAME: Pubkey - - data: - TYPENAME: PruneData - 4: - Ping: - NEWTYPE: - TYPENAME: Ping - 5: - Pong: - NEWTYPE: - TYPENAME: Ping -Ping: - STRUCT: - - from: - TYPENAME: Pubkey - - token: - TYPENAME: Hash - - signature: - TYPENAME: Signature - -# ------------------------ -# CRDS -# ------------------------ -CrdsData: - ENUM: - 0: - ContactInfo: - NEWTYPE: - TYPENAME: ContactInfo - 1: - Vote: - TUPLE: - - U8 - - TYPENAME: Vote - 2: - LowestSlot: - TUPLE: - - U8 - - TYPENAME: LowestSlot - 3: - SnapshotHashes: - NEWTYPE: - TYPENAME: SnapshotHashes - 4: - AccountsHashes: - NEWTYPE: - TYPENAME: SnapshotHashes - 5: - EpochSlots: - TUPLE: - - U8 - - TYPENAME: EpochSlots - 6: - LegacyVersion: - STRUCT: - - from: - TYPENAME: Pubkey - - wallclock: U64 - - major: U16 - - minor: U16 - - patch: U16 - - commit: - OPTION: U32 - 7: - Version: - STRUCT: - - from: - TYPENAME: Pubkey - - wallclock: U64 - - major: U16 - - minor: U16 - - patch: U16 - - commit: - OPTION: U32 - - feature_set: U32 - 8: - NodeInstance: - STRUCT: - - from: - TYPENAME: Pubkey - - wallclock: U64 - - timestamp: U64 - - token: U64 - 9: - DuplicateShred: - TUPLE: - - U16 - - TYPENAME: DuplicateShred - 10: - IncrementalSnapshotHashes: - NEWTYPE: - TYPENAME: IncrementalSnapshotHashes - -# ------------------------ -# Auxiliary stuff -# ------------------------ -BitVecU8: - STRUCT: - - bits: - TYPENAME: BitVecU8Inner - - len: U64 -BitVecU8Inner: - NEWTYPESTRUCT: - OPTION: - SEQ: U8 -BitVecU64: - STRUCT: - - bits: - TYPENAME: BitVecU64Inner - - len: U64 -BitVecU64Inner: - NEWTYPESTRUCT: - OPTION: - SEQ: U64 - -# ------------------------ -# CRDTs -# ------------------------ -ContactInfo: - STRUCT: - - id: - TYPENAME: Pubkey - - gossip: - TYPENAME: SocketAddr - - tvu: - TYPENAME: SocketAddr - - tvu_forwards: - TYPENAME: SocketAddr - - repair: - TYPENAME: SocketAddr - - tpu: - TYPENAME: SocketAddr - - tpu_forwards: - TYPENAME: SocketAddr - - tpu_vote: - TYPENAME: SocketAddr - - rpc: - TYPENAME: SocketAddr - - rpc_pubsub: - TYPENAME: SocketAddr - - serve_repair: - TYPENAME: SocketAddr - - wallclock: U64 - - shred_version: U16 -CompressedSlots: - ENUM: - 0: - Flate2: - NEWTYPE: - TYPENAME: SlotsFlate2 - 1: - Uncompressed: - NEWTYPE: - TYPENAME: SlotsUncompressed -DuplicateShred: - STRUCT: - - from: - TYPENAME: Pubkey - - wallclock: U64 - - slot: U64 - - shred_index: U32 - - shred_type: U8 - - num_chunks: U8 - - chunk_index: U8 - - chunk: - SEQ: U8 -EpochSlots: - STRUCT: - - from: - TYPENAME: Pubkey - - slots: - SEQ: - TYPENAME: CompressedSlots - - wallclock: U64 -IncrementalSnapshotHashes: - STRUCT: - - from: - TYPENAME: Pubkey - - base: - TYPENAME: SlotHash - - hashes: - SEQ: - TYPENAME: SlotHash - - wallclock: U64 -SlotHash: - STRUCT: - - slot: U64 - - hash: - TYPENAME: Hash -SlotsFlate2: - STRUCT: - - first_slot: U64 - - num: U64 - - compressed: - SEQ: U8 -SlotsUncompressed: - STRUCT: - - first_slot: U64 - - num: U64 - - slots: - TYPENAME: BitVecU8 -LowestSlot: - STRUCT: - - from: - TYPENAME: Pubkey - - root: U64 - - lowest: U64 - - slots: - SEQ: U64 - - stash: - SEQ: UNIT - - wallclock: U64 -SnapshotHashes: - STRUCT: - - from: - TYPENAME: Pubkey - - hashes: - SEQ: - TYPENAME: SlotHash - - wallclock: U64 -Vote: - STRUCT: - - from: - TYPENAME: Pubkey - - transaction: - TYPENAME: Transaction - - wallclock: U64 - - slot: - OPTION: U64 - -# ------------------------ -# Message Types -# ------------------------ -Bloom: - STRUCT: - - keys: - SEQ: U64 - - bits: - TYPENAME: BitVecU64 - - num_bits_set: U64 -CrdsFilter: - STRUCT: - - filter: - TYPENAME: Bloom - - mask: U64 - - mask_bits: U32 -CrdsValue: - STRUCT: - - signature: - TYPENAME: Signature - - data: - TYPENAME: CrdsData -PruneData: - STRUCT: - - pubkey: - TYPENAME: Pubkey - - prunes: - SEQ: - TYPENAME: Pubkey - - signature: - TYPENAME: Signature - - destination: - TYPENAME: Pubkey - - wallclock: U64 diff --git a/pkg/gossip/socketaddr.go b/pkg/gossip/socketaddr.go deleted file mode 100644 index 83062b33..00000000 --- a/pkg/gossip/socketaddr.go +++ /dev/null @@ -1,66 +0,0 @@ -package gossip - -import ( - "net/netip" - - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/serde" -) - -type SocketAddr struct { - netip.AddrPort -} - -func DeserializeSocketAddr(deserializer serde.Deserializer) (sa SocketAddr, err error) { - var raw RawSocketAddr - raw, err = DeserializeRawSocketAddr(deserializer) - if err != nil { - return - } - sa.AddrPort = netip.AddrPortFrom(raw.Addr.Addr, raw.Port) - return -} - -func (sa SocketAddr) Serialize(serializer serde.Serializer) error { - raw := RawSocketAddr{ - Addr: Addr{sa.Addr()}, - Port: sa.Port(), - } - return raw.Serialize(serializer) -} - -type Addr struct { - netip.Addr -} - -func DeserializeAddr(deserializer serde.Deserializer) (sa Addr, err error) { - var raw RawAddr - raw, err = DeserializeRawAddr(deserializer) - if err != nil { - return - } - switch x := raw.(type) { - case *RawAddr__V4: - sa.Addr = netip.AddrFrom4(*x) - if sa.As4() == [4]byte{0, 0, 0, 0} { - // All zero IP serves as a placeholder - sa.Addr = netip.Addr{} - } - case *RawAddr__V6: - sa.Addr = netip.AddrFrom16(*x) - default: - panic("unexpected RawSocketAddr") - } - return -} - -func (a Addr) Serialize(serializer serde.Serializer) error { - var raw RawAddr - if a.Is4() { - v4 := RawAddr__V4(a.As4()) - raw = &v4 - } else { - v6 := RawAddr__V6(a.As16()) - raw = &v6 - } - return raw.Serialize(serializer) -} diff --git a/pkg/gossip/transaction.go b/pkg/gossip/transaction.go deleted file mode 100644 index 09c7d0f1..00000000 --- a/pkg/gossip/transaction.go +++ /dev/null @@ -1,115 +0,0 @@ -package gossip - -import ( - "github.com/gagliardetto/solana-go" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/serde" -) - -// TODO write codegen for this - -type Transaction solana.Transaction - -func DeserializeTransaction(deserializer serde.Deserializer) (Transaction, error) { - var obj Transaction - numSigs, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - for i := uint8(0); i < numSigs; i++ { - sig, err := DeserializeSignature(deserializer) - if err != nil { - return obj, err - } - obj.Signatures = append(obj.Signatures, solana.Signature(sig)) - } - obj.Message, err = DeserializeTxMessage(deserializer) - if err != nil { - return obj, err - } - return obj, nil -} - -func (obj *Transaction) Serialize(serializer serde.Serializer) error { - panic("not implemented") -} - -func DeserializeTxMessage(deserializer serde.Deserializer) (solana.Message, error) { - var obj solana.Message - numRequiredSigs, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - obj.Header.NumRequiredSignatures = numRequiredSigs - numReadonlySignedAccs, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - obj.Header.NumReadonlySignedAccounts = numReadonlySignedAccs - numReadonlyUnsignedAccs, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - obj.Header.NumReadonlyUnsignedAccounts = numReadonlyUnsignedAccs - numAccountKeys, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - for i := uint8(0); i < numAccountKeys; i++ { - address, err := DeserializePubkey(deserializer) - if err != nil { - return obj, err - } - obj.AccountKeys = append(obj.AccountKeys, solana.PublicKey(address)) - } - recentBlockHash, err := DeserializeHash(deserializer) - if err != nil { - return obj, err - } - obj.RecentBlockhash = solana.Hash(recentBlockHash) - numInsns, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - for i := uint8(0); i < numInsns; i++ { - insn, err := DeserializeInstruction(deserializer) - if err != nil { - return obj, err - } - obj.Instructions = append(obj.Instructions, insn) - } - return obj, nil -} - -func DeserializeInstruction(deserializer serde.Deserializer) (solana.CompiledInstruction, error) { - var obj solana.CompiledInstruction - programIdIdx, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - obj.ProgramIDIndex = uint16(programIdIdx) - numAccs, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - for i := uint8(0); i < numAccs; i++ { - idx, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - obj.Accounts = append(obj.Accounts, uint16(idx)) - } - // This is brain-dead - dataLen, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - obj.Data = make([]byte, dataLen) - for i := uint8(0); i < dataLen; i++ { - _byte, err := deserializer.DeserializeU8() - if err != nil { - return obj, err - } - obj.Data[i] = _byte - } - return obj, nil -} diff --git a/pkg/gossip/types.go b/pkg/gossip/types.go deleted file mode 100644 index 47481eac..00000000 --- a/pkg/gossip/types.go +++ /dev/null @@ -1,15 +0,0 @@ -package gossip - -import "github.com/gagliardetto/solana-go" - -func (p Pubkey) MarshalText() ([]byte, error) { - return solana.PublicKey(p).MarshalText() -} - -func (h Hash) MarshalText() ([]byte, error) { - return solana.Hash(h).MarshalText() -} - -func (s Signature) MarshalText() ([]byte, error) { - return solana.Signature(s).MarshalText() -} diff --git a/pkg/repair/protocol.go b/pkg/repair/protocol.go new file mode 100644 index 00000000..9a7040ff --- /dev/null +++ b/pkg/repair/protocol.go @@ -0,0 +1,164 @@ +package repair + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "fmt" + "time" + + "github.com/Overclock-Validator/mithril/pkg/gossip" +) + +const ( + repairProtocolPingResponse = uint32(0) + repairProtocolPong = uint32(7) + + repairProtocolWindowIndex = uint32(8) + repairProtocolHighestWindowIndex = uint32(9) + + repairSignatureOffset = 4 + repairSignatureSize = 64 + repairPingSize = 4 + 32 + 32 + 64 +) + +type Ping struct { + From gossip.Pubkey + Token [32]byte + Signature gossip.Signature +} + +func NewWindowIndexRequest(identity ed25519.PrivateKey, recipient gossip.Pubkey, slot uint64, shredIndex uint64) ([]byte, uint32, error) { + nonce, err := randomNonce() + if err != nil { + return nil, 0, err + } + packet, err := BuildWindowIndexRequest(identity, recipient, slot, shredIndex, nonce) + return packet, nonce, err +} + +func BuildWindowIndexRequest(identity ed25519.PrivateKey, recipient gossip.Pubkey, slot uint64, shredIndex uint64, nonce uint32) ([]byte, error) { + return buildRequest(identity, recipient, repairProtocolWindowIndex, slot, shredIndex, nonce, uint64(time.Now().UnixMilli())) +} + +func NewHighestWindowIndexRequest(identity ed25519.PrivateKey, recipient gossip.Pubkey, slot uint64, shredIndex uint64) ([]byte, uint32, error) { + nonce, err := randomNonce() + if err != nil { + return nil, 0, err + } + packet, err := BuildHighestWindowIndexRequest(identity, recipient, slot, shredIndex, nonce) + return packet, nonce, err +} + +func BuildHighestWindowIndexRequest(identity ed25519.PrivateKey, recipient gossip.Pubkey, slot uint64, shredIndex uint64, nonce uint32) ([]byte, error) { + return buildRequest(identity, recipient, repairProtocolHighestWindowIndex, slot, shredIndex, nonce, uint64(time.Now().UnixMilli())) +} + +func ResponseNonce(packet []byte) (uint32, bool) { + if len(packet) < 4 { + return 0, false + } + return binary.LittleEndian.Uint32(packet[len(packet)-4:]), true +} + +func DecodePing(packet []byte) (Ping, bool) { + var ping Ping + if len(packet) != repairPingSize { + return ping, false + } + if binary.LittleEndian.Uint32(packet[0:4]) != repairProtocolPingResponse { + return ping, false + } + copy(ping.From[:], packet[4:36]) + copy(ping.Token[:], packet[36:68]) + copy(ping.Signature[:], packet[68:132]) + if !ed25519.Verify(ed25519.PublicKey(ping.From[:]), ping.Token[:], ping.Signature[:]) { + return Ping{}, false + } + return ping, true +} + +func BuildPong(identity ed25519.PrivateKey, ping Ping) ([]byte, error) { + sender, err := senderPubkey(identity) + if err != nil { + return nil, err + } + hash := hashPingToken(ping.Token) + signature := ed25519.Sign(identity, hash[:]) + + var packet []byte + packet = binary.LittleEndian.AppendUint32(packet, repairProtocolPong) + packet = append(packet, sender[:]...) + packet = append(packet, hash[:]...) + packet = append(packet, signature...) + return packet, nil +} + +func buildRequest(identity ed25519.PrivateKey, recipient gossip.Pubkey, variant uint32, slot uint64, shredIndex uint64, nonce uint32, timestamp uint64) ([]byte, error) { + sender, err := senderPubkey(identity) + if err != nil { + return nil, err + } + + var packet []byte + packet = binary.LittleEndian.AppendUint32(packet, variant) + packet = append(packet, make([]byte, repairSignatureSize)...) + packet = append(packet, sender[:]...) + packet = append(packet, recipient[:]...) + packet = binary.LittleEndian.AppendUint64(packet, timestamp) + packet = binary.LittleEndian.AppendUint32(packet, nonce) + packet = binary.LittleEndian.AppendUint64(packet, slot) + packet = binary.LittleEndian.AppendUint64(packet, shredIndex) + + signRepairPacket(identity, packet) + return packet, nil +} + +func senderPubkey(identity ed25519.PrivateKey) (gossip.Pubkey, error) { + var out gossip.Pubkey + if len(identity) != ed25519.PrivateKeySize { + return out, fmt.Errorf("invalid repair identity size %d", len(identity)) + } + pub, ok := identity.Public().(ed25519.PublicKey) + if !ok || len(pub) != ed25519.PublicKeySize { + return out, fmt.Errorf("invalid repair identity public key") + } + copy(out[:], pub) + return out, nil +} + +func signRepairPacket(identity ed25519.PrivateKey, packet []byte) { + signable := make([]byte, 0, len(packet)-repairSignatureSize) + signable = append(signable, packet[:repairSignatureOffset]...) + signable = append(signable, packet[repairSignatureOffset+repairSignatureSize:]...) + signature := ed25519.Sign(identity, signable) + copy(packet[repairSignatureOffset:repairSignatureOffset+repairSignatureSize], signature) +} + +func VerifySignedRequest(packet []byte, sender gossip.Pubkey) bool { + if len(packet) < repairSignatureOffset+repairSignatureSize { + return false + } + signable := make([]byte, 0, len(packet)-repairSignatureSize) + signable = append(signable, packet[:repairSignatureOffset]...) + signable = append(signable, packet[repairSignatureOffset+repairSignatureSize:]...) + return ed25519.Verify(ed25519.PublicKey(sender[:]), signable, packet[repairSignatureOffset:repairSignatureOffset+repairSignatureSize]) +} + +func hashPingToken(token [32]byte) gossip.Hash { + h := sha256.New() + _, _ = h.Write([]byte("SOLANA_PING_PONG")) + _, _ = h.Write(token[:]) + var out gossip.Hash + copy(out[:], h.Sum(nil)) + return out +} + +func randomNonce() (uint32, error) { + var raw [4]byte + if _, err := rand.Read(raw[:]); err != nil { + return 0, err + } + return binary.LittleEndian.Uint32(raw[:]), nil +} diff --git a/pkg/repair/protocol_test.go b/pkg/repair/protocol_test.go new file mode 100644 index 00000000..fbcf2c20 --- /dev/null +++ b/pkg/repair/protocol_test.go @@ -0,0 +1,133 @@ +package repair + +import ( + "crypto/ed25519" + "encoding/binary" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/gossip" +) + +func TestBuildWindowIndexRequest(t *testing.T) { + pub, identity, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatalf("GenerateKey returned error: %v", err) + } + var sender gossip.Pubkey + copy(sender[:], pub) + var recipient gossip.Pubkey + for idx := range recipient { + recipient[idx] = byte(idx + 1) + } + + packet, err := buildRequest(identity, recipient, repairProtocolWindowIndex, 123, 456, 789, 101112) + if err != nil { + t.Fatalf("buildRequest returned error: %v", err) + } + if len(packet) != 160 { + t.Fatalf("packet length = %d, want 160", len(packet)) + } + if got := binary.LittleEndian.Uint32(packet[0:4]); got != repairProtocolWindowIndex { + t.Fatalf("variant = %d, want %d", got, repairProtocolWindowIndex) + } + if string(packet[68:100]) != string(sender[:]) { + t.Fatalf("sender was not encoded") + } + if string(packet[100:132]) != string(recipient[:]) { + t.Fatalf("recipient was not encoded") + } + if got := binary.LittleEndian.Uint64(packet[132:140]); got != 101112 { + t.Fatalf("timestamp = %d, want 101112", got) + } + if got := binary.LittleEndian.Uint32(packet[140:144]); got != 789 { + t.Fatalf("nonce = %d, want 789", got) + } + if got := binary.LittleEndian.Uint64(packet[144:152]); got != 123 { + t.Fatalf("slot = %d, want 123", got) + } + if got := binary.LittleEndian.Uint64(packet[152:160]); got != 456 { + t.Fatalf("shred index = %d, want 456", got) + } + if !VerifySignedRequest(packet, sender) { + t.Fatalf("request signature did not verify") + } +} + +func TestBuildHighestWindowIndexRequest(t *testing.T) { + _, identity, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatalf("GenerateKey returned error: %v", err) + } + var recipient gossip.Pubkey + packet, err := buildRequest(identity, recipient, repairProtocolHighestWindowIndex, 123, 456, 789, 101112) + if err != nil { + t.Fatalf("buildRequest returned error: %v", err) + } + if got := binary.LittleEndian.Uint32(packet[0:4]); got != repairProtocolHighestWindowIndex { + t.Fatalf("variant = %d, want %d", got, repairProtocolHighestWindowIndex) + } +} + +func TestDecodeRepairPingAndBuildPong(t *testing.T) { + peerPub, peerIdentity, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatalf("GenerateKey peer returned error: %v", err) + } + ourPub, ourIdentity, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatalf("GenerateKey identity returned error: %v", err) + } + + var token [32]byte + for idx := range token { + token[idx] = byte(idx + 11) + } + signature := ed25519.Sign(peerIdentity, token[:]) + packet := make([]byte, 0, repairPingSize) + packet = binary.LittleEndian.AppendUint32(packet, repairProtocolPingResponse) + packet = append(packet, peerPub...) + packet = append(packet, token[:]...) + packet = append(packet, signature...) + + ping, ok := DecodePing(packet) + if !ok { + t.Fatalf("DecodePing rejected valid repair ping") + } + if string(ping.From[:]) != string(peerPub) { + t.Fatalf("ping sender was not decoded") + } + + pong, err := BuildPong(ourIdentity, ping) + if err != nil { + t.Fatalf("BuildPong returned error: %v", err) + } + if len(pong) != repairPingSize { + t.Fatalf("pong length = %d, want %d", len(pong), repairPingSize) + } + if got := binary.LittleEndian.Uint32(pong[0:4]); got != repairProtocolPong { + t.Fatalf("pong variant = %d, want %d", got, repairProtocolPong) + } + if string(pong[4:36]) != string(ourPub) { + t.Fatalf("pong sender was not encoded") + } + hash := hashPingToken(token) + if string(pong[36:68]) != string(hash[:]) { + t.Fatalf("pong hash was not encoded") + } + if !ed25519.Verify(ourPub, hash[:], pong[68:132]) { + t.Fatalf("pong signature did not verify") + } +} + +func TestDecodeRepairPingRejectsInvalidSignature(t *testing.T) { + pub, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatalf("GenerateKey returned error: %v", err) + } + packet := make([]byte, repairPingSize) + binary.LittleEndian.PutUint32(packet[0:4], repairProtocolPingResponse) + copy(packet[4:36], pub) + if _, ok := DecodePing(packet); ok { + t.Fatalf("DecodePing accepted invalid signature") + } +} diff --git a/pkg/replay/block.go b/pkg/replay/block.go index c5b4072a..26202a52 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -1188,10 +1188,16 @@ func ReplayBlocks( startSlot, endSlot uint64, rpcEndpoints []string, // RPC endpoints in priority order (first = primary, rest = fallbacks) lightbringerEndpoint string, + turbineBindAddr string, + turbineGossipEntrypoint string, + turbineGossipBindAddr string, + turbineAdvertisedIP string, + turbineShredVersion uint16, blockDir string, txParallelism int, isLive bool, useLightbringer bool, + useTurbine bool, dbgOpts *DebugOptions, metricsWriter io.Writer, rpcServer SlotCtxSetter, @@ -1335,7 +1341,15 @@ func ReplayBlocks( } // Resolve consensus config defaults before forkchoice init so we can // check whether enforcement requires authorized voters. - consensusCfg := resolveConsensusConfig(consensusOpts, useLightbringer, isLive) + useLiveShredStream := useLightbringer || useTurbine + consensusCfg := resolveConsensusConfig(consensusOpts, useLightbringer, useTurbine, isLive) + consensusManagedLiveStream := consensusCfg.enforceActive && + isLive && + consensusManagesLiveShredStream(consensusCfg.enforceSource, useLightbringer, useTurbine) + consensusLiveStreamName := "Lightbringer" + if useTurbine { + consensusLiveStreamName = "TURBINE" + } epochAuthVoters := global.EpochAuthorizedVoters() if epochAuthVoters == nil { @@ -1380,16 +1394,30 @@ func ReplayBlocks( var opts *blockstream.BlockSourceOpts if useLightbringer { opts = &blockstream.BlockSourceOpts{ - SourceType: blockstream.BlockSourceLightbringer, - RpcClient: rpcc, - LightbringerEndpoint: lightbringerEndpoint, - BackupRpcEndpoints: rpcBackups, - StartSlot: startSlot, - EndSlot: endSlot, - BlockDir: blockDir, - ConsensusManagedLightbringer: consensusCfg.enforceActive && - isLive && - consensusCfg.enforceSource == "lightbringer", + SourceType: blockstream.BlockSourceLightbringer, + RpcClient: rpcc, + LightbringerEndpoint: lightbringerEndpoint, + BackupRpcEndpoints: rpcBackups, + StartSlot: startSlot, + EndSlot: endSlot, + BlockDir: blockDir, + ConsensusManagedLightbringer: consensusManagedLiveStream, + } + } else if useTurbine { + opts = &blockstream.BlockSourceOpts{ + SourceType: blockstream.BlockSourceTurbine, + RpcClient: rpcc, + TurbineBindAddr: turbineBindAddr, + TurbineGossipEntrypoint: turbineGossipEntrypoint, + TurbineGossipBindAddr: turbineGossipBindAddr, + TurbineAdvertisedIP: turbineAdvertisedIP, + TurbineShredVersion: turbineShredVersion, + LeaderForSlot: global.LeaderForSlot, + BackupRpcEndpoints: rpcBackups, + StartSlot: startSlot, + EndSlot: endSlot, + BlockDir: blockDir, + ConsensusManagedLightbringer: consensusManagedLiveStream, } } else { opts = &blockstream.BlockSourceOpts{ @@ -1458,7 +1486,7 @@ func ReplayBlocks( } syncConsensusBufferedExecutionMode := func(triggerSlot uint64) { - if !consensusCfg.enforceActive || !isLive || consensusCfg.enforceSource != "lightbringer" { + if !consensusManagedLiveStream { return } @@ -1485,7 +1513,7 @@ func ReplayBlocks( return nil } - if !consensusBufferedExecutionActive && isLive && consensusCfg.enforceSource == "lightbringer" { + if !consensusBufferedExecutionActive && consensusManagedLiveStream { if block == nil || !block.FromLightbringer { return nil } @@ -1493,7 +1521,7 @@ func ReplayBlocks( readyConsensusPath = nil observeConsensusAnchor() pruneObservedConsensusBlocks(observedConsensusBlocks, currentConsensusAnchorSlot()) - mlog.Log.Warnf("forkchoice: enabling buffered execution at slot %d after block source switched to Lightbringer", block.Slot) + mlog.Log.Warnf("forkchoice: enabling buffered execution at slot %d after block source switched to %s", block.Slot, consensusLiveStreamName) } if !consensusBufferedExecutionActive { @@ -1617,13 +1645,13 @@ func ReplayBlocks( if block.FromLightbringer { stats := blockStream.GetFetchStats() - if shouldDiscardLightbringerObservationAfterFallback(isLive, useLightbringer, block, stats) { + if shouldDiscardLightbringerObservationAfterFallback(isLive, useLiveShredStream, block, stats) { modeStr := "catchup" if stats.IsNearTip { modeStr = "near-tip" } - mlog.Log.Warnf("forkchoice: discarding stale Lightbringer observation for slot %d after source fallback (mode=%s current_source=%s anchor=%d next_emitted_slot=%d)", - block.Slot, modeStr, stats.CurrentSource, currentConsensusAnchorSlot(), stats.NextSlot) + mlog.Log.Warnf("forkchoice: discarding stale %s observation for slot %d after source fallback (mode=%s current_source=%s anchor=%d next_emitted_slot=%d)", + consensusLiveStreamName, block.Slot, modeStr, stats.CurrentSource, currentConsensusAnchorSlot(), stats.NextSlot) continue } } @@ -1644,6 +1672,22 @@ func ReplayBlocks( case errors.Is(err, forkchoice.ErrNeedWait), errors.Is(err, forkchoice.ErrPathIncomplete): continue case errors.Is(err, forkchoice.ErrDepthExceeded): + if consensusManagedLiveStream && isLive && useLiveShredStream { + anchorSlot := currentConsensusAnchorSlot() + discardedObservedBlocks := len(observedConsensusBlocks) + readyDecisionCount := 0 + if readyConsensusPath != nil { + readyDecisionCount = len(readyConsensusPath.decisions) + } + mlog.Log.Warnf("forkchoice: unable to resolve %s consensus path within %d slots from anchor %d after observing slot %d; falling back to RPC catchup (discarded_observed_blocks=%d discarded_ready_decisions=%d)", + consensusLiveStreamName, consensusCfg.maxDepth, anchorSlot, block.Slot, discardedObservedBlocks, readyDecisionCount) + consensusBufferedExecutionActive = false + readyConsensusPath = nil + clearObservedConsensusBlocks(observedConsensusBlocks) + observeConsensusAnchor() + blockStream.ForceRPCFallback("consensus_depth_exceeded") + continue + } if consensusCoordinator.Policy() == "halt" { result.Error = fmt.Errorf("forkchoice: unable to resolve a confirmed path within %d slots from anchor %d", consensusCfg.maxDepth, currentConsensusAnchorSlot()) @@ -2127,6 +2171,16 @@ func ReplayBlocks( // Line 2: Current block source mlog.Log.InfofPrecise(" block source: %s", formatBlockSourceStatus(fetchStats)) + if consensusBufferedExecutionActive { + readyDecisionCount := 0 + readyLeafSlot := uint64(0) + if readyConsensusPath != nil { + readyDecisionCount = len(readyConsensusPath.decisions) + readyLeafSlot = readyConsensusPath.leafSlot + } + mlog.Log.InfofPrecise(" consensus buffer: observed=%d ready_decisions=%d anchor=%d ready_leaf=%d", + len(observedConsensusBlocks), readyDecisionCount, currentConsensusAnchorSlot(), readyLeafSlot) + } // Line 3: CU and transaction stats (median/min/max) mlog.Log.InfofPrecise(" cu: median %d, min %d, max %d | txns: median vote %d, median non-vote %d", @@ -2157,11 +2211,12 @@ func ReplayBlocks( var mem runtime.MemStats runtime.ReadMemStats(&mem) const gib = 1024 * 1024 * 1024 - mlog.Log.InfofPrecise(" memory: alloc %.1fGiB | inuse %.1fGiB | idle %.1fGiB | released %.1fGiB | objs %d | gc %d | queue=%d", + mlog.Log.InfofPrecise(" memory: alloc %.1fGiB | inuse %.1fGiB | idle %.1fGiB | released %.1fGiB | next_gc %.1fGiB | objs %d | gc %d | queue=%d", float64(mem.HeapAlloc)/gib, float64(mem.HeapInuse)/gib, float64(mem.HeapIdle)/gib, float64(mem.HeapReleased)/gib, + float64(mem.NextGC)/gib, mem.HeapObjects, mem.NumGC, acctsDb.StoreQueueLen(), diff --git a/pkg/replay/consensus.go b/pkg/replay/consensus.go index 24ea48b6..f6324bf9 100644 --- a/pkg/replay/consensus.go +++ b/pkg/replay/consensus.go @@ -11,7 +11,7 @@ import ( const ( defaultConsensusMaxDepth = 64 defaultConsensusPolicy = "halt" - defaultConsensusEnforceSource = "lightbringer" + defaultConsensusEnforceSource = "stream" ) // ConsensusOpts contains vote-anchored consensus configuration. @@ -19,7 +19,7 @@ const ( type ConsensusOpts struct { SkipPathMaxDepth int // Max slots for skip-path solver (default: 64) UnresolvedPolicy string // "halt" or "warn" (default: "halt") - EnforceOnSource string // "lightbringer" or "all" (default: "lightbringer") + EnforceOnSource string // "lightbringer", "turbine", "stream", or "all" (default: "stream") } type consensusConfig struct { @@ -40,7 +40,7 @@ type pendingConsensusPath struct { originalDecisions []forkchoice.SlotDecision } -func resolveConsensusConfig(opts *ConsensusOpts, useLightbringer, isLive bool) consensusConfig { +func resolveConsensusConfig(opts *ConsensusOpts, useLightbringer, useTurbine, isLive bool) consensusConfig { cfg := consensusConfig{ maxDepth: defaultConsensusMaxDepth, policy: defaultConsensusPolicy, @@ -59,18 +59,52 @@ func resolveConsensusConfig(opts *ConsensusOpts, useLightbringer, isLive bool) c } } + if isLive && useTurbine && !useLightbringer && cfg.enforceSource == "lightbringer" { + mlog.Log.Warnf("forkchoice: consensus.enforce_on_source=%q is legacy Lightbringer-only while block source is native turbine; treating it as %q for this run", + cfg.enforceSource, "turbine") + cfg.enforceSource = "turbine" + } + switch cfg.enforceSource { - case "lightbringer", "all": + case "lightbringer", "turbine", "stream", "all": default: mlog.Log.Warnf("forkchoice: invalid EnforceOnSource=%q, defaulting to %q", cfg.enforceSource, defaultConsensusEnforceSource) cfg.enforceSource = defaultConsensusEnforceSource } - cfg.enforceActive = cfg.enforceSource == "all" || useLightbringer + cfg.enforceActive = consensusAppliesToRun(cfg.enforceSource, useLightbringer, useTurbine) cfg.bufferedExecutionActive = !isLive || cfg.enforceSource == "all" return cfg } +func consensusAppliesToRun(enforceSource string, useLightbringer, useTurbine bool) bool { + switch enforceSource { + case "all": + return true + case "stream": + return useLightbringer || useTurbine + case "lightbringer": + return useLightbringer + case "turbine": + return useTurbine + default: + return false + } +} + +func consensusManagesLiveShredStream(enforceSource string, useLightbringer, useTurbine bool) bool { + switch enforceSource { + case "stream", "all": + return useLightbringer || useTurbine + case "lightbringer": + return useLightbringer + case "turbine": + return useTurbine + default: + return false + } +} + func newPendingConsensusPath(anchorSlot uint64, resolvedPath *forkchoice.ResolvedPath) *pendingConsensusPath { if resolvedPath == nil { return nil @@ -107,5 +141,5 @@ func shouldDiscardLightbringerObservationAfterFallback(isLive, useLightbringer b useLightbringer && block != nil && block.FromLightbringer && - (!stats.IsNearTip || stats.CurrentSource != "lightbringer") + (!stats.IsNearTip || (stats.CurrentSource != "lightbringer" && stats.CurrentSource != "turbine")) } diff --git a/pkg/replay/consensus_fallback_test.go b/pkg/replay/consensus_fallback_test.go index c03ccfae..6874d39f 100644 --- a/pkg/replay/consensus_fallback_test.go +++ b/pkg/replay/consensus_fallback_test.go @@ -31,6 +31,13 @@ func TestShouldDiscardLightbringerObservationAfterFallback(t *testing.T) { t.Fatalf("expected active Lightbringer observations to be retained") } + if shouldDiscardLightbringerObservationAfterFallback(true, true, lightbringerBlock, blockstream.FetchStatsSnapshot{ + IsNearTip: true, + CurrentSource: "turbine", + }) { + t.Fatalf("expected active native turbine observations to be retained") + } + if shouldDiscardLightbringerObservationAfterFallback(true, true, &b.Block{Slot: 123}, blockstream.FetchStatsSnapshot{ IsNearTip: false, CurrentSource: "rpc", @@ -38,3 +45,47 @@ func TestShouldDiscardLightbringerObservationAfterFallback(t *testing.T) { t.Fatalf("expected RPC block to be retained") } } + +func TestResolveConsensusConfigAppliesToNativeTurbineByDefault(t *testing.T) { + cfg := resolveConsensusConfig(nil, false, true, true) + if !cfg.enforceActive { + t.Fatalf("default stream consensus should apply to native turbine") + } + if cfg.enforceSource != "stream" { + t.Fatalf("default consensus source = %q, want stream", cfg.enforceSource) + } + if cfg.bufferedExecutionActive { + t.Fatalf("live turbine consensus should arm buffered execution only after handoff") + } + + cfg = resolveConsensusConfig(&ConsensusOpts{EnforceOnSource: "lightbringer"}, false, true, true) + if !cfg.enforceActive || cfg.enforceSource != "turbine" { + t.Fatalf("legacy lightbringer consensus should be upgraded for native turbine, got active=%v source=%q", cfg.enforceActive, cfg.enforceSource) + } + if cfg.bufferedExecutionActive { + t.Fatalf("live turbine consensus should arm buffered execution only after handoff") + } + + cfg = resolveConsensusConfig(&ConsensusOpts{EnforceOnSource: "turbine"}, false, true, true) + if !cfg.enforceActive { + t.Fatalf("explicit turbine consensus should apply to native turbine") + } + if cfg.bufferedExecutionActive { + t.Fatalf("live turbine consensus should arm buffered execution only after handoff") + } + + cfg = resolveConsensusConfig(&ConsensusOpts{EnforceOnSource: "stream"}, false, true, true) + if !cfg.enforceActive { + t.Fatalf("stream consensus should apply to native turbine") + } + + cfg = resolveConsensusConfig(&ConsensusOpts{EnforceOnSource: "lightbringer"}, true, false, true) + if !cfg.enforceActive { + t.Fatalf("lightbringer consensus should apply to lightbringer") + } + + cfg = resolveConsensusConfig(&ConsensusOpts{EnforceOnSource: "all"}, false, false, true) + if !cfg.enforceActive || !cfg.bufferedExecutionActive { + t.Fatalf("all consensus should apply immediately") + } +} diff --git a/pkg/replay/transaction.go b/pkg/replay/transaction.go index 775fb744..451ddb53 100644 --- a/pkg/replay/transaction.go +++ b/pkg/replay/transaction.go @@ -20,6 +20,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/metrics" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/txverify" "github.com/Overclock-Validator/mithril/pkg/util" bin "github.com/gagliardetto/binary" "github.com/gagliardetto/solana-go" @@ -316,7 +317,7 @@ type sigverifySnapshot struct { } func buildSigverifySnapshot(tx *solana.Transaction, slot uint64) (*sigverifySnapshot, error) { - message, err := tx.Message.MarshalBinary() + message, err := txverify.MessageBytes(tx) if err != nil { return nil, err } diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 1035bc49..3f0b74ea 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -1,8 +1,11 @@ package rpcserver import ( + "bytes" "context" + "encoding/json" "fmt" + "io" "net" "net/http" "net/http/httptest" @@ -41,6 +44,18 @@ type RpcServer struct { sendTransactionLeaderForwardCount uint64 } +const maxQuietMethodProbeBody = 64 << 10 + +var supportedRPCMethods = map[string]struct{}{ + "getAccountInfo": {}, + "getBankHash": {}, + "getBlockHeight": {}, + "getEpochInfo": {}, + "getLatestBlockhash": {}, + "sendTransaction": {}, + "simulateTransaction": {}, +} + func NewRpcServer(acctsDb *accountsdb.AccountsDb, port uint16, epochSchedule *sealevel.SysvarEpochSchedule) *RpcServer { var err error rpcServer := &RpcServer{} @@ -103,7 +118,159 @@ func (rpcServer *RpcServer) getSlotCtx() *sealevel.SlotCtx { func (rpcServer *RpcServer) Start() { rpcServer.startClusterNodesRefreshLoop() - go http.Serve(rpcServer.listener, rpcServer.rpcService) + go http.Serve(rpcServer.listener, rpcServer) +} + +func (rpcServer *RpcServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if quietNonRPCProbe(w, r) || writeQuietRPCProbeError(w, r) { + return + } + rpcServer.rpcService.ServeHTTP(w, r) +} + +type rpcMethodProbe struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` +} + +type rpcProbeErrorResponse struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Error rpcProbeError `json:"error"` +} + +type rpcProbeError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func quietNonRPCProbe(w http.ResponseWriter, r *http.Request) bool { + if strings.Contains(strings.ToLower(r.Header.Get("Connection")), "upgrade") { + return false + } + if r.Method == http.MethodPost { + return false + } + http.NotFound(w, r) + return true +} + +func writeQuietRPCProbeError(w http.ResponseWriter, r *http.Request) bool { + body, ok := readQuietMethodProbeBody(r) + if !ok && r.ContentLength != 0 { + return false + } + body = bytes.TrimSpace(body) + if len(body) == 0 { + writeRPCProbeError(w, nil, http.StatusBadRequest, -32600, "Invalid request") + return true + } + if body[0] == '[' { + return writeQuietBatchProbeError(w, body) + } + + var req rpcMethodProbe + if err := json.Unmarshal(body, &req); err != nil { + writeRPCProbeError(w, nil, http.StatusInternalServerError, -32700, "Parse error") + return true + } + if req.Method == "" { + writeRPCProbeError(w, req.ID, http.StatusBadRequest, -32600, "Invalid request") + return true + } + if _, ok := supportedRPCMethods[req.Method]; ok { + return false + } + + writeRPCProbeError(w, req.ID, http.StatusInternalServerError, -32601, fmt.Sprintf("method '%s' not found", req.Method)) + return true +} + +func writeQuietBatchProbeError(w http.ResponseWriter, body []byte) bool { + var reqs []rpcMethodProbe + if err := json.Unmarshal(body, &reqs); err != nil { + writeRPCProbeError(w, nil, http.StatusInternalServerError, -32700, "Parse error") + return true + } + if len(reqs) == 0 { + writeRPCProbeError(w, nil, http.StatusBadRequest, -32600, "Invalid request") + return true + } + allQuiet := true + for _, req := range reqs { + if _, ok := supportedRPCMethods[req.Method]; ok { + allQuiet = false + break + } + } + if !allQuiet { + return false + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + resps := make([]rpcProbeErrorResponse, 0, len(reqs)) + for _, req := range reqs { + code := -32601 + message := fmt.Sprintf("method '%s' not found", req.Method) + if req.Method == "" { + code = -32600 + message = "Invalid request" + } + resps = append(resps, rpcProbeErrorResponse{ + JSONRPC: "2.0", + ID: normalizedRPCProbeID(req.ID), + Error: rpcProbeError{ + Code: code, + Message: message, + }, + }) + } + _ = json.NewEncoder(w).Encode(resps) + return true +} + +func writeRPCProbeError(w http.ResponseWriter, id json.RawMessage, status int, code int, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(rpcProbeErrorResponse{ + JSONRPC: "2.0", + ID: normalizedRPCProbeID(id), + Error: rpcProbeError{ + Code: code, + Message: message, + }, + }) +} + +func normalizedRPCProbeID(id json.RawMessage) json.RawMessage { + if len(id) == 0 { + return json.RawMessage("null") + } + return id +} + +func readQuietMethodProbeBody(r *http.Request) ([]byte, bool) { + if r.Body == nil || r.Body == http.NoBody || r.ContentLength == 0 || r.ContentLength > maxQuietMethodProbeBody { + return nil, false + } + if r.ContentLength > 0 { + body, err := io.ReadAll(r.Body) + r.Body = io.NopCloser(bytes.NewReader(body)) + return body, err == nil + } + + body, err := io.ReadAll(io.LimitReader(r.Body, maxQuietMethodProbeBody+1)) + if err != nil { + r.Body = io.NopCloser(io.MultiReader(bytes.NewReader(body), r.Body)) + return nil, false + } + if len(body) > maxQuietMethodProbeBody { + r.Body = io.NopCloser(io.MultiReader(bytes.NewReader(body), r.Body)) + return nil, false + } + r.Body = io.NopCloser(bytes.NewReader(body)) + return body, true } func (rpcServer *RpcServer) startClusterNodesRefreshLoop() { diff --git a/pkg/rpcserver/rpcserver_test.go b/pkg/rpcserver/rpcserver_test.go new file mode 100644 index 00000000..8ec2c4de --- /dev/null +++ b/pkg/rpcserver/rpcserver_test.go @@ -0,0 +1,81 @@ +package rpcserver + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestServeHTTPQuietlyHandlesUnsupportedMethod(t *testing.T) { + rpcServer := &RpcServer{} + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"jsonrpc":"2.0","method":"getSlot","id":7}`)) + rec := httptest.NewRecorder() + + rpcServer.ServeHTTP(rec, req) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("expected status %d, got %d", http.StatusInternalServerError, rec.Code) + } + + var resp struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Error struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if resp.JSONRPC != "2.0" || resp.ID != 7 { + t.Fatalf("unexpected response identity: %+v", resp) + } + if resp.Error.Code != -32601 || resp.Error.Message != "method 'getSlot' not found" { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestServeHTTPQuietlyHandlesInvalidRequest(t *testing.T) { + rpcServer := &RpcServer{} + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("")) + rec := httptest.NewRecorder() + + rpcServer.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("expected status %d, got %d", http.StatusBadRequest, rec.Code) + } + + var resp struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Error struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if resp.JSONRPC != "2.0" || resp.ID != nil { + t.Fatalf("unexpected response identity: %+v", resp) + } + if resp.Error.Code != -32600 || resp.Error.Message != "Invalid request" { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestServeHTTPQuietlyHandlesNonRPCProbe(t *testing.T) { + rpcServer := &RpcServer{} + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + + rpcServer.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("expected status %d, got %d", http.StatusNotFound, rec.Code) + } +} diff --git a/pkg/sbpf/program.go b/pkg/sbpf/program.go index 5f1c14dd..c112be14 100644 --- a/pkg/sbpf/program.go +++ b/pkg/sbpf/program.go @@ -15,6 +15,20 @@ type Program struct { SbpfVersion sbpfver.SbpfVersion } +func (p *Program) MemoryBytes() uint64 { + if p == nil { + return 0 + } + total := uint64(len(p.RO)) + uint64(len(p.Text))*8 + if len(p.RO) == 0 { + total += uint64(len(p.TextBytes)) + } + if len(p.Funcs) > 0 { + total += uint64(len(p.Funcs)) * 32 + } + return total +} + // Verify runs the static bytecode verifier. func (p *Program) Verify() error { return NewVerifier(p).VerifyProgram() diff --git a/pkg/sealevel/vote_program.go b/pkg/sealevel/vote_program.go index cbdc183a..4fa81535 100644 --- a/pkg/sealevel/vote_program.go +++ b/pkg/sealevel/vote_program.go @@ -207,6 +207,9 @@ func (vote *VoteInstrVote) UnmarshalWithDecoder(decoder *bin.Decoder) error { if err != nil { return err } + if slotsLen > uint64(decoder.Remaining()/8) { + return InstrErrInvalidInstructionData + } for count := uint64(0); count < slotsLen; count++ { slot, err := decoder.ReadUint64(bin.LE) @@ -303,6 +306,9 @@ func (updateVoteState *VoteInstrUpdateVoteState) UnmarshalWithDecoder(decoder *b if err != nil { return err } + if numLockouts > MaxLockoutHistory || numLockouts > uint64(decoder.Remaining()/12) { + return InstrErrInvalidInstructionData + } updateVoteState.Lockouts.Clear() updateVoteState.Lockouts.SetBaseCap(int(numLockouts)) @@ -502,6 +508,9 @@ func (cuvs *CompactUpdateVoteState) UnmarshalWithDecoder(decoder *bin.Decoder) e if err != nil { return err } + if lockoutsLen > MaxLockoutHistory { + return InstrErrInvalidInstructionData + } for count := 0; count < lockoutsLen; count++ { var lockoutOffset LockoutOffset @@ -548,6 +557,9 @@ func (towerSync *VoteInstrTowerSync) UnmarshalWithDecoder(decoder *bin.Decoder) if err != nil { return err } + if lockoutOffsetsLen > MaxLockoutHistory { + return InstrErrInvalidInstructionData + } var lastSlot uint64 if towerSync.Root != nil { diff --git a/pkg/sealevel/vote_program_decode_test.go b/pkg/sealevel/vote_program_decode_test.go new file mode 100644 index 00000000..dd3f8b70 --- /dev/null +++ b/pkg/sealevel/vote_program_decode_test.go @@ -0,0 +1,32 @@ +package sealevel + +import ( + "encoding/binary" + "testing" + + bin "github.com/gagliardetto/binary" +) + +func TestVoteInstrUpdateVoteStateRejectsHugeLockoutCount(t *testing.T) { + raw := make([]byte, 8) + binary.LittleEndian.PutUint64(raw, ^uint64(0)) + decoder := bin.NewBinDecoder(raw) + + var vote VoteInstrUpdateVoteState + if err := vote.UnmarshalWithDecoder(decoder); err == nil { + t.Fatalf("expected huge lockout count to fail") + } +} + +func TestVoteInstrTowerSyncRejectsHugeLockoutCount(t *testing.T) { + raw := []byte{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x03, + } + decoder := bin.NewBinDecoder(raw) + + var vote VoteInstrTowerSync + if err := vote.UnmarshalWithDecoder(decoder); err == nil { + t.Fatalf("expected huge lockout count to fail") + } +} diff --git a/pkg/shred/entry.go b/pkg/shred/entry.go deleted file mode 100644 index 6db129f4..00000000 --- a/pkg/shred/entry.go +++ /dev/null @@ -1,42 +0,0 @@ -package shred - -import ( - "fmt" - - "github.com/gagliardetto/binary" - "github.com/gagliardetto/solana-go" -) - -type Entry struct { - NumHashes uint64 - Hash solana.Hash - Txns []solana.Transaction -} - -func (en *Entry) UnmarshalWithDecoder(decoder *bin.Decoder) (err error) { - // read the number of hashes: - if en.NumHashes, err = decoder.ReadUint64(bin.LE); err != nil { - return fmt.Errorf("failed to read number of hashes: %w", err) - } - // read the hash: - _, err = decoder.Read(en.Hash[:]) - if err != nil { - return fmt.Errorf("failed to read hash: %w", err) - } - // read the number of transactions: - numTxns, err := decoder.ReadUint64(bin.LE) - if err != nil { - return fmt.Errorf("failed to read number of transactions: %w", err) - } - if numTxns > uint64(decoder.Remaining()) { - return fmt.Errorf("not enough bytes to read %d transactions", numTxns) - } - // read the transactions: - en.Txns = make([]solana.Transaction, numTxns) - for i := uint64(0); i < numTxns; i++ { - if err = en.Txns[i].UnmarshalWithDecoder(decoder); err != nil { - return fmt.Errorf("failed to read transaction %d: %w", i, err) - } - } - return -} diff --git a/pkg/shred/shred.go b/pkg/shred/shred.go deleted file mode 100644 index f981f25d..00000000 --- a/pkg/shred/shred.go +++ /dev/null @@ -1,174 +0,0 @@ -package shred - -import ( - "encoding/base64" - "encoding/binary" - "encoding/hex" - "fmt" - - "github.com/gagliardetto/solana-go" -) - -type Shred struct { - CommonHeader - //CodeHeader - DataHeader - Payload []byte - MerklePath [][20]byte -} - -const ( - LegacyCodeID = uint8(0b0101_1010) - LegacyDataID = uint8(0b1010_0101) - MerkleTypeMask = uint8(0xF0) - MerkleDepthMask = uint8(0x0F) - MerkleCodeID = uint8(0x40) - MerkleDataID = uint8(0x80) -) - -const ( - FlagDataTickMask = uint8(0b0011_1111) - FlagDataEndOfBatch = uint8(0b0100_0000) - FlagDataEndOfBlock = uint8(0b1100_0000) -) - -const ( - RevisionV1 = 1 - RevisionV2 = 2 -) - -const ( - LegacyDataV1HeaderSize = 86 - LegacyDataV2HeaderSize = 88 - LegacyDataV1PayloadSize = 1057 // TODO where does this number come from? -) - -// NewShredFromSerialized creates a shred object from the given buffer. -// -// The original slice may be deallocated after this function returns. -func NewShredFromSerialized(shred []byte, revision int) (s Shred) { - if len(shred) < 88 { - return - } - variant := shred[64] - switch { - case variant == LegacyCodeID: - //s.loadCode() - panic("todo legacy code shred") - case variant == LegacyDataID: - var payloadOff, payloadSize int - switch revision { - case 1: - s.DataHeader.ParentOffset = binary.LittleEndian.Uint16(shred[0x53:0x55]) - s.DataHeader.Flags = shred[0x55] - s.DataHeader.Size = LegacyDataV1HeaderSize + LegacyDataV1PayloadSize - payloadOff = LegacyDataV1HeaderSize - payloadSize = LegacyDataV1PayloadSize - case 2: - s.DataHeader.ParentOffset = binary.LittleEndian.Uint16(shred[0x53:0x55]) - s.DataHeader.Flags = shred[0x55] - s.DataHeader.Size = binary.LittleEndian.Uint16(shred[0x56:0x58]) - payloadOff = LegacyDataV2HeaderSize - payloadSize = int(s.DataHeader.Size) - LegacyDataV2HeaderSize - default: - panic(fmt.Sprintf("unsupported shred revision %d", revision)) - } - if payloadSize < 0 { - return - } - if len(shred) < int(s.DataHeader.Size) { - return - } - s.Payload = make([]byte, payloadSize) - copy(s.Payload, shred[payloadOff:payloadOff+payloadSize]) - case variant&MerkleTypeMask == MerkleCodeID: - panic("todo merkle code shred") - //return MerkleCodeFromPayload(shred) - case variant&MerkleTypeMask == MerkleDataID: - s.DataHeader.ParentOffset = binary.LittleEndian.Uint16(shred[0x53:0x55]) - s.DataHeader.Flags = shred[0x55] - s.DataHeader.Size = binary.LittleEndian.Uint16(shred[0x56:0x58]) - payloadOff := LegacyDataV2HeaderSize - merkleDepth := int(variant & MerkleDepthMask) - merkleProofSize := merkleDepth * 20 - payloadSize := int(s.DataHeader.Size) - LegacyDataV2HeaderSize - if payloadSize < 0 { - return - } - if len(shred) < int(s.DataHeader.Size)+merkleProofSize { - return - } - s.Payload = make([]byte, payloadSize) - copy(s.Payload, shred[payloadOff:payloadOff+payloadSize]) - s.MerklePath = make([][20]byte, merkleDepth) - for i := range s.MerklePath { - copy(s.MerklePath[i][:], shred[len(shred)-(merkleDepth-i)*20:len(shred)-(merkleDepth-i-1)*20]) - } - default: - return - } - copy(s.Signature[:], shred[0x00:0x40]) - s.Variant = variant - s.Slot = binary.LittleEndian.Uint64(shred[0x41:0x49]) - s.Index = binary.LittleEndian.Uint32(shred[0x49:0x4d]) - s.Version = binary.LittleEndian.Uint16(shred[0x4d:0x4f]) - s.FECSetIndex = binary.LittleEndian.Uint32(shred[0x4f:0x53]) - return -} - -func (s Shred) MarshalYAML() (any, error) { - merklePath := make([]string, len(s.MerklePath)) - for i, x := range s.MerklePath { - merklePath[i] = hex.EncodeToString(x[:]) - } - return struct { - CommonHeader - DataHeader - Payload string - MerklePath []string `json:",omitempty"` - }{ - CommonHeader: s.CommonHeader, - DataHeader: s.DataHeader, - Payload: base64.StdEncoding.EncodeToString(nil), - MerklePath: merklePath, - }, nil -} - -type CommonHeader struct { - Signature solana.Signature - Variant uint8 - Slot uint64 - Index uint32 - Version uint16 - FECSetIndex uint32 -} - -func (c *CommonHeader) Ok() bool { - return c.IsData() || c.IsCode() -} - -func (c *CommonHeader) IsData() bool { - return c.Variant == LegacyDataID || (c.Variant&MerkleTypeMask) == MerkleDataID -} - -func (c *CommonHeader) IsCode() bool { - return c.Variant == LegacyCodeID || (c.Variant&MerkleTypeMask) == MerkleCodeID -} - -type DataHeader struct { - ParentOffset uint16 - Flags uint8 - Size uint16 -} - -func (d *DataHeader) EndOfBlock() bool { - return d.Flags&FlagDataEndOfBlock != 0 -} - -func (s *DataHeader) EndOfBatch() bool { - return s.Flags&FlagDataEndOfBatch == 1 -} - -func (s *DataHeader) Tick() uint8 { - return s.Flags & FlagDataTickMask -} diff --git a/pkg/shred/shredder.go b/pkg/shred/shredder.go deleted file mode 100644 index 2e9de00c..00000000 --- a/pkg/shred/shredder.go +++ /dev/null @@ -1,21 +0,0 @@ -package shred - -func Concat(shreds []Shred) []byte { - var total int - for i := range shreds { - if !shreds[i].IsData() { - continue - } - total += len(shreds[i].Payload) - } - buf := make([]byte, 0, total) - for i := range shreds { - if !shreds[i].IsData() { - continue - } - target := buf[len(buf) : len(buf)+len(shreds[i].Payload)] - copy(target, shreds[i].Payload) - buf = buf[:len(buf)+len(shreds[i].Payload)] - } - return buf -} diff --git a/pkg/snapshot/build_db.go b/pkg/snapshot/build_db.go index 0f516bb5..bc3f9da7 100644 --- a/pkg/snapshot/build_db.go +++ b/pkg/snapshot/build_db.go @@ -22,9 +22,19 @@ import ( ) const ( - maxIndexEntryCommitter = 512 - maxIndexEntryBuilder = 500 - maxAppendVecCopying = 500 + DefaultSnapshotIndexEntryCommitterWorkers = 64 + DefaultSnapshotIndexEntryBuilderWorkers = 64 + DefaultSnapshotAppendVecCopyingWorkers = 32 + DefaultSnapshotIndexShards = 64 + DefaultSnapshotMaxConcurrentFlushers = 8 +) + +var ( + SnapshotIndexEntryCommitterWorkers = DefaultSnapshotIndexEntryCommitterWorkers + SnapshotIndexEntryBuilderWorkers = DefaultSnapshotIndexEntryBuilderWorkers + SnapshotAppendVecCopyingWorkers = DefaultSnapshotAppendVecCopyingWorkers + SnapshotIndexShards = DefaultSnapshotIndexShards + SnapshotIndexTempDir string ) // CleanAccountsDbDir removes all artifacts from a previous incomplete snapshot run. @@ -163,6 +173,74 @@ var ( appendVecCopyingInProgress = &atomic.Int64{} ) +func positiveOrDefault(value int, fallback int) int { + if value > 0 { + return value + } + return fallback +} + +func snapshotIndexEntryCommitterWorkers() int { + return positiveOrDefault(SnapshotIndexEntryCommitterWorkers, DefaultSnapshotIndexEntryCommitterWorkers) +} + +func snapshotIndexEntryBuilderWorkers() int { + return positiveOrDefault(SnapshotIndexEntryBuilderWorkers, DefaultSnapshotIndexEntryBuilderWorkers) +} + +func snapshotAppendVecCopyingWorkers() int { + return positiveOrDefault(SnapshotAppendVecCopyingWorkers, DefaultSnapshotAppendVecCopyingWorkers) +} + +func snapshotIndexShards() int { + return positiveOrDefault(SnapshotIndexShards, DefaultSnapshotIndexShards) +} + +func snapshotMaxConcurrentFlushers() int { + return positiveOrDefault(MaxConcurrentFlushers, DefaultSnapshotMaxConcurrentFlushers) +} + +func logSnapshotBootstrapTuning() { + indexTempDir := SnapshotIndexTempDir + if indexTempDir == "" { + indexTempDir = "(accountsdb)" + } + mlog.Log.Infof("Snapshot bootstrap tuning: append_vec_workers=%d index_builder_workers=%d index_committer_workers=%d index_shards=%d max_concurrent_flushers=%d zstd_decoder_concurrency=%d index_temp_dir=%s", + snapshotAppendVecCopyingWorkers(), + snapshotIndexEntryBuilderWorkers(), + snapshotIndexEntryCommitterWorkers(), + snapshotIndexShards(), + snapshotMaxConcurrentFlushers(), + ZstdDecoderConcurrency, + indexTempDir) +} + +func prepareSnapshotIndexWorkDir(accountsDbDir string) (string, func(), error) { + if SnapshotIndexTempDir == "" { + logsDir := filepath.Join(accountsDbDir, "mithril_db_log_shards") + if err := os.MkdirAll(logsDir, 0775); err != nil { + return "", nil, err + } + return logsDir, func() {}, nil + } + + if err := os.MkdirAll(SnapshotIndexTempDir, 0775); err != nil { + return "", nil, fmt.Errorf("creating snapshot index temp dir %s: %w", SnapshotIndexTempDir, err) + } + logsDir, err := os.MkdirTemp(SnapshotIndexTempDir, "mithril-db-log-shards-*") + if err != nil { + return "", nil, fmt.Errorf("creating snapshot index work dir in %s: %w", SnapshotIndexTempDir, err) + } + mlog.Log.Infof("Snapshot index shard logs/SST staging: %s", logsDir) + + cleanup := func() { + if err := os.RemoveAll(logsDir); err != nil { + mlog.Log.Warnf("failed to remove snapshot index temp dir %s: %v", logsDir, err) + } + } + return logsDir, cleanup, nil +} + func BuildAccountsDbPaths( ctx context.Context, snapshotFile string, @@ -196,17 +274,19 @@ func BuildAccountsDbPaths( if err = os.MkdirAll(appendVecsOutputDir, 0775); err != nil { return nil, nil, err } + logSnapshotBootstrapTuning() defer ants.Release() var largestFileId atomic.Uint64 wg := &sync.WaitGroup{} - logsDir := filepath.Join(accountsDbDir, "mithril_db_log_shards") - if err = os.MkdirAll(logsDir, 0775); err != nil { + logsDir, cleanupIndexWorkDir, err := prepareSnapshotIndexWorkDir(accountsDbDir) + if err != nil { return nil, nil, err } - numShards := 256 + defer cleanupIndexWorkDir() + numShards := snapshotIndexShards() sl := NewShardLogger(numShards, logsDir) // Create stake pubkey collector for building stake index during appendvec processing @@ -463,9 +543,13 @@ func initWorkerPools( largestFileId *atomic.Uint64, stakeCollector *stakeIndexCollector, ) (*snapshotWorkerPools, error) { - indexEntryCommitterPool, err := ants.NewPoolWithFunc(maxIndexEntryCommitter, func(i any) { + indexEntryCommitterWorkers := snapshotIndexEntryCommitterWorkers() + indexEntryBuilderWorkers := snapshotIndexEntryBuilderWorkers() + appendVecCopyingWorkers := snapshotAppendVecCopyingWorkers() + + indexEntryCommitterPool, err := ants.NewPoolWithFunc(indexEntryCommitterWorkers, func(i any) { tasks := indexEntryCommitterInProgress.Add(1) - statsd.Gauge(statsd.SnapshotWorkerPoolUtilization, float64(tasks)/float64(maxIndexEntryCommitter), []string{"index_entry_committer"}) + statsd.Gauge(statsd.SnapshotWorkerPoolUtilization, float64(tasks)/float64(indexEntryCommitterWorkers), []string{"index_entry_committer"}) start := time.Now() defer wg.Done() task := i.(indexEntryCommitterTask) @@ -480,9 +564,9 @@ func initWorkerPools( return nil, err } - indexEntryBuilderPool, err := ants.NewPoolWithFunc(maxIndexEntryBuilder, func(i any) { + indexEntryBuilderPool, err := ants.NewPoolWithFunc(indexEntryBuilderWorkers, func(i any) { tasks := indexEntryBuilderInProgress.Add(1) - statsd.Gauge(statsd.SnapshotWorkerPoolUtilization, float64(tasks)/float64(maxIndexEntryBuilder), []string{"index_entry_builder"}) + statsd.Gauge(statsd.SnapshotWorkerPoolUtilization, float64(tasks)/float64(indexEntryBuilderWorkers), []string{"index_entry_builder"}) start := time.Now() defer wg.Done() task := i.(indexEntryBuilderTask) @@ -508,9 +592,9 @@ func initWorkerPools( return nil, err } - appendVecCopyingPool, err := ants.NewPoolWithFunc(maxAppendVecCopying, func(i any) { + appendVecCopyingPool, err := ants.NewPoolWithFunc(appendVecCopyingWorkers, func(i any) { tasks := appendVecCopyingInProgress.Add(1) - statsd.Gauge(statsd.SnapshotWorkerPoolUtilization, float64(tasks)/float64(maxAppendVecCopying), []string{"append_vec_copying"}) + statsd.Gauge(statsd.SnapshotWorkerPoolUtilization, float64(tasks)/float64(appendVecCopyingWorkers), []string{"append_vec_copying"}) start := time.Now() defer wg.Done() task := i.(appendVecCopyingTask) diff --git a/pkg/snapshot/build_db_with_incr.go b/pkg/snapshot/build_db_with_incr.go index 3ef31844..2499b0fc 100644 --- a/pkg/snapshot/build_db_with_incr.go +++ b/pkg/snapshot/build_db_with_incr.go @@ -62,6 +62,7 @@ func BuildAccountsDbAuto( if err = os.MkdirAll(appendVecsOutputDir, 0775); err != nil { return nil, nil, err } + logSnapshotBootstrapTuning() defer ants.Release() @@ -69,11 +70,12 @@ func BuildAccountsDbAuto( var largestFileId atomic.Uint64 wg := &sync.WaitGroup{} - numShards := 256 - logsDir := filepath.Join(accountsDbDir, "mithril_db_log_shards") - if err = os.MkdirAll(logsDir, 0775); err != nil { + numShards := snapshotIndexShards() + logsDir, cleanupIndexWorkDir, err := prepareSnapshotIndexWorkDir(accountsDbDir) + if err != nil { return nil, nil, err } + defer cleanupIndexWorkDir() sl := NewShardLogger(numShards, logsDir) // Create stake pubkey collector for building stake index during appendvec processing @@ -147,7 +149,7 @@ func BuildAccountsDbAuto( } else if strings.Contains(err.Error(), "no rpc nodes") || strings.Contains(err.Error(), "no nodes found") { errMsg += "\n Hint: Check RPC endpoints connectivity or try again later" } - return nil, nil, fmt.Errorf(errMsg) + return nil, nil, fmt.Errorf("%s", errMsg) } mlog.Log.Debugf("found incremental snapshot URL in %s: %s", fmtDuration(time.Since(incrSnapshotDlStart)), incrementalSnapshotPath) diff --git a/pkg/snapshot/shard.go b/pkg/snapshot/shard.go index 2290eb93..2f5e77a6 100644 --- a/pkg/snapshot/shard.go +++ b/pkg/snapshot/shard.go @@ -24,7 +24,7 @@ import ( "golang.org/x/sync/semaphore" ) -var MaxConcurrentFlushers int = 16 +var MaxConcurrentFlushers int = DefaultSnapshotMaxConcurrentFlushers type shardRequest struct { k solana.PublicKey @@ -65,14 +65,18 @@ type shard struct { // shards for logging entries. Entries are flushed to shardedSetter // when log reaches a certain size or on shard closure. func NewShardLogger(numShards int, filePrefix string) *ShardLogger { + if numShards <= 0 { + numShards = DefaultSnapshotIndexShards + } if numShards > 1000 { panic(fmt.Sprintf("numShards=%d > 1000 is too many shards", numShards)) } + flushers := snapshotMaxConcurrentFlushers() sl := &ShardLogger{ shards: make([]*shard, numShards), filePrefix: filePrefix, wg: &sync.WaitGroup{}, - flushSem: semaphore.NewWeighted(int64(MaxConcurrentFlushers)), + flushSem: semaphore.NewWeighted(int64(flushers)), } sl.wg.Add(numShards) diff --git a/pkg/snapshotdl/snapshotdl.go b/pkg/snapshotdl/snapshotdl.go index 76ed43d1..054e6c1c 100644 --- a/pkg/snapshotdl/snapshotdl.go +++ b/pkg/snapshotdl/snapshotdl.go @@ -5,6 +5,8 @@ import ( "fmt" "io" stdlog "log" + "net" + "net/url" "strconv" "strings" "sync" @@ -79,6 +81,7 @@ type SnapshotConfig struct { TCPTimeoutMs int MinNodeVersion string AllowedNodeVersions []string + NodeBlacklist []string // RPC URLs, host:port pairs, or hosts/IPs to avoid as snapshot sources // Snapshot age thresholds (slots) FullThreshold int @@ -198,8 +201,8 @@ func (sc SnapshotConfig) toInternalConfig(path string) config.Config { TCPTimeoutMs: sc.TCPTimeoutMs, MinNodeVersion: sc.MinNodeVersion, AllowedNodeVersions: sc.AllowedNodeVersions, - MaxFullSnapshots: sc.MaxFullSnapshots, - SafetyMarginSlots: sc.SafetyMarginSlots, + MaxFullSnapshots: sc.MaxFullSnapshots, + SafetyMarginSlots: sc.SafetyMarginSlots, } } @@ -225,14 +228,148 @@ func filterByMaxSlot(results []rpc.NodeResult, maxSlot int64) ([]rpc.NodeResult, return filtered, filteredOut } +type snapshotNodeBlacklist struct { + keys map[string]struct{} +} + +func newSnapshotNodeBlacklist(entries []string) snapshotNodeBlacklist { + keys := make(map[string]struct{}) + for _, entry := range entries { + for _, key := range snapshotEndpointKeys(entry) { + keys[key] = struct{}{} + } + } + return snapshotNodeBlacklist{keys: keys} +} + +func (b snapshotNodeBlacklist) empty() bool { + return len(b.keys) == 0 +} + +func (b snapshotNodeBlacklist) contains(endpoint string) bool { + if b.empty() { + return false + } + for _, key := range snapshotEndpointKeys(endpoint) { + if _, ok := b.keys[key]; ok { + return true + } + } + return false +} + +func snapshotEndpointKeys(raw string) []string { + cleaned := strings.TrimSpace(raw) + if cleaned == "" { + return nil + } + + var keys []string + seen := make(map[string]struct{}) + add := func(key string) { + key = strings.TrimRight(strings.TrimSpace(strings.ToLower(key)), "/") + if key == "" { + return + } + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + keys = append(keys, key) + } + + normalized := strings.TrimRight(strings.ToLower(cleaned), "/") + add(normalized) + + if u, err := url.Parse(normalized); err == nil && u.Host != "" { + hostPort := strings.ToLower(u.Host) + host := strings.ToLower(u.Hostname()) + port := u.Port() + add(hostPort) + if u.Scheme != "" { + add(u.Scheme + "://" + hostPort) + } + add(host) + if host != "" && port != "" { + add(host + ":" + port) + add(net.JoinHostPort(host, port)) + } + return keys + } + + hostPort := normalized + if slash := strings.IndexByte(hostPort, '/'); slash >= 0 { + hostPort = hostPort[:slash] + } + add(hostPort) + + if host, port, err := net.SplitHostPort(hostPort); err == nil { + host = strings.Trim(host, "[]") + add(host) + if host != "" && port != "" { + add(host + ":" + port) + add(net.JoinHostPort(host, port)) + } + return keys + } + + if strings.Count(hostPort, ":") == 1 { + parts := strings.SplitN(hostPort, ":", 2) + add(parts[0]) + } + + return keys +} + +func filterSnapshotRPCNodes(nodes []rpc.RPCNode, blacklist snapshotNodeBlacklist) ([]rpc.RPCNode, int) { + if blacklist.empty() { + return nodes, 0 + } + + filtered := make([]rpc.RPCNode, 0, len(nodes)) + skipped := 0 + for _, node := range nodes { + if blacklist.contains(node.Address) { + skipped++ + continue + } + filtered = append(filtered, node) + } + return filtered, skipped +} + +func filterSnapshotNodeResults(results []rpc.NodeResult, blacklist snapshotNodeBlacklist) ([]rpc.NodeResult, int) { + if blacklist.empty() { + return results, 0 + } + + filtered := make([]rpc.NodeResult, 0, len(results)) + skipped := 0 + for _, result := range results { + if blacklist.contains(result.RPC) { + skipped++ + continue + } + filtered = append(filtered, result) + } + return filtered, skipped +} + +func logSnapshotBlacklist(stage string, skipped int, remaining int) { + if skipped == 0 { + return + } + mlog.Log.Infof("Snapshot node blacklist excluded %d %s candidate(s); %d remain", skipped, stage, remaining) +} + // incBaseMatchStats tracks statistics for incremental base matching filter type incBaseMatchStats struct { - totalWithFull int // nodes with a full snapshot - totalWithInc int // nodes with any incremental - afterIncBaseMatch int // nodes remaining after incremental base match filter - uniqueFullSlots int // number of unique full snapshot slots - uniqueIncBases int // number of unique incremental base slots - matchingFullSlots int // full slots that have at least one matching inc base + totalWithFull int // nodes with a full snapshot + totalWithInc int // nodes with any incremental + afterIncBaseMatch int // nodes remaining after incremental base match filter + uniqueFullSlots int // number of unique full snapshot slots + uniqueIncBases int // number of unique incremental base slots + matchingFullSlots int // full slots that have at least one matching inc base } // filterByIncrementalBaseMatch filters results to only include nodes whose FullSlot @@ -315,6 +452,7 @@ func DownloadSnapshot(ctx context.Context, rpcEndpoints []string, path string) ( func GetSnapshotURL(ctx context.Context, snapCfg SnapshotConfig) (string, int, int, error) { defer suppressStdlogOutput()() cfg := snapCfg.toInternalConfig("") + blacklist := newSnapshotNodeBlacklist(snapCfg.NodeBlacklist) // Step 1: Get reference slot from multiple RPCs for reliability mlog.Log.Infof("Getting reference slot from RPC(s)...") @@ -330,11 +468,19 @@ func GetSnapshotURL(ctx context.Context, snapCfg SnapshotConfig) (string, int, i if len(nodes) == 0 { return "", 0, 0, fmt.Errorf("no rpc nodes available from cluster") } + var skipped int + nodes, skipped = filterSnapshotRPCNodes(nodes, blacklist) + logSnapshotBlacklist("discovered", skipped, len(nodes)) + if len(nodes) == 0 { + return "", 0, 0, fmt.Errorf("all discovered rpc nodes were excluded by snapshot.node_blacklist") + } mlog.Log.Infof("Found %d potential snapshot sources", len(nodes)) // Step 3: Evaluate nodes with version tracking and statistics mlog.Log.Infof("Evaluating nodes for snapshot availability and speed...") results, stats := rpc.EvaluateNodesWithVersionsAndStats(nodes, cfg, referenceSlot) + results, skipped = filterSnapshotNodeResults(results, blacklist) + logSnapshotBlacklist("evaluated", skipped, len(results)) // Step 3.5: Filter to only full snapshots that have matching incrementals somewhere results, _ = filterByIncrementalBaseMatch(results) @@ -424,6 +570,7 @@ func GetSnapshotURL(ctx context.Context, snapCfg SnapshotConfig) (string, int, i func GetSnapshotURLWithInfo(ctx context.Context, snapCfg SnapshotConfig) (*SnapshotInfo, error) { defer suppressStdlogOutput()() cfg := snapCfg.toInternalConfig("") + blacklist := newSnapshotNodeBlacklist(snapCfg.NodeBlacklist) // Step 1: Get reference slot from multiple RPCs for reliability referenceSlot, preferredRPC, err := rpc.GetReferenceSlotFromMultiple(cfg.RPCAddresses) @@ -439,10 +586,18 @@ func GetSnapshotURLWithInfo(ctx context.Context, snapCfg SnapshotConfig) (*Snaps if len(nodes) == 0 { return nil, fmt.Errorf("no rpc nodes available from cluster") } + var skipped int + nodes, skipped = filterSnapshotRPCNodes(nodes, blacklist) + logSnapshotBlacklist("discovered", skipped, len(nodes)) + if len(nodes) == 0 { + return nil, fmt.Errorf("all discovered rpc nodes were excluded by snapshot.node_blacklist") + } // Step 3: Evaluate nodes with version tracking and statistics mlog.Log.Infof("Probing %d nodes for snapshot availability...", len(nodes)) results, stats := rpc.EvaluateNodesWithVersionsAndStats(nodes, cfg, referenceSlot) + results, skipped = filterSnapshotNodeResults(results, blacklist) + logSnapshotBlacklist("evaluated", skipped, len(results)) // Step 3.5: Filter to only full snapshots that have matching incrementals somewhere // This prevents selecting a fast full snapshot that has no compatible incrementals @@ -569,6 +724,7 @@ func GetSnapshotURLWithInfo(ctx context.Context, snapCfg SnapshotConfig) (*Snaps func DownloadSnapshotWithConfig(ctx context.Context, path string, snapCfg SnapshotConfig) (string, int, int, error) { defer suppressStdlogOutput()() cfg := snapCfg.toInternalConfig(path) + blacklist := newSnapshotNodeBlacklist(snapCfg.NodeBlacklist) // Step 1: Get reference slot from multiple RPCs for reliability mlog.Log.Infof("Getting reference slot from RPC(s)...") @@ -584,11 +740,19 @@ func DownloadSnapshotWithConfig(ctx context.Context, path string, snapCfg Snapsh if len(nodes) == 0 { return "", 0, 0, fmt.Errorf("no rpc nodes available from cluster") } + var skipped int + nodes, skipped = filterSnapshotRPCNodes(nodes, blacklist) + logSnapshotBlacklist("discovered", skipped, len(nodes)) + if len(nodes) == 0 { + return "", 0, 0, fmt.Errorf("all discovered rpc nodes were excluded by snapshot.node_blacklist") + } mlog.Log.Infof("Found %d potential snapshot sources", len(nodes)) // Step 3: Evaluate nodes with version tracking and statistics mlog.Log.Infof("Evaluating nodes for snapshot availability and speed...") results, stats := rpc.EvaluateNodesWithVersionsAndStats(nodes, cfg, referenceSlot) + results, skipped = filterSnapshotNodeResults(results, blacklist) + logSnapshotBlacklist("evaluated", skipped, len(results)) // Step 3.5: Filter to only full snapshots that have matching incrementals somewhere results, _ = filterByIncrementalBaseMatch(results) @@ -689,6 +853,7 @@ func DownloadIncrementalSnapshot(rpcEndpoints []string, path string, referenceSl func DownloadIncrementalSnapshotWithConfig(path string, referenceSlot int, fullSnapshotSlot int, snapCfg SnapshotConfig) (string, int, int, error) { defer suppressStdlogOutput()() cfg := snapCfg.toInternalConfig(path) + blacklist := newSnapshotNodeBlacklist(snapCfg.NodeBlacklist) ctx := context.Background() mlog.Log.Infof("Searching for incremental snapshot matching full slot %d...", fullSnapshotSlot) @@ -702,9 +867,17 @@ func DownloadIncrementalSnapshotWithConfig(path string, referenceSlot int, fullS if len(nodes) == 0 { return "", 0, 0, fmt.Errorf("no rpc nodes available from cluster") } + var skipped int + nodes, skipped = filterSnapshotRPCNodes(nodes, blacklist) + logSnapshotBlacklist("discovered", skipped, len(nodes)) + if len(nodes) == 0 { + return "", 0, 0, fmt.Errorf("all discovered rpc nodes were excluded by snapshot.node_blacklist") + } // Step 2: Evaluate nodes results, stats := rpc.EvaluateNodesWithVersionsAndStats(nodes, cfg, referenceSlot) + results, skipped = filterSnapshotNodeResults(results, blacklist) + logSnapshotBlacklist("evaluated", skipped, len(results)) if snapCfg.Verbose && stats != nil { filterCfg := rpc.FilterConfig{ @@ -853,15 +1026,16 @@ func extractFullSnapshotSlot(path string) int { // Returns: (httpURL, baseSlot, endSlot, error) // // Fallback strategy (different from full snapshot): -// 1. Try the same node that provided the full snapshot (fastest, most likely match) -// 2. If that fails, find ALL nodes with matching base slot (more flexible than full snapshot) -// 3. Among matching nodes, prioritize by: -// a. Freshness (highest end slot = most recent incremental) -// b. Speed (faster downloads preferred when end slots are equal) -// 4. Try multiple candidates for resilience (uses MaxSnapshotURLAttempts) +// 1. Try the same node that provided the full snapshot (fastest, most likely match) +// 2. If that fails, find ALL nodes with matching base slot (more flexible than full snapshot) +// 3. Among matching nodes, prioritize by: +// a. Freshness (highest end slot = most recent incremental) +// b. Speed (faster downloads preferred when end slots are equal) +// 4. Try multiple candidates for resilience (uses MaxSnapshotURLAttempts) func GetIncrementalSnapshotURL(fullSnapshotURL string, referenceSlot int, fullSnapshotSlot int, snapCfg SnapshotConfig) (string, int, int, error) { defer suppressStdlogOutput()() cfg := snapCfg.toInternalConfig("") + blacklist := newSnapshotNodeBlacklist(snapCfg.NodeBlacklist) ctx := context.Background() // Extract the source node RPC from the full snapshot URL @@ -875,7 +1049,9 @@ func GetIncrementalSnapshotURL(fullSnapshotURL string, referenceSlot int, fullSn } // Step 1: Try to get incremental from the same source as the full snapshot - if sourceNodeRPC != "" { + if sourceNodeRPC != "" && blacklist.contains(sourceNodeRPC) { + mlog.Log.Infof("Skipping same-source incremental check because %s is in snapshot.node_blacklist", sourceNodeRPC) + } else if sourceNodeRPC != "" { mlog.Log.Infof("Checking same source for incremental (base slot %d): %s", fullSnapshotSlot, sourceNodeRPC) urlInfo, err := snapshot.GetSnapshotURL(ctx, sourceNodeRPC, "incremental") @@ -909,9 +1085,17 @@ func GetIncrementalSnapshotURL(fullSnapshotURL string, referenceSlot int, fullSn if len(nodes) == 0 { return "", 0, 0, fmt.Errorf("no rpc nodes available from cluster") } + var skipped int + nodes, skipped = filterSnapshotRPCNodes(nodes, blacklist) + logSnapshotBlacklist("discovered", skipped, len(nodes)) + if len(nodes) == 0 { + return "", 0, 0, fmt.Errorf("all discovered rpc nodes were excluded by snapshot.node_blacklist") + } // Evaluate nodes results, stats := rpc.EvaluateNodesWithVersionsAndStats(nodes, cfg, referenceSlot) + results, skipped = filterSnapshotNodeResults(results, blacklist) + logSnapshotBlacklist("evaluated", skipped, len(results)) if snapCfg.Verbose && stats != nil { filterCfg := rpc.FilterConfig{ diff --git a/pkg/snapshotdl/snapshotdl_test.go b/pkg/snapshotdl/snapshotdl_test.go new file mode 100644 index 00000000..6b37e860 --- /dev/null +++ b/pkg/snapshotdl/snapshotdl_test.go @@ -0,0 +1,80 @@ +package snapshotdl + +import ( + "testing" + + snaprpc "github.com/Overclock-Validator/solana-snapshot-finder-go/pkg/rpc" +) + +func TestSnapshotNodeBlacklistMatchesCommonEndpointForms(t *testing.T) { + blacklist := newSnapshotNodeBlacklist([]string{ + "http://203.0.113.10:8899/", + "198.51.100.24:8899", + "bad-snapshot-node.example.com", + }) + + tests := []struct { + name string + endpoint string + want bool + }{ + { + name: "full rpc url", + endpoint: "http://203.0.113.10:8899", + want: true, + }, + { + name: "snapshot url from same source", + endpoint: "https://203.0.113.10:8899/snapshot-123-abc.tar.zst", + want: true, + }, + { + name: "host port", + endpoint: "http://198.51.100.24:8899", + want: true, + }, + { + name: "hostname", + endpoint: "http://bad-snapshot-node.example.com:8899", + want: true, + }, + { + name: "allowed node", + endpoint: "http://good-snapshot-node.example.com:8899", + want: false, + }, + { + name: "empty endpoint", + endpoint: "", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := blacklist.contains(tt.endpoint); got != tt.want { + t.Fatalf("contains(%q) = %v, want %v", tt.endpoint, got, tt.want) + } + }) + } +} + +func TestFilterSnapshotRPCNodes(t *testing.T) { + blacklist := newSnapshotNodeBlacklist([]string{"203.0.113.10", "bad.example.com"}) + nodes := []snaprpc.RPCNode{ + {Address: "http://203.0.113.10:8899"}, + {Address: "http://good.example.com:8899"}, + {Address: "http://bad.example.com:8899"}, + } + + filtered, skipped := filterSnapshotRPCNodes(nodes, blacklist) + if skipped != 2 { + t.Fatalf("skipped = %d, want 2", skipped) + } + if len(filtered) != 1 { + t.Fatalf("len(filtered) = %d, want 1", len(filtered)) + } + if filtered[0].Address != "http://good.example.com:8899" { + t.Fatalf("remaining node = %q, want good.example.com", filtered[0].Address) + } +} diff --git a/pkg/turbine/assembler.go b/pkg/turbine/assembler.go new file mode 100644 index 00000000..191e4fd7 --- /dev/null +++ b/pkg/turbine/assembler.go @@ -0,0 +1,654 @@ +package turbine + +import ( + "errors" + "fmt" + "sort" + "sync" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/klauspost/reedsolomon" +) + +var ( + ErrDuplicateShred = errors.New("duplicate data shred") + ErrSlotIncomplete = errors.New("slot incomplete") + ErrSlotOverflow = errors.New("slot has too many data shreds") +) + +const ( + maxDataShredsPerSlot = 64 * 1024 + maxRetainedIncompleteSlotLag = uint64(512) + maxRetainedIncompleteSlotCap = 1024 + maxRetainedCompletedSlotLag = uint64(512) + repairObservedSlotLag = uint64(1) + repairScanSlotWindow = uint64(96) +) + +type SlotAssembler struct { + mu sync.Mutex + slots map[uint64]*slotState + completedSlots map[uint64]struct{} + encoders map[fecLayout]reedsolomon.Encoder + maxObservedSlot uint64 + recoveredDataShreds uint64 + evictedSlots uint64 + ignoredOldShreds uint64 +} + +type SlotRepairRequest struct { + Slot uint64 + MissingDataShreds []uint32 + NeedHighestDataShred bool + HighestDataShredIndex uint32 +} + +type slotState struct { + slot uint64 + parentSlot uint64 + shreds map[uint32]*Shred + fecSets map[uint32]*fecState + lastIndex uint32 + haveLast bool + shredVer uint16 + firstParent bool +} + +type fecLayout struct { + dataShreds uint16 + codingShreds uint16 + shardSize int +} + +type fecState struct { + slot uint64 + fecSetIndex uint32 + data map[uint32]*Shred + coding map[uint16]*Shred + layout fecLayout + haveLayout bool + signature [64]byte + haveSig bool + dataVariant byte + codeVariant byte +} + +func NewSlotAssembler() *SlotAssembler { + return &SlotAssembler{ + slots: make(map[uint64]*slotState), + completedSlots: make(map[uint64]struct{}), + encoders: make(map[fecLayout]reedsolomon.Encoder), + } +} + +func (a *SlotAssembler) AddPacket(packet []byte) (*block.Block, error) { + shred, err := ParseShred(packet) + if err != nil { + if errors.Is(err, ErrCodingShredIgnored) { + return nil, nil + } + return nil, err + } + return a.AddShred(shred) +} + +func (a *SlotAssembler) AddShred(shred *Shred) (*block.Block, error) { + if shred == nil { + return nil, nil + } + a.mu.Lock() + defer a.mu.Unlock() + + if shred.Slot > a.maxObservedSlot { + a.maxObservedSlot = shred.Slot + } + a.pruneOldSlotsLocked() + if a.slotTooOldLocked(shred.Slot) { + a.ignoredOldShreds++ + return nil, nil + } + if _, completed := a.completedSlots[shred.Slot]; completed { + a.ignoredOldShreds++ + return nil, nil + } + + state := a.slotState(shred.Slot, shred.Version) + var err error + switch shred.Type { + case ShredTypeData: + err = state.addDataShred(shred) + case ShredTypeCode: + err = state.addCodingShred(shred) + default: + return nil, nil + } + if err != nil { + if errors.Is(err, ErrDuplicateShred) { + return nil, nil + } + return nil, err + } + + recovered, err := a.recoverFEC(state, shred.FECSetIndex) + if err != nil { + return nil, err + } + for _, recoveredShred := range recovered { + err := state.addDataShred(recoveredShred) + if err != nil && !errors.Is(err, ErrDuplicateShred) { + return nil, err + } + if err == nil { + a.recoveredDataShreds++ + } + } + + if !state.complete() { + return nil, nil + } + + blk, err := state.block() + if err != nil { + return nil, err + } + delete(a.slots, shred.Slot) + a.completedSlots[shred.Slot] = struct{}{} + return blk, nil +} + +func (a *SlotAssembler) slotState(slot uint64, version uint16) *slotState { + state := a.slots[slot] + if state != nil { + return state + } + state = &slotState{ + slot: slot, + shreds: make(map[uint32]*Shred), + fecSets: make(map[uint32]*fecState), + shredVer: version, + lastIndex: ^uint32(0), + } + a.slots[slot] = state + return state +} + +func (a *SlotAssembler) slotTooOldLocked(slot uint64) bool { + if a.maxObservedSlot <= maxRetainedIncompleteSlotLag { + return false + } + return slot < a.maxObservedSlot-maxRetainedIncompleteSlotLag +} + +func (a *SlotAssembler) pruneOldSlotsLocked() { + if len(a.slots) > 0 && a.maxObservedSlot > maxRetainedIncompleteSlotLag { + minSlot := a.maxObservedSlot - maxRetainedIncompleteSlotLag + for slot := range a.slots { + if slot < minSlot { + delete(a.slots, slot) + a.evictedSlots++ + } + } + } + if a.maxObservedSlot > maxRetainedCompletedSlotLag { + minSlot := a.maxObservedSlot - maxRetainedCompletedSlotLag + for slot := range a.completedSlots { + if slot < minSlot { + delete(a.completedSlots, slot) + } + } + } + + if len(a.slots) == 0 { + return + } + for len(a.slots) > maxRetainedIncompleteSlotCap { + var oldest uint64 + first := true + for slot := range a.slots { + if first || slot < oldest { + oldest = slot + first = false + } + } + if first { + return + } + delete(a.slots, oldest) + a.evictedSlots++ + } +} + +func (s *slotState) addDataShred(shred *Shred) error { + if _, exists := s.shreds[shred.Index]; exists { + return ErrDuplicateShred + } + if shred.Index >= maxDataShredsPerSlot { + return fmt.Errorf("%w: slot %d shred index %d", ErrSlotOverflow, shred.Slot, shred.Index) + } + if s.shredVer != shred.Version { + return fmt.Errorf("mixed shred versions for slot %d: %d and %d", shred.Slot, s.shredVer, shred.Version) + } + if !s.firstParent { + s.parentSlot = shred.ParentSlot() + s.firstParent = true + } else if s.parentSlot != shred.ParentSlot() { + return fmt.Errorf("mixed parent slots for slot %d: %d and %d", shred.Slot, s.parentSlot, shred.ParentSlot()) + } + if err := s.addDataToFEC(shred); err != nil { + return err + } + s.shreds[shred.Index] = shred + if shred.LastInSlot() { + s.haveLast = true + s.lastIndex = shred.Index + } + return nil +} + +func (s *slotState) repairRequest(maxMissing int) (SlotRepairRequest, bool) { + req := SlotRepairRequest{Slot: s.slot} + + var maxObserved uint32 + haveData := false + for index := range s.shreds { + if !haveData || index > maxObserved { + maxObserved = index + haveData = true + } + } + + req.NeedHighestDataShred = !s.haveLast + if haveData && maxObserved < maxDataShredsPerSlot-1 { + req.HighestDataShredIndex = maxObserved + 1 + } + + missingThrough := maxObserved + if s.haveLast { + missingThrough = s.lastIndex + } + for index := uint32(0); index <= missingThrough && len(req.MissingDataShreds) < maxMissing; index++ { + if s.shreds[index] == nil { + req.MissingDataShreds = append(req.MissingDataShreds, index) + } + if index == maxDataShredsPerSlot-1 { + break + } + } + + if len(req.MissingDataShreds) == 0 && !req.NeedHighestDataShred { + return SlotRepairRequest{}, false + } + return req, true +} + +func (s *slotState) addCodingShred(shred *Shred) error { + if shred.NumDataShreds == 0 || shred.NumCodingShreds == 0 { + return fmt.Errorf("invalid coding shred FEC layout for slot %d fec_set=%d: data=%d coding=%d", shred.Slot, shred.FECSetIndex, shred.NumDataShreds, shred.NumCodingShreds) + } + if shred.Position >= shred.NumCodingShreds { + return fmt.Errorf("invalid coding shred position for slot %d fec_set=%d: position=%d coding=%d", shred.Slot, shred.FECSetIndex, shred.Position, shred.NumCodingShreds) + } + if s.shredVer != shred.Version { + return fmt.Errorf("mixed shred versions for slot %d: %d and %d", shred.Slot, s.shredVer, shred.Version) + } + fec := s.fecSet(shred.FECSetIndex) + if _, exists := fec.coding[shred.Position]; exists { + return ErrDuplicateShred + } + if err := fec.acceptShred(shred); err != nil { + return err + } + layout, err := shred.fecLayout() + if err != nil { + return err + } + if err := fec.setLayout(layout); err != nil { + return err + } + fec.coding[shred.Position] = shred + return nil +} + +func (a *SlotAssembler) CompleteSlot(slot uint64) (*block.Block, error) { + a.mu.Lock() + defer a.mu.Unlock() + + state := a.slots[slot] + if state == nil || !state.complete() { + return nil, ErrSlotIncomplete + } + blk, err := state.block() + if err != nil { + return nil, err + } + delete(a.slots, slot) + a.completedSlots[slot] = struct{}{} + return blk, nil +} + +func (a *SlotAssembler) ActiveSlots() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.slots) +} + +func (a *SlotAssembler) RecoveredDataShreds() uint64 { + a.mu.Lock() + defer a.mu.Unlock() + return a.recoveredDataShreds +} + +func (a *SlotAssembler) EvictedSlots() uint64 { + a.mu.Lock() + defer a.mu.Unlock() + return a.evictedSlots +} + +func (a *SlotAssembler) IgnoredOldShreds() uint64 { + a.mu.Lock() + defer a.mu.Unlock() + return a.ignoredOldShreds +} + +func (a *SlotAssembler) RepairRequests(maxSlots int, maxMissingPerSlot int) []SlotRepairRequest { + if maxSlots <= 0 { + return nil + } + if maxMissingPerSlot <= 0 { + maxMissingPerSlot = 1 + } + + a.mu.Lock() + defer a.mu.Unlock() + + if a.maxObservedSlot <= repairObservedSlotLag { + return nil + } + repairThrough := a.maxObservedSlot - repairObservedSlotLag + start := uint64(0) + if repairThrough > repairScanSlotWindow { + start = repairThrough - repairScanSlotWindow + } + if a.maxObservedSlot > maxRetainedIncompleteSlotLag { + minRetained := a.maxObservedSlot - maxRetainedIncompleteSlotLag + if start < minRetained { + start = minRetained + } + } + + requests := make([]SlotRepairRequest, 0, maxSlots) + for slot := start; slot <= repairThrough && len(requests) < maxSlots; slot++ { + if _, completed := a.completedSlots[slot]; completed { + continue + } + state := a.slots[slot] + if state == nil { + requests = append(requests, SlotRepairRequest{ + Slot: slot, + NeedHighestDataShred: true, + HighestDataShredIndex: 0, + }) + continue + } + if req, ok := state.repairRequest(maxMissingPerSlot); ok { + requests = append(requests, req) + } + } + return requests +} + +func (a *SlotAssembler) recoverFEC(state *slotState, fecSetIndex uint32) ([]*Shred, error) { + fec := state.fecSets[fecSetIndex] + if fec == nil || !fec.haveLayout { + return nil, nil + } + layout := fec.layout + if int(layout.dataShreds)+int(layout.codingShreds) == len(fec.data)+len(fec.coding) { + return nil, nil + } + if len(fec.data)+len(fec.coding) < int(layout.dataShreds) { + return nil, nil + } + + shards := make([][]byte, int(layout.dataShreds)+int(layout.codingShreds)) + for idx, shred := range fec.data { + if idx >= uint32(layout.dataShreds) { + continue + } + shard, err := shred.erasureShard() + if err != nil { + return nil, err + } + shards[int(idx)] = append([]byte(nil), shard...) + } + for pos, shred := range fec.coding { + if pos >= layout.codingShreds { + continue + } + shard, err := shred.erasureShard() + if err != nil { + return nil, err + } + shards[int(layout.dataShreds)+int(pos)] = append([]byte(nil), shard...) + } + encoder, err := a.fecEncoder(layout) + if err != nil { + return nil, err + } + required := make([]bool, int(layout.dataShreds)+int(layout.codingShreds)) + var missingData int + for idx := 0; idx < int(layout.dataShreds); idx++ { + if fec.data[uint32(idx)] == nil { + required[idx] = true + missingData++ + } + } + if missingData == 0 { + return nil, nil + } + if err := encoder.ReconstructSome(shards, required); err != nil { + if errors.Is(err, reedsolomon.ErrTooFewShards) { + return nil, nil + } + return nil, fmt.Errorf("recover FEC set slot %d fec_set=%d: %w", state.slot, fecSetIndex, err) + } + + var recovered []*Shred + for idx := 0; idx < int(layout.dataShreds); idx++ { + if !required[idx] { + continue + } + shard := shards[idx] + if len(shard) != layout.shardSize { + return nil, fmt.Errorf("recover FEC set slot %d fec_set=%d: recovered shard %d size %d, want %d", state.slot, fecSetIndex, idx, len(shard), layout.shardSize) + } + shred, err := fec.recoveredDataShred(uint32(idx), shard) + if err != nil { + return nil, err + } + recovered = append(recovered, shred) + } + return recovered, nil +} + +func (a *SlotAssembler) fecEncoder(layout fecLayout) (reedsolomon.Encoder, error) { + encoder := a.encoders[layout] + if encoder != nil { + return encoder, nil + } + encoder, err := reedsolomon.New( + int(layout.dataShreds), + int(layout.codingShreds), + reedsolomon.WithInversionCache(false), + ) + if err != nil { + return nil, err + } + a.encoders[layout] = encoder + return encoder, nil +} + +func (s *slotState) fecSet(fecSetIndex uint32) *fecState { + fec := s.fecSets[fecSetIndex] + if fec != nil { + return fec + } + fec = &fecState{ + slot: s.slot, + fecSetIndex: fecSetIndex, + data: make(map[uint32]*Shred), + coding: make(map[uint16]*Shred), + } + s.fecSets[fecSetIndex] = fec + return fec +} + +func (s *slotState) addDataToFEC(shred *Shred) error { + if !isMerkleVariant(shred.Variant) || len(shred.Payload) < dataPayloadSize { + return nil + } + if shred.Index < shred.FECSetIndex { + return nil + } + fec := s.fecSet(shred.FECSetIndex) + if err := fec.acceptShred(shred); err != nil { + return err + } + fec.data[shred.Index-shred.FECSetIndex] = shred + return nil +} + +func (f *fecState) acceptShred(shred *Shred) error { + if !f.haveSig { + copy(f.signature[:], shred.Signature[:]) + f.haveSig = true + } else if f.signature != shred.Signature { + return fmt.Errorf("mixed FEC signatures for slot %d fec_set=%d", f.slot, f.fecSetIndex) + } + + switch shred.Type { + case ShredTypeData: + expected, ok := merkleCounterpartVariant(shred.Variant, ShredTypeCode) + if !ok { + return fmt.Errorf("unsupported data shred variant for slot %d fec_set=%d: 0x%02x", f.slot, f.fecSetIndex, shred.Variant) + } + if f.dataVariant == 0 { + f.dataVariant = shred.Variant + } else if f.dataVariant != shred.Variant { + return fmt.Errorf("mixed data shred variants for slot %d fec_set=%d: 0x%02x and 0x%02x", f.slot, f.fecSetIndex, f.dataVariant, shred.Variant) + } + if f.codeVariant != 0 && f.codeVariant != expected { + return fmt.Errorf("mixed data/code shred variants for slot %d fec_set=%d: data=0x%02x code=0x%02x", f.slot, f.fecSetIndex, shred.Variant, f.codeVariant) + } + case ShredTypeCode: + expected, ok := merkleCounterpartVariant(shred.Variant, ShredTypeData) + if !ok { + return fmt.Errorf("unsupported coding shred variant for slot %d fec_set=%d: 0x%02x", f.slot, f.fecSetIndex, shred.Variant) + } + if f.codeVariant == 0 { + f.codeVariant = shred.Variant + } else if f.codeVariant != shred.Variant { + return fmt.Errorf("mixed coding shred variants for slot %d fec_set=%d: 0x%02x and 0x%02x", f.slot, f.fecSetIndex, f.codeVariant, shred.Variant) + } + if f.dataVariant != 0 && f.dataVariant != expected { + return fmt.Errorf("mixed data/code shred variants for slot %d fec_set=%d: data=0x%02x code=0x%02x", f.slot, f.fecSetIndex, f.dataVariant, shred.Variant) + } + } + return nil +} + +func (f *fecState) setLayout(layout fecLayout) error { + if !f.haveLayout { + f.layout = layout + f.haveLayout = true + return nil + } + if f.layout != layout { + return fmt.Errorf("mixed FEC layouts for slot %d fec_set=%d: %+v and %+v", f.slot, f.fecSetIndex, f.layout, layout) + } + return nil +} + +func (s *Shred) fecLayout() (fecLayout, error) { + shard, err := s.erasureShard() + if err != nil { + return fecLayout{}, err + } + return fecLayout{ + dataShreds: s.NumDataShreds, + codingShreds: s.NumCodingShreds, + shardSize: len(shard), + }, nil +} + +func (f *fecState) recoveredDataShred(dataIndex uint32, shard []byte) (*Shred, error) { + code := f.firstCodingShred() + if code == nil { + return nil, fmt.Errorf("recover FEC set slot %d fec_set=%d: no coding shred template", f.slot, f.fecSetIndex) + } + payload := make([]byte, dataPayloadSize) + copy(payload[:shredSignatureSize], code.Signature[:]) + copy(payload[shredSignatureSize:], shard) + shred, err := ParseShred(payload) + if err != nil { + return nil, fmt.Errorf("parse recovered data shred slot %d fec_set=%d data_index=%d: %w", f.slot, f.fecSetIndex, dataIndex, err) + } + expectedVariant, ok := merkleCounterpartVariant(code.Variant, ShredTypeData) + if !ok { + return nil, fmt.Errorf("recover FEC set slot %d fec_set=%d: unsupported coding variant 0x%02x", f.slot, f.fecSetIndex, code.Variant) + } + if shred.Type != ShredTypeData || + shred.Variant != expectedVariant || + shred.Slot != f.slot || + shred.FECSetIndex != f.fecSetIndex || + shred.Index != f.fecSetIndex+dataIndex || + shred.Version != code.Version { + return nil, fmt.Errorf("recover FEC set slot %d fec_set=%d: invalid recovered data shred index=%d slot=%d variant=0x%02x version=%d", f.slot, f.fecSetIndex, shred.Index, shred.Slot, shred.Variant, shred.Version) + } + shred.Recovered = true + return shred, nil +} + +func (f *fecState) firstCodingShred() *Shred { + for _, shred := range f.coding { + return shred + } + return nil +} + +func (s *slotState) complete() bool { + if !s.haveLast { + return false + } + for idx := uint32(0); idx <= s.lastIndex; idx++ { + if s.shreds[idx] == nil { + return false + } + } + return true +} + +func (s *slotState) orderedShreds() []*Shred { + indexes := make([]int, 0, len(s.shreds)) + for idx := range s.shreds { + indexes = append(indexes, int(idx)) + } + sort.Ints(indexes) + out := make([]*Shred, 0, len(indexes)) + for _, idx := range indexes { + out = append(out, s.shreds[uint32(idx)]) + } + return out +} + +func (s *slotState) block() (*block.Block, error) { + entries, err := DecodeEntriesFromDataShreds(s.orderedShreds()) + if err != nil { + return nil, err + } + blk := BlockFromEntries(s.slot, s.parentSlot, entries) + if err := validateBlockTransactions(blk); err != nil { + return nil, err + } + return blk, nil +} diff --git a/pkg/turbine/assembler_test.go b/pkg/turbine/assembler_test.go new file mode 100644 index 00000000..c65f5754 --- /dev/null +++ b/pkg/turbine/assembler_test.go @@ -0,0 +1,584 @@ +package turbine + +import ( + "context" + "crypto/ed25519" + "encoding/binary" + "errors" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/fixtures" + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/txverify" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" +) + +func TestParseMerkleDataShred(t *testing.T) { + raw := fixtures.DataShreds(t, "mainnet", 102815960)[0] + + shred, err := ParseShred(raw) + if err != nil { + t.Fatalf("ParseShred returned error: %v", err) + } + + if shred.Slot != 102815960 { + t.Fatalf("slot = %d, want 102815960", shred.Slot) + } + if shred.Index != 0 { + t.Fatalf("index = %d, want 0", shred.Index) + } + if shred.ParentSlot() != 102815959 { + t.Fatalf("parent slot = %d, want 102815959", shred.ParentSlot()) + } + if len(shred.Data) == 0 { + t.Fatalf("expected parsed data payload") + } +} + +func TestClassifyCurrentMerkleVariants(t *testing.T) { + for _, variant := range []byte{0x80, 0x8f, 0x90, 0x9f, 0xb0, 0xbf} { + shredType, err := classifyVariant(variant) + if err != nil { + t.Fatalf("classifyVariant(0x%02x) returned error: %v", variant, err) + } + if shredType != ShredTypeData { + t.Fatalf("classifyVariant(0x%02x) = %v, want data", variant, shredType) + } + } + for _, variant := range []byte{0x40, 0x4f, 0x60, 0x6f, 0x70, 0x7f} { + shredType, err := classifyVariant(variant) + if err != nil { + t.Fatalf("classifyVariant(0x%02x) returned error: %v", variant, err) + } + if shredType != ShredTypeCode { + t.Fatalf("classifyVariant(0x%02x) = %v, want code", variant, shredType) + } + } + for _, variant := range []byte{0x00, 0x50, 0xa0, 0xc0} { + if _, err := classifyVariant(variant); !errors.Is(err, ErrUnsupportedShred) { + t.Fatalf("classifyVariant(0x%02x) error = %v, want ErrUnsupportedShred", variant, err) + } + } +} + +func TestMerkleShredSignatureVerification(t *testing.T) { + packet := make([]byte, dataPayloadSize) + packet[shredVariantOffset] = merkleDataVariant + packet[dataFlagsOffset] = shredFlagLastShredInSlot + binary.LittleEndian.PutUint64(packet[shredSlotOffset:shredSlotOffset+8], 10) + binary.LittleEndian.PutUint32(packet[shredIndexOffset:shredIndexOffset+4], 0) + binary.LittleEndian.PutUint16(packet[shredVersionOffset:shredVersionOffset+2], 1) + binary.LittleEndian.PutUint32(packet[shredFECSetIndexOffset:shredFECSetIndexOffset+4], 0) + binary.LittleEndian.PutUint16(packet[dataParentOffsetOffset:dataParentOffsetOffset+2], 1) + binary.LittleEndian.PutUint16(packet[dataSizeOffset:dataSizeOffset+2], dataHeaderSize+4) + copy(packet[dataHeaderSize:], []byte("test")) + + shred, err := ParseShred(packet) + if err != nil { + t.Fatalf("ParseShred returned error: %v", err) + } + root, err := shred.MerkleRoot() + if err != nil { + t.Fatalf("MerkleRoot returned error: %v", err) + } + + var seed [32]byte + for i := range seed { + seed[i] = byte(i + 1) + } + privateKey := ed25519.NewKeyFromSeed(seed[:]) + signature := ed25519.Sign(privateKey, root[:]) + copy(packet[shredSignatureOffset:shredSignatureSize], signature) + publicKey := privateKey.Public().(ed25519.PublicKey) + var leader solana.PublicKey + copy(leader[:], publicKey) + + shred, err = ParseShred(packet) + if err != nil { + t.Fatalf("ParseShred signed packet returned error: %v", err) + } + if err := shred.VerifySignature(leader); err != nil { + t.Fatalf("VerifySignature returned error: %v", err) + } + + packet[dataHeaderSize] ^= 0xff + corrupt, err := ParseShred(packet) + if err != nil { + t.Fatalf("ParseShred corrupt packet returned error: %v", err) + } + if err := corrupt.VerifySignature(leader); !errors.Is(err, ErrInvalidSignature) { + t.Fatalf("VerifySignature corrupt error = %v, want ErrInvalidSignature", err) + } +} + +func TestSlotAssemblerBuildsBlockFromCompleteDataShreds(t *testing.T) { + rawShreds := fixtures.DataShreds(t, "mainnet", 102815960) + assembler := NewSlotAssembler() + + var blockSeen bool + for idx, raw := range rawShreds { + blk, err := assembler.AddPacket(raw) + if err != nil { + t.Fatalf("AddPacket(%d) returned error: %v", idx, err) + } + if blk == nil { + continue + } + blockSeen = true + if idx != len(rawShreds)-1 { + t.Fatalf("assembled block before final shred: idx=%d total=%d", idx, len(rawShreds)) + } + if blk.Slot != 102815960 { + t.Fatalf("block slot = %d, want 102815960", blk.Slot) + } + if blk.SourceParentSlot != 102815959 { + t.Fatalf("source parent slot = %d, want 102815959", blk.SourceParentSlot) + } + if !blk.FromLightbringer { + t.Fatalf("expected block to be marked as live shred-stream sourced") + } + if len(blk.Transactions) != 3177 { + t.Fatalf("transactions = %d, want 3177", len(blk.Transactions)) + } + if len(blk.Transactions[0].Signatures) == 0 { + t.Fatalf("first transaction has no signatures") + } + } + + if !blockSeen { + t.Fatalf("assembler did not emit a completed block") + } +} + +func TestValidateBlockTransactionsRejectsInvalidSignature(t *testing.T) { + rawShreds := fixtures.DataShreds(t, "mainnet", 102815960) + assembler := NewSlotAssembler() + + var blk *block.Block + for _, raw := range rawShreds { + var err error + blk, err = assembler.AddPacket(raw) + if err != nil { + t.Fatalf("AddPacket returned error: %v", err) + } + } + if blk == nil || len(blk.Transactions) == 0 { + t.Fatalf("assembler did not emit a block with transactions") + } + + blk.Transactions[0].Message.RecentBlockhash[0] ^= 0xff + if err := validateBlockTransactions(blk); err == nil { + t.Fatalf("validateBlockTransactions accepted a mutated transaction") + } +} + +func TestFixtureRawTransactionSignatureBytes(t *testing.T) { + rawShreds := fixtures.DataShreds(t, "mainnet", 102815960) + var batchBytes []byte + var tx solana.Transaction + var rawTx []byte + var txIndex uint64 + for _, raw := range rawShreds { + shred, err := ParseShred(raw) + if err != nil { + t.Fatalf("ParseShred returned error: %v", err) + } + batchBytes = append(batchBytes, shred.Data...) + if !shred.DataComplete() { + continue + } + + var decoder bin.Decoder + decoder.SetEncoding(bin.EncodingBin) + decoder.Reset(batchBytes) + numEntries, err := decoder.ReadUint64(bin.LE) + if err != nil { + t.Fatalf("read entry count: %v", err) + } + for entryIdx := uint64(0); entryIdx < numEntries; entryIdx++ { + if _, err := decoder.ReadUint64(bin.LE); err != nil { + t.Fatalf("read num hashes: %v", err) + } + if _, err := decoder.ReadBytes(32); err != nil { + t.Fatalf("read hash: %v", err) + } + numTxns, err := decoder.ReadUint64(bin.LE) + if err != nil { + t.Fatalf("read tx count: %v", err) + } + for i := uint64(0); i < numTxns; i++ { + start := decoder.Position() + var current solana.Transaction + if err := current.UnmarshalWithDecoder(&decoder); err != nil { + t.Fatalf("read tx: %v", err) + } + if txIndex == 1 { + tx = current + rawTx = append([]byte(nil), batchBytes[start:decoder.Position()]...) + break + } + txIndex++ + } + if rawTx != nil { + break + } + } + if rawTx != nil { + break + } + batchBytes = batchBytes[:0] + } + if rawTx == nil { + t.Fatalf("did not find transaction 1") + } + + rawDecoder := bin.NewBinDecoder(rawTx) + numSigs, err := rawDecoder.ReadCompactU16() + if err != nil { + t.Fatalf("read raw signature count: %v", err) + } + if numSigs != len(tx.Signatures) { + t.Fatalf("raw signature count = %d, decoded signatures = %d", numSigs, len(tx.Signatures)) + } + for i := 0; i < numSigs; i++ { + if _, err := rawDecoder.ReadBytes(64); err != nil { + t.Fatalf("read raw signature %d: %v", i, err) + } + } + rawMessage := rawTx[rawDecoder.Position():] + remarshaled, err := tx.Message.MarshalBinary() + if err != nil { + t.Fatalf("MarshalBinary returned error: %v", err) + } + if string(remarshaled) != string(rawMessage) { + limit := len(remarshaled) + if len(rawMessage) < limit { + limit = len(rawMessage) + } + diff := -1 + for i := 0; i < limit; i++ { + if remarshaled[i] != rawMessage[i] { + diff = i + break + } + } + t.Fatalf("remarshaled message differs from raw at %d raw_len=%d remarshaled_len=%d raw_prefix=%x remarshaled_prefix=%x", diff, len(rawMessage), len(remarshaled), rawMessage[:min(24, len(rawMessage))], remarshaled[:min(24, len(remarshaled))]) + } + signers := tx.Message.Signers() + if len(signers) != len(tx.Signatures) { + t.Fatalf("signers=%d signatures=%d", len(signers), len(tx.Signatures)) + } + for i, sig := range tx.Signatures { + if !sig.Verify(signers[i], rawMessage) { + t.Fatalf("raw message signature %d failed for %s", i, signers[i]) + } + } + if err := txverify.VerifyTransaction(&tx); err != nil { + t.Fatalf("txverify failed for raw transaction: %v", err) + } + + var parsed []*Shred + for _, raw := range rawShreds { + shred, err := ParseShred(raw) + if err != nil { + t.Fatalf("ParseShred for block decode returned error: %v", err) + } + parsed = append(parsed, shred) + } + entries, err := DecodeEntriesFromDataShreds(parsed) + if err != nil { + t.Fatalf("DecodeEntriesFromDataShreds returned error: %v", err) + } + var entryTx *solana.Transaction + var entryTxIndex uint64 + for entryIdx := range entries { + for txIdx := range entries[entryIdx].Txns { + if entryTxIndex == 1 { + entryTx = &entries[entryIdx].Txns[txIdx] + break + } + entryTxIndex++ + } + if entryTx != nil { + break + } + } + if entryTx == nil { + t.Fatalf("did not find decoded entry transaction 1") + } + entryMsg, _ := txverify.MessageBytes(entryTx) + if string(entryMsg) != string(rawMessage) { + t.Fatalf("decoded transaction message does not match raw message") + } + if err := txverify.VerifyTransaction(entryTx); err != nil { + t.Fatalf("txverify failed for decoded entry transaction 1: %v", err) + } + blk := BlockFromEntries(102815960, 102815959, entries) + if len(blk.Transactions) < 2 { + t.Fatalf("block transactions = %d", len(blk.Transactions)) + } + if err := txverify.VerifyTransaction(blk.Transactions[1]); err != nil { + t.Fatalf("txverify failed for block transaction 1: %v", err) + } +} + +func TestSlotAssemblerIgnoresCodingShreds(t *testing.T) { + raw := fixtures.CodeShreds(t, "mainnet", 102815960)[0] + + blk, err := NewSlotAssembler().AddPacket(raw) + if err != nil { + t.Fatalf("AddPacket returned error: %v", err) + } + if blk != nil { + t.Fatalf("expected coding shred to be ignored") + } + + if _, err := ParseShred(raw); !errors.Is(err, ErrCodingShredIgnored) { + t.Fatalf("ParseShred error = %v, want ErrCodingShredIgnored", err) + } +} + +func TestParseMerkleCodingShred(t *testing.T) { + raw := localnetMerkleShreds(t, "c")[0] + + shred, err := ParseShred(raw) + if err != nil { + t.Fatalf("ParseShred returned error: %v", err) + } + if shred.Type != ShredTypeCode { + t.Fatalf("type = %v, want code", shred.Type) + } + if shred.NumDataShreds == 0 || shred.NumCodingShreds == 0 { + t.Fatalf("invalid FEC layout: data=%d coding=%d", shred.NumDataShreds, shred.NumCodingShreds) + } + if _, err := shred.erasureShard(); err != nil { + t.Fatalf("erasureShard returned error: %v", err) + } +} + +func TestSlotAssemblerRecoversMissingMerkleDataShredFromCodingShreds(t *testing.T) { + dataShreds := localnetMerkleShreds(t, "d") + codeShreds := localnetMerkleShreds(t, "c") + if len(dataShreds) < 2 || len(codeShreds) == 0 { + t.Fatalf("fixture needs data and coding shreds") + } + + missing, err := ParseShred(dataShreds[1]) + if err != nil { + t.Fatalf("ParseShred missing data returned error: %v", err) + } + assembler := NewSlotAssembler() + for idx, raw := range dataShreds { + if idx == 1 { + continue + } + if _, err := assembler.AddPacket(raw); err != nil { + t.Fatalf("AddPacket data %d returned error: %v", idx, err) + } + } + var blockSeen bool + for idx, raw := range codeShreds { + blk, err := assembler.AddPacket(raw) + if err != nil { + t.Fatalf("AddPacket code %d returned error: %v", idx, err) + } + if blk != nil { + blockSeen = true + } + } + if blockSeen { + return + } + + assembler.mu.Lock() + defer assembler.mu.Unlock() + state := assembler.slots[missing.Slot] + if state == nil { + t.Fatalf("slot state missing for slot %d", missing.Slot) + } + recovered := state.shreds[missing.Index] + if recovered == nil { + t.Fatalf("missing data shred index %d was not recovered", missing.Index) + } + if !recovered.Recovered { + t.Fatalf("data shred index %d present but not marked recovered", missing.Index) + } + if string(recovered.Data) != string(missing.Data) { + t.Fatalf("recovered data does not match original data") + } +} + +func TestSlotAssemblerRejectsOversizedShredIndex(t *testing.T) { + raw := fixtures.DataShreds(t, "mainnet", 102815960)[0] + shred, err := ParseShred(raw) + if err != nil { + t.Fatalf("ParseShred returned error: %v", err) + } + shred.Index = maxDataShredsPerSlot + + if _, err := NewSlotAssembler().AddShred(shred); !errors.Is(err, ErrSlotOverflow) { + t.Fatalf("AddShred error = %v, want ErrSlotOverflow", err) + } +} + +func TestSlotAssemblerPrunesOldIncompleteSlots(t *testing.T) { + assembler := NewSlotAssembler() + maxSlot := maxRetainedIncompleteSlotLag + 20 + for slot := uint64(1); slot <= maxSlot; slot++ { + _, err := assembler.AddShred(&Shred{ + Variant: legacyDataVariant, + Type: ShredTypeData, + Slot: slot, + Index: 0, + Version: 1, + ParentOffset: 1, + Data: []byte{byte(slot)}, + }) + if err != nil { + t.Fatalf("AddShred(%d) returned error: %v", slot, err) + } + } + + if got := assembler.ActiveSlots(); got > int(maxRetainedIncompleteSlotLag)+1 { + t.Fatalf("active slots = %d, want at most %d", got, maxRetainedIncompleteSlotLag+1) + } + if got := assembler.EvictedSlots(); got == 0 { + t.Fatalf("expected old incomplete slots to be evicted") + } + assembler.mu.Lock() + _, oldExists := assembler.slots[1] + assembler.mu.Unlock() + if oldExists { + t.Fatalf("expected oldest incomplete slot to be evicted") + } + + _, err := assembler.AddShred(&Shred{ + Variant: legacyDataVariant, + Type: ShredTypeData, + Slot: 1, + Index: 1, + Version: 1, + ParentOffset: 1, + Data: []byte{1}, + }) + if err != nil { + t.Fatalf("AddShred for stale slot returned error: %v", err) + } + assembler.mu.Lock() + _, oldExists = assembler.slots[1] + assembler.mu.Unlock() + if oldExists { + t.Fatalf("expected stale shred not to recreate evicted slot") + } + if got := assembler.IgnoredOldShreds(); got == 0 { + t.Fatalf("expected stale shred counter to increment") + } +} + +func TestSlotAssemblerRepairRequestsIncludeAbsentAndIncompleteSlots(t *testing.T) { + assembler := NewSlotAssembler() + for _, shred := range []*Shred{ + { + Variant: legacyDataVariant, + Type: ShredTypeData, + Slot: 10, + Index: 1, + Version: 1, + ParentOffset: 1, + Data: []byte{1}, + }, + { + Variant: legacyDataVariant, + Type: ShredTypeData, + Slot: 14, + Index: 0, + Version: 1, + ParentOffset: 1, + Data: []byte{1}, + }, + } { + if _, err := assembler.AddShred(shred); err != nil { + t.Fatalf("AddShred returned error: %v", err) + } + } + + requests := assembler.RepairRequests(32, 8) + bySlot := make(map[uint64]SlotRepairRequest, len(requests)) + for _, req := range requests { + bySlot[req.Slot] = req + } + + if req, ok := bySlot[9]; !ok || !req.NeedHighestDataShred || req.HighestDataShredIndex != 0 { + t.Fatalf("slot 9 absent repair request = %+v, ok=%v", req, ok) + } + req, ok := bySlot[10] + if !ok { + t.Fatalf("expected repair request for incomplete slot 10") + } + if !req.NeedHighestDataShred || req.HighestDataShredIndex != 2 { + t.Fatalf("slot 10 highest request = need %v index %d, want true index 2", req.NeedHighestDataShred, req.HighestDataShredIndex) + } + if len(req.MissingDataShreds) != 1 || req.MissingDataShreds[0] != 0 { + t.Fatalf("slot 10 missing shreds = %v, want [0]", req.MissingDataShreds) + } +} + +func TestUDPReceiverSignalsReadyAfterBind(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + receiver := NewUDPReceiver("127.0.0.1:0") + done := make(chan error, 1) + go func() { + done <- receiver.Run(ctx) + }() + + select { + case err := <-receiver.Ready(): + if err != nil { + t.Fatalf("Ready returned error: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for receiver readiness") + } + + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error after cancel: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for receiver shutdown") + } +} + +func localnetMerkleShreds(t testing.TB, prefix string) [][]byte { + t.Helper() + dir := fixtures.Path(t, "shreds", "localnet", "merkle") + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("cannot read localnet merkle shreds: %v", err) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].Name() < entries[j].Name() + }) + var shreds [][]byte + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, entry.Name())) + if err != nil { + t.Fatalf("cannot read shred %s: %v", entry.Name(), err) + } + shreds = append(shreds, raw) + } + return shreds +} diff --git a/pkg/turbine/entries.go b/pkg/turbine/entries.go new file mode 100644 index 00000000..b622cafb --- /dev/null +++ b/pkg/turbine/entries.go @@ -0,0 +1,157 @@ +package turbine + +import ( + "fmt" + "sort" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/txverify" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" +) + +type Entry struct { + NumHashes uint64 + Hash solana.Hash + Txns []solana.Transaction +} + +func (e *Entry) UnmarshalWithDecoder(decoder *bin.Decoder) error { + var err error + if e.NumHashes, err = decoder.ReadUint64(bin.LE); err != nil { + return fmt.Errorf("read num_hashes: %w", err) + } + if _, err = decoder.Read(e.Hash[:]); err != nil { + return fmt.Errorf("read hash: %w", err) + } + numTxns, err := decoder.ReadUint64(bin.LE) + if err != nil { + return fmt.Errorf("read transaction count: %w", err) + } + if numTxns > uint64(decoder.Remaining()) { + return fmt.Errorf("transaction count %d exceeds remaining bytes %d", numTxns, decoder.Remaining()) + } + e.Txns = make([]solana.Transaction, numTxns) + for i := uint64(0); i < numTxns; i++ { + if err = e.Txns[i].UnmarshalWithDecoder(decoder); err != nil { + return fmt.Errorf("read transaction %d: %w", i, err) + } + } + return nil +} + +type entryBatch struct { + Entries []Entry +} + +func (b *entryBatch) UnmarshalWithDecoder(decoder *bin.Decoder) error { + numEntries, err := decoder.ReadUint64(bin.LE) + if err != nil { + return fmt.Errorf("read entry count: %w", err) + } + if numEntries > uint64(decoder.Remaining()) { + return fmt.Errorf("entry count %d exceeds remaining bytes %d", numEntries, decoder.Remaining()) + } + b.Entries = make([]Entry, numEntries) + for i := uint64(0); i < numEntries; i++ { + if err = b.Entries[i].UnmarshalWithDecoder(decoder); err != nil { + return fmt.Errorf("read entry %d: %w", i, err) + } + } + return nil +} + +func DecodeEntriesFromDataShreds(shreds []*Shred) ([]Entry, error) { + if len(shreds) == 0 { + return nil, nil + } + sort.Slice(shreds, func(i, j int) bool { + return shreds[i].Index < shreds[j].Index + }) + + var entries []Entry + var batchBytes []byte + for _, shred := range shreds { + if shred == nil || shred.Type != ShredTypeData { + continue + } + batchBytes = append(batchBytes, shred.Data...) + if !shred.DataComplete() { + continue + } + batchEntries, err := decodeEntryBatch(batchBytes) + if err != nil { + return nil, fmt.Errorf("decode entry batch ending at shred %d: %w", shred.Index, err) + } + entries = append(entries, batchEntries...) + // Decoded transactions retain slices into the batch buffer for instruction data. + // Keep the backing array alive instead of reusing and overwriting it. + batchBytes = nil + } + if len(batchBytes) != 0 { + return nil, fmt.Errorf("slot ended with %d undecoded entry bytes", len(batchBytes)) + } + return entries, nil +} + +func decodeEntryBatch(data []byte) ([]Entry, error) { + var decoder bin.Decoder + decoder.SetEncoding(bin.EncodingBin) + decoder.Reset(data) + var batch entryBatch + if err := batch.UnmarshalWithDecoder(&decoder); err != nil { + return nil, err + } + return batch.Entries, nil +} + +func BlockFromEntries(slot uint64, parentSlot uint64, entries []Entry) *block.Block { + blk := &block.Block{ + Slot: slot, + SourceParentSlot: parentSlot, + Transactions: make([]*solana.Transaction, 0, len(entries)*4), + Entries: make([]*block.TxEntry, len(entries)), + FromLightbringer: true, + } + + var txOffset uint64 + for entryIdx, entry := range entries { + txEntry := &block.TxEntry{ + NumHashes: entry.NumHashes, + Hash: append([]byte(nil), entry.Hash[:]...), + Indices: make([]uint64, len(entry.Txns)), + } + for txIdx := range entry.Txns { + tx := &entry.Txns[txIdx] + blk.Transactions = append(blk.Transactions, tx) + blk.NumSignatures += uint64(tx.Message.Header.NumRequiredSignatures) + blk.Versions = append(blk.Versions, uint8(tx.Message.GetVersion())) + txEntry.Indices[txIdx] = txOffset + uint64(txIdx) + } + blk.Entries[entryIdx] = txEntry + txOffset += uint64(len(entry.Txns)) + } + if len(entries) > 0 { + blk.Blockhash = entries[len(entries)-1].Hash + } + return blk +} + +func validateBlockTransactions(blk *block.Block) error { + if blk == nil { + return nil + } + for txIdx, tx := range blk.Transactions { + if tx == nil { + return fmt.Errorf("slot %d transaction %d is nil", blk.Slot, txIdx) + } + if err := txverify.VerifyTransaction(tx); err != nil { + txSig := "" + if len(tx.Signatures) > 0 { + txSig = tx.Signatures[0].String() + } + return fmt.Errorf("slot %d transaction %d %s version=%d failed signature verification: %w", blk.Slot, txIdx, txSig, tx.Message.GetVersion(), err) + } + } + return nil +} diff --git a/pkg/turbine/receiver.go b/pkg/turbine/receiver.go new file mode 100644 index 00000000..8ce095c4 --- /dev/null +++ b/pkg/turbine/receiver.go @@ -0,0 +1,241 @@ +package turbine + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/gossip" + "github.com/gagliardetto/solana-go" +) + +type LeaderForSlotFunc func(slot uint64) (solana.PublicKey, bool) + +type UDPReceiver struct { + Addr string + + assembler *SlotAssembler + leaderForSlot LeaderForSlotFunc + repairClient *repairClient + blocks chan *block.Block + errs chan error + ready chan error + once sync.Once + readyOnce sync.Once + + packets atomic.Uint64 + dataShreds atomic.Uint64 + codingShreds atomic.Uint64 + parseErrors atomic.Uint64 + signatureErrors atomic.Uint64 + missingLeaders atomic.Uint64 + assemblyErrors atomic.Uint64 + blocksEmitted atomic.Uint64 + lastPacketUnix atomic.Int64 + lastDataSlot atomic.Uint64 + lastBlockSlot atomic.Uint64 +} + +type ReceiverStats struct { + Packets uint64 + DataShreds uint64 + CodingShreds uint64 + ParseErrors uint64 + SignatureErrors uint64 + MissingLeaders uint64 + AssemblyErrors uint64 + BlocksEmitted uint64 + RecoveredData uint64 + EvictedSlots uint64 + IgnoredOldShreds uint64 + Repair RepairStats + LastPacketUnix int64 + LastDataSlot uint64 + LastBlockSlot uint64 + ActiveSlots int +} + +func NewUDPReceiver(addr string) *UDPReceiver { + return &UDPReceiver{ + Addr: addr, + assembler: NewSlotAssembler(), + blocks: make(chan *block.Block, 1024), + errs: make(chan error, 16), + ready: make(chan error, 1), + } +} + +func (r *UDPReceiver) SetLeaderForSlot(fn LeaderForSlotFunc) { + r.leaderForSlot = fn +} + +func (r *UDPReceiver) SetRepairPeerSource(identity ed25519.PrivateKey, source func() []gossip.RepairPeer) error { + client, err := newRepairClient(identity, source) + if err != nil { + return err + } + r.repairClient = client + return nil +} + +func (r *UDPReceiver) Blocks() <-chan *block.Block { + return r.blocks +} + +func (r *UDPReceiver) Errors() <-chan error { + return r.errs +} + +func (r *UDPReceiver) Ready() <-chan error { + return r.ready +} + +func (r *UDPReceiver) Stats() ReceiverStats { + return ReceiverStats{ + Packets: r.packets.Load(), + DataShreds: r.dataShreds.Load(), + CodingShreds: r.codingShreds.Load(), + ParseErrors: r.parseErrors.Load(), + SignatureErrors: r.signatureErrors.Load(), + MissingLeaders: r.missingLeaders.Load(), + AssemblyErrors: r.assemblyErrors.Load(), + BlocksEmitted: r.blocksEmitted.Load(), + RecoveredData: r.assembler.RecoveredDataShreds(), + EvictedSlots: r.assembler.EvictedSlots(), + IgnoredOldShreds: r.assembler.IgnoredOldShreds(), + Repair: r.repairStats(), + LastPacketUnix: r.lastPacketUnix.Load(), + LastDataSlot: r.lastDataSlot.Load(), + LastBlockSlot: r.lastBlockSlot.Load(), + ActiveSlots: r.assembler.ActiveSlots(), + } +} + +func (r *UDPReceiver) repairStats() RepairStats { + if r.repairClient == nil { + return RepairStats{} + } + return r.repairClient.stats() +} + +func (r *UDPReceiver) signalReady(err error) { + r.readyOnce.Do(func() { + r.ready <- err + close(r.ready) + }) +} + +func (r *UDPReceiver) Run(ctx context.Context) error { + defer r.once.Do(func() { + close(r.blocks) + close(r.errs) + }) + + udpAddr, err := net.ResolveUDPAddr("udp", r.Addr) + if err != nil { + r.signalReady(err) + return err + } + conn, err := net.ListenUDP("udp", udpAddr) + if err != nil { + r.signalReady(err) + return err + } + defer conn.Close() + r.signalReady(nil) + + go func() { + <-ctx.Done() + _ = conn.Close() + }() + if r.repairClient != nil { + go r.repairClient.run(ctx, conn, r.assembler) + } + + buf := make([]byte, packetDataSize) + for { + n, addr, err := conn.ReadFromUDP(buf) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + r.packets.Add(1) + r.lastPacketUnix.Store(time.Now().Unix()) + packet := buf[:n] + if r.repairClient != nil && r.repairClient.handleRepairPing(conn, packet, addr) { + continue + } + shred, err := ParseShred(packet) + if err != nil { + if errors.Is(err, ErrCodingShredIgnored) { + r.codingShreds.Add(1) + continue + } + r.parseErrors.Add(1) + select { + case r.errs <- err: + default: + } + continue + } + switch shred.Type { + case ShredTypeData: + r.dataShreds.Add(1) + r.lastDataSlot.Store(shred.Slot) + case ShredTypeCode: + r.codingShreds.Add(1) + } + if r.leaderForSlot != nil { + leader, ok := r.leaderForSlot(shred.Slot) + if !ok { + r.missingLeaders.Add(1) + select { + case r.errs <- fmt.Errorf("missing leader for turbine shred slot %d", shred.Slot): + default: + } + continue + } + if err := shred.VerifySignature(leader); err != nil { + r.signatureErrors.Add(1) + select { + case r.errs <- err: + default: + } + continue + } + } + if r.repairClient != nil { + r.repairClient.observeShredResponse(conn, packet, addr, shred) + } + blk, err := r.assembler.AddShred(shred) + if err != nil { + if errors.Is(err, ErrDuplicateShred) { + continue + } + r.assemblyErrors.Add(1) + select { + case r.errs <- err: + default: + } + continue + } + if blk == nil { + continue + } + r.blocksEmitted.Add(1) + r.lastBlockSlot.Store(blk.Slot) + select { + case r.blocks <- blk: + case <-ctx.Done(): + return nil + } + } +} diff --git a/pkg/turbine/repair.go b/pkg/turbine/repair.go new file mode 100644 index 00000000..4d14c727 --- /dev/null +++ b/pkg/turbine/repair.go @@ -0,0 +1,387 @@ +package turbine + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "fmt" + "net" + "sync" + "sync/atomic" + "time" + + "github.com/Overclock-Validator/mithril/pkg/gossip" + repairproto "github.com/Overclock-Validator/mithril/pkg/repair" +) + +const ( + repairScanInterval = 100 * time.Millisecond + repairRequestTimeout = 750 * time.Millisecond + repairPeerRefreshInterval = 2 * time.Second + repairMaxSlotsPerScan = 32 + repairMaxMissingPerSlot = 256 + repairMaxFollowupRequests = 256 + repairMaxOutstanding = 2048 +) + +type RepairPeerSource func() []gossip.RepairPeer + +type RepairStats struct { + Requests uint64 + Responses uint64 + Timeouts uint64 + Pings uint64 + Pongs uint64 + Errors uint64 + Outstanding int + Peers int +} + +type repairRequestKind uint8 + +const ( + repairRequestWindowIndex repairRequestKind = iota + repairRequestHighestWindowIndex +) + +type repairRequestKey struct { + kind repairRequestKind + slot uint64 + index uint32 +} + +type repairAddressKey struct { + ip [16]byte + port int +} + +type repairResponseKey struct { + addr repairAddressKey + nonce uint32 +} + +type outstandingRepairRequest struct { + key repairRequestKey + nonce uint32 + addr repairAddressKey + sentAt time.Time +} + +type repairClient struct { + identity ed25519.PrivateKey + peerSource RepairPeerSource + + mu sync.Mutex + outstanding map[repairRequestKey]outstandingRepairRequest + byResponse map[repairResponseKey]repairRequestKey + peerCursor uint64 + + peerCacheMu sync.Mutex + peerCache []gossip.RepairPeer + peerCacheAt time.Time + + requests atomic.Uint64 + responses atomic.Uint64 + timeouts atomic.Uint64 + pings atomic.Uint64 + pongs atomic.Uint64 + errors atomic.Uint64 +} + +func newRepairClient(identity ed25519.PrivateKey, peerSource RepairPeerSource) (*repairClient, error) { + var err error + if len(identity) == 0 { + _, identity, err = ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate repair identity: %w", err) + } + } + if len(identity) != ed25519.PrivateKeySize { + return nil, fmt.Errorf("invalid repair identity size %d", len(identity)) + } + if peerSource == nil { + return nil, fmt.Errorf("repair peer source is required") + } + return &repairClient{ + identity: append(ed25519.PrivateKey(nil), identity...), + peerSource: peerSource, + outstanding: make(map[repairRequestKey]outstandingRepairRequest), + byResponse: make(map[repairResponseKey]repairRequestKey), + }, nil +} + +func (c *repairClient) run(ctx context.Context, conn *net.UDPConn, assembler *SlotAssembler) { + ticker := time.NewTicker(repairScanInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.expireOutstanding(time.Now()) + c.repairOnce(conn, assembler) + } + } +} + +func (c *repairClient) repairOnce(conn *net.UDPConn, assembler *SlotAssembler) { + peers := c.peerSnapshot(time.Now()) + if len(peers) == 0 { + return + } + requests := assembler.RepairRequests(repairMaxSlotsPerScan, repairMaxMissingPerSlot) + if len(requests) == 0 { + return + } + + budget := repairMaxSlotsPerScan * (repairMaxMissingPerSlot + 1) + if budget > repairMaxOutstanding { + budget = repairMaxOutstanding + } + for _, req := range requests { + for _, index := range req.MissingDataShreds { + if budget <= 0 { + return + } + if c.sendRequest(conn, peers, repairRequestWindowIndex, req.Slot, index) { + budget-- + } + } + if req.NeedHighestDataShred { + if budget <= 0 { + return + } + if c.sendRequest(conn, peers, repairRequestHighestWindowIndex, req.Slot, req.HighestDataShredIndex) { + budget-- + } + } + } +} + +func (c *repairClient) handleRepairPing(conn *net.UDPConn, packet []byte, from *net.UDPAddr) bool { + ping, ok := repairproto.DecodePing(packet) + if !ok { + return false + } + c.pings.Add(1) + pong, err := repairproto.BuildPong(c.identity, ping) + if err != nil { + c.errors.Add(1) + return true + } + if _, err := conn.WriteToUDP(pong, from); err != nil { + c.errors.Add(1) + return true + } + c.pongs.Add(1) + return true +} + +func (c *repairClient) observeShredResponse(conn *net.UDPConn, packet []byte, from *net.UDPAddr, shred *Shred) { + if from == nil || shred == nil { + return + } + nonce, ok := repairproto.ResponseNonce(packet) + if !ok { + return + } + addrKey, ok := repairAddressKeyFromUDP(from) + if !ok { + return + } + responseKey := repairResponseKey{addr: addrKey, nonce: nonce} + + c.mu.Lock() + reqKey, ok := c.byResponse[responseKey] + if !ok { + c.mu.Unlock() + return + } + outstanding := c.outstanding[reqKey] + delete(c.byResponse, responseKey) + delete(c.outstanding, reqKey) + c.mu.Unlock() + + if outstanding.key.slot != shred.Slot { + return + } + c.responses.Add(1) + if outstanding.key.kind != repairRequestHighestWindowIndex || shred.Type != ShredTypeData { + return + } + + peers := c.peerSnapshot(time.Now()) + if len(peers) == 0 { + return + } + start := outstanding.key.index + followups := 0 + for index := start; index < shred.Index && followups < repairMaxFollowupRequests; index++ { + if c.sendRequest(conn, peers, repairRequestWindowIndex, shred.Slot, index) { + followups++ + } + } + if !shred.LastInSlot() && followups < repairMaxFollowupRequests && shred.Index < maxDataShredsPerSlot-1 { + c.sendRequest(conn, peers, repairRequestHighestWindowIndex, shred.Slot, shred.Index+1) + } +} + +func (c *repairClient) sendRequest(conn *net.UDPConn, peers []gossip.RepairPeer, kind repairRequestKind, slot uint64, index uint32) bool { + if conn == nil || len(peers) == 0 { + return false + } + key := repairRequestKey{kind: kind, slot: slot, index: index} + + c.mu.Lock() + if _, exists := c.outstanding[key]; exists { + c.mu.Unlock() + return false + } + if len(c.outstanding) >= repairMaxOutstanding { + c.mu.Unlock() + return false + } + peer, ok := c.nextPeerLocked(peers) + c.mu.Unlock() + if !ok { + return false + } + + var ( + packet []byte + nonce uint32 + err error + ) + switch kind { + case repairRequestWindowIndex: + packet, nonce, err = repairproto.NewWindowIndexRequest(c.identity, peer.Pubkey, slot, uint64(index)) + case repairRequestHighestWindowIndex: + packet, nonce, err = repairproto.NewHighestWindowIndexRequest(c.identity, peer.Pubkey, slot, uint64(index)) + default: + return false + } + if err != nil { + c.errors.Add(1) + return false + } + + addrKey, ok := repairAddressKeyFromUDP(peer.Addr) + if !ok { + return false + } + responseKey := repairResponseKey{addr: addrKey, nonce: nonce} + c.mu.Lock() + if _, exists := c.outstanding[key]; exists { + c.mu.Unlock() + return false + } + if len(c.outstanding) >= repairMaxOutstanding { + c.mu.Unlock() + return false + } + c.outstanding[key] = outstandingRepairRequest{ + key: key, + nonce: nonce, + addr: addrKey, + sentAt: time.Now(), + } + c.byResponse[responseKey] = key + c.mu.Unlock() + + if _, err := conn.WriteToUDP(packet, peer.Addr); err != nil { + c.mu.Lock() + delete(c.outstanding, key) + delete(c.byResponse, responseKey) + c.mu.Unlock() + c.errors.Add(1) + return false + } + + c.requests.Add(1) + return true +} + +func (c *repairClient) nextPeerLocked(peers []gossip.RepairPeer) (gossip.RepairPeer, bool) { + for attempts := 0; attempts < len(peers); attempts++ { + index := int(c.peerCursor % uint64(len(peers))) + c.peerCursor++ + peer := peers[index] + if peer.Addr != nil && peer.Addr.Port != 0 { + return peer, true + } + } + return gossip.RepairPeer{}, false +} + +func (c *repairClient) expireOutstanding(now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + for key, outstanding := range c.outstanding { + if now.Sub(outstanding.sentAt) < repairRequestTimeout { + continue + } + delete(c.outstanding, key) + delete(c.byResponse, repairResponseKey{addr: outstanding.addr, nonce: outstanding.nonce}) + c.timeouts.Add(1) + } +} + +func (c *repairClient) peerSnapshot(now time.Time) []gossip.RepairPeer { + c.peerCacheMu.Lock() + if !c.peerCacheAt.IsZero() && now.Sub(c.peerCacheAt) < repairPeerRefreshInterval { + peers := c.peerCache + c.peerCacheMu.Unlock() + return peers + } + c.peerCacheMu.Unlock() + + peers := c.peerSource() + + c.peerCacheMu.Lock() + c.peerCache = peers + c.peerCacheAt = now + c.peerCacheMu.Unlock() + return peers +} + +func (c *repairClient) cachedPeerCount() int { + c.peerCacheMu.Lock() + count := len(c.peerCache) + cacheFresh := !c.peerCacheAt.IsZero() && time.Since(c.peerCacheAt) < repairPeerRefreshInterval + c.peerCacheMu.Unlock() + if cacheFresh || c.peerSource == nil { + return count + } + return len(c.peerSnapshot(time.Now())) +} + +func repairAddressKeyFromUDP(addr *net.UDPAddr) (repairAddressKey, bool) { + var key repairAddressKey + if addr == nil { + return key, false + } + ip := addr.IP.To16() + if ip == nil || addr.Port == 0 { + return key, false + } + copy(key.ip[:], ip) + key.port = addr.Port + return key, true +} + +func (c *repairClient) stats() RepairStats { + peers := c.cachedPeerCount() + c.mu.Lock() + outstanding := len(c.outstanding) + c.mu.Unlock() + return RepairStats{ + Requests: c.requests.Load(), + Responses: c.responses.Load(), + Timeouts: c.timeouts.Load(), + Pings: c.pings.Load(), + Pongs: c.pongs.Load(), + Errors: c.errors.Load(), + Outstanding: outstanding, + Peers: peers, + } +} diff --git a/pkg/turbine/shred.go b/pkg/turbine/shred.go new file mode 100644 index 00000000..ee4a85d8 --- /dev/null +++ b/pkg/turbine/shred.go @@ -0,0 +1,381 @@ +package turbine + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + + "github.com/gagliardetto/solana-go" +) + +const ( + packetDataSize = 1232 + + shredSignatureOffset = 0 + shredSignatureSize = 64 + shredVariantOffset = 64 + shredSlotOffset = 65 + shredIndexOffset = 73 + shredVersionOffset = 77 + shredFECSetIndexOffset = 79 + + dataParentOffsetOffset = 83 + dataFlagsOffset = 85 + dataSizeOffset = 86 + dataHeaderSize = 88 + dataPayloadSize = 1203 + + codingNumDataOffset = 83 + codingNumCodingOffset = 85 + codingPositionOffset = 87 + codingHeaderSize = 89 + codingPayloadSize = packetDataSize - 4 + + merkleRootSize = 32 + merkleProofEntrySize = 20 + + merkleHashPrefixLeaf = "\x00SOLANA_MERKLE_SHREDS_LEAF" + merkleHashPrefixNode = "\x01SOLANA_MERKLE_SHREDS_NODE" + + legacyDataVariant = byte(0b1010_0101) + legacyCodeVariant = byte(0b0101_1010) + + merkleVariantTypeMask = byte(0xf0) + merkleCodeVariant = byte(0x40) + merkleCodeChained = byte(0x60) + merkleCodeResigned = byte(0x70) + merkleDataVariant = byte(0x80) + merkleDataChained = byte(0x90) + merkleDataResigned = byte(0xb0) + + shredFlagTickMask = byte(0b0011_1111) + shredFlagDataComplete = byte(0b0100_0000) + shredFlagLastShredInSlot = byte(0b1100_0000) +) + +var ( + ErrShortShred = errors.New("short shred packet") + ErrUnsupportedShred = errors.New("unsupported shred variant") + ErrInvalidDataShred = errors.New("invalid data shred") + ErrCodingShredIgnored = errors.New("coding shred ignored") + ErrInvalidSignature = errors.New("invalid shred signature") +) + +type ShredType uint8 + +const ( + ShredTypeData ShredType = iota + ShredTypeCode +) + +type Shred struct { + Signature solana.Signature + Variant byte + Type ShredType + Slot uint64 + Index uint32 + Version uint16 + FECSetIndex uint32 + + ParentOffset uint16 + Flags byte + Data []byte + Payload []byte + + NumDataShreds uint16 + NumCodingShreds uint16 + Position uint16 + Recovered bool +} + +func ParseShred(packet []byte) (*Shred, error) { + if len(packet) <= dataHeaderSize { + return nil, fmt.Errorf("%w: got %d bytes", ErrShortShred, len(packet)) + } + + variant := packet[shredVariantOffset] + shredType, err := classifyVariant(variant) + if err != nil { + return nil, err + } + if shredType == ShredTypeCode { + return parseCodingShred(packet, variant) + } + return parseDataShred(packet, variant) +} + +func parseDataShred(packet []byte, variant byte) (*Shred, error) { + size := int(binary.LittleEndian.Uint16(packet[dataSizeOffset : dataSizeOffset+2])) + if size < dataHeaderSize || size > len(packet) || size > packetDataSize { + return nil, fmt.Errorf("%w: data size %d packet size %d", ErrInvalidDataShred, size, len(packet)) + } + + shred := &Shred{ + Variant: variant, + Type: ShredTypeData, + Slot: binary.LittleEndian.Uint64(packet[shredSlotOffset : shredSlotOffset+8]), + Index: binary.LittleEndian.Uint32(packet[shredIndexOffset : shredIndexOffset+4]), + Version: binary.LittleEndian.Uint16(packet[shredVersionOffset : shredVersionOffset+2]), + FECSetIndex: binary.LittleEndian.Uint32(packet[shredFECSetIndexOffset : shredFECSetIndexOffset+4]), + ParentOffset: binary.LittleEndian.Uint16(packet[dataParentOffsetOffset : dataParentOffsetOffset+2]), + Flags: packet[dataFlagsOffset], + Data: make([]byte, size-dataHeaderSize), + } + copy(shred.Signature[:], packet[shredSignatureOffset:shredSignatureSize]) + copy(shred.Data, packet[dataHeaderSize:size]) + payloadSize := len(packet) + if isMerkleVariant(variant) && payloadSize >= dataPayloadSize { + payloadSize = dataPayloadSize + } + shred.Payload = make([]byte, payloadSize) + copy(shred.Payload, packet[:payloadSize]) + return shred, nil +} + +func parseCodingShred(packet []byte, variant byte) (*Shred, error) { + if !isMerkleVariant(variant) { + return nil, ErrCodingShredIgnored + } + if len(packet) < codingPayloadSize { + return nil, fmt.Errorf("%w: coding payload size %d", ErrShortShred, len(packet)) + } + + shred := &Shred{ + Variant: variant, + Type: ShredTypeCode, + Slot: binary.LittleEndian.Uint64(packet[shredSlotOffset : shredSlotOffset+8]), + Index: binary.LittleEndian.Uint32(packet[shredIndexOffset : shredIndexOffset+4]), + Version: binary.LittleEndian.Uint16(packet[shredVersionOffset : shredVersionOffset+2]), + FECSetIndex: binary.LittleEndian.Uint32(packet[shredFECSetIndexOffset : shredFECSetIndexOffset+4]), + NumDataShreds: binary.LittleEndian.Uint16(packet[codingNumDataOffset : codingNumDataOffset+2]), + NumCodingShreds: binary.LittleEndian.Uint16(packet[codingNumCodingOffset : codingNumCodingOffset+2]), + Position: binary.LittleEndian.Uint16(packet[codingPositionOffset : codingPositionOffset+2]), + Payload: make([]byte, codingPayloadSize), + } + copy(shred.Signature[:], packet[shredSignatureOffset:shredSignatureSize]) + copy(shred.Payload, packet[:codingPayloadSize]) + return shred, nil +} + +func classifyVariant(variant byte) (ShredType, error) { + switch { + case variant == legacyDataVariant: + return ShredTypeData, nil + case variant == legacyCodeVariant: + return ShredTypeCode, nil + case variant&merkleVariantTypeMask == merkleCodeVariant, + variant&merkleVariantTypeMask == merkleCodeChained, + variant&merkleVariantTypeMask == merkleCodeResigned: + return ShredTypeCode, nil + case variant&merkleVariantTypeMask == merkleDataVariant, + variant&merkleVariantTypeMask == merkleDataChained, + variant&merkleVariantTypeMask == merkleDataResigned: + return ShredTypeData, nil + default: + return 0, fmt.Errorf("%w: 0x%02x", ErrUnsupportedShred, variant) + } +} + +func isMerkleVariant(variant byte) bool { + switch variant & merkleVariantTypeMask { + case merkleCodeVariant, merkleCodeChained, merkleCodeResigned, merkleDataVariant, merkleDataChained, merkleDataResigned: + return true + default: + return false + } +} + +func merkleVariantInfo(variant byte) (proofSize byte, chained bool, resigned bool, ok bool) { + switch variant & merkleVariantTypeMask { + case merkleCodeVariant, merkleDataVariant: + return variant & 0x0f, false, false, true + case merkleCodeChained, merkleDataChained: + return variant & 0x0f, true, false, true + case merkleCodeResigned, merkleDataResigned: + return variant & 0x0f, true, true, true + default: + return 0, false, false, false + } +} + +func merkleCounterpartVariant(variant byte, shredType ShredType) (byte, bool) { + proofSize, chained, resigned, ok := merkleVariantInfo(variant) + if !ok { + return 0, false + } + var prefix byte + switch shredType { + case ShredTypeData: + if resigned { + prefix = merkleDataResigned + } else if chained { + prefix = merkleDataChained + } else { + prefix = merkleDataVariant + } + case ShredTypeCode: + if resigned { + prefix = merkleCodeResigned + } else if chained { + prefix = merkleCodeChained + } else { + prefix = merkleCodeVariant + } + default: + return 0, false + } + return prefix | proofSize, true +} + +func merkleCapacity(payloadSize, headerSize int, proofSize byte, chained bool, resigned bool) (int, error) { + capacity := payloadSize - headerSize - int(proofSize)*merkleProofEntrySize + if chained { + capacity -= merkleRootSize + } + if resigned { + capacity -= shredSignatureSize + } + if capacity < 0 { + return 0, fmt.Errorf("%w: proof size %d", ErrUnsupportedShred, proofSize) + } + return capacity, nil +} + +func (s *Shred) erasureShard() ([]byte, error) { + if s == nil || !isMerkleVariant(s.Variant) { + return nil, ErrUnsupportedShred + } + proofSize, chained, resigned, ok := merkleVariantInfo(s.Variant) + if !ok { + return nil, ErrUnsupportedShred + } + + var start, headerSize, payloadSize int + switch s.Type { + case ShredTypeData: + start = shredSignatureSize + headerSize = dataHeaderSize + payloadSize = dataPayloadSize + case ShredTypeCode: + start = codingHeaderSize + headerSize = codingHeaderSize + payloadSize = codingPayloadSize + default: + return nil, ErrUnsupportedShred + } + capacity, err := merkleCapacity(payloadSize, headerSize, proofSize, chained, resigned) + if err != nil { + return nil, err + } + end := headerSize + capacity + if end > len(s.Payload) || start > end { + return nil, fmt.Errorf("%w: erasure slice %d:%d payload %d", ErrShortShred, start, end, len(s.Payload)) + } + return s.Payload[start:end], nil +} + +func (s *Shred) MerkleRoot() (solana.Hash, error) { + if s == nil || !isMerkleVariant(s.Variant) { + return solana.Hash{}, ErrUnsupportedShred + } + proofSize, chained, resigned, ok := merkleVariantInfo(s.Variant) + if !ok { + return solana.Hash{}, ErrUnsupportedShred + } + + var headerSize, payloadSize, index int + switch s.Type { + case ShredTypeData: + if s.Index < s.FECSetIndex { + return solana.Hash{}, fmt.Errorf("%w: data shred index %d before fec_set_index %d", ErrInvalidDataShred, s.Index, s.FECSetIndex) + } + headerSize = dataHeaderSize + payloadSize = dataPayloadSize + index = int(s.Index - s.FECSetIndex) + case ShredTypeCode: + headerSize = codingHeaderSize + payloadSize = codingPayloadSize + index = int(s.NumDataShreds) + int(s.Position) + default: + return solana.Hash{}, ErrUnsupportedShred + } + + capacity, err := merkleCapacity(payloadSize, headerSize, proofSize, chained, resigned) + if err != nil { + return solana.Hash{}, err + } + proofOffset := headerSize + capacity + if chained { + proofOffset += merkleRootSize + } + proofEnd := proofOffset + int(proofSize)*merkleProofEntrySize + if proofEnd > len(s.Payload) || shredSignatureSize > proofOffset { + return solana.Hash{}, fmt.Errorf("%w: merkle proof slice %d:%d payload %d", ErrShortShred, proofOffset, proofEnd, len(s.Payload)) + } + + root := merkleHashLeaf(s.Payload[shredSignatureSize:proofOffset]) + proof := s.Payload[proofOffset:proofEnd] + for len(proof) > 0 { + entry := proof[:merkleProofEntrySize] + if index%2 == 0 { + root = merkleHashNode(root[:merkleProofEntrySize], entry) + } else { + root = merkleHashNode(entry, root[:merkleProofEntrySize]) + } + index >>= 1 + proof = proof[merkleProofEntrySize:] + } + if index != 0 { + return solana.Hash{}, fmt.Errorf("%w: invalid merkle proof path", ErrInvalidSignature) + } + return root, nil +} + +func (s *Shred) VerifySignature(leader solana.PublicKey) error { + root, err := s.MerkleRoot() + if err != nil { + return err + } + if !leader.Verify(root[:], s.Signature) { + return fmt.Errorf("%w: slot %d shred %d", ErrInvalidSignature, s.Slot, s.Index) + } + return nil +} + +func merkleHashLeaf(data []byte) solana.Hash { + return hashv([][]byte{[]byte(merkleHashPrefixLeaf), data}) +} + +func merkleHashNode(left []byte, right []byte) solana.Hash { + return hashv([][]byte{[]byte(merkleHashPrefixNode), left, right}) +} + +func hashv(parts [][]byte) solana.Hash { + h := sha256.New() + for _, part := range parts { + _, _ = h.Write(part) + } + var out solana.Hash + copy(out[:], h.Sum(nil)) + return out +} + +func (s *Shred) DataComplete() bool { + return s.Flags&shredFlagDataComplete != 0 +} + +func (s *Shred) LastInSlot() bool { + return s.Flags&shredFlagLastShredInSlot == shredFlagLastShredInSlot +} + +func (s *Shred) ReferenceTick() byte { + return s.Flags & shredFlagTickMask +} + +func (s *Shred) ParentSlot() uint64 { + if uint64(s.ParentOffset) > s.Slot { + return 0 + } + return s.Slot - uint64(s.ParentOffset) +} diff --git a/pkg/txverify/txverify.go b/pkg/txverify/txverify.go new file mode 100644 index 00000000..789cc7c1 --- /dev/null +++ b/pkg/txverify/txverify.go @@ -0,0 +1,48 @@ +package txverify + +import ( + "fmt" + + "github.com/gagliardetto/solana-go" +) + +func MessageBytes(tx *solana.Transaction) ([]byte, error) { + if tx == nil { + return nil, fmt.Errorf("nil transaction") + } + msg, err := tx.Message.MarshalBinary() + if err != nil { + return nil, err + } + if tx.Message.IsVersioned() { + if len(msg) == 0 { + return nil, fmt.Errorf("empty versioned message") + } + version := byte(tx.Message.GetVersion()) + if version == 0 { + msg[0] = 0x80 + } else { + msg[0] = 0x7f + version + } + } + return msg, nil +} + +func VerifyTransaction(tx *solana.Transaction) error { + msg, err := MessageBytes(tx) + if err != nil { + return err + } + + signers := tx.Message.Signers() + if len(signers) != len(tx.Signatures) { + return fmt.Errorf("got %d signers, but %d signatures", len(signers), len(tx.Signatures)) + } + + for i, sig := range tx.Signatures { + if !sig.Verify(signers[i], msg) { + return fmt.Errorf("invalid signature by %s", signers[i]) + } + } + return nil +}