From 37a7c6ed9c6a4f40ee93cc91a32bd65971842c6a Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:22:11 -0700 Subject: [PATCH 1/3] Add option for specify per-edge delay for pubnet data --- src/App/Program.fs | 11 ++ src/FSLibrary.Tests/Tests.fs | 6 +- src/FSLibrary/MaxTPSTest.fs | 10 +- src/FSLibrary/MinBlockTimeTest.fs | 7 +- src/FSLibrary/StellarCoreSet.fs | 5 + src/FSLibrary/StellarMissionContext.fs | 1 + src/FSLibrary/StellarNetworkData.fs | 168 ++++++++++++++---- src/FSLibrary/StellarNetworkDelays.fs | 95 +++++++--- .../sample-network-data-delay.json | 77 ++++++++ 9 files changed, 314 insertions(+), 66 deletions(-) create mode 100644 src/FSLibrary/json-type-samples/sample-network-data-delay.json diff --git a/src/App/Program.fs b/src/App/Program.fs index 83ade839..1d121f92 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -71,6 +71,7 @@ type MissionOptions tolerateNodeTaints: seq, apiRateLimit: int, pubnetData: string option, + pubnetDataDelay: bool, flatQuorum: bool option, tier1Keys: string option, maxConnections: int option, @@ -298,6 +299,12 @@ type MissionOptions [] member self.PubnetData = pubnetData + [] + member self.PubnetDataDelay = pubnetDataDelay + [] member self.FlatQuorum = flatQuorum @@ -689,6 +696,9 @@ let main argv = 0 | :? MissionOptions as mission -> + if mission.PubnetData.IsNone && mission.PubnetDataDelay then + failwith "Error: --pubnet-data-delay requires --pubnet-data to be set" + let _ = logToConsoleAndFile (sprintf "%s/stellar-supercluster.log" mission.Destination) let ll = @@ -789,6 +799,7 @@ let main argv = tolerateNodeTaints = List.map splitLabel (List.ofSeq mission.TolerateNodeTaints) apiRateLimit = mission.ApiRateLimit pubnetData = mission.PubnetData + pubnetDataDelay = mission.PubnetDataDelay flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys maxConnections = mission.MaxConnections diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index ce280d8b..fda40592 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -68,6 +68,7 @@ let ctx : MissionContext = tolerateNodeTaints = [] apiRateLimit = 10 pubnetData = None + pubnetDataDelay = false flatQuorum = None tier1Keys = None maxConnections = None @@ -373,7 +374,10 @@ type Tests(output: ITestOutputHelper) = let Chennai = { lat = 13.08784; lon = 80.27847 } let dns1 = PeerDnsName "www.foo.com" let dns2 = PeerDnsName "www.bar.com" - let cmd = getNetworkDelayCommands Ashburn [| (Beauharnois, dns1); (Chennai, dns2) |] None + + let cmd = + getNetworkDelayCommands (getPeerDelays Ashburn [| (Beauharnois, dns1); (Chennai, dns2) |]) None + let cmdStr = cmd.ToString() Assert.Contains(dns1.StringName, cmdStr) diff --git a/src/FSLibrary/MaxTPSTest.fs b/src/FSLibrary/MaxTPSTest.fs index dec4072b..6ce5ea89 100644 --- a/src/FSLibrary/MaxTPSTest.fs +++ b/src/FSLibrary/MaxTPSTest.fs @@ -154,9 +154,15 @@ let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadG List.find (fun (cs: CoreSet) -> cs.name.StringName = "stellar" || cs.name.StringName = "sdf") allNodes let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) allNodes + let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad = Some true) allNodes + + let loadGenNodes = + if List.isEmpty loadGenNodes then + // On smaller networks, run loadgen on all nodes to better balance the overhead of load generation + if List.length allNodes > smallNetworkSize then tier1 else allNodes + else + loadGenNodes - // On smaller networks, run loadgen on all nodes to better balance the overhead of load generation - let loadGenNodes = if List.length allNodes > smallNetworkSize then tier1 else allNodes let isLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) loadGenNodes // Assign pre-generated transaction information to each load generator node. diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index 3401f5eb..819efe19 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -333,8 +333,13 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: None } let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) allNodes + let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad = Some true) allNodes - let loadGenNodes = if List.length allNodes > smallNetworkSize then tier1 else allNodes + let loadGenNodes = + if List.isEmpty loadGenNodes then + if List.length allNodes > smallNetworkSize then tier1 else allNodes + else + loadGenNodes let isLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) loadGenNodes diff --git a/src/FSLibrary/StellarCoreSet.fs b/src/FSLibrary/StellarCoreSet.fs index 34a2d84f..8069851a 100644 --- a/src/FSLibrary/StellarCoreSet.fs +++ b/src/FSLibrary/StellarCoreSet.fs @@ -192,6 +192,7 @@ type QuorumSetConfiguration = type CoreSetOptions = { nodeCount: int nodeLocs: GeoLoc list option + edgeDelays: Map option dbType: DBType emptyDirType: EmptyDirType syncStartupDelay: int option @@ -211,6 +212,8 @@ type CoreSetOptions = validate: bool homeDomain: string option tier1: bool option + // This should only be set in the pubnet data with delay case + generatesLoad: bool option catchupMode: CatchupMode image: string initialization: CoreSetInitialization @@ -233,6 +236,7 @@ type CoreSetOptions = static member GetDefault(image: string) = { nodeCount = 3 nodeLocs = None + edgeDelays = None dbType = Sqlite emptyDirType = MemoryBackedEmptyDir syncStartupDelay = Some(5) @@ -252,6 +256,7 @@ type CoreSetOptions = validate = true homeDomain = Some "stellar.org" tier1 = None + generatesLoad = None catchupMode = CatchupComplete image = image initialization = CoreSetInitialization.Default diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index c64b4cca..2b34c6cb 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -69,6 +69,7 @@ type MissionContext = tolerateNodeTaints: ((string * string option) list) apiRateLimit: int pubnetData: string option + pubnetDataDelay: bool flatQuorum: bool option tier1Keys: string option maxConnections: int option diff --git a/src/FSLibrary/StellarNetworkData.fs b/src/FSLibrary/StellarNetworkData.fs index 61f3d6b1..6fb10f1b 100644 --- a/src/FSLibrary/StellarNetworkData.fs +++ b/src/FSLibrary/StellarNetworkData.fs @@ -20,7 +20,42 @@ let PubnetLatestHistoryArchiveState = let TestnetLatestHistoryArchiveState = "http://history.stellar.org/prd/core-testnet/core_testnet_001/.well-known/stellar-history.json" -type PubnetNode = JsonProvider<"json-type-samples/sample-network-data.json", SampleIsList=false, ResolutionFolder=cwd> +type PubnetNodeJSON = + JsonProvider<"json-type-samples/sample-network-data.json", SampleIsList=false, ResolutionFolder=cwd> + +type PubnetNodeDelayJSON = + JsonProvider<"json-type-samples/sample-network-data-delay.json", SampleIsList=false, ResolutionFolder=cwd> + +type PubnetNode = + { PublicKey: string + Peers: string array + GeneratesLoad: bool option + RadarHomeDomain: string option + RadarIsValidating: bool option + RadarGeoData: {| Latitude: decimal; Longitude: decimal |} option + RadarName: string option } + + static member ofJSON(node: PubnetNodeJSON.Root) : PubnetNode = + { PublicKey = node.PublicKey + Peers = node.Peers + GeneratesLoad = None + RadarHomeDomain = node.RadarHomeDomain + RadarIsValidating = node.RadarIsValidating + RadarGeoData = + match node.RadarGeoData with + | Some geoData -> Some {| Latitude = geoData.Latitude; Longitude = geoData.Longitude |} + | None -> None + RadarName = node.RadarName } + + static member ofJSONDelay(node: PubnetNodeDelayJSON.Root) : PubnetNode = + { PublicKey = node.PublicKey + Peers = node.Peers |> Array.map (fun p -> p.Key) + GeneratesLoad = node.GeneratesLoad + RadarHomeDomain = node.RadarHomeDomain + RadarIsValidating = node.RadarIsValidating + RadarGeoData = None + RadarName = node.RadarName } + type Tier1PublicKey = JsonProvider<"json-type-samples/sample-keys.json", SampleIsList=false, ResolutionFolder=cwd> // Adjacency map for peers @@ -147,8 +182,8 @@ let locations = // Each edge connecting a and b is represented as (a, b) if a < b, and (b, a) otherwise. // This makes sense as the graph is undirected, and it also makes it easier to handle a set of edges. -let extractEdges (graph: PubnetNode.Root array) : (string * string) array = - let getEdgesFromNode (node: PubnetNode.Root) : (string * string) array = +let extractEdges (graph: PubnetNode array) : (string * string) array = + let getEdgesFromNode (node: PubnetNode) : (string * string) array = node.Peers |> Array.filter (fun peer -> peer < node.PublicKey) // This filter ensures that we add each edge exactly once. |> Array.map (fun peer -> (peer, node.PublicKey)) @@ -260,7 +295,7 @@ let private pruneAdjacencyMap (maxConnections: int) (noPrune: Set) (m: P // then we pick a random edge (a, b), remove (a, b) and add (a, u) and (u, b). // We continue this process until u has a desired degree. let addEdges - (graph: PubnetNode.Root array) + (graph: PubnetNode array) (newNodes: string array) (tier1KeySet: Set) (random: System.Random) @@ -334,14 +369,12 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin if context.pubnetData.IsNone then failwith "pubnet simulation requires --pubnet-data=" - let allPubnetNodes : PubnetNode.Root array = PubnetNode.Load(context.pubnetData.Value) - // A Random object with a fixed seed. let random = System.Random context.randomSeed let newTier1Nodes = [ for i in 1 .. context.tier1OrgsToAdd * tier1OrgSize -> - PubnetNode.Parse( + PubnetNodeJSON.Parse( sprintf """ [{ "publicKey": "%s", "radar_homeDomain": "home.domain.%d" }] """ (KeyPair.Random().Address) @@ -351,19 +384,19 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let newNonTier1Nodes = [ for i in 1 .. context.nonTier1NodesToAdd -> - PubnetNode.Parse(sprintf """ [{ "publicKey": "%s" }] """ (KeyPair.Random().Address)).[0] ] + PubnetNodeJSON.Parse(sprintf """ [{ "publicKey": "%s" }] """ (KeyPair.Random().Address)).[0] ] |> Array.ofList let tier1KeySet : Set = if context.tier1Keys.IsSome then - let newTier1Keys = Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) newTier1Nodes in + let newTier1Keys = Array.map (fun (n: PubnetNodeJSON.Root) -> n.PublicKey) newTier1Nodes in Tier1PublicKey.Load(context.tier1Keys.Value) |> Array.map (fun n -> n.PublicKey) |> Array.append newTier1Keys |> Set.ofArray else - PubnetNode.Load(context.pubnetData.Value) + PubnetNodeJSON.Load(context.pubnetData.Value) // Any node with a home domain is considered tier1. |> Array.filter (fun n -> n.RadarHomeDomain.IsSome) |> Array.map (fun n -> n.PublicKey) @@ -376,7 +409,40 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin Array.append newTier1Nodes newNonTier1Nodes |> Array.sortBy (fun _ -> random.Next()) - let allPubnetNodes = allPubnetNodes |> Array.append newNodes + let (allPubnetNodes: PubnetNode array, edgeDelays: Map option) = + if context.pubnetDataDelay then + if newNodes.Length > 0 then + failwith "--pubnet-data-delay cannot be used with --tier1-orgs-to-add or --non-tier1-nodes-to-add" + + let nodes = PubnetNodeDelayJSON.Load context.pubnetData.Value + + let edgeDelays = + nodes + |> Array.map (fun n -> n.Peers |> Array.map (fun p -> (n.PublicKey, p.Key), p.OwdMs)) + |> Array.concat + |> Map.ofArray + // Check that edgeDelays is connection-wise symmetric (but allow asymmetric delays) + for key1, key2 in edgeDelays.Keys do + if not (edgeDelays.ContainsKey(key2, key1)) then + failwithf + "Edge delay data is not symmetric: (%s, %s) is present but (%s, %s) is not" + key1 + key2 + key2 + key1 + + let nodes = nodes |> Array.map PubnetNode.ofJSONDelay + // Check that all load-generating nodes have a home domain. + for n in nodes do + if n.GeneratesLoad = Some true && n.RadarHomeDomain.IsNone then + failwithf "Load generator node %s does not have a home domain" n.PublicKey + + nodes, Some edgeDelays + else + PubnetNodeJSON.Load context.pubnetData.Value + |> Array.append newNodes + |> Array.map PubnetNode.ofJSON, + None // For each pubkey in the pubnet, we map it to an actual KeyPair (with a private // key) to use in the simulation. It's important to keep these straight! The keys @@ -384,7 +450,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // throughout the rest of this function, as strings called "pubkey", but should not // appear in the final CoreSets we're building. let mutable pubnetKeyToSimKey : Map = - Array.map (fun (n: PubnetNode.Root) -> (n.PublicKey, KeyPair.Random())) allPubnetNodes + Array.map (fun (n: PubnetNode) -> (n.PublicKey, KeyPair.Random())) allPubnetNodes |> Map.ofArray // Not every pubkey used in the qsets is represented in the base set of pubnet nodes, @@ -401,8 +467,14 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // This is because `networkSizeLimit` may be smaller than the degree of some new node // and cause an issue to the scaling algorithm. let adjacencyMap = - addEdges allPubnetNodes (Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) newNodes) tier1KeySet random - |> if context.fullyConnectTier1 then fullyConnectTier1 tier1KeySet else id + addEdges allPubnetNodes (Array.map (fun (n: PubnetNodeJSON.Root) -> n.PublicKey) newNodes) tier1KeySet random + |> if context.fullyConnectTier1 then + if context.pubnetDataDelay then + failwith "--pubnet-data-delay cannot be used with --fully-connect-tier1" + else + fullyConnectTier1 tier1KeySet + else + id |> match context.maxConnections with | Some maxConnections -> // Prune map to ensure that no node has more than @@ -420,8 +492,8 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let orgNodes, miscNodes = allPubnetNodes - |> Array.filter (fun (n: PubnetNode.Root) -> minAllowedConnectionCount <= numPeers adjacencyMap n.PublicKey) - |> Array.partition (fun (n: PubnetNode.Root) -> n.RadarHomeDomain.IsSome) + |> Array.filter (fun (n: PubnetNode) -> minAllowedConnectionCount <= numPeers adjacencyMap n.PublicKey) + |> Array.partition (fun (n: PubnetNode) -> n.RadarHomeDomain.IsSome) // We then trim down the set of misc nodes so that they fit within simulation // size limit passed. If we can't even fit the org nodes, we fail here. @@ -439,18 +511,27 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let allPubnetNodes = Array.append orgNodes miscNodes let _ = assert ((Array.length allPubnetNodes) <= context.networkSizeLimit) - let allPubnetNodeKeys = - Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) allPubnetNodes - |> Set.ofArray + let allPubnetNodeKeys = Array.map (fun (n: PubnetNode) -> n.PublicKey) allPubnetNodes |> Set.ofArray LogInfo "SimulatePubnet will run with %d nodes" (Array.length allPubnetNodes) + let edgeDelays : Map option = + match edgeDelays with + | None -> None + | Some edgeDelays -> + edgeDelays + |> Map.toSeq + |> Seq.filter (fun ((a, b), _) -> Set.contains a allPubnetNodeKeys && Set.contains b allPubnetNodeKeys) + |> Seq.map (fun ((a, b), delay) -> ((getSimKey a).PublicKey, (getSimKey b).PublicKey), delay) + |> Map.ofSeq + |> Some + // We then group the org nodes by their home domains. The domain names are drawn // from the HomeDomains of the public network but with periods replaced with dashes, // and lowercased, so for example keybase.io turns into keybase-io. - let groupedOrgNodes : (HomeDomainName * PubnetNode.Root array) array = + let groupedOrgNodes : (HomeDomainName * PubnetNode array) array = Array.groupBy - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> let domain = n.RadarHomeDomain.Value // We turn 'www.stellar.org' into 'stellar' // and 'stellar.blockdaemon.com' into 'blockdaemon' @@ -478,12 +559,22 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin HomeDomainName lowercase) orgNodes + // Check that load generators each exist in size-1 orgs + for hdn, nodes in groupedOrgNodes do + let loadGenerators = nodes |> Array.filter (fun n -> n.GeneratesLoad = Some true) + + if loadGenerators.Length > 0 && nodes.Length > 1 then + failwithf + "Load generator node(s) found in org %s with %d nodes. Each node that generates load should have its own home domain" + hdn.StringName + nodes.Length + // Then build a map from accountID to HomeDomainName and index-within-domain // for each org node. let orgNodeHomeDomains : Map = Array.collect - (fun (hdn: HomeDomainName, nodes: PubnetNode.Root array) -> - Array.mapi (fun (i: int) (n: PubnetNode.Root) -> (n.PublicKey, (hdn, i))) nodes) + (fun (hdn: HomeDomainName, nodes: PubnetNode array) -> + Array.mapi (fun (i: int) (n: PubnetNode) -> (n.PublicKey, (hdn, i))) nodes) groupedOrgNodes |> Map.ofArray @@ -538,14 +629,12 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let defaultQuorum : QuorumSetSpec = let tier1Nodes = allPubnetNodes - |> Array.filter - (fun (n: PubnetNode.Root) -> (Set.contains n.PublicKey tier1KeySet) && n.RadarHomeDomain.IsSome) + |> Array.filter (fun (n: PubnetNode) -> (Set.contains n.PublicKey tier1KeySet) && n.RadarHomeDomain.IsSome) let tier1NodesGroupedByHomeDomain : (string array) array = tier1Nodes - |> Array.groupBy (fun (n: PubnetNode.Root) -> n.RadarHomeDomain.Value) - |> Array.map - (fun (_, nodes: PubnetNode.Root []) -> Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) nodes) + |> Array.groupBy (fun (n: PubnetNode) -> n.RadarHomeDomain.Value) + |> Array.map (fun (_, nodes: PubnetNode []) -> Array.map (fun (n: PubnetNode) -> n.PublicKey) nodes) let orgToExplicitQSet (org: string array) : ExplicitQuorumSet = { thresholdPercent = Some(51) // Simple majority @@ -555,7 +644,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let tier1Orgs = tier1Nodes |> Array.map - (fun (n: PubnetNode.Root) -> { name = (homeDomainNameForKey n.PublicKey).StringName; quality = High }) + (fun (n: PubnetNode) -> { name = (homeDomainNameForKey n.PublicKey).StringName; quality = High }) |> Set.ofArray let flatQset = @@ -596,9 +685,9 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // as long as the random function is persistent. let geoLocations : GeoLoc array = allPubnetNodes - |> Array.filter (fun (n: PubnetNode.Root) -> n.RadarGeoData.IsSome) + |> Array.filter (fun (n: PubnetNode) -> n.RadarGeoData.IsSome) |> Array.map - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> { lat = float n.RadarGeoData.Value.Latitude lon = float n.RadarGeoData.Value.Longitude }) |> Seq.ofArray @@ -611,7 +700,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // The assignment is deterministic as it depends on the public key of the // node. This ensures that geolocations persist across runs, even if the // total number of nodes changes via the *-orgs-to-add flags. - let getGeoLocOrDefault (n: PubnetNode.Root) : GeoLoc = + let getGeoLocOrDefault (n: PubnetNode) : GeoLoc = match n.RadarGeoData with | Some geoData -> { lat = float geoData.Latitude; lon = float geoData.Longitude } | None -> @@ -636,7 +725,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin allPubnetNodes |> Array.map - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> let key = getSimPubKey n.PublicKey let peers = @@ -659,7 +748,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // Given a node, returns a tuple where the first element is a boolean // indicating whether the node is a validator, and the second element is an // appropriate quorum set configuration for that node. - let computeQset (n: PubnetNode.Root) = + let computeQset (n: PubnetNode) = let tier1 = Set.contains n.PublicKey tier1KeySet let hdn = homeDomainNameForKey n.PublicKey @@ -746,7 +835,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let miscCoreSets : CoreSet array = Array.mapi - (fun (_: int) (n: PubnetNode.Root) -> + (fun (_: int) (n: PubnetNode) -> let hdn = homeDomainNameForKey n.PublicKey let keys = [| getSimKey n.PublicKey |] let tier1 = Set.contains n.PublicKey tier1KeySet @@ -760,7 +849,9 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin tier1 = Some tier1 validate = validate homeDomain = if validate then Some hdn.StringName else None + generatesLoad = None nodeLocs = Some [ getGeoLocOrDefault n ] + edgeDelays = edgeDelays preferredPeersMap = Some(keysToPreferredPeersMap keys) } let shouldWaitForConsensus = manualclose @@ -770,11 +861,12 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let orgCoreSets : CoreSet array = Array.map - (fun (hdn: HomeDomainName, nodes: PubnetNode.Root array) -> + (fun (hdn: HomeDomainName, nodes: PubnetNode array) -> assert (nodes.Length <> 0) let nodeList = List.ofArray nodes - let keys = Array.map (fun (n: PubnetNode.Root) -> getSimKey n.PublicKey) nodes + let keys = Array.map (fun (n: PubnetNode) -> getSimKey n.PublicKey) nodes let tier1 = Set.contains nodes.[0].PublicKey tier1KeySet + let generatesLoad = nodes.[0].GeneratesLoad let validate, qset = mergeQSets (List.map computeQset nodeList) @@ -785,7 +877,9 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin tier1 = Some tier1 validate = validate homeDomain = if validate then Some hdn.StringName else None + generatesLoad = generatesLoad nodeLocs = Some(List.map getGeoLocOrDefault nodeList) + edgeDelays = edgeDelays preferredPeersMap = Some(keysToPreferredPeersMap keys) } let shouldWaitForConsensus = manualclose diff --git a/src/FSLibrary/StellarNetworkDelays.fs b/src/FSLibrary/StellarNetworkDelays.fs index c934f669..468f70e5 100644 --- a/src/FSLibrary/StellarNetworkDelays.fs +++ b/src/FSLibrary/StellarNetworkDelays.fs @@ -55,7 +55,12 @@ let networkPingInMs (loc1: GeoLoc) (loc2: GeoLoc) : double = // A ping is a round trip, so double one-way delay. 2.0 * (networkDelayInMs loc1 loc2) -let getNetworkDelayCommands (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) array) (delay: int option) : ShCmd = +let getPeerDelays (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) array) : (int * PeerDnsName) array = + // Get the one way delays from loc1 to the locationss in locsAndNames + locsAndNames + |> Array.map (fun (loc2, name) -> int (networkDelayInMs loc1 loc2), name) + +let getNetworkDelayCommands (delaysAndNames: (int * PeerDnsName) array) (delay: int option) : ShCmd = // Traffic shaping happens using the 'tc' command on linux. This is a // complicated command. We build up the commands in pieces. @@ -264,19 +269,19 @@ let getNetworkDelayCommands (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) let perPeerResolveCmds : ShCmd array = Array.mapi - (fun (i: int) (loc2: GeoLoc, peer: PeerDnsName) -> + (fun (i: int) (_, peer: PeerDnsName) -> let classNo = 1 + i [| resolveName classNo peer |]) - locsAndNames + delaysAndNames |> Array.concat let perPeerCmds : ShCmd array = Array.mapi - (fun (i: int) (loc2: GeoLoc, peer: PeerDnsName) -> + (fun (i: int) (edgeDelay: int, peer: PeerDnsName) -> let msDelay = match delay with | Some d -> d - | None -> int (networkDelayInMs loc1 loc2) + | None -> edgeDelay let classNo = 1 + i @@ -284,7 +289,7 @@ let getNetworkDelayCommands (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) addFilter classNo addNetemQdisc classNo msDelay addFqLeaf classNo |]) - locsAndNames + delaysAndNames |> Array.concat let seq = @@ -302,7 +307,6 @@ let getNetworkDelayCommands (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) ShSeq seq - type NetworkCfg with member self.LocAndDnsName (cs: CoreSet) (i: int) : (GeoLoc * PeerDnsName) Option = @@ -319,27 +323,68 @@ type NetworkCfg with match self.missionContext.installNetworkDelay with | Some true -> let atLeastOneLocation = Map.exists (fun _ cs -> cs.options.nodeLocs.IsSome) self.coreSets - - if atLeastOneLocation then - true - else - // If there's _some_ geo info, we can extrapolate. - // However, if there's _no_ geo info, we can't really do anything. - // Don't install network delay if your topology has no geo info. - failwith "Network delays can't be installed if no geo info provided." + let haveDelays = Map.forall (fun _ cs -> cs.options.edgeDelays.IsSome) self.coreSets + + match self.missionContext.pubnetDataDelay with + | true -> + if haveDelays then + true + else + failwith "Per-edge network delays can't be installed if edge data is missing" + | false -> + if atLeastOneLocation then + true + else + // If there's _some_ geo info, we can extrapolate. + // However, if there's _no_ geo info, we can't really do anything. + // Don't install network delay if your topology has no geo info. + failwith "Network delays can't be installed if no geo info provided." | _ -> false member self.NetworkDelayScript (cs: CoreSet) (i: int) : ShCmd = - match cs.options.nodeLocs with - | None -> ShCmd.True() - | Some (locs) -> - let selfLoc = locs.[i] - - let otherLocsAndNames : (GeoLoc * PeerDnsName) array = + match self.missionContext.pubnetDataDelay with + | true -> + let otherDelaysAndNames : (int * PeerDnsName) array = match cs.options.preferredPeersMap with | Some (otherMap) -> let otherKeys = Array.ofList otherMap.[cs.keys.[i].PublicKey] - Array.choose self.LocAndDnsNameForKey otherKeys - | None -> Array.choose id (self.MapAllPeers self.LocAndDnsName) - - getNetworkDelayCommands selfLoc otherLocsAndNames self.missionContext.flatNetworkDelay + let otherKeysSet = Set.ofArray otherKeys + + let names = + self.MapAllPeers + (fun cs i -> + let pk = cs.keys.[i].PublicKey + if otherKeysSet.Contains pk then Some(pk, self.PeerDnsName cs i) else None) + |> Array.choose id + |> Map.ofArray + + let delays = cs.options.edgeDelays.Value + let selfKey = cs.keys.[i].PublicKey + + otherKeys + |> Array.map (fun peerKey -> delays.[selfKey, peerKey], names.[peerKey]) + | None -> + if self.missionContext.flatNetworkDelay.IsNone then + failwith + "Failed to construct network delay script: no preferred peers map or flat network delay" + else + [||] + + getNetworkDelayCommands otherDelaysAndNames self.missionContext.flatNetworkDelay + | false -> + match cs.options.nodeLocs with + | None -> ShCmd.True() + | Some (locs) -> + let selfLoc = locs.[i] + + let otherDelaysAndNames : (int * PeerDnsName) array = + let locsAndNames = + match cs.options.preferredPeersMap with + | Some (otherMap) -> + let otherKeys = Array.ofList otherMap.[cs.keys.[i].PublicKey] + Array.choose self.LocAndDnsNameForKey otherKeys + | None -> Array.choose id (self.MapAllPeers self.LocAndDnsName) + + locsAndNames |> getPeerDelays selfLoc + + getNetworkDelayCommands otherDelaysAndNames self.missionContext.flatNetworkDelay diff --git a/src/FSLibrary/json-type-samples/sample-network-data-delay.json b/src/FSLibrary/json-type-samples/sample-network-data-delay.json new file mode 100644 index 00000000..becb7256 --- /dev/null +++ b/src/FSLibrary/json-type-samples/sample-network-data-delay.json @@ -0,0 +1,77 @@ +[ + { + "isTier1": true, + "numTotalInboundPeers": 2, + "numTotalOutboundPeers": 2, + "peers": [ + {"key": "GBOOLF6VITJWPTYYTWGJST35XHFUCTPTOJGZBPA7FLNINX6YD3AQHUBL", "owdMs": 3}, + {"key": "GDUBKMXWLSQNK33XQNRPSDEYODROPPRM4XLJGXFEZPWCLKIOFOTTCX3B", "owdMs": 10} + ], + "publicKey": "GCP7LQUW5UXXDNQI4ENXTUULZOIEIR6UDRC5S3VN6ZGKMOZ6TL3YB6IS", + "radar_homeDomain": "www.someone.org", + "radar_index": 1, + "radar_ip": "192.168.1.1", + "radar_isValidating": true, + "radar_isp": "Some Datacenter Inc.", + "radar_name": "SomeoneNet-1", + "radar_organizationId": "764efa883dda1e11db47671c4a3bbd9e", + "radar_quorumSet": { + "hashKey": "MzI3MzUyODNjODk2NzQ2YmQyMzNjOThkNDZkMDM4OThkOGJmNDdjZiAgLQo=", + "innerQuorumSets": [ + { + "hashKey": "OTI1NzIxZjllYzZiYjE1MGVjODQ0YWJjN2MwNjVkZDI5ODY0NjhiZiAgLQo=", + "innerQuorumSets": [], + "threshold": 2, + "validators": [ + "GDMSQ7KDJKDVQCVDMZ7VW2FXDUZLHILTEA7JEE7TBFADYG6PNJSABZTM", + "GA4LI76TTHQDK3QL6DAUCOUMBB3UHDU3IWT3HAVNZ36HRWQQS2CMPVB5", + "GBBAF6QXXYXG2RC2ZWSIX6UR3LQRDLFNCRFJ226L5DMFK3O3AEVFOCP6" + ] + }, + { + "hashKey": "NTVjYTYyODZlM2U0ZjRmYmE1ZDA0NDgzMzNmYTk5ZmM1YTQwNGE3MyAgLQo=", + "innerQuorumSets": [], + "threshold": 2, + "validators": [ + "GDVIDS5TYLQJ5THYGB6ULNJJMTUIZYROPZOGZGLVFO336KBGYSJCYA4N", + "GA7SDMIG3CNKEE6CP45IJSK3VOATMIZIXGHVNC2HRQACWC3KHD7EA5S3", + "GB37UNY2X5X4DSOUVUAXPXTOVKWF4NHTXTTPW2FCVHGNSGYLPQKBUBXW" + ] + } + ], + "threshold": 1, + "validators": [] + }, + "version": "v13.1.0" + }, + { + "numTotalInboundPeers": 1, + "numTotalOutboundPeers": 1, + "peers": [ + {"key": "GDQHFKIEF3NPKZVDC2WO5E5P54H3UG3QU2AZZCDINVF4THK5BZV6E4AK", "owdMs": 5} + ], + "publicKey": "GBDRGD7ZSIBXYLRHNO2LRI2BLOLOLD4THAMDCMCVK3JX2GFE4KQWABUE", + "radar_index": 0.5, + "radar_ip": "192.168.1.1", + "radar_isValidating": false, + "radar_isp": "Some ISP", + "radar_quorumSet": { + "innerQuorumSets": [], + "threshold": 9999999, + "validators": ["GBN5UP5XG2NKBGSBXGV632XFUQYYU62XYWR47GH7KQZUQDTVT7ROGK5C", + "GDQHFKIEF3NPKZVDC2WO5E5P54H3UG3QU2AZZCDINVF4THK5BZV6E4AK" + ] + }, + "generatesLoad": false, + "version": "stellar-core 15.0.0 (8d6e6d74cfae65943211bdcc5ba083976d525a04)" + }, + { + "peers": [ + {"key": "GBN5UP5XG2NKBGSBXGV632XFUQYYU62XYWR47GH7KQZUQDTVT7ROGK5C", "owdMs": 9} + ], + "radar_homeDomain": "horizon", + "generatesLoad": true, + "publicKey": "GATTOMMXJ4TITUGIP4MQ635Z6FJYK57WOWGDQ7AM6DUWYBM5TUJ2BLFV", + "version": "v15.0.0" + } +] From f7af1db7d113b27eee0881bd09e350b411b496fe Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:26:53 -0700 Subject: [PATCH 2/3] Add flag for enabling e2e metric --- src/App/Program.fs | 11 +++++++++++ src/FSLibrary.Tests/Tests.fs | 1 + src/FSLibrary/StellarCoreCfg.fs | 6 ++++++ src/FSLibrary/StellarMissionContext.fs | 1 + 4 files changed, 19 insertions(+) diff --git a/src/App/Program.fs b/src/App/Program.fs index 1d121f92..7097cb1f 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -72,6 +72,7 @@ type MissionOptions apiRateLimit: int, pubnetData: string option, pubnetDataDelay: bool, + measureE2eLatency: bool, flatQuorum: bool option, tier1Keys: string option, maxConnections: int option, @@ -305,6 +306,12 @@ type MissionOptions Default = false)>] member self.PubnetDataDelay = pubnetDataDelay + [] + member self.MeasureE2eLatency = measureE2eLatency + [] member self.FlatQuorum = flatQuorum @@ -699,6 +706,9 @@ let main argv = if mission.PubnetData.IsNone && mission.PubnetDataDelay then failwith "Error: --pubnet-data-delay requires --pubnet-data to be set" + if mission.MeasureE2eLatency && not mission.PubnetDataDelay then + failwith "Error: --measure-e2e-latency requires --pubnet-data-delay to be set" + let _ = logToConsoleAndFile (sprintf "%s/stellar-supercluster.log" mission.Destination) let ll = @@ -800,6 +810,7 @@ let main argv = apiRateLimit = mission.ApiRateLimit pubnetData = mission.PubnetData pubnetDataDelay = mission.PubnetDataDelay + measureE2eLatency = mission.MeasureE2eLatency flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys maxConnections = mission.MaxConnections diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index fda40592..89388eed 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -69,6 +69,7 @@ let ctx : MissionContext = apiRateLimit = 10 pubnetData = None pubnetDataDelay = false + measureE2eLatency = false flatQuorum = None tier1Keys = None maxConnections = None diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index e4c2fd4a..56831ff7 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -155,6 +155,7 @@ type StellarCoreCfg = automaticMaintenanceCount: int accelerateTime: bool generateLoad: bool + measureE2eLatency: bool updateSorobanCosts: bool option manualClose: bool invariantChecks: InvariantChecksSpec @@ -294,6 +295,9 @@ type StellarCoreCfg = t.Add("ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING", self.accelerateTime) |> ignore t.Add("ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING", self.generateLoad) |> ignore + if self.measureE2eLatency && self.network.missionContext.measureE2eLatency then + t.Add("LOADGEN_MEASURE_TX_LATENCY_FOR_TESTING", true) |> ignore + if self.updateSorobanCosts.IsSome then t.Add("UPDATE_SOROBAN_COSTS_DURING_PROTOCOL_UPGRADE_FOR_TESTING", self.updateSorobanCosts.Value) |> ignore @@ -630,6 +634,7 @@ type NetworkCfg with automaticMaintenanceCount = if opts.performMaintenance then 50000 else 0 accelerateTime = opts.accelerateTime generateLoad = true + measureE2eLatency = opts.generatesLoad = Some true updateSorobanCosts = opts.updateSorobanCosts manualClose = false invariantChecks = opts.invariantChecks @@ -671,6 +676,7 @@ type NetworkCfg with automaticMaintenanceCount = if c.options.performMaintenance then 50000 else 0 accelerateTime = c.options.accelerateTime generateLoad = true + measureE2eLatency = c.options.generatesLoad = Some true updateSorobanCosts = c.options.updateSorobanCosts manualClose = false invariantChecks = c.options.invariantChecks diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 2b34c6cb..c6aec02a 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -70,6 +70,7 @@ type MissionContext = apiRateLimit: int pubnetData: string option pubnetDataDelay: bool + measureE2eLatency: bool flatQuorum: bool option tier1Keys: string option maxConnections: int option From 015d9e5feaed3d056203240077e161c416cd64a8 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:27:37 -0700 Subject: [PATCH 3/3] Add flag for setting PEER_AUTHENTICATION_TIMEOUT --- src/App/Program.fs | 7 +++++++ src/FSLibrary.Tests/Tests.fs | 1 + src/FSLibrary/StellarCoreCfg.fs | 4 ++++ src/FSLibrary/StellarMissionContext.fs | 1 + 4 files changed, 13 insertions(+) diff --git a/src/App/Program.fs b/src/App/Program.fs index 7097cb1f..bb679ede 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -73,6 +73,7 @@ type MissionOptions pubnetData: string option, pubnetDataDelay: bool, measureE2eLatency: bool, + peerAuthenticationTimeout: int option, flatQuorum: bool option, tier1Keys: string option, maxConnections: int option, @@ -312,6 +313,11 @@ type MissionOptions Default = false)>] member self.MeasureE2eLatency = measureE2eLatency + [] + member self.PeerAuthenticationTimeout = peerAuthenticationTimeout + [] member self.FlatQuorum = flatQuorum @@ -811,6 +817,7 @@ let main argv = pubnetData = mission.PubnetData pubnetDataDelay = mission.PubnetDataDelay measureE2eLatency = mission.MeasureE2eLatency + peerAuthenticationTimeout = mission.PeerAuthenticationTimeout flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys maxConnections = mission.MaxConnections diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 89388eed..b0d3672d 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -70,6 +70,7 @@ let ctx : MissionContext = pubnetData = None pubnetDataDelay = false measureE2eLatency = false + peerAuthenticationTimeout = None flatQuorum = None tier1Keys = None maxConnections = None diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index 56831ff7..87eed94f 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -346,6 +346,10 @@ type StellarCoreCfg = t.Add("MAX_ADDITIONAL_PEER_CONNECTIONS", self.targetPeerConnections * 3) |> ignore + match self.network.missionContext.peerAuthenticationTimeout with + | Some timeout -> t.Add("PEER_AUTHENTICATION_TIMEOUT", timeout) |> ignore + | None -> () + t.Add("QUORUM_INTERSECTION_CHECKER", false) |> ignore t.Add("MANUAL_CLOSE", self.manualClose) |> ignore diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index c6aec02a..a48ac2c4 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -71,6 +71,7 @@ type MissionContext = pubnetData: string option pubnetDataDelay: bool measureE2eLatency: bool + peerAuthenticationTimeout: int option flatQuorum: bool option tier1Keys: string option maxConnections: int option