diff --git a/Directory.Build.props b/Directory.Build.props index fa5d7aead8..2d52b8a8c0 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,6 +1,6 @@  - 9.0.2.1 + 9.0.2.2 latest true enable diff --git a/README.md b/README.md index f9fa98fdac..d56d910042 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,4 @@ -# Npgsql - the .NET data provider for PostgreSQL - -[![stable](https://img.shields.io/nuget/v/Npgsql.svg?label=stable)](https://www.nuget.org/packages/Npgsql/) -[![next patch](https://img.shields.io/myget/npgsql/v/npgsql.svg?label=next%20patch)](https://www.myget.org/feed/npgsql/package/nuget/Npgsql) -[![daily builds (vnext)](https://img.shields.io/myget/npgsql-vnext/v/npgsql.svg?label=vnext)](https://www.myget.org/feed/npgsql-vnext/package/nuget/Npgsql) -[![build](https://github.com/npgsql/npgsql/actions/workflows/build.yml/badge.svg)](https://github.com/npgsql/npgsql/actions/workflows/build.yml) -[![gitter](https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg)](https://gitter.im/npgsql/npgsql) - -## What is Npgsql? - -Npgsql is the open source .NET data provider for PostgreSQL. It allows you to connect and interact with PostgreSQL server using .NET. - -For the full documentation, please visit [the Npgsql website](https://www.npgsql.org). For the Entity Framework Core provider that works with this provider, see [Npgsql.EntityFrameworkCore.PostgreSQL](https://github.com/npgsql/efcore.pg). - -## Quickstart - -Here's a basic code snippet to get you started: - -```csharp -using Npgsql; - -var connString = "Host=myserver;Username=mylogin;Password=mypass;Database=mydatabase"; - -var dataSourceBuilder = new NpgsqlDataSourceBuilder(connString); -var dataSource = dataSourceBuilder.Build(); - -var conn = await dataSource.OpenConnectionAsync(); - -// Insert some data -await using (var cmd = new NpgsqlCommand("INSERT INTO data (some_field) VALUES (@p)", conn)) -{ - cmd.Parameters.AddWithValue("p", "Hello world"); - await cmd.ExecuteNonQueryAsync(); -} - -// Retrieve all rows -await using (var cmd = new NpgsqlCommand("SELECT some_field FROM data", conn)) -await using (var reader = await cmd.ExecuteReaderAsync()) -{ - while (await reader.ReadAsync()) - Console.WriteLine(reader.GetString(0)); -} -``` - -## Key features - -* High-performance PostgreSQL driver. Regularly figures in the top contenders on the [TechEmpower Web Framework Benchmarks](https://www.techempower.com/benchmarks/). -* Full support of most PostgreSQL types, including advanced ones such as arrays, enums, ranges, multiranges, composites, JSON, PostGIS and others. -* Highly-efficient bulk import/export API. -* Failover, load balancing and general multi-host support. -* Great integration with Entity Framework Core via [Npgsql.EntityFrameworkCore.PostgreSQL](https://www.nuget.org/packages/Npgsql.EntityFrameworkCore.PostgreSQL). - -For the full documentation, please visit the Npgsql website at [https://www.npgsql.org](https://www.npgsql.org). - -## YugabyteDB Npgsql Features +# Npgsql YugabyteDB - the .NET data provider for YugabyteDB Yugabyte Npgsql driver is a distributed .NET driver for YSQL built on the PostgreSQL Npgsql driver. Although the upstream PostgreSQL driver works with YugabyteDB, the Yugabyte driver enhances YugabyteDB by eliminating the need for external load balancers. @@ -79,12 +25,12 @@ Load balancing connection properties: The following connection properties are added to enable load balancing: * Load Balance Hosts - Starting with version 8.0.3.2, it expects one of False, Any (same as true), OnlyPrimary, OnlyRR, PreferPrimary and PreferRR as its possible values. - * False - No connection load balancing. Behaviour is similar to vanilla Npgsql driver - * Any - Same as value true. Distribute connections equally across all nodes in the cluster, irrespective of its type (primary or read-replica) - * OnlyPrimary - Create connections equally across only the primary nodes of the cluster - * OnlyRR - Create connections equally across only the read-replica nodes of the cluster - * PreferPrimary - Create connections equally across primary cluster nodes. If none available, on any available read replica node in the cluster - * PreferRR - Create connections equally across read replica nodes of the cluster. If none available, on any available primary cluster node + * False - No connection load balancing. Behaviour is similar to vanilla Npgsql driver + * Any - Same as value true. Distribute connections equally across all nodes in the cluster, irrespective of its type (primary or read-replica) + * OnlyPrimary - Create connections equally across only the primary nodes of the cluster + * OnlyRR - Create connections equally across only the read-replica nodes of the cluster + * PreferPrimary - Create connections equally across primary cluster nodes. If none available, on any available read replica node in the cluster + * PreferRR - Create connections equally across read replica nodes of the cluster. If none available, on any available primary cluster node * Topology Keys - provide comma-separated geo-location values to enable topology-aware load balancing. Geo-locations can be provided as cloud.region.zone. * YB Servers Refresh Interval - The list of servers, to balance the connection load on, are refreshed periodically every 5 minutes by default. This time can be regulated by this property. * Fallback To Topology Keys Only - Decides if the driver can fall back to nodes outside of the given placements for new connections, if the nodes in the given placements are not available. Value true means stick to explicitly given placements for fallback, else fail. Value false means fall back to entire cluster nodes when nodes in the given placements are unavailable. Default is false. It is ignored if topology-keys is not specified or load-balance is set to either prefer-primary or prefer-rr. @@ -116,4 +62,60 @@ Multiple topologies can also be passed to the Topology Keys property, and each o ```csharp var connString = "host=127.0.0.3;port=5433;database=yugabyte;userid=yugabyte;password=yugsbyte;Load Balance Hosts=true;Timeout=0;Topology Keys=cloud1.region1.zone1:1,cloud2.region2.zone2:2"; -``` \ No newline at end of file +``` + +### ----------------------------------- Upstream ReadMe Follows -------------------------------------- + +# Npgsql - the .NET data provider for PostgreSQL + +[![stable](https://img.shields.io/nuget/v/Npgsql.svg?label=stable)](https://www.nuget.org/packages/Npgsql/) +[![next patch](https://img.shields.io/myget/npgsql/v/npgsql.svg?label=next%20patch)](https://www.myget.org/feed/npgsql/package/nuget/Npgsql) +[![daily builds (vnext)](https://img.shields.io/myget/npgsql-vnext/v/npgsql.svg?label=vnext)](https://www.myget.org/feed/npgsql-vnext/package/nuget/Npgsql) +[![build](https://github.com/npgsql/npgsql/actions/workflows/build.yml/badge.svg)](https://github.com/npgsql/npgsql/actions/workflows/build.yml) +[![gitter](https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg)](https://gitter.im/npgsql/npgsql) + +## What is Npgsql? + +Npgsql is the open source .NET data provider for PostgreSQL. It allows you to connect and interact with PostgreSQL server using .NET. + +For the full documentation, please visit [the Npgsql website](https://www.npgsql.org). For the Entity Framework Core provider that works with this provider, see [Npgsql.EntityFrameworkCore.PostgreSQL](https://github.com/npgsql/efcore.pg). + +## Quickstart + +Here's a basic code snippet to get you started: + +```csharp +using Npgsql; + +var connString = "Host=myserver;Username=mylogin;Password=mypass;Database=mydatabase"; + +var dataSourceBuilder = new NpgsqlDataSourceBuilder(connString); +var dataSource = dataSourceBuilder.Build(); + +var conn = await dataSource.OpenConnectionAsync(); + +// Insert some data +await using (var cmd = new NpgsqlCommand("INSERT INTO data (some_field) VALUES (@p)", conn)) +{ + cmd.Parameters.AddWithValue("p", "Hello world"); + await cmd.ExecuteNonQueryAsync(); +} + +// Retrieve all rows +await using (var cmd = new NpgsqlCommand("SELECT some_field FROM data", conn)) +await using (var reader = await cmd.ExecuteReaderAsync()) +{ + while (await reader.ReadAsync()) + Console.WriteLine(reader.GetString(0)); +} +``` + +## Key features + +* High-performance PostgreSQL driver. Regularly figures in the top contenders on the [TechEmpower Web Framework Benchmarks](https://www.techempower.com/benchmarks/). +* Full support of most PostgreSQL types, including advanced ones such as arrays, enums, ranges, multiranges, composites, JSON, PostGIS and others. +* Highly-efficient bulk import/export API. +* Failover, load balancing and general multi-host support. +* Great integration with Entity Framework Core via [Npgsql.EntityFrameworkCore.PostgreSQL](https://www.nuget.org/packages/Npgsql.EntityFrameworkCore.PostgreSQL). + +For the full documentation, please visit the Npgsql website at [https://www.npgsql.org](https://www.npgsql.org). \ No newline at end of file diff --git a/src/Npgsql/ClusterAwareDataSource.cs b/src/Npgsql/ClusterAwareDataSource.cs index e94e8e4519..1e13062f20 100644 --- a/src/Npgsql/ClusterAwareDataSource.cs +++ b/src/Npgsql/ClusterAwareDataSource.cs @@ -70,6 +70,11 @@ public class ClusterAwareDataSource: NpgsqlDataSource /// protected static Dictionary poolToNumConnMapRR = new Dictionary(); + /// + /// Preserves connection counts across map rebuilds (e.g. during Refresh). + /// + protected static Dictionary _savedConnectionCounts = new Dictionary(StringComparer.OrdinalIgnoreCase); + /// /// Stores a map of host to their priority /// @@ -78,7 +83,12 @@ public class ClusterAwareDataSource: NpgsqlDataSource /// /// Stores a map of host to their priority /// - protected static Dictionary priorityToPoolIndexMap = new Dictionary(); + protected static Dictionary> priorityToPoolIndexMapPrimary = new Dictionary>(); + + /// + /// Stores a map of host to their priority + /// + protected static Dictionary> priorityToPoolIndexMapRR = new Dictionary>(); /// /// Connection settings @@ -130,6 +140,19 @@ public class ClusterAwareDataSource: NpgsqlDataSource /// protected ConcurrentDictionary> fallbackPublicIPs; + /// + /// Contains a dictionary of IPs for all Read replica IPs + /// Key = Read replica IPs + /// Value = Dictionary (IP , nodeType) + /// + protected Dictionary AllRRIps = new Dictionary(); + /// + /// Contains a dictionary of IPs for PRimary nodes of cluster + /// Key = Primary node IPs + /// Value = Dictionary (IP , nodeType) + /// + protected Dictionary AllPrimaryIps = new Dictionary(); + /// /// To set refresh value explicitly /// @@ -223,7 +246,7 @@ protected virtual void CreatePool(Dictionary hostsmap) var poolSettings = settings.Clone(); poolSettings.Host = host.Key; _connectionLogger.LogDebug("Adding {host} to connection pool", poolSettings.Host); - NpgsqlDataSource poolnew = settings.Pooling? new PoolingDataSource(poolSettings, dataSourceConfig): new UnpooledDataSource(poolSettings, dataSourceConfig); + NpgsqlDataSource poolnew = settings.Pooling? new YBPoolingWrapperDataSource(poolSettings, dataSourceConfig): new UnpooledDataSource(poolSettings, dataSourceConfig); _pools.Add(poolnew); if (host.Value.Equals("primary", StringComparison.OrdinalIgnoreCase)) { @@ -268,6 +291,11 @@ public static int GetLoad(string server) } } + if (_savedConnectionCounts.TryGetValue(server, out var savedCount) && savedCount > 0) + { + return savedCount; + } + return 0; } @@ -339,20 +367,43 @@ void UpdateConnectionMap(int poolIndex, int incDec) lock (lockObject) { int currentCount; + var host = _pools[poolIndex].Settings.Host; if (poolToNumConnMapPrimary.ContainsKey(currPool)) { currentCount = poolToNumConnMapPrimary[currPool]; - poolToNumConnMapPrimary[currPool] += incDec; + poolToNumConnMapPrimary[currPool] += incDec; _connectionLogger.LogTrace("Updated the current count for {host} from {currentCount} to {newCount}", - _pools[poolIndex].Settings.Host, currentCount, poolToNumConnMapPrimary[currPool]); + host, currentCount, poolToNumConnMapPrimary[currPool]); } else if (poolToNumConnMapRR.ContainsKey(currPool)) { currentCount = poolToNumConnMapRR[currPool]; poolToNumConnMapRR[currPool] += incDec; _connectionLogger.LogTrace("Updated the current count for {host} from {currentCount} to {newCount}", - _pools[poolIndex].Settings.Host, currentCount, poolToNumConnMapRR[currPool]); + host, currentCount, poolToNumConnMapRR[currPool]); + } + else if (incDec > 0) + { + if (_hostsToNodeTypeMap != null && host != null && _hostsToNodeTypeMap.TryGetValue(host, out var nodeType)) + { + if (nodeType.Equals("read_replica", StringComparison.OrdinalIgnoreCase)) + { + poolToNumConnMapRR[currPool] = Math.Max(0, incDec); + } + else + { + poolToNumConnMapPrimary[currPool] = Math.Max(0, incDec); + } + } + } + else if (incDec < 0 && host != null && _savedConnectionCounts.TryGetValue(host, out var savedCount)) + { + var newCount = savedCount + incDec; + if (newCount > 0) + _savedConnectionCounts[host] = newCount; + else + _savedConnectionCounts.Remove(host); } } } @@ -406,7 +457,7 @@ internal override async ValueTask Get(NpgsqlConnection conn, Np { NpgsqlConnector? connector = null; var exceptions = new List(); - connector = await getConnector(conn, timeout,async, cancellationToken, exceptions).ConfigureAwait(false); + connector = await getConnector(conn, timeout,async, cancellationToken, exceptions, false).ConfigureAwait(false); if (this is TopologyAwareDataSource) { @@ -424,7 +475,7 @@ internal override async ValueTask Get(NpgsqlConnection conn, Np { exceptions.Clear(); CreatePool(fallbackPrivateIPs[i]); - connector = await getConnector(conn, timeout,async, cancellationToken, exceptions).ConfigureAwait(false); + connector = await getConnector(conn, timeout,async, cancellationToken, exceptions, false).ConfigureAwait(false); if (connector != null) break; } @@ -432,7 +483,7 @@ internal override async ValueTask Get(NpgsqlConnection conn, Np { exceptions.Clear(); CreatePool(fallbackPublicIPs[i]); - connector = await getConnector(conn, timeout,async, cancellationToken, exceptions).ConfigureAwait(false); + connector = await getConnector(conn, timeout,async, cancellationToken, exceptions, false).ConfigureAwait(false); if (connector != null) break; } @@ -449,24 +500,38 @@ internal override async ValueTask Get(NpgsqlConnection conn, Np { exceptions.Clear(); CreatePool(fallbackPrivateIPs[REST_OF_CLUSTER]); - connector = await getConnector(conn, timeout,async, cancellationToken, exceptions).ConfigureAwait(false); + connector = await getConnector(conn, timeout,async, cancellationToken, exceptions, false).ConfigureAwait(false); } else if (public_ip != null) { exceptions.Clear(); CreatePool(fallbackPublicIPs[REST_OF_CLUSTER]); - connector = await getConnector(conn, timeout,async, cancellationToken, exceptions).ConfigureAwait(false); + connector = await getConnector(conn, timeout,async, cancellationToken, exceptions, false).ConfigureAwait(false); } } } + if (connector == null) + { + if (Settings.LoadBalanceHosts == LoadBalanceHosts.PreferPrimary) + { + CreatePool(AllRRIps); + connector = await getConnector(conn, timeout,async, cancellationToken, exceptions, true).ConfigureAwait(false); + } + if (Settings.LoadBalanceHosts == LoadBalanceHosts.PreferRR) + { + CreatePool(AllPrimaryIps); + connector = await getConnector(conn, timeout,async, cancellationToken, exceptions, true).ConfigureAwait(false); + } + } + if (connector != null) { return connector; } _connectionLogger.LogDebug("Failed to apply Load balance. Trying normal connection"); conn.Settings.LoadBalanceHosts = LoadBalanceHosts.False; - connector = await getConnector(conn, timeout,async, cancellationToken, exceptions).ConfigureAwait(false); + connector = await getConnector(conn, timeout,async, cancellationToken, exceptions, true).ConfigureAwait(false); return connector ?? throw NoSuitableHostsException(exceptions); } @@ -476,15 +541,36 @@ internal override async ValueTask Get(NpgsqlConnection conn, Np if (priority == chosenHostPriority) return null; NpgsqlConnector? connector = null; - var poolIndex = -1; - if (priorityToPoolIndexMap.ContainsKey(priority)) + Dictionary>? priorityToPoolIndexMap = null; + if (Settings.LoadBalanceHosts == LoadBalanceHosts.OnlyPrimary || Settings.LoadBalanceHosts == LoadBalanceHosts.PreferPrimary) { - poolIndex = priorityToPoolIndexMap[priority]; + priorityToPoolIndexMap = priorityToPoolIndexMapPrimary; + } + else if (Settings.LoadBalanceHosts == LoadBalanceHosts.OnlyRR || Settings.LoadBalanceHosts == LoadBalanceHosts.PreferRR) + { + priorityToPoolIndexMap = priorityToPoolIndexMapRR; + } + else if (Settings.LoadBalanceHosts == LoadBalanceHosts.Any || Settings.LoadBalanceHosts == LoadBalanceHosts.True) + { + priorityToPoolIndexMap = priorityToPoolIndexMapPrimary + .Concat(priorityToPoolIndexMapRR) + .GroupBy(p => p.Key) + .ToDictionary( + g => g.Key, + g => g.SelectMany(x => x.Value).ToList() + ); + } + Debug.Assert(priorityToPoolIndexMap != null, nameof(priorityToPoolIndexMap) + " = null"); + if (!priorityToPoolIndexMap.TryGetValue(priority, out var poolIndices)){ + priority++; + connector = await getConnector(chosenHostPriority, priority, conn, timeout, async, cancellationToken, exceptions) + .ConfigureAwait(false); + return connector; } - if (poolIndex == -1) - return null; - UpdateConnectionMap(poolIndex, 1); + foreach (var poolIndex in poolIndices) + { + UpdateConnectionMap(poolIndex, 1); var timeoutPerHost = timeout.IsSet ? timeout.CheckAndGetTimeLeft() : TimeSpan.Zero; var preferredType = GetTargetSessionAttributes(conn); var checkUnpreferred = preferredType is TargetSessionAttributes.PreferPrimary or TargetSessionAttributes.PreferStandby; @@ -498,10 +584,16 @@ await TryGet(conn, timeoutPerHost, async, preferredType, IsOnline, poolIndex, ex : null); if (connector == null) { + if (!unreachableHostsIndices.Contains(poolIndex)) unreachableHostsIndices.Add(poolIndex); + var settingsHost = _pools[poolIndex].Settings.Host; + if (settingsHost != null && !unreachableHosts.Contains(settingsHost)) unreachableHosts.Add(settingsHost); + var pool = _pools[poolIndex]; + if (poolToNumConnMapPrimary.ContainsKey(pool)) + poolToNumConnMapPrimary.Remove(pool); + else if (poolToNumConnMapRR.ContainsKey(pool)) + poolToNumConnMapRR.Remove(pool); UpdateConnectionMap(poolIndex, -1); - priority++; - connector = await getConnector(chosenHostPriority, priority, conn, timeout, async, cancellationToken, exceptions) - .ConfigureAwait(false); + continue; } if (connector != null) @@ -511,18 +603,28 @@ await TryGet(conn, timeoutPerHost, async, preferredType, IsOnline, poolIndex, ex { unreachableHosts.Remove(host); } + + return connector; } + } + + priority++; + connector = await getConnector(chosenHostPriority, priority, conn, timeout, async, cancellationToken, exceptions) + .ConfigureAwait(false); return connector; } async Task getConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, - CancellationToken cancellationToken, List exceptions) + CancellationToken cancellationToken, List exceptions, bool isFinalFallback) { NpgsqlConnector? connector = null; - for (var i = 0; i < _pools.Count; i++) + while (true) { CheckDisposed(); - + if (AreAllEligiblePoolsExhausted() && !isFinalFallback) + { + return null; + } var poolIndex = conn.Settings.LoadBalanceHosts != LoadBalanceHosts.False ? GetRoundRobinIndex() : -2; if (poolIndex == -1) return null; @@ -568,10 +670,9 @@ await TryGet(conn, timeoutPerHost, async, preferredType, IsOnline, poolIndex, ex { break; } - unreachableHostsIndices.Add(poolIndex); + if (!unreachableHostsIndices.Contains(poolIndex)) unreachableHostsIndices.Add(poolIndex); var settingsHost = _pools[poolIndex].Settings.Host; - if (settingsHost != null) unreachableHosts.Add(settingsHost); - // poolToNumConnMap.Remove(poolIndex); + if (settingsHost != null && !unreachableHosts.Contains(settingsHost)) unreachableHosts.Add(settingsHost); var pool = _pools[poolIndex]; if (poolToNumConnMapPrimary.ContainsKey(pool)) poolToNumConnMapPrimary.Remove(pool); @@ -589,8 +690,14 @@ await TryGet(conn, timeoutPerHost, async, preferredType, IsOnline, poolIndex, ex internal override bool TryGetIdleConnector([NotNullWhen(true)] out NpgsqlConnector? connector) => throw new NpgsqlException("Npgsql bug: trying to get an idle connector from " + nameof(ClusterAwareDataSource)); + internal override bool TryGetIdleConnector(NpgsqlConnectionStringBuilder originalConnString, out NpgsqlConnector? connector) => throw new NotImplementedException(); + internal override ValueTask OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken) => throw new NotImplementedException(); + internal override ValueTask OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken, + NpgsqlConnectionStringBuilder settings) => + throw new NotImplementedException(); + internal override void Return(NpgsqlConnector connector) { var host = connector.Host; @@ -713,6 +820,25 @@ int GetRoundRobinIndex() } } + bool AreAllEligiblePoolsExhausted() + { + return settings.LoadBalanceHosts switch + { + LoadBalanceHosts.OnlyPrimary or LoadBalanceHosts.PreferPrimary=> + poolToNumConnMapPrimary.Count == 0, + + LoadBalanceHosts.OnlyRR or LoadBalanceHosts.PreferRR => + poolToNumConnMapRR.Count == 0, + + LoadBalanceHosts.Any or LoadBalanceHosts.True => + poolToNumConnMapPrimary.Count == 0 && + poolToNumConnMapRR.Count == 0, + + _ => true + }; + } + + int getHosts(Dictionary poolToNumConnMap) { var PoolIndex = -1; @@ -770,23 +896,27 @@ static TargetSessionAttributes GetTargetSessionAttributes(NpgsqlConnection conne NpgsqlConnector? connector = null; try { - if (pool.TryGetIdleConnector(out connector)) + if (pool.TryGetIdleConnector(settings, out connector)) { if (databaseState == DatabaseState.Unknown) { - databaseState = await connector.QueryDatabaseState(new NpgsqlTimeout(timeoutPerHost), async, cancellationToken).ConfigureAwait(false); - Debug.Assert(databaseState != DatabaseState.Unknown); - if (!stateValidator(databaseState, preferredType)) + if (connector != null) { - pool.Return(connector); - return null; + databaseState = await connector.QueryDatabaseState(new NpgsqlTimeout(timeoutPerHost), async, cancellationToken) + .ConfigureAwait(false); + Debug.Assert(databaseState != DatabaseState.Unknown); + if (!stateValidator(databaseState, preferredType)) + { + pool.Return(connector); + return null; + } } } return connector; } else { - connector = await pool.OpenNewConnector(conn, new NpgsqlTimeout(timeoutPerHost), async, cancellationToken).ConfigureAwait(false); + connector = await pool.OpenNewConnector(conn, new NpgsqlTimeout(timeoutPerHost), async, cancellationToken, settings).ConfigureAwait(false); if (connector is not null) { if (databaseState == DatabaseState.Unknown) diff --git a/src/Npgsql/MultiHostDataSourceWrapper.cs b/src/Npgsql/MultiHostDataSourceWrapper.cs index c347e81bb8..7e869586f1 100644 --- a/src/Npgsql/MultiHostDataSourceWrapper.cs +++ b/src/Npgsql/MultiHostDataSourceWrapper.cs @@ -36,8 +36,16 @@ internal override ValueTask Get(NpgsqlConnection conn, NpgsqlTi => wrappedSource.Get(conn, timeout, async, cancellationToken); internal override bool TryGetIdleConnector([NotNullWhen(true)] out NpgsqlConnector? connector) => throw new NpgsqlException("Npgsql bug: trying to get an idle connector from " + nameof(MultiHostDataSourceWrapper)); + + internal override bool TryGetIdleConnector(NpgsqlConnectionStringBuilder originalConnString, out NpgsqlConnector? connector) => throw new System.NotImplementedException(); + internal override ValueTask OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken) => throw new NpgsqlException("Npgsql bug: trying to open a new connector from " + nameof(MultiHostDataSourceWrapper)); + + internal override ValueTask OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken, + NpgsqlConnectionStringBuilder settings) => + throw new System.NotImplementedException(); + internal override void Return(NpgsqlConnector connector) => wrappedSource.Return(connector); diff --git a/src/Npgsql/NpgsqlDataSource.cs b/src/Npgsql/NpgsqlDataSource.cs index bd66d6e230..622b6ba688 100644 --- a/src/Npgsql/NpgsqlDataSource.cs +++ b/src/Npgsql/NpgsqlDataSource.cs @@ -25,7 +25,7 @@ public abstract class NpgsqlDataSource : DbDataSource /// Contains the connection string returned to the user from /// after the connection has been opened. Does not contain the password unless Persist Security Info=true. /// - internal NpgsqlConnectionStringBuilder Settings { get; } + internal NpgsqlConnectionStringBuilder Settings { get; set; } internal NpgsqlDataSourceConfiguration Configuration { get; } internal NpgsqlLoggingConfiguration LoggingConfiguration { get; } @@ -399,9 +399,15 @@ internal abstract ValueTask Get( internal abstract bool TryGetIdleConnector([NotNullWhen(true)] out NpgsqlConnector? connector); + internal abstract bool TryGetIdleConnector(NpgsqlConnectionStringBuilder originalConnString, out NpgsqlConnector? connector); + + internal abstract ValueTask OpenNewConnector( NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken); + internal abstract ValueTask OpenNewConnector( + NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken, NpgsqlConnectionStringBuilder settings); + internal abstract void Return(NpgsqlConnector connector); internal abstract bool OwnsConnectors { get; } @@ -556,4 +562,5 @@ sealed class DatabaseStateInfo(DatabaseState state, NpgsqlTimeout timeout, DateT public DatabaseStateInfo() : this(default, default, default) { } } + } diff --git a/src/Npgsql/NpgsqlMultiHostDataSource.cs b/src/Npgsql/NpgsqlMultiHostDataSource.cs index 61a6810133..49c11799bd 100644 --- a/src/Npgsql/NpgsqlMultiHostDataSource.cs +++ b/src/Npgsql/NpgsqlMultiHostDataSource.cs @@ -360,12 +360,18 @@ int GetRoundRobinIndex() } } + internal override ValueTask OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken, + NpgsqlConnectionStringBuilder settings) => + throw new NotImplementedException(); + internal override void Return(NpgsqlConnector connector) => throw new NpgsqlException("Npgsql bug: a connector was returned to " + nameof(NpgsqlMultiHostDataSource)); internal override bool TryGetIdleConnector([NotNullWhen(true)] out NpgsqlConnector? connector) => throw new NpgsqlException("Npgsql bug: trying to get an idle connector from " + nameof(NpgsqlMultiHostDataSource)); + internal override bool TryGetIdleConnector(NpgsqlConnectionStringBuilder originalConnString, out NpgsqlConnector? connector) => throw new NotImplementedException(); + internal override ValueTask OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken) => throw new NpgsqlException("Npgsql bug: trying to open a new connector from " + nameof(NpgsqlMultiHostDataSource)); diff --git a/src/Npgsql/PoolingDataSource.cs b/src/Npgsql/PoolingDataSource.cs index 6910ee60da..732ae1af74 100644 --- a/src/Npgsql/PoolingDataSource.cs +++ b/src/Npgsql/PoolingDataSource.cs @@ -218,8 +218,10 @@ internal sealed override bool TryGetIdleConnector([NotNullWhen(true)] out Npgsql return false; } + internal override bool TryGetIdleConnector(NpgsqlConnectionStringBuilder originalConnString, out NpgsqlConnector? connector) => throw new NotImplementedException(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool CheckIdleConnector([NotNullWhen(true)] NpgsqlConnector? connector) + protected bool CheckIdleConnector([NotNullWhen(true)] NpgsqlConnector? connector) { if (connector is null) return false; @@ -320,7 +322,11 @@ bool CheckIdleConnector([NotNullWhen(true)] NpgsqlConnector? connector) return null; } - internal sealed override void Return(NpgsqlConnector connector) + internal override ValueTask OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken, + NpgsqlConnectionStringBuilder settings) => + throw new NotImplementedException(); + + internal override void Return(NpgsqlConnector connector) { Debug.Assert(!connector.InTransaction); Debug.Assert(connector.MultiplexAsyncWritingLock == 0 || connector.IsBroken || connector.IsClosed, @@ -366,7 +372,7 @@ public override void Clear() } } - void CloseConnector(NpgsqlConnector connector) + protected void CloseConnector(NpgsqlConnector connector) { try { diff --git a/src/Npgsql/TopologyAwareDataSource.cs b/src/Npgsql/TopologyAwareDataSource.cs index 0fb70ed9c7..48f9ff311f 100644 --- a/src/Npgsql/TopologyAwareDataSource.cs +++ b/src/Npgsql/TopologyAwareDataSource.cs @@ -16,13 +16,11 @@ namespace YBNpgsql; public sealed class TopologyAwareDataSource: ClusterAwareDataSource { ConcurrentDictionary?> allowedPlacements; - Dictionary AllRRIps = new Dictionary(); - Dictionary AllPrimaryIps = new Dictionary(); internal TopologyAwareDataSource(NpgsqlConnectionStringBuilder settings, NpgsqlDataSourceConfiguration dataSourceConfig) : base(settings,dataSourceConfig,false) { allowedPlacements = new ConcurrentDictionary?>(); - ParseGeoLocations(); + ParseGeoLocations(); _connectionLogger.LogDebug("Allowed Placements: {allowedPlacements}", allowedPlacements); Debug.Assert(initialHosts != null, nameof(initialHosts) + " != null"); foreach (var host in initialHosts.ToList()) @@ -144,22 +142,56 @@ protected override void CreatePool(Dictionary hostsmap) } if (flag == 1) + { + var existingPool = _pools.First(p => host.Key.Equals(p.Settings.Host, StringComparison.OrdinalIgnoreCase)); + if (host.Value.Equals("primary", StringComparison.OrdinalIgnoreCase)) + { + if (!poolToNumConnMapPrimary.ContainsKey(existingPool)) + { + var restoredCount = _savedConnectionCounts.TryGetValue(host.Key, out var sc) ? sc : 0; + poolToNumConnMapPrimary[existingPool] = restoredCount; + _savedConnectionCounts.Remove(host.Key); + } + } + else if (host.Value.Equals("read_replica", StringComparison.OrdinalIgnoreCase)) + { + if (!poolToNumConnMapRR.ContainsKey(existingPool)) + { + var restoredCount = _savedConnectionCounts.TryGetValue(host.Key, out var sc) ? sc : 0; + poolToNumConnMapRR[existingPool] = restoredCount; + _savedConnectionCounts.Remove(host.Key); + } + } continue; + } var poolSettings = settings.Clone(); poolSettings.Host = host.Key; _connectionLogger.LogDebug("Adding {host} to connection pool", poolSettings.Host); - NpgsqlDataSource poolnew = settings.Pooling? new PoolingDataSource(poolSettings, dataSourceConfig): new UnpooledDataSource(poolSettings, dataSourceConfig); + NpgsqlDataSource poolnew = settings.Pooling? new YBPoolingWrapperDataSource(poolSettings, dataSourceConfig): new UnpooledDataSource(poolSettings, dataSourceConfig); _pools.Add(poolnew); int index; index = _pools.IndexOf(poolnew); var priority = hostToPriorityMap[host.Key]; - priorityToPoolIndexMap[priority] = index; if (host.Value.Equals("primary", StringComparison.OrdinalIgnoreCase)) { + if (!priorityToPoolIndexMapPrimary.TryGetValue(priority, out var list)) + { + list = new List(); + priorityToPoolIndexMapPrimary[priority] = list; + } + + list.Add(index);; poolToNumConnMapPrimary[poolnew] = 0; } else if (host.Value.Equals("read_replica", StringComparison.OrdinalIgnoreCase)) { + if (!priorityToPoolIndexMapRR.TryGetValue(priority, out var list)) + { + list = new List(); + priorityToPoolIndexMapRR[priority] = list; + } + + list.Add(index); poolToNumConnMapRR[poolnew] = 0; } } @@ -280,7 +312,6 @@ Dictionary GetRelevantServerToNodeTypeMap(Dictionary GetPrivateOrPublicServers(Dictionary privateHosts, Dictionary publicHosts) @@ -329,7 +360,6 @@ Dictionary GetRelevantServerToNodeTypeMap(Dictionary GetRelevantServerToNodeTypeMap(Dictionary currentPrivateIPs, string internal override bool Refresh() { _connectionLogger.LogDebug("Refreshing connection"); + foreach (var kvp in poolToNumConnMapPrimary) + if (kvp.Key.Settings.Host != null) + _savedConnectionCounts[kvp.Key.Settings.Host] = kvp.Value; + foreach (var kvp in poolToNumConnMapRR) + if (kvp.Key.Settings.Host != null) + _savedConnectionCounts[kvp.Key.Settings.Host] = kvp.Value; poolToNumConnMapPrimary.Clear(); poolToNumConnMapRR.Clear(); Debug.Assert(initialHosts != null, nameof(initialHosts) + " != null"); diff --git a/src/Npgsql/UnpooledDataSource.cs b/src/Npgsql/UnpooledDataSource.cs index b6f7d1b4ff..a811de3b2a 100644 --- a/src/Npgsql/UnpooledDataSource.cs +++ b/src/Npgsql/UnpooledDataSource.cs @@ -39,10 +39,16 @@ internal override bool TryGetIdleConnector([NotNullWhen(true)] out NpgsqlConnect return false; } + internal override bool TryGetIdleConnector(NpgsqlConnectionStringBuilder originalConnString, out NpgsqlConnector? connector) => throw new System.NotImplementedException(); + internal override ValueTask OpenNewConnector( NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken) => new((NpgsqlConnector?)null); + internal override ValueTask OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken, + NpgsqlConnectionStringBuilder settings) => + throw new System.NotImplementedException(); + internal override void Return(NpgsqlConnector connector) { Interlocked.Decrement(ref _numConnectors); diff --git a/src/Npgsql/YBPoolingWrapperDataSource.cs b/src/Npgsql/YBPoolingWrapperDataSource.cs new file mode 100644 index 0000000000..ce03806dfc --- /dev/null +++ b/src/Npgsql/YBPoolingWrapperDataSource.cs @@ -0,0 +1,172 @@ +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using YBNpgsql.Internal; +using YBNpgsql.Util; + +namespace YBNpgsql; + +/// +/// +/// +sealed class YBPoolingWrapperDataSource: PoolingDataSource +{ + static readonly object lockObject = new object(); + ConcurrentDictionary connStringToConnectorsMap = null!; + ConcurrentDictionary connStringToIdleConnectorsMap = null!; + + internal YBPoolingWrapperDataSource(NpgsqlConnectionStringBuilder settings, NpgsqlDataSourceConfiguration dataSourceConfiguration) : + base(settings, dataSourceConfiguration) + { + connStringToConnectorsMap = new ConcurrentDictionary(); + connStringToIdleConnectorsMap = new ConcurrentDictionary(); + + } + + internal override async ValueTask OpenNewConnector( + NpgsqlConnection conn, NpgsqlTimeout timeout, bool async, CancellationToken cancellationToken, NpgsqlConnectionStringBuilder originalConnString) + { + ConfiguredValueTaskAwaitable awaitableconnector; + NpgsqlConnector? connector = null; + lock (lockObject) + { + var originalConnStringCopy = originalConnString.Clone(); + /* + * The connectors are created using the Settings initialized in the pool. + * When connection strings are different, it will still use the first connection string which was initialized in the pool to create the connection. + * So check if connection string is same as the original connection string + * if not replace the host in the original conn string with the chosen host in the pool Settings + */ + if (!Settings.Equals(originalConnStringCopy)) + { + originalConnStringCopy.Host = Settings.Host; + Settings = originalConnStringCopy; + } + // 1. Call base logic → this increments _numConnectors and populates Connectors[] + awaitableconnector = base.OpenNewConnector(conn, timeout, async, cancellationToken).ConfigureAwait(false); + } + + connector = await awaitableconnector; + + if (connector is not null) + { + lock (lockObject) + { + if (connStringToConnectorsMap.TryGetValue(originalConnString, out var list)) + { + for (var i = 0; i < MaxConnections; i++) + if (Interlocked.CompareExchange(ref list[i], connector, null) == null) + break; + connStringToConnectorsMap[originalConnString] = list; + } + else + { + list = new NpgsqlConnector[MaxConnections]; + for (var i = 0; i < MaxConnections; i++) + if (Interlocked.CompareExchange(ref list[i], connector, null) == null) + break; + connStringToConnectorsMap[originalConnString] = list; + } + } + + } + + return connector; + } + + internal override bool TryGetIdleConnector(NpgsqlConnectionStringBuilder originalConnString, [NotNullWhen(true)] out NpgsqlConnector? connector) + + { + connector = null; + + // Try to get the array for this conn string + if (!connStringToIdleConnectorsMap.TryGetValue(originalConnString, out var connectors)) + return false; + + // Fast scan for the first non-null, *idle* connector + // (assuming "idle" means connector.State == Idle or similar) + lock (lockObject) + { + for (var i = 0; i < connectors.Length; i++) + { + var c = connectors[i]; + if (c is null) + continue; // skip nulls quickly + + if (CheckIdleConnector(c)) + { + connector = c; + for (var j = 0; j < MaxConnections; j++) + if (Interlocked.CompareExchange(ref connectors[j], null, connector) == connector) + break; + connStringToIdleConnectorsMap[originalConnString] = connectors; + if (connStringToConnectorsMap.TryGetValue(originalConnString, out var list)) + { + for (var j = 0; j < MaxConnections; j++) + if (Interlocked.CompareExchange(ref list[j], connector, null) == null) + break; + connStringToConnectorsMap[originalConnString] = list; + } + return true; + } + } + } + + return false; + } + + internal override void Return(NpgsqlConnector connector) + { + var flag = 0; + lock (lockObject) + { + foreach (var connStringToConnectors in connStringToConnectorsMap) + { + for (var i = 0; i < connStringToConnectors.Value.Length; i++) + { + if (connStringToConnectors.Value[i] == null) + continue; + if (ReferenceEquals(connStringToConnectors.Value[i], connector)) + { + + if (connStringToConnectorsMap.TryGetValue(connStringToConnectors.Key, out var connectorslist)) + { + for (var j = 0; j < MaxConnections; j++) + if (Interlocked.CompareExchange(ref connectorslist[j], null, connector) == connector) + break; + connStringToConnectorsMap[connStringToConnectors.Key] = connectorslist; + + } + if (connStringToIdleConnectorsMap.TryGetValue(connStringToConnectors.Key, out var list)) + { + for (var j = 0; j < MaxConnections; j++) + if (Interlocked.CompareExchange(ref list[j], connector, null) == null) + break; + connStringToIdleConnectorsMap[connStringToConnectors.Key] = list; + } + else + { + list = new NpgsqlConnector[MaxConnections]; + for (var j = 0; j < MaxConnections; j++) + if (Interlocked.CompareExchange(ref list[j], connector, null) == null) + break; + connStringToIdleConnectorsMap[connStringToConnectors.Key] = list; + } + + flag = 1; + break; + } + } + if (flag == 1) + break; + } + } + + base.Return(connector); + + } + + +} diff --git a/test/Npgsql.Tests/YBClusterAwareRRSupportTests.cs b/test/Npgsql.Tests/YBClusterAwareRRSupportTests.cs index 3b40e9f2fe..60c8230183 100644 --- a/test/Npgsql.Tests/YBClusterAwareRRSupportTests.cs +++ b/test/Npgsql.Tests/YBClusterAwareRRSupportTests.cs @@ -23,11 +23,6 @@ public async Task TestOnlyPrimary() conns = await CreateConnections(connStringBuilder, numConns, new []{numConns / 3, numConns / 3, numConns / 3, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } finally { CloseConnections(conns); @@ -47,11 +42,6 @@ public async Task TestPreferPrimary() conns = await CreateConnections(connStringBuilder, numConns, new []{numConns / 3, numConns / 3, numConns / 3, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } finally { CloseConnections(conns); @@ -65,29 +55,15 @@ public async Task TestPreferPrimaryAllNodesDown() List conns = new List(); CreateRRCluster(); // Stop node: 127.0.0.1, 127.0.0.2, 127.0.0.3 - - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 1"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); - cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); - cmd = "/bin/yb-ctl stop_node 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 1", "stop node 1"); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 3", "stop node 3"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{-1, -1, -1, numConns / 3, numConns / 3, numConns / 3}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } finally { CloseConnections(conns); @@ -106,11 +82,7 @@ public async Task TestOnlyRR() conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, numConns / 3, numConns / 3, numConns / 3}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -130,11 +102,6 @@ public async Task TestPreferRR() conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, numConns / 3, numConns / 3, numConns / 3}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } finally { CloseConnections(conns); @@ -149,28 +116,14 @@ public async Task TestPreferRRAllNodesDown() List conns = new List(); CreateRRCluster(); // Stop node : 127.0.0.4, 127.0.0.5, 127.0.0.6 - - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); - cmd = "/bin/yb-ctl stop_node 5"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); - cmd = "/bin/yb-ctl stop_node 6"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); + ExecuteShellCommand("/bin/yb-ctl stop_node 6", "stop node 6"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{numConns / 3, numConns / 3, numConns / 3, -1, -1, -1}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } finally { CloseConnections(conns); @@ -189,11 +142,6 @@ public async Task TestAny() conns = await CreateConnections(connStringBuilder, numConns, new []{numConns / 6, numConns / 6, numConns / 6, numConns / 6, numConns / 6, numConns / 6}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } finally { CloseConnections(conns); @@ -237,31 +185,18 @@ static async Task> CreateConnections(string connString, i void CreateRRCluster() { - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl create --rf 3 --placement_info cloud1.datacenter1.rack1,cloud1.datacenter2.rack1,cloud1.datacenter3.rack1 --tserver_flags \"placement_uuid=live,max_stale_read_bound_time_ms=60000000\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/build/latest/bin/yb-admin --master_addresses 127.0.0.1:7100,127.0.0.2:7100,127.0.0.3:7100 modify_placement_info cloud1.datacenter1.rack1,cloud1.datacenter2.rack1,cloud1.datacenter3.rack1 3 live"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl add_node --placement_info cloud1.datacenter2.rack1 --tserver_flags placement_uuid=rr"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl add_node --placement_info cloud1.datacenter3.rack1 --tserver_flags placement_uuid=rr"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl add_node --placement_info cloud1.datacenter4.rack1 --tserver_flags placement_uuid=rr"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); + ExecuteShellCommand("/bin/yb-ctl create --rf 3 --placement_info cloud1.datacenter1.rack1,cloud1.datacenter2.rack1,cloud1.datacenter3.rack1 --tserver_flags \"placement_uuid=live,max_stale_read_bound_time_ms=60000000\"", "create cluster"); + ExecuteShellCommand("/bin/yb-admin --master_addresses 127.0.0.1:7100,127.0.0.2:7100,127.0.0.3:7100 modify_placement_info cloud1.datacenter1.rack1,cloud1.datacenter2.rack1,cloud1.datacenter3.rack1 3 live", "modify placement info"); + ExecuteShellCommand("/bin/yb-ctl add_node --placement_info cloud1.datacenter2.rack1 --tserver_flags placement_uuid=rr", "add RR node (datacenter2)"); + ExecuteShellCommand("/bin/yb-ctl add_node --placement_info cloud1.datacenter3.rack1 --tserver_flags placement_uuid=rr", "add RR node (datacenter3)"); + ExecuteShellCommand("/bin/yb-ctl add_node --placement_info cloud1.datacenter4.rack1 --tserver_flags placement_uuid=rr", "add RR node (datacenter4)"); + System.Threading.Thread.Sleep(5000); } protected void DestroyCluster() { - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl destroy"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); } void CloseConnections(List conns) diff --git a/test/Npgsql.Tests/YBFallBackTopologyExtended.cs b/test/Npgsql.Tests/YBFallBackTopologyExtended.cs index 705ba59bb2..7015fe65ea 100644 --- a/test/Npgsql.Tests/YBFallBackTopologyExtended.cs +++ b/test/Npgsql.Tests/YBFallBackTopologyExtended.cs @@ -10,45 +10,37 @@ namespace YBNpgsql.Tests; public class YBFallBackTopologyExtended : YBFallbackTopolgyTests { string connStringBuilder = "host=127.0.0.1,127.0.0.5,127.0.0.7;port=5433;database=yugabyte;userid=yugabyte;password=yugsbyte;Load Balance Hosts=true;Timeout=0;YB Servers Refresh Interval=10;Topology Keys="; - int numConnections = 12; + int numConnections = 18; new void CreateCluster() { - string? _Output = null; - string? _Error = null; + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); var cmd = "/bin/yb-ctl start --rf 3 --placement_info \"aws.us-west.us-west-1a,aws.us-west.us-west-1a,aws.us-west.us-west-1a\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "start cluster"); cmd = "/bin/yb-ctl add_node --placement_info \"aws.us-east.us-east-2a\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "add node (us-east-2a)"); cmd = "/bin/yb-ctl add_node --placement_info \"aws.us-east.us-east-2b\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "add node (us-east-2b)"); cmd = "/bin/yb-ctl add_node --placement_info \"aws.us-east.us-east-2c\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "add node (us-east-2c)"); } void startYBDBClusterWithNineNodes() { - string? _Output = null; - string? _Error = null; - ExecuteShellCommand( "/bin/yb-ctl destroy", ref _Output, ref _Error ); + ExecuteShellCommand( "/bin/yb-ctl destroy", "destroy cluster"); ExecuteShellCommand( "/bin/yb-ctl --rf 3 start --placement_info \"aws.us-west.us-west-1a\" ", - ref _Output, ref _Error ); + "start cluster"); ExecuteShellCommand( "/bin/yb-ctl add_node --placement_info \"aws.us-east.us-east-2a\"", - ref _Output, ref _Error ); + "add node (us-east-2a)"); ExecuteShellCommand( "/bin/yb-ctl add_node --placement_info \"aws.us-east.us-east-2a\"", - ref _Output, ref _Error ); + "add node (us-east-2a)"); ExecuteShellCommand( "/bin/yb-ctl add_node --placement_info \"aws.eu-north.eu-north-2a\"", - ref _Output, ref _Error ); - + "add node (eu-north-2a)"); ExecuteShellCommand( "/bin/yb-ctl add_node --placement_info \"aws.eu-west.eu-west-2a\"", - ref _Output, ref _Error ); + "add node (eu-west-2a)"); ExecuteShellCommand( "/bin/yb-ctl add_node --placement_info \"aws.eu-west.eu-west-2a\"", - ref _Output, ref _Error ); + "add node (eu-west-2a)"); ExecuteShellCommand( "/bin/yb-ctl add_node --placement_info \"aws.eu-north.eu-north-2a\"", - ref _Output, ref _Error ); + "add node (eu-north-2a)"); Thread.Sleep(5000); } @@ -76,134 +68,127 @@ async Task createConnectionsWithoutCloseAndVerify(string tkValue, int[] counts) public async Task TestFallback() { - string? _Output = null; - string? _Error = null; - CreateCluster(); int[] count = { 4, 4, 4, 0, 0, 0 }; var conns = await CreateConnections(connStringBuilder+"aws.us-west.us-west-1a:1,aws.us-east.us-east-2a:2,aws.us-east.us-east-2b:3,aws.us-east.us-east-2c:4", count); var cmd = "/bin/yb-ctl stop_node 1"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "stop node 1"); cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "stop node 2"); cmd = "/bin/yb-ctl stop_node 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "stop node 3"); cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "stop node 4"); + + Thread.Sleep(15000); count = new[] { -1, -1, -1, -1, 12, 0 }; conns = await CreateConnections(connStringBuilder+"aws.us-west.us-west-1a:1,aws.us-east.us-east-2a:2,aws.us-east.us-east-2b:3,aws.us-east.us-east-2c:4", count); cmd = "/bin/yb-ctl start_node 4 --placement_info \"aws.us-east.us-east-2a\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "start node 4 (aws.us-east.us-east-2a)"); Thread.Sleep(15000); count = new[] { -1, -1, -1, 12, 0, 0 }; conns = await CreateConnections(connStringBuilder+"aws.us-west.us-west-1a:1,aws.us-east.us-east-2a:2,aws.us-east.us-east-2b:3,aws.us-east.us-east-2c:4", count); cmd = "/bin/yb-ctl start_node 1 --placement_info \"aws.us-west.us-west-1a\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "start node 1 (aws.us-west.us-west-1a)"); cmd = "/bin/yb-ctl start_node 2 --placement_info \"aws.us-west.us-west-1a\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "start node 2 (aws.us-west.us-west-1a)"); Thread.Sleep(15000); - count = new[] { 6, 6, -1, -1, -1, -1 }; + count = new[] { 6, 6, -1, 0, 0, 0 }; conns = await CreateConnections(connStringBuilder+"aws.us-west.us-west-1a:1,aws.us-east.us-east-2a:2,aws.us-east.us-east-2b:3,aws.us-east.us-east-2c:4", count); DestroyCluster(); } - [Test] + [Test, Timeout(240000)] public async Task CheckMultiNodeDown(){ // Start RF=3 cluster with 9 nodes and with placements (127.0.0.1, 127.0.0.2, 127.0.0.3) -> us-west-1a, // and 127.0.0.4 -> us-east-2a, 127.0.0.5 -> us-east-2a and 127.0.0.6 -> eu-north-2a, 127.0.0.9 -> eu-north-2a, // and 127.0.0.7 -> eu-west-2a, 127.0.0.8 -> eu-west-2a. startYBDBClusterWithNineNodes(); - string? _Output = null; - string? _Error = null; try { await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1,aws.us-east.*:2,aws.eu-west" + ".*:3,aws.eu-north.*:4", new[] { 6, 6, 6, 0, 0, 0, 0, 0, 0 }); - ExecuteShellCommand("/bin/yb-ctl stop_node 1", ref _Output, ref _Error); - ExecuteShellCommand("/bin/yb-ctl stop_node 2", ref _Output, ref _Error); - ExecuteShellCommand("/bin/yb-ctl stop_node 3", ref _Output, ref _Error); - ExecuteShellCommand("/bin/yb-ctl stop_node 4", ref _Output, ref _Error); - ExecuteShellCommand("/bin/yb-ctl stop_node 5", ref _Output, ref _Error); - ExecuteShellCommand("/bin/yb-ctl stop_node 7", ref _Output, ref _Error); - ExecuteShellCommand("/bin/yb-ctl stop_node 8", ref _Output, ref _Error); + ExecuteShellCommand("/bin/yb-ctl stop_node 1", "stop node 1"); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 3", "stop node 3"); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); + ExecuteShellCommand("/bin/yb-ctl stop_node 7", "stop node 7"); + ExecuteShellCommand("/bin/yb-ctl stop_node 8", "stop node 8"); await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1,aws.us-east.*:2,aws.eu-west" + ".*:3,aws.eu-north.*:4", new[] { -1, -1, -1, -1, -1, 9, -1, -1, 9 }); - ExecuteShellCommand("/bin/yb-ctl stop_node 9", ref _Output, ref _Error); + ExecuteShellCommand("/bin/yb-ctl stop_node 9", "stop node 9"); await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1,aws.us-east.*:2,aws.eu-west" + ".*:3,aws.eu-north.*:4", new[] { -1, -1, -1, -1, -1, 27, -1, -1, -1 }); ExecuteShellCommand("/bin/yb-ctl start_node 2 --placement_info \"aws.us-west.us-west-1a\"", - ref _Output, ref _Error); + "start node 2 (aws.us-west.us-west-1a)"); Thread.Sleep(15000); await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1,aws.us-east.*:2,aws.eu-west" + ".*:3,aws.eu-north.*:4", new[] { -1, 18, -1, -1, -1, 27, -1, -1, -1 }); - ExecuteShellCommand("/bin/yb-ctl stop_node 2", ref _Output, ref _Error); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1,aws.us-east.*:2,aws.eu-west" + ".*:3,aws.eu-north.*:4", new[]{-1, -1, -1, -1, -1, 45, -1, -1, -1 }); - ExecuteShellCommand("/bin/yb-ctl start_node 5 --placement_info \"aws.us-east.us-east-2a\"", ref _Output, ref _Error ); + ExecuteShellCommand("/bin/yb-ctl start_node 5 --placement_info \"aws.us-east.us-east-2a\"", "start node 5 (aws.us-east.us-east-2a)"); Thread.Sleep(15000); await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1,aws.us-east.*:2,aws.eu-west" + ".*:3,aws.eu-north.*:4", new[]{-1, -1, -1, -1, 18, 45, -1, -1, -1}); - ExecuteShellCommand("/bin/yb-ctl stop_node 5", ref _Output, ref _Error ); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1,aws.us-east.*:2,aws.eu-west" + ".*:3,aws.eu-north.*:4", new[]{-1, -1, -1, -1, -1, 63, -1, -1, -1}); } finally { - ExecuteShellCommand("/bin/yb-ctl destroy", ref _Output, ref _Error ); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); } } - [Test] - private async Task checkNodeDownPrimary() { - string? _Output = null; - string? _Error = null; + [Test, Timeout(240000)] + public async Task checkNodeDownPrimary() { + + var savedConnString = connStringBuilder; + connStringBuilder = "host=127.0.0.1;port=5433;database=yugabyte;userid=yugabyte;password=yugsbyte;Load Balance Hosts=true;Timeout=0;YB Servers Refresh Interval=10;Topology Keys="; - ExecuteShellCommand("/bin/yb-ctl destroy", ref _Output, ref _Error); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); ExecuteShellCommand("/bin/yb-ctl --rf 3 start --placement_info \"aws.us-west.us-west-1a\" ", - ref _Output, ref _Error); + "start cluster"); + Thread.Sleep(15000); try { await createConnectionsWithoutCloseAndVerify( "aws.us-west.*:1", new[]{6, 6, 6}); - ExecuteShellCommand("/bin/yb-ctl stop_node 1", ref _Output, ref _Error); + ExecuteShellCommand("/bin/yb-ctl stop_node 1", "stop node 1"); await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1", new[]{-1, 15, 15}); ExecuteShellCommand("/bin/yb-ctl start_node 1 --placement_info \"aws.us-west.us-west-1a\"", - ref _Output, ref _Error); + "start node 1 (aws.us-west.us-west-1a)"); ClusterAwareDataSource.forceRefresh = true; Thread.Sleep(5000); await createConnectionsWithoutCloseAndVerify("aws.us-west.*:1", new[]{16, 16, 16}); } finally { - ExecuteShellCommand("/bin/yb-ctl destroy", ref _Output, ref _Error); + connStringBuilder = savedConnString; + ExecuteShellCommand("/bin/yb-ctl status", "cluster status"); } } } diff --git a/test/Npgsql.Tests/YBFallbackTopolgyTests.cs b/test/Npgsql.Tests/YBFallbackTopolgyTests.cs index a0a780ac0c..e69f0e011e 100644 --- a/test/Npgsql.Tests/YBFallbackTopolgyTests.cs +++ b/test/Npgsql.Tests/YBFallbackTopolgyTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Net; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -12,7 +13,7 @@ public class YBFallbackTopolgyTests : YBTestUtils static int mlock = 0; string connStringBuilder = "host=127.0.0.1;port=5433;database=yugabyte;userid=yugabyte;password=yugsbyte;Load Balance Hosts=true;Timeout=0;Topology Keys="; - [Test] + [Test, Timeout(60000)] public async Task TestFallback1() { CreateCluster(); @@ -20,7 +21,7 @@ public async Task TestFallback1() CloseConnections(conns); DestroyCluster(); } - [Test] + [Test, Timeout(60000)] public async Task TestFallback2() { CreateCluster(); @@ -29,7 +30,7 @@ public async Task TestFallback2() DestroyCluster(); } - [Test] + [Test, Timeout(60000)] public async Task TestFallback3() { CreateCluster(); @@ -38,7 +39,7 @@ public async Task TestFallback3() DestroyCluster(); } - [Test] + [Test, Timeout(60000)] public async Task TestFallback4() { CreateCluster(); @@ -47,7 +48,7 @@ public async Task TestFallback4() DestroyCluster(); } - [Test] + [Test, Timeout(60000)] public async Task TestFallback5() { CreateCluster(); @@ -58,11 +59,7 @@ public async Task TestFallback5() await VerifyOn("127.0.0.1", 1); - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 1"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 1", "stop node 1"); var conns = await CreateConnections(connString, new[]{-1, 12, 0}); @@ -70,7 +67,7 @@ public async Task TestFallback5() DestroyCluster(); } - [Test] + [Test, Timeout(60000)] public async Task TestFallback6() { CreateCluster(); @@ -81,11 +78,7 @@ public async Task TestFallback6() await VerifyOn("127.0.0.1", 1); - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 1"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 1", "stop node 1"); var conns = await CreateConnections(connString, new[]{-1, 6, 6}); CloseConnections(conns); @@ -143,6 +136,12 @@ static void CloseConnections(Listconns) conn.Close(); } } + // conn.Close() with pooling enabled returns the connection to the pool + // rather than closing the physical TCP connection. ClearAllPools() + // forces physical disconnection, and the sleep lets the server + // register the disconnections before we verify counts via rpcz. + NpgsqlConnection.ClearAllPools(); + Thread.Sleep(1000); VerifyLocal("127.0.0.1", 0); VerifyLocal("127.0.0.2", 0); VerifyLocal("127.0.0.3", 0); @@ -150,19 +149,15 @@ static void CloseConnections(Listconns) protected void CreateCluster() { - string? _Output = null; - string? _Error = null; + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); var cmd = "/bin/yb-ctl start --rf 3 --placement_info \"aws.us-west.us-west-2a,aws.us-west.us-west-2b,aws.us-west.us-west-2c\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand(cmd, "start cluster"); + System.Threading.Thread.Sleep(5000); } protected void DestroyCluster() { - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl destroy"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); } } diff --git a/test/Npgsql.Tests/YBLoadBalancerTests.cs b/test/Npgsql.Tests/YBLoadBalancerTests.cs index 69831667cf..da26b00fb9 100644 --- a/test/Npgsql.Tests/YBLoadBalancerTests.cs +++ b/test/Npgsql.Tests/YBLoadBalancerTests.cs @@ -12,7 +12,7 @@ public class YBLoadBalancerTests : YBTestUtils { int numConns = 6; - [Test] + [Test, Timeout(60000)] public async Task TestLoadBalance1() { var connStringBuilder = "host=127.0.0.1;database=yugabyte;userid=yugabyte;password=yugsbyte;Load Balance Hosts=any;Timeout=0"; @@ -28,11 +28,7 @@ public async Task TestLoadBalance1() await VerifyOn("127.0.0.3", numConns / 3); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { foreach (var conn in conns) @@ -47,7 +43,7 @@ public async Task TestLoadBalance1() } } - [Test] + [Test, Timeout(60000)] public async Task TestLoadBalance2() { var connStringBuilder = "host=127.0.0.1;port=5433;database=yugabyte;userid=yugabyte;password=yugsbyte;Load Balance Hosts=true;YB Servers Refresh Interval=30;Timeout=0"; @@ -59,13 +55,7 @@ public async Task TestLoadBalance2() var conn1 = CreateConnections(connStringBuilder, numConns); conns.AddRange(conn1); - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 1"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine(_Output); - - System.Threading.Thread.Sleep(30000); + ExecuteShellCommand("/bin/yb-ctl stop_node 1", "stop node 1"); var conn2 = CreateConnections(connStringBuilder, numConns); conns.AddRange(conn2); @@ -73,11 +63,7 @@ public async Task TestLoadBalance2() await VerifyOn("127.0.0.2", 5); await VerifyOn("127.0.0.3", 5); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { foreach (var conn in conns) @@ -93,7 +79,7 @@ public async Task TestLoadBalance2() } } - [Test] + [Test, Timeout(60000)] public async Task TestLoadBalance3() { var connStringBuilder = "host=127.0.0.1;port=5433;database=yugabyte;userid=yugabyte;password=yugsbyte;Load Balance Hosts=true;Timeout=0"; @@ -130,11 +116,7 @@ public async Task TestLoadBalance3() await VerifyOn("127.0.0.2", numThreads * numConns/3); await VerifyOn("127.0.0.3", numThreads * numConns / 3); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { Console.WriteLine("Conns count" + allConns.Count); @@ -151,19 +133,14 @@ public async Task TestLoadBalance3() void CreateCluster() { - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl create --rf 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); + ExecuteShellCommand("/bin/yb-ctl create --rf 3", "create cluster"); + System.Threading.Thread.Sleep(5000); } void DestroyCluster() { - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl destroy"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); } static List CreateConnections(string connString, int numConns) diff --git a/test/Npgsql.Tests/YBPoolingWrapperTests.cs b/test/Npgsql.Tests/YBPoolingWrapperTests.cs new file mode 100644 index 0000000000..9764d75fd3 --- /dev/null +++ b/test/Npgsql.Tests/YBPoolingWrapperTests.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace YBNpgsql.Tests; + +public class YBPoolingWrapperTests : YBTestUtils +{ + [Test, Timeout(60000)] + public void TestPoolingForMultipleConnStrings() + { + var connStringBuilder1 = "host=127.0.0.1;database=yugabyte;userid=postgres;password=postgres;Load Balance Hosts=any;Timeout=0;"; + var connStringBuilder2 = "host=127.0.0.1;database=yugabyte;userid=tester;password=abc123;Load Balance Hosts=any;Timeout=0;"; + + + List conns = new List(); + CreateCluster(); + + NpgsqlConnection conn1 = new NpgsqlConnection(connStringBuilder1); + NpgsqlConnection conn2 = new NpgsqlConnection(connStringBuilder2); + + try + { + NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder1); + conn.Open(); + conn.Close(); + conn1.Open(); + for (var i = 0; i < 6; i++) + { + NpgsqlConnection conn3 = new NpgsqlConnection(connStringBuilder1); + conn3.Open(); + conns.Add(conn3); + } + conn2.Open(); + + NpgsqlCommand cmd1 = new NpgsqlCommand("SELECT current_user;", conn1); + NpgsqlDataReader reader1 = cmd1.ExecuteReader(); + while (reader1.Read()) + { + Assert.AreEqual(reader1.GetString(0), "postgres"); + } + + NpgsqlCommand cmd2 = new NpgsqlCommand("SELECT current_user;", conn2); + NpgsqlDataReader reader2 = cmd2.ExecuteReader(); + while (reader2.Read()) + { + Assert.AreEqual(reader2.GetString(0), "tester"); + } + + Console.WriteLine("Connections Created"); + } + finally + { + foreach (var conn in conns) + { + conn.Close(); + } + + conn1.Close(); + conn2.Close(); + DestroyCluster(); + } + } + + [Test, Timeout(60000)] + public void TestPoolingForMultipleConnStringsWithTopologyKeys() + { + var connStringBuilder1 = "host=127.0.0.1;database=yugabyte;userid=postgres;password=postgres;Load Balance Hosts=any;Topology Keys=cloud1.datacenter1.rack1:1;Timeout=0;"; + var connStringBuilder2 = "host=127.0.0.1;database=yugabyte;userid=tester;password=abc123;Load Balance Hosts=any;Topology Keys=cloud1.datacenter1.rack1:1;Timeout=0;"; + + + List conns = new List(); + CreateCluster(); + + NpgsqlConnection conn1 = new NpgsqlConnection(connStringBuilder1); + NpgsqlConnection conn2 = new NpgsqlConnection(connStringBuilder2); + + try + { + NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder1); + conn.Open(); + conn.Close(); + conn1.Open(); + for (var i = 0; i < 6; i++) + { + NpgsqlConnection conn3 = new NpgsqlConnection(connStringBuilder1); + conn3.Open(); + conns.Add(conn3); + } + conn2.Open(); + + NpgsqlCommand cmd1 = new NpgsqlCommand("SELECT current_user;", conn1); + NpgsqlDataReader reader1 = cmd1.ExecuteReader(); + while (reader1.Read()) + { + Assert.AreEqual(reader1.GetString(0), "postgres"); + } + + NpgsqlCommand cmd2 = new NpgsqlCommand("SELECT current_user;", conn2); + NpgsqlDataReader reader2 = cmd2.ExecuteReader(); + while (reader2.Read()) + { + Assert.AreEqual(reader2.GetString(0), "tester"); + } + + Console.WriteLine("Connections Created"); + } + finally + { + foreach (var conn in conns) + { + conn.Close(); + } + + conn1.Close(); + conn2.Close(); + DestroyCluster(); + } + } + + [Test, Timeout(60000)] + public void TestPoolingForMultipleConnStringsMultiThread() + { + var connStringBuilder1 = "host=127.0.0.1;database=yugabyte;userid=postgres;password=postgres;Load Balance Hosts=any;Timeout=0;"; + var connStringBuilder2 = "host=127.0.0.1;database=yugabyte;userid=tester;password=abc123;Load Balance Hosts=any;Timeout=0;"; + + List conns1 = new List(); + List conns2 = new List(); + + CreateCluster(); + try + { + List threads = new List(); + List conn = new List(); + var numThreads = 5; + for (var i = 0; i < numThreads; i++) + { + Thread thread = new Thread(() => { + var threadConns = CreateConnections(connStringBuilder1, 6); // Each thread uses its own list + lock (conns1) + { + conns1.AddRange(threadConns); + } + }); + threads.Add(thread); + } + for (var i = 0; i < numThreads; i++) + { + Thread thread = new Thread(() => { + var threadConns = CreateConnections(connStringBuilder2, 6); // Each thread uses its own list + lock (conns2) + { + conns2.AddRange(threadConns); // Safely add to the shared list + } + }); + threads.Add(thread); + } + + foreach (var thread in threads) + { + thread.Start(); + } + + foreach (var thread in threads) + { + thread.Join(); + } + + foreach (var conn1 in conns1) + { + NpgsqlCommand cmd1 = new NpgsqlCommand("SELECT current_user;", conn1); + NpgsqlDataReader reader1 = cmd1.ExecuteReader(); + while (reader1.Read()) + { + Assert.AreEqual("postgres", reader1.GetString(0)); + } + } + + foreach (var conn2 in conns2) + { + NpgsqlCommand cmd2 = new NpgsqlCommand("SELECT current_user;", conn2); + NpgsqlDataReader reader2 = cmd2.ExecuteReader(); + while (reader2.Read()) + { + Assert.AreEqual("tester",reader2.GetString(0) ); + } + } + + } + finally + { + foreach (var conn in conns1) + { + conn.Close(); + } + foreach (var conn in conns2) + { + conn.Close(); + } + DestroyCluster(); + } + } + static List CreateConnections(string connString, int numConns) + { + List conns = new List(); + try + { + for (var i = 1; i <= numConns; i++) + { + NpgsqlConnection conn = new NpgsqlConnection(connString); + conn.Open(); + conns.Add(conn); + } + + Console.WriteLine("Connections Created"); + } + catch (Exception ex) + { + Console.WriteLine("Failure:" + ex.Message); + Console.WriteLine("Failure stacktrace: " + ex.StackTrace); + return conns; + } + + return conns; + + } + void CreateCluster() + { + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); + var cmd = "/bin/yb-ctl create --rf 3"; + ExecuteShellCommand(cmd, "create cluster"); + System.Threading.Thread.Sleep(5000); + cmd = "/bin/ysqlsh -c \\\"CREATE USER tester WITH PASSWORD 'abc123'\\\""; + ExecuteShellCommand(cmd, "create user tester"); + cmd = "/bin/ysqlsh -c \\\"GRANT ALL PRIVILEGES ON DATABASE \\\"yugabyte\\\" to tester;\\\""; + ExecuteShellCommand(cmd, "grant privileges to tester"); + } + + void DestroyCluster() + { + var cmd = "/bin/yb-ctl destroy"; + ExecuteShellCommand(cmd, "destroy cluster"); + } +} diff --git a/test/Npgsql.Tests/YBPreparedStatementsTest.cs b/test/Npgsql.Tests/YBPreparedStatementsTest.cs index d9e8a245bf..bb1b313499 100644 --- a/test/Npgsql.Tests/YBPreparedStatementsTest.cs +++ b/test/Npgsql.Tests/YBPreparedStatementsTest.cs @@ -1,14 +1,35 @@ using System; +using System.Threading; using System.Threading.Tasks; using YBNpgsqlTypes; using NUnit.Framework; namespace YBNpgsql.Tests; -public class YBPreparedStatementsTest +public class YBPreparedStatementsTest : YBTestUtils { - [Test] + [OneTimeSetUp] + public void SetUp() + { + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); + ExecuteShellCommand("/bin/yb-ctl create --rf 3", "create cluster"); + Thread.Sleep(5000); + + var connStringBuilder = "host=127.0.0.1;port=5433;database=yugabyte;userid=yugabyte;password=yugabyte"; + using var conn = new NpgsqlConnection(connStringBuilder); + conn.Open(); + using var cmd = new NpgsqlCommand("CREATE DATABASE northwind", conn); + try { cmd.ExecuteNonQuery(); } + catch (Exception ex) { Console.WriteLine("northwind DB may already exist: " + ex.Message); } + } + + [OneTimeTearDown] + public void TearDown() + { + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); + } + [Test] public async Task PreparedStatementsTestWithFlagsEnabled() { var connStringBuilder = "host=localhost;port=5433;database=yugabyte;userid=yugabyte;password=yugabyte;Enable Discard Sequences=false;Enable Discard Temp= false;Enable Close All=false;Load Balance Hosts=true;"; @@ -16,6 +37,8 @@ public async Task PreparedStatementsTestWithFlagsEnabled() try { conn.Open(); + using var dropCmd = new NpgsqlCommand("DROP TABLE IF EXISTS employee", conn); + dropCmd.ExecuteNonQuery(); NpgsqlCommand empCreateCmd = new NpgsqlCommand("CREATE TABLE employee (id int PRIMARY KEY,age int);", conn); empCreateCmd.ExecuteNonQuery(); Console.WriteLine("Created table Employee"); @@ -34,16 +57,14 @@ public async Task PreparedStatementsTestWithFlagsEnabled() await empInsertCommand.ExecuteNonQueryAsync(); } - conn.Close(); } - catch (PostgresException e) + finally { - Console.WriteLine(e); + conn.Close(); } } [Test] - public void TypeLoadingTimeTest() { var connStringBuilderWithNoTypeLoading = "host=localhost;port=5433;database=northwind;userid=yugabyte;Enable Discard Sequences=false;Enable Discard Temp= false;Load Balance Hosts=true; Server Compatibility Mode=NoTypeLoading"; diff --git a/test/Npgsql.Tests/YBTestUtils.cs b/test/Npgsql.Tests/YBTestUtils.cs index 72847107ab..df04aa094d 100644 --- a/test/Npgsql.Tests/YBTestUtils.cs +++ b/test/Npgsql.Tests/YBTestUtils.cs @@ -10,51 +10,48 @@ namespace YBNpgsql.Tests; public class YBTestUtils { - public void ExecuteShellCommand(string argument, ref string? _outputMessage, ref string? _errorMessage) -{ - var path = Environment.GetEnvironmentVariable("YBDB_PATH"); - var arguments = path + argument; - // Set process variable - // Provides access to local and remote processes and enables you to start and stop local system processes. - Process? _Process = null; - try + static readonly string YbdbPath = Environment.GetEnvironmentVariable("YBDB_PATH") + ?? throw new ArgumentException("YBDB_PATH not initialized"); + + public void ExecuteShellCommand(string argument, string message) { - ProcessStartInfo startInfo = new ProcessStartInfo() - { - FileName = "/bin/bash", - Arguments = " -c \"" + arguments + " \"", - CreateNoWindow = true, - RedirectStandardOutput = true, - RedirectStandardInput = true, - RedirectStandardError = true, - }; - _Process = new Process() + var arguments = YbdbPath + argument; + Process? process = null; + try { - StartInfo = startInfo, - }; - _Process.Start(); + var startInfo = new ProcessStartInfo() + { + FileName = "/bin/bash", + Arguments = " -c \"" + arguments + " \"", + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardInput = true, + RedirectStandardError = true, + }; + process = new Process() { StartInfo = startInfo }; + Console.WriteLine("Executing command to " + message); + process.Start(); - // Instructs the Process component to wait indefinitely for the associated process to exit. - _errorMessage = _Process.StandardError.ReadToEnd(); - _Process.WaitForExit(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + var output = process.StandardOutput.ReadToEnd(); + process.WaitForExit(); - // Instructs the Process component to wait indefinitely for the associated process to exit. - _outputMessage = _Process.StandardOutput.ReadToEnd(); - _Process.WaitForExit(); - } - catch (Exception _Exception) - { - // Error - Console.WriteLine("Exception caught in process: {0}", _Exception.ToString()); - } - finally - { - // close process and do cleanup - _Process?.Close(); - _Process?.Dispose(); - _Process = null!; + Console.WriteLine("Output:" + output); + if (!string.IsNullOrWhiteSpace(error)) + Console.WriteLine("Error:" + error); + } + catch (Exception ex) + { + Console.WriteLine("Exception caught in process: {0}", ex); + throw; + } + finally + { + process?.Close(); + process?.Dispose(); + } } -} protected static async Task VerifyOn(string server, int ExpectedCount) { diff --git a/test/Npgsql.Tests/YBTopologyAwareRRSupportTests.cs b/test/Npgsql.Tests/YBTopologyAwareRRSupportTests.cs index 9c942dbd43..33e87ca2c5 100644 --- a/test/Npgsql.Tests/YBTopologyAwareRRSupportTests.cs +++ b/test/Npgsql.Tests/YBTopologyAwareRRSupportTests.cs @@ -23,11 +23,6 @@ public async Task TestOnlyPrimary() conns = await CreateConnections(connStringBuilder, numConns, new []{0, numConns, 0, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } finally { CloseConnections(conns); @@ -42,25 +37,14 @@ public async Task TestOnlyPrimaryAllNodesDownAllPlacement() List conns = new List(); CreateRRCluster(); // Stop node : 127.0.0.2, 127.0.0.3 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 3", "stop node 3"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{numConns, -1, -1, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } finally { CloseConnections(conns); @@ -75,14 +59,8 @@ public void TestOnlyPrimaryAllNodesDownInAllPlacementsFallBackToTopologyOnly() List conns = new List(); CreateRRCluster(); // Stop node : 127.0.0.2, 127.0.0.3 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 3", "stop node 3"); try { @@ -94,10 +72,9 @@ public void TestOnlyPrimaryAllNodesDownInAllPlacementsFallBackToTopologyOnly() } } - catch (NpgsqlException ex) + catch (NpgsqlException e) { - if (ex.Message.Equals("No suitable host was found", StringComparison.OrdinalIgnoreCase)) - Console.WriteLine("Expected Failure:" + ex.Message); + Console.WriteLine("Caught expected exception:" + e.Message); } finally { @@ -113,22 +90,14 @@ public async Task TestOnlyPrimaryAllNodesDownPrimarylacement() List conns = new List(); CreateRRCluster(); // Stop node : 127.0.0.2 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{0, -1, numConns, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -148,11 +117,7 @@ public async Task TestPreferPrimary() conns = await CreateConnections(connStringBuilder, numConns, new []{0, numConns, 0, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -167,21 +132,13 @@ public async Task TestPreferPrimaryAllNodesDownPrimaryPlacement() List conns = new List(); CreateRRCluster(); // Stop node : 127.0.0.2 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{0, -1, numConns, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -190,43 +147,30 @@ public async Task TestPreferPrimaryAllNodesDownPrimaryPlacement() [Test, Timeout(60000)] public async Task? TestPreferPrimaryAllPrimaryNodesDown() { - var connStringBuilder = "host=127.0.0.1;database=yugabyte;userid=yugabyte;password=yugabyte;Load Balance Hosts=preferrr;Topology Keys=cloud1.datacenter2.rack1:1,cloud1.datacenter3.rack1:2;Timeout=0"; + var connStringBuilder = "host=127.0.0.1;database=yugabyte;userid=yugabyte;password=yugabyte;Load Balance Hosts=preferprimary;Topology Keys=cloud1.datacenter2.rack1:1,cloud1.datacenter3.rack1:2;Timeout=0"; List conns = new List(); CreateRRCluster(); + + conns = await CreateConnections(connStringBuilder, numConns, new []{0, numConns, 0, 0, 0, 0}); + // Stop Node: 127.0.0.1, 127.0.0.2, 127.0.0.3 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 1"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 1", "stop node 1"); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 3", "stop node 3"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{-1, -1, -1, numConns / 3, numConns / 3, numConns / 3}); // Start Node 1 - _Output = null; - _Error = null; - cmd = "/bin/yb-ctl start_node 1"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl start_node 1", "start node 1"); Thread.Sleep(10000); - conns.Concat(await CreateConnections(connStringBuilder, numConns, new []{numConns, -1, -1, numConns / 3, numConns / 3, numConns / 3})); + conns.AddRange(await CreateConnections(connStringBuilder, numConns, new []{numConns, -1, -1, numConns / 3, numConns / 3, numConns / 3})); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -241,25 +185,15 @@ public async Task TestPreferPrimaryAllNodesDownAllPlacement() List conns = new List(); CreateRRCluster(); // Stop node : 127.0.0.2, 127.0.0.3 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 3", "stop node 3"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{numConns, -1, -1, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -277,11 +211,7 @@ public async Task TestOnlyRR() { conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, numConns, 0, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -296,21 +226,13 @@ public async Task TestOnlyRRAllNodesDownInPrimaryPlacement() List conns = new List(); CreateRRCluster(); // Stop Node: 127.0.0.4 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, -1, numConns, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -325,24 +247,14 @@ public async Task TestOnlyRRAllNodesDownInAllPlacement() List conns = new List(); CreateRRCluster(); // Stop Node: 127.0.0.4, 127.0.0.5 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 5"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, -1, -1, numConns}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -357,14 +269,8 @@ public void TestOnlyRRAllNodesDownInAllPlacementsFallBackToTopologyOnly() List conns = new List(); CreateRRCluster(); // Stop Node: 127.0.0.4, 127.0.0.5 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 5"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); try { @@ -376,11 +282,11 @@ public void TestOnlyRRAllNodesDownInAllPlacementsFallBackToTopologyOnly() } } - catch (NpgsqlException ex) + catch (NpgsqlException e) { - if (ex.Message.Equals("No suitable host was found", StringComparison.OrdinalIgnoreCase)) - Console.WriteLine("Expected Failure:" + ex.Message); + Console.WriteLine("Caught expected Exception:" + e.Message); } + finally { CloseConnections(conns); @@ -399,11 +305,7 @@ public async Task TestPreferRR() { conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, numConns, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -418,21 +320,13 @@ public async Task TestPreferRRAllNodesDownInPrimaryPlacement() List conns = new List(); CreateRRCluster(); // Stop Node: 127.0.0.4 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, -1, numConns, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -447,24 +341,14 @@ public async Task TestPreferRRAllNodesDownInAllPlacements() List conns = new List(); CreateRRCluster(); // Stop Node: 127.0.0.4, 127.0.0.5 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 5"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, -1, -1, numConns}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -474,40 +358,30 @@ public async Task TestPreferRRAllNodesDownInAllPlacements() [Test, Timeout(60000)] public async Task? TestPreferRRAllRRNodesDown() { - var connStringBuilder = "host=127.0.0.1;database=yugabyte;userid=yugabyte;password=yugabyte;Load Balance Hosts=preferrr;Topology Keys=cloud1.datacenter2.rack1:1,cloud1.datacenter3.rack1:2;Timeout=0"; + var connStringBuilder = "host=127.0.0.1;database=yugabyte;userid=yugabyte;password=yugabyte;Load Balance Hosts=preferrr;Topology Keys=cloud1.datacenter2.rack1:1,cloud1.datacenter3.rack1:2;Timeout=0;YB Servers Refresh Interval=10"; List conns = new List(); CreateRRCluster(); + + conns = await CreateConnections(connStringBuilder, numConns, new []{0, 0, 0, numConns, 0, 0}); + // Stop Node: 127.0.0.4, 127.0.0.5, 127.0.0.6 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 5"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 6"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); + ExecuteShellCommand("/bin/yb-ctl stop_node 6", "stop node 6"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{numConns / 3, numConns / 3, numConns / 3, -1, -1, -1}); // Start RR node: 127.0.0.4 - cmd = "/bin/yb-ctl start_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl start_node 4", "start node 4"); + Thread.Sleep(15000); - conns.Concat(await CreateConnections(connStringBuilder, numConns, new []{numConns / 3, numConns / 3, numConns / 3, numConns, -1, -1})); + conns.AddRange(await CreateConnections(connStringBuilder, numConns, new []{numConns / 3, numConns / 3, numConns / 3, numConns, -1, -1})); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -525,11 +399,7 @@ public async Task TestAny() { conns = await CreateConnections(connStringBuilder, numConns, new []{0, numConns / 2, 0, numConns / 2, 0, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -544,24 +414,14 @@ public async Task TestAnyAllNodesDownPrimaryPlacement() List conns = new List(); CreateRRCluster(); // Stop Node: 127.0.0.2, 127.0.0.4 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{0, -1, numConns /2, -1, numConns / 2, 0}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -576,20 +436,10 @@ public void TestAnyAllNodesDownAllPlacementFallBackToTopologyOnly() List conns = new List(); CreateRRCluster(); // Stop Node: 127.0.0.2, 127.0.0.3, 127.0.0.4, 127.0.0.5 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 5"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 3", "stop node 3"); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); try { @@ -601,10 +451,9 @@ public void TestAnyAllNodesDownAllPlacementFallBackToTopologyOnly() } } - catch (NpgsqlException ex) + catch (NpgsqlException e) { - if (ex.Message.Equals("No suitable host was found", StringComparison.OrdinalIgnoreCase)) - Console.WriteLine("Expected Failure:" + ex.Message); + Console.WriteLine("Caught Expected Exception:" + e.Message); } finally { @@ -620,30 +469,16 @@ public async Task TestAnyAllNodesDownAllPlacement() List conns = new List(); CreateRRCluster(); // Stop Node: 127.0.0.2, 127.0.0.3, 127.0.0.4, 127.0.0.5 - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl stop_node 2"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 3"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 4"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl stop_node 5"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl stop_node 2", "stop node 2"); + ExecuteShellCommand("/bin/yb-ctl stop_node 3", "stop node 3"); + ExecuteShellCommand("/bin/yb-ctl stop_node 4", "stop node 4"); + ExecuteShellCommand("/bin/yb-ctl stop_node 5", "stop node 5"); try { conns = await CreateConnections(connStringBuilder, numConns, new []{numConns /2 , -1, -1, -1, -1, numConns / 2}); } - catch (Exception ex) - { - Console.WriteLine("Failure:" + ex.Message); - Console.WriteLine("Failure stacktrace: " + ex.StackTrace); - } + finally { CloseConnections(conns); @@ -688,31 +523,18 @@ static async Task> CreateConnections(string connString, i void CreateRRCluster() { - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl create --rf 3 --placement_info cloud1.datacenter1.rack1,cloud1.datacenter2.rack1,cloud1.datacenter3.rack1 --tserver_flags \"placement_uuid=live,max_stale_read_bound_time_ms=60000000\""; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/build/latest/bin/yb-admin --master_addresses 127.0.0.1:7100,127.0.0.2:7100,127.0.0.3:7100 modify_placement_info cloud1.datacenter1.rack1,cloud1.datacenter2.rack1,cloud1.datacenter3.rack1 3 live"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl add_node --placement_info cloud1.datacenter2.rack1 --tserver_flags placement_uuid=rr"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl add_node --placement_info cloud1.datacenter3.rack1 --tserver_flags placement_uuid=rr"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); - cmd = "/bin/yb-ctl add_node --placement_info cloud1.datacenter4.rack1 --tserver_flags placement_uuid=rr"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); - Console.WriteLine("Output:" + _Output); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); + ExecuteShellCommand("/bin/yb-ctl create --rf 3 --placement_info cloud1.datacenter1.rack1,cloud1.datacenter2.rack1,cloud1.datacenter3.rack1 --tserver_flags \"placement_uuid=live,max_stale_read_bound_time_ms=60000000\"", "create cluster"); + ExecuteShellCommand("/bin/yb-admin --master_addresses 127.0.0.1:7100,127.0.0.2:7100,127.0.0.3:7100 modify_placement_info cloud1.datacenter1.rack1,cloud1.datacenter2.rack1,cloud1.datacenter3.rack1 3 live", "modify placement info"); + ExecuteShellCommand("/bin/yb-ctl add_node --placement_info cloud1.datacenter2.rack1 --tserver_flags placement_uuid=rr", "add RR node (datacenter2)"); + ExecuteShellCommand("/bin/yb-ctl add_node --placement_info cloud1.datacenter3.rack1 --tserver_flags placement_uuid=rr", "add RR node (datacenter3)"); + ExecuteShellCommand("/bin/yb-ctl add_node --placement_info cloud1.datacenter4.rack1 --tserver_flags placement_uuid=rr", "add RR node (datacenter4)"); + System.Threading.Thread.Sleep(5000); } protected void DestroyCluster() { - string? _Output = null; - string? _Error = null; - var cmd = "/bin/yb-ctl destroy"; - ExecuteShellCommand(cmd, ref _Output, ref _Error ); + ExecuteShellCommand("/bin/yb-ctl destroy", "destroy cluster"); } void CloseConnections(List conns)