From 067941f3148331bcc2bb06361f697df2e8568cfb Mon Sep 17 00:00:00 2001 From: Hemant Dangi Date: Mon, 22 Jun 2026 17:37:06 +0530 Subject: [PATCH] MDEV-28233: rsync SST script silently runs unencrypted if stunnel is not installed Issue: When ssl-mode required encryption but the means to perform it was missing, the SST scripts silently fell back to a cleartext transfer: - wsrep_sst_rsync: ran over plain TCP when the 'stunnel' binary was absent. - wsrep_sst_mariabackup: socat used a cleartext socket when ssl-mode was set but no usable cert/key was found (encrypt stayed 0). Solution: Abort the SST instead of falling back to an unencrypted transfer when ssl-mode is not DISABLED but encryption cannot be set up: - wsrep_sst_rsync: derive the implicit ssl-mode from the SSL config even when stunnel is absent, then abort with ENOENT if ssl-mode is active and the stunnel binary is not found. - wsrep_sst_mariabackup: after reading the SSL configuration, abort with EINVAL if ssl-mode is not DISABLED but encrypt resolved to 0 (no usable cert/key). --- .../galera_sst_mariabackup_missing_ssl.result | 27 ++++++++ .../r/galera_sst_rsync_missing_stunnel.result | 27 ++++++++ .../t/galera_sst_mariabackup_missing_ssl.cnf | 11 +++ .../t/galera_sst_mariabackup_missing_ssl.test | 69 +++++++++++++++++++ .../t/galera_sst_rsync_missing_stunnel.cnf | 10 +++ .../t/galera_sst_rsync_missing_stunnel.test | 66 ++++++++++++++++++ scripts/wsrep_sst_mariabackup.sh | 15 ++++ scripts/wsrep_sst_rsync.sh | 23 +++++-- 8 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_sst_mariabackup_missing_ssl.result create mode 100644 mysql-test/suite/galera/r/galera_sst_rsync_missing_stunnel.result create mode 100644 mysql-test/suite/galera/t/galera_sst_mariabackup_missing_ssl.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_mariabackup_missing_ssl.test create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_missing_stunnel.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_missing_stunnel.test diff --git a/mysql-test/suite/galera/r/galera_sst_mariabackup_missing_ssl.result b/mysql-test/suite/galera/r/galera_sst_mariabackup_missing_ssl.result new file mode 100644 index 0000000000000..0424988061ea7 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_mariabackup_missing_ssl.result @@ -0,0 +1,27 @@ +connection node_2; +connection node_1; +connection node_1; +connection node_2; +connection node_1; +call mtr.add_suppression("WSREP: .*State transfer to.* failed:"); +call mtr.add_suppression("WSREP: Will never receive state. Need to abort."); +connection node_2; +# Force a fresh SST on node_2 +# Start node_2 with SSL cert/key simulated as missing +# node_2 failed to start, as expected +# Joiner refused the SST instead of running unencrypted +FOUND 1 /ssl-mode is set to .REQUIRED., but no usable SSL/ in mysqld.2.err +# Restart node_2 normally so it rejoins +connection node_2; +call mtr.add_suppression("WSREP_SST:"); +call mtr.add_suppression("WSREP: Process completed with error:"); +call mtr.add_suppression("WSREP: Failed to read .ready ."); +call mtr.add_suppression("WSREP: Failed to read uuid:seqno and wsrep_gtid_domain_id from joiner script."); +call mtr.add_suppression("WSREP: Failed to read state from: .*"); +call mtr.add_suppression("WSREP: Failed to prepare for .*"); +call mtr.add_suppression("WSREP: SST failed:.*"); +call mtr.add_suppression("WSREP: SST request callback failed. This is unrecoverable, restart required."); +call mtr.add_suppression("WSREP: .*State transfer to.* failed:"); +call mtr.add_suppression("WSREP: Will never receive state. Need to abort."); +call mtr.add_suppression("WSREP: Requesting state transfer failed:.*"); +connection node_1; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_missing_stunnel.result b/mysql-test/suite/galera/r/galera_sst_rsync_missing_stunnel.result new file mode 100644 index 0000000000000..771d6e0fd7583 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_rsync_missing_stunnel.result @@ -0,0 +1,27 @@ +connection node_2; +connection node_1; +connection node_1; +connection node_2; +connection node_1; +call mtr.add_suppression("WSREP: .*State transfer to.* failed:"); +call mtr.add_suppression("WSREP: Will never receive state. Need to abort."); +connection node_2; +# Force a fresh SST on node_2 +# Start node_2 with stunnel simulated as missing +# node_2 failed to start, as expected +# Joiner refused the SST instead of running unencrypted +FOUND 1 /ssl-mode is set to .REQUIRED., but the .stunnel. binary was not found/ in mysqld.2.err +# Restart node_2 normally so it rejoins +connection node_2; +call mtr.add_suppression("WSREP_SST:"); +call mtr.add_suppression("WSREP: Process completed with error:"); +call mtr.add_suppression("WSREP: Failed to read .ready ."); +call mtr.add_suppression("WSREP: Failed to read uuid:seqno and wsrep_gtid_domain_id from joiner script."); +call mtr.add_suppression("WSREP: Failed to read state from: .*"); +call mtr.add_suppression("WSREP: Failed to prepare for .*"); +call mtr.add_suppression("WSREP: SST failed:.*"); +call mtr.add_suppression("WSREP: SST request callback failed. This is unrecoverable, restart required."); +call mtr.add_suppression("WSREP: .*State transfer to.* failed:"); +call mtr.add_suppression("WSREP: Will never receive state. Need to abort."); +call mtr.add_suppression("WSREP: Requesting state transfer failed:.*"); +connection node_1; diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_missing_ssl.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup_missing_ssl.cnf new file mode 100644 index 0000000000000..5c9ea2149a3c7 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_missing_ssl.cnf @@ -0,0 +1,11 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=mariabackup +wsrep_sst_auth="root:" +ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem +ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem +ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem + +[sst] +ssl-mode=REQUIRED diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_missing_ssl.test b/mysql-test/suite/galera/t/galera_sst_mariabackup_missing_ssl.test new file mode 100644 index 0000000000000..a636634cd5025 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_missing_ssl.test @@ -0,0 +1,69 @@ +# MDEV-28233: with ssl-mode requiring encryption but no usable cert/key, +# wsrep_sst_mariabackup used to fall back to an unencrypted transfer +# WITHOUT any error or warning. Before the fix the joiner only logged the +# informational line: +# WSREP_SST: [INFO] SSL configuration: ... MODE='REQUIRED' ... encrypt='0' +# and then opened a cleartext socket. +# +# Steps: +# 1. Form a 2-node cluster with ssl-mode=REQUIRED and SSL cert/key. +# 2. Shut node_2 down and force a fresh SST. +# 3. Restart node_2 with cert/key simulated as missing +# (MTR_SST_SIMULATE_NO_SSL_CERT=1). +# 4. Check the joiner refuses the SST and logs the error. +# 5. Restart node_2 normally so it rejoins. +# +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_mariabackup.inc + +# Save auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +# Donor-side noise when the joiner aborts the SST. +call mtr.add_suppression("WSREP: .*State transfer to.* failed:"); +call mtr.add_suppression("WSREP: Will never receive state. Need to abort."); + +--connection node_2 +--source include/shutdown_mysqld.inc + +--echo # Force a fresh SST on node_2 +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat + +--echo # Start node_2 with SSL cert/key simulated as missing +# Abort exit code varies by platform. +--error 1,134 +--exec MTR_SST_SIMULATE_NO_SSL_CERT=1 $MYSQLD_LAST_CMD +--echo # node_2 failed to start, as expected + +--echo # Joiner refused the SST instead of running unencrypted +--let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.2.err +--let SEARCH_PATTERN = ssl-mode is set to .REQUIRED., but no usable SSL +--source include/search_pattern_in_file.inc + +--echo # Restart node_2 normally so it rejoins +--source include/start_mysqld.inc +--source include/wait_until_connected_again.inc + +--connection node_2 +# Joiner-side noise from the failed SST attempt. +call mtr.add_suppression("WSREP_SST:"); +call mtr.add_suppression("WSREP: Process completed with error:"); +call mtr.add_suppression("WSREP: Failed to read .ready ."); +call mtr.add_suppression("WSREP: Failed to read uuid:seqno and wsrep_gtid_domain_id from joiner script."); +call mtr.add_suppression("WSREP: Failed to read state from: .*"); +call mtr.add_suppression("WSREP: Failed to prepare for .*"); +call mtr.add_suppression("WSREP: SST failed:.*"); +call mtr.add_suppression("WSREP: SST request callback failed. This is unrecoverable, restart required."); +call mtr.add_suppression("WSREP: .*State transfer to.* failed:"); +call mtr.add_suppression("WSREP: Will never receive state. Need to abort."); +call mtr.add_suppression("WSREP: Requesting state transfer failed:.*"); + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_missing_stunnel.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_missing_stunnel.cnf new file mode 100644 index 0000000000000..39fbab6cb5cc4 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_missing_stunnel.cnf @@ -0,0 +1,10 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=rsync +ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem +ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem +ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem + +[sst] +ssl-mode=REQUIRED diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_missing_stunnel.test b/mysql-test/suite/galera/t/galera_sst_rsync_missing_stunnel.test new file mode 100644 index 0000000000000..1310c27c6db09 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_missing_stunnel.test @@ -0,0 +1,66 @@ +# MDEV-28233: the rsync SST script must NOT silently fall back to an +# unencrypted transfer when encryption is requested (ssl-mode is set to a +# value other than DISABLED) but the stunnel binary is not installed. +# +# Steps: +# 1. Form a 2-node cluster with ssl-mode=REQUIRED (stunnel present). +# 2. Shut node_2 down and force a fresh SST. +# 3. Restart node_2 with stunnel hidden (MTR_SST_SIMULATE_NO_STUNNEL=1). +# 4. Check the joiner refuses the SST and logs the error. +# 5. Restart node_2 normally so it rejoins. +# +--source include/galera_cluster.inc +--source include/have_innodb.inc +# Needed for the initial SST; only simulated as missing below. +--source include/have_stunnel.inc + +# Save auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +# Donor-side noise when the joiner aborts the SST. +call mtr.add_suppression("WSREP: .*State transfer to.* failed:"); +call mtr.add_suppression("WSREP: Will never receive state. Need to abort."); + +--connection node_2 +--source include/shutdown_mysqld.inc + +--echo # Force a fresh SST on node_2 +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat + +--echo # Start node_2 with stunnel simulated as missing +# Abort exit code +--error 1,134 +--exec MTR_SST_SIMULATE_NO_STUNNEL=1 $MYSQLD_LAST_CMD +--echo # node_2 failed to start, as expected + +--echo # Joiner refused the SST instead of running unencrypted +--let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.2.err +--let SEARCH_PATTERN = ssl-mode is set to .REQUIRED., but the .stunnel. binary was not found +--source include/search_pattern_in_file.inc + +--echo # Restart node_2 normally so it rejoins +--source include/start_mysqld.inc +--source include/wait_until_connected_again.inc + +--connection node_2 +# Joiner-side noise from the failed SST attempt. +call mtr.add_suppression("WSREP_SST:"); +call mtr.add_suppression("WSREP: Process completed with error:"); +call mtr.add_suppression("WSREP: Failed to read .ready ."); +call mtr.add_suppression("WSREP: Failed to read uuid:seqno and wsrep_gtid_domain_id from joiner script."); +call mtr.add_suppression("WSREP: Failed to read state from: .*"); +call mtr.add_suppression("WSREP: Failed to prepare for .*"); +call mtr.add_suppression("WSREP: SST failed:.*"); +call mtr.add_suppression("WSREP: SST request callback failed. This is unrecoverable, restart required."); +call mtr.add_suppression("WSREP: .*State transfer to.* failed:"); +call mtr.add_suppression("WSREP: Will never receive state. Need to abort."); +call mtr.add_suppression("WSREP: Requesting state transfer failed:.*"); + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--source include/auto_increment_offset_restore.inc diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index e3e52d92fcbc3..004c8c21135be 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -565,6 +565,11 @@ read_cnf() if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then check_server_ssl_config fi + # MTR test hook: simulate a missing SSL certificate and key. + if [ -n "${MTR_SST_SIMULATE_NO_SSL_CERT:-}" ]; then + tpem="" + tkey="" + fi if [ "$tmode" != 'DISABLED' ]; then if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ] then @@ -593,6 +598,16 @@ read_cnf() "CERT='$tpem', KEY='$tkey', MODE='$tmode'," \ "encrypt='$encrypt'" + # ssl-mode requires encryption but none could be set up (no usable + # cert/key): abort instead of silently transferring in cleartext. + if [ "$tmode" != 'DISABLED' -a $encrypt -eq 0 ]; then + wsrep_log_error "ssl-mode is set to '$tmode', but no usable SSL" \ + "certificate and key were found. Cannot perform an" \ + "encrypted transfer. Please configure ssl-cert and" \ + "ssl-key, or set ssl-mode to DISABLED." + exit 22 # EINVAL + fi + if [ $encrypt -ge 2 ]; then ssl_dhparams=$(parse_cnf "$encgroups" 'ssl-dhparams') fi diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 53ff8cc196308..6c683a8fdf85d 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -192,14 +192,11 @@ SSTCAP="$tcap" SSLMODE=$(parse_cnf "$encgroups" 'ssl-mode' | tr '[[:lower:]]' '[[:upper:]]') if [ -z "$SSLMODE" ]; then - # Implicit verification if CA is set and the SSL mode - # is not specified by user: + # ssl-mode not set: derive it from the SSL config. Set it even when + # stunnel is absent, so the check below aborts instead of silently + # falling back to an unencrypted transfer. if [ -n "$SSTCA$SSTCAP" ]; then - STUNNEL_BIN=$(commandex 'stunnel') - if [ -n "$STUNNEL_BIN" ]; then - SSLMODE='VERIFY_CA' - fi - # Require SSL by default if SSL key and cert are present: + SSLMODE='VERIFY_CA' elif [ -n "$SSTKEY" -a -n "$SSTCERT" ]; then SSLMODE='REQUIRED' fi @@ -267,10 +264,22 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then if [ -z "${STUNNEL_BIN+x}" ]; then STUNNEL_BIN=$(commandex 'stunnel') fi + # MTR test hook: simulate a missing stunnel binary. + if [ -n "${MTR_SST_SIMULATE_NO_STUNNEL:-}" ]; then + STUNNEL_BIN="" + fi if [ -n "$STUNNEL_BIN" ]; then wsrep_log_info "Using stunnel for SSL encryption: CA: '$SSTCA'," \ "CAPATH='$SSTCAP', ssl-mode='$SSLMODE'" STUNNEL="$STUNNEL_BIN $STUNNEL_CONF" + else + # Encryption required but stunnel missing: abort instead of + # silently falling back to an unencrypted transfer. + wsrep_log_error "ssl-mode is set to '$SSLMODE', but the 'stunnel'" \ + "binary was not found in the path. Cannot perform" \ + "an encrypted transfer. Please install stunnel or" \ + "set ssl-mode to DISABLED." + exit 2 # ENOENT fi fi