MDEV-37605 Extend mariadb-binlog to convert innodb based binary logs to legacy

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result

@@ -0,0 +1,25 @@
+include/reset_master.inc
+set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22");
+*** Generate a small workload into binlog-000000.ibb
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0);
+UPDATE t1 SET b=1 WHERE a=1;
+DELETE FROM t1 WHERE a=2;
+REPLACE INTO t1 VALUES (3, 3);
+SELECT * FROM t1 ORDER BY a;
+a	b
+1	1
+3	3
+FLUSH BINARY LOGS;
+*** Convert binlog-000000.ibb to legacy format
+*** Converted file contains the synthesized header events
+FOUND 1 /Gtid list/ in conv_listing.txt
+FOUND 1 /Binlog checkpoint/ in conv_listing.txt
+FOUND 1 /Start: binlog v 4/ in conv_listing.txt
+*** Round-trip: replay the converted file and compare data
+DROP TABLE t1;
+SELECT * FROM t1 ORDER BY a;
+a	b
+1	1
+3	3
+DROP TABLE t1;

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test

@@ -0,0 +1,61 @@
+# mysqlbinlog_convert.test
+#
+# Purpose:
+#   Basic test for MDEV-37605: mariadb-binlog conversion of InnoDB-format
+#   binlogs (.ibb) into legacy-format binary log 
+#
+
+--source include/have_binlog_format_row.inc
+--source include/have_innodb_binlog.inc
+
+--let $datadir= `SELECT @@datadir`
+
+--source include/reset_master.inc
+
+# Fixed timestamp for deterministic event listings (with --timezone=GMT-3
+# from the -master.opt file, matching binlog_in_engine.mysqlbinlog).
+set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22");
+
+--echo *** Generate a small workload into binlog-000000.ibb
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0);
+UPDATE t1 SET b=1 WHERE a=1;
+DELETE FROM t1 WHERE a=2;
+REPLACE INTO t1 VALUES (3, 3);
+SELECT * FROM t1 ORDER BY a;
+
+# Rotate so binlog-000000.ibb is complete on disk. Waiting for the *next*
+# pre-allocated file (binlog-000002.ibb, fully pre-allocated and empty)
+# guarantees both 000000 and the new active 000001 exist.
+FLUSH BINARY LOGS;
+--let $binlog_name= binlog-000002.ibb
+--let $binlog_size= 262144
+--source include/wait_for_engine_binlog.inc
+
+--echo *** Convert binlog-000000.ibb to legacy format
+--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/conv $datadir/binlog-000000.ibb
+
+
+--echo *** Converted file contains the synthesized header events
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/conv.000001 > $MYSQL_TMP_DIR/conv_listing.txt
+--let SEARCH_FILE= $MYSQL_TMP_DIR/conv_listing.txt
+--let SEARCH_PATTERN= Gtid list
+--source include/search_pattern_in_file.inc
+--let SEARCH_PATTERN= Binlog checkpoint
+--source include/search_pattern_in_file.inc
+--let SEARCH_PATTERN= Start: binlog v 4
+--source include/search_pattern_in_file.inc
+--remove_file $MYSQL_TMP_DIR/conv_listing.txt
+
+--echo *** Round-trip: replay the converted file and compare data
+# The replayed GTIDs duplicate the originals, so the replay must not use
+# GTID strict mode (same as in binlog_in_engine.mysqlbinlog).
+--exec $MYSQL_BINLOG --gtid-strict-mode=0 $MYSQL_TMP_DIR/conv.000001 > $MYSQLTEST_VARDIR/tmp/convert_replay.sql
+DROP TABLE t1;
+--exec $MYSQL --abort-source-on-error -e "source $MYSQLTEST_VARDIR/tmp/convert_replay.sql;" test
+--remove_file $MYSQLTEST_VARDIR/tmp/convert_replay.sql
+SELECT * FROM t1 ORDER BY a;
+
+# Cleanup
+--remove_file $MYSQL_TMP_DIR/conv.000001
+DROP TABLE t1;

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result

@@ -0,0 +1,46 @@
+SET @@session.sql_log_bin= 0;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t4 (a INT PRIMARY KEY) ENGINE=InnoDB;
+SET @@session.sql_log_bin= 1;
+include/reset_master.inc
+*** Generate GTIDs across several InnoDB binlog files
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (2);
+SET @@session.gtid_domain_id= 1;
+SET @@session.server_id= 2;
+INSERT INTO t2 VALUES (1);
+FLUSH BINARY LOGS;
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (1);
+SET @@session.gtid_domain_id= 2;
+SET @@session.server_id= 3;
+INSERT INTO t3 VALUES (1);
+FLUSH BINARY LOGS;
+SET @@session.gtid_domain_id= 1;
+SET @@session.server_id= 1;
+INSERT INTO t2 VALUES (2);
+SET @@session.gtid_domain_id= 3;
+SET @@session.server_id= 4;
+INSERT INTO t4 VALUES (1);
+# restart
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (3);
+FLUSH BINARY LOGS;
+*** Convert the generated InnoDB binlogs to legacy format
+*** Verify synthesized Gtid_list events in converted legacy binlogs
+FOUND 1 /Gtid list \[\]/ in gtid_conv_listing.txt
+FOUND 1 /Gtid list \[0-1-1,\n# 1-2-1\]/ in gtid_conv_listing.txt
+FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\]/ in gtid_conv_listing.txt
+FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt
+*** Convert the generated InnoDB binlog(binlog-000002.ibb) to legacy format in random order
+*** Verify synthesized Gtid_list events in converted legacy binlogs
+NOT FOUND /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;
+DROP TABLE t4;

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test

@@ -0,0 +1,129 @@
+#
+# Purpose:
+#   Verify that mariadb-binlog populates synthesized GTID_LIST_EVENTs correctly
+#   when converting InnoDB-format binlogs (.ibb) to legacy binlog files.
+#
+# Methodology:
+#   Generate several .ibb files with GTIDs across multiple domains and server
+#   ids, including GTIDs that are not ordered by server id. Restart the server
+#   in the middle so the converter sees another FORMAT_DESCRIPTION_EVENT and
+#   rotates the generated legacy output. Convert all completed .ibb files and
+#   verify each generated legacy binlog contains the expected Gtid_list state.
+#   Convert the .ibb files again in random order and verify the generated 
+#   legacy binlogs contain the expected Gtid_list state.
+#
+
+--source include/have_binlog_format_row.inc
+--source include/have_innodb_binlog.inc
+
+--let $datadir= `SELECT @@datadir`
+
+# start of binlog-000000.ibb
+
+SET @@session.sql_log_bin= 0;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t4 (a INT PRIMARY KEY) ENGINE=InnoDB;
+SET @@session.sql_log_bin= 1;
+
+--source include/reset_master.inc
+
+--echo *** Generate GTIDs across several InnoDB binlog files
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (2);
+
+SET @@session.gtid_domain_id= 1;
+SET @@session.server_id= 2;
+INSERT INTO t2 VALUES (1);
+
+# end of binlog-000000.ibb
+
+FLUSH BINARY LOGS;
+
+# start of binlog-000001.ibb
+
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (1);
+
+SET @@session.gtid_domain_id= 2;
+SET @@session.server_id= 3;
+INSERT INTO t3 VALUES (1);
+
+# end of binlog-000001.ibb
+
+FLUSH BINARY LOGS;
+
+# start of binlog-000002.ibb
+
+SET @@session.gtid_domain_id= 1;
+SET @@session.server_id= 1;
+INSERT INTO t2 VALUES (2);
+
+SET @@session.gtid_domain_id= 3;
+SET @@session.server_id= 4;
+INSERT INTO t4 VALUES (1);
+
+--source include/restart_mysqld.inc
+
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (3);
+
+# end of binlog-000002.ibb
+
+FLUSH BINARY LOGS;
+
+--let $binlog_name= binlog-000003.ibb
+--let $binlog_size= 262144
+--source include/wait_for_engine_binlog.inc
+
+
+--echo *** Convert the generated InnoDB binlogs to legacy format
+--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/gtid_conv $datadir/binlog-000000.ibb $datadir/binlog-000001.ibb $datadir/binlog-000002.ibb
+
+--echo *** Verify synthesized Gtid_list events in converted legacy binlogs
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000001 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_FILE= $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[\]
+--source include/search_pattern_in_file.inc
+
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000002 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[0-1-1,\n# 1-2-1\]
+--source include/search_pattern_in_file.inc
+
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000003 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\]
+--source include/search_pattern_in_file.inc
+
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000004 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]
+--source include/search_pattern_in_file.inc
+
+--remove_file $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000001
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000002
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000003
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000004
+
+
+--echo *** Convert the generated InnoDB binlog(binlog-000002.ibb) to legacy format in random order
+--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/gtid_conv $datadir/binlog-000002.ibb
+
+--echo *** Verify synthesized Gtid_list events in converted legacy binlogs
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000001 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_FILE= $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]
+# TODO: Tarun this search will fail now as this functionality is not yet implemented
+--source include/search_pattern_in_file.inc
+
+--remove_file $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000001
+
+
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;
+DROP TABLE t4;

sql/log_event.cc

@@ -2370,6 +2370,60 @@ Format_description_log_event(const uchar *buf, uint event_len,
   DBUG_VOID_RETURN;
 }
 
+bool
+Format_description_log_event::to_packet(String *packet)
+{
+  uchar *p;
+  uint32 needed_length=
+    packet->length() + START_V3_HEADER_LEN + 1 + number_of_event_types + 1;
+  if (packet->reserve(needed_length))
+    return true;
+  p= (uchar *)packet->ptr() + packet->length();
+  packet->length(needed_length);
+  int2store(p, binlog_version);
+  p += 2;
+  memcpy(p, server_version, ST_SERVER_VER_LEN);
+  p+= ST_SERVER_VER_LEN;
+/* TODO: Tarun get this reviewed */
+#ifdef MYSQL_SERVER
+  if (!dont_set_created)
+    created= get_time();
+#endif
+  int4store(p, created);
+  p+= 4;
+  *p++= common_header_len;
+  memcpy(p, post_header_len, number_of_event_types);
+  p+= number_of_event_types;
+
+  /*
+    if checksum is requested
+    record the checksum-algorithm descriptor next to
+    post_header_len vector which will be followed by the checksum value.
+    Master is supposed to trigger checksum computing by binlog_checksum_options,
+    slave does it via marking the event according to
+    FD_queue checksum_alg value.
+  */
+  compile_time_assert(BINLOG_CHECKSUM_ALG_DESC_LEN == 1);
+  uint8 checksum_byte= (uint8) (used_checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF ?
+                                used_checksum_alg : BINLOG_CHECKSUM_ALG_OFF);
+  DBUG_ASSERT(used_checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF);
+  /* 
+     FD of checksum-aware server is always checksum-equipped, (V) is in,
+     regardless of @@global.binlog_checksum policy.
+     Thereby a combination of (A) == 0, (V) != 0 means
+     it's the checksum-aware server's FD event that heads checksum-free binlog
+     file. 
+     Here 0 stands for checksumming OFF to evaluate (V) as 0 is that case.
+     A combination of (A) != 0, (V) != 0 denotes FD of the checksum-aware server
+     heading the checksummed binlog.
+     (A), (V) presence in FD of the checksum-aware server makes the event
+     1 + 4 bytes bigger comparing to the former FD.
+  */
+  *p++= checksum_byte;
+
+  return false;
+}
+
 bool Format_description_log_event::start_decryption(Start_encryption_log_event* sele)
 {
   DBUG_ASSERT(crypto_data.scheme == 0);
@@ -2560,6 +2614,16 @@ Rotate_log_event::Rotate_log_event(const uchar *buf, uint event_len,
   Binlog_checkpoint_log_event methods
 **************************************************************************/
 
+Binlog_checkpoint_log_event::Binlog_checkpoint_log_event(
+    const char *binlog_file_name_arg, uint binlog_file_len_arg)
+    : Log_event(),
+      binlog_file_name(my_strndup(PSI_INSTRUMENT_ME, binlog_file_name_arg,
+                                  binlog_file_len_arg, MYF(MY_WME))),
+      binlog_file_len(binlog_file_len_arg)
+{
+  cache_type= EVENT_NO_CACHE;
+}
+
 Binlog_checkpoint_log_event::Binlog_checkpoint_log_event(
        const uchar *buf, uint event_len,
        const Format_description_log_event *description_event)
@@ -2771,6 +2835,35 @@ Gtid_list_log_event::Gtid_list_log_event(const uchar *buf, uint event_len,
 #endif
 }
 
+bool
+Gtid_list_log_event::to_packet(String *packet)
+{
+  uint32 i;
+  uchar *p;
+  uint32 needed_length;
+
+  DBUG_ASSERT(count < 1<<28);
+
+  needed_length= packet->length() + get_data_size();
+  if (packet->reserve(needed_length))
+    return true;
+  p= (uchar *)packet->ptr() + packet->length();;
+  packet->length(needed_length);
+  int4store(p, (count & ((1<<28)-1)) | gl_flags);
+  p += 4;
+  /* Initialise the padding for empty Gtid_list. */
+  if (count == 0)
+    int2store(p, 0);
+  for (i= 0; i < count; ++i)
+  {
+    int4store(p, list[i].domain_id);
+    int4store(p+4, list[i].server_id);
+    int8store(p+8, list[i].seq_no);
+    p += 16;
+  }
+
+  return false;
+}
 
 /*
   Used to record gtid_list event while sending binlog to slave, without having to