diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/pom.xml b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/pom.xml index ec528cde751..efa26b87a6c 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/pom.xml +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/pom.xml @@ -187,6 +187,7 @@ limitations under the License. io.debezium:debezium-core io.debezium:debezium-ddl-parser io.debezium:debezium-connector-mysql + io.debezium:debezium-connector-binlog org.apache.flink:flink-connector-debezium org.apache.flink:flink-connector-mysql-cdc org.antlr:antlr4-runtime diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlMetadataAccessor.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlMetadataAccessor.java index 4ef336bcd84..9d919edf305 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlMetadataAccessor.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlMetadataAccessor.java @@ -41,7 +41,14 @@ public class MySqlMetadataAccessor implements MetadataAccessor { public MySqlMetadataAccessor(MySqlSourceConfig sourceConfig) { this.sourceConfig = sourceConfig; this.partition = - new MySqlPartition(sourceConfig.getMySqlConnectorConfig().getLogicalName()); + new MySqlPartition( + sourceConfig.getMySqlConnectorConfig().getLogicalName(), + sourceConfig + .getMySqlConnectorConfig() + .getConfig() + .getString( + io.debezium.relational.RelationalDatabaseConnectorConfig + .DATABASE_NAME)); } /** diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/parser/CustomAlterTableParserListener.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/parser/CustomAlterTableParserListener.java index db2c1c8840e..dcf175a1ab9 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/parser/CustomAlterTableParserListener.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/parser/CustomAlterTableParserListener.java @@ -96,7 +96,8 @@ public void exitCopyCreateTable(MySqlParser.CopyCreateTableContext ctx) { tableId, original.columns(), original.primaryKeyColumnNames(), - original.defaultCharsetName()); + original.defaultCharsetName(), + Collections.emptyList()); parser.signalCreateTable(tableId, ctx); Schema.Builder builder = Schema.newBuilder(); original.columns().forEach(column -> builder.column(toCdcColumn(column))); @@ -170,7 +171,9 @@ public void exitColumnCreateTable(MySqlParser.ColumnCreateTableContext ctx) { public void enterColumnDeclaration(MySqlParser.ColumnDeclarationContext ctx) { parser.runIfNotNull( () -> { - String columnName = parser.parseName(ctx.uid()); + // Debezium 2.1 changed the MySQL grammar: columnDeclaration now exposes the + // column name via fullColumnName() instead of uid(). + String columnName = parser.parseName(ctx.fullColumnName().uid()); ColumnEditor columnEditor = Column.editor().name(columnName); if (columnDefinitionListener == null) { columnDefinitionListener = diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/parser/CustomMySqlAntlrDdlParser.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/parser/CustomMySqlAntlrDdlParser.java index d92329094be..a81d24c7515 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/parser/CustomMySqlAntlrDdlParser.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/parser/CustomMySqlAntlrDdlParser.java @@ -22,6 +22,7 @@ import io.debezium.antlr.AntlrDdlParserListener; import io.debezium.antlr.DataTypeResolver; import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser; +import io.debezium.connector.mysql.charset.MySqlCharsetRegistry; import io.debezium.ddl.parser.mysql.generated.MySqlParser; import io.debezium.relational.Tables; @@ -40,7 +41,17 @@ public class CustomMySqlAntlrDdlParser extends MySqlAntlrDdlParser { public CustomMySqlAntlrDdlParser( boolean includeComments, boolean tinyInt1isBit, boolean isTableIdCaseInsensitive) { - super(true, false, includeComments, null, Tables.TableFilter.includeAll()); + // Debezium 2.7 added the BinlogCharsetRegistry argument. It must not be null: the + // parser dereferences it in extractCharset() whenever a COLLATE clause appears without + // an accompanying CHARACTER SET, which is how MySQL renders a collation. This is the + // same implementation Debezium's own MySqlDatabaseSchema resolves from its registry. + super( + true, + false, + includeComments, + null, + Tables.TableFilter.includeAll(), + new MySqlCharsetRegistry()); this.parsedEvents = new LinkedList<>(); this.tinyInt1isBit = tinyInt1isBit; this.isTableIdCaseInsensitive = isTableIdCaseInsensitive; diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlPipelineRecordEmitter.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlPipelineRecordEmitter.java index 41befa1f3d6..b7e0ae76f36 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlPipelineRecordEmitter.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlPipelineRecordEmitter.java @@ -37,6 +37,7 @@ import org.apache.flink.connector.base.source.reader.RecordEmitter; import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser; +import io.debezium.connector.mysql.charset.MySqlCharsetRegistry; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.Column; import io.debezium.relational.RelationalDatabaseConnectorConfig; @@ -315,7 +316,14 @@ private synchronized MySqlAntlrDdlParser getParser() { false); mySqlAntlrDdlParser = new MySqlAntlrDdlParser( - true, false, includeComments, null, Tables.TableFilter.includeAll()); + true, + false, + includeComments, + null, + Tables.TableFilter.includeAll(), + // Debezium 2.7 added the BinlogCharsetRegistry argument; it is + // dereferenced when parsing a COLLATE clause, so it must not be null. + new MySqlCharsetRegistry()); } return mySqlAntlrDdlParser; } diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/utils/MySqlSchemaUtils.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/utils/MySqlSchemaUtils.java index ca1b972af83..46c7728b4a9 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/utils/MySqlSchemaUtils.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/utils/MySqlSchemaUtils.java @@ -23,8 +23,8 @@ import org.apache.flink.cdc.connectors.mysql.schema.MySqlSchema; import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig; -import io.debezium.connector.mysql.MySqlConnection; import io.debezium.connector.mysql.MySqlPartition; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.Table; import io.debezium.relational.history.TableChanges; diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MysqlPipelineNewlyAddedTableITCase.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MysqlPipelineNewlyAddedTableITCase.java index 6226941038b..e5970e6b823 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MysqlPipelineNewlyAddedTableITCase.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MysqlPipelineNewlyAddedTableITCase.java @@ -55,7 +55,7 @@ import org.apache.flink.table.planner.factories.TestValuesTableFactory; import org.apache.flink.util.ExceptionUtils; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.jdbc.JdbcConnection; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oracle/src/test/java/org/apache/flink/cdc/connectors/oracle/source/OraclePipelineITCase.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oracle/src/test/java/org/apache/flink/cdc/connectors/oracle/source/OraclePipelineITCase.java index 700e87395e2..303c5154e06 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oracle/src/test/java/org/apache/flink/cdc/connectors/oracle/source/OraclePipelineITCase.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oracle/src/test/java/org/apache/flink/cdc/connectors/oracle/source/OraclePipelineITCase.java @@ -17,6 +17,7 @@ package org.apache.flink.cdc.connectors.oracle.source; +import org.apache.flink.api.common.JobStatus; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.cdc.common.configuration.Configuration; @@ -146,7 +147,7 @@ public void testAlterAddAllColumnTypeStatement() throws Exception { dbzProperties.setProperty("database.connection.adapter", "logminer"); dbzProperties.setProperty("log.mining.strategy", "online_catalog"); dbzProperties.setProperty("snapshot.locking.mode", "none"); - dbzProperties.setProperty("database.history.store.only.captured.tables.ddl", "true"); + dbzProperties.setProperty("schema.history.internal.store.only.captured.tables.ddl", "true"); dbzProperties.setProperty("include.schema.changes", "true"); OracleSourceConfigFactory configFactory = new OracleSourceConfigFactory(); configFactory.username(CONNECTOR_USER); @@ -605,7 +606,7 @@ public void testParseAlterStatement() throws Exception { dbzProperties.setProperty("database.connection.adapter", "logminer"); dbzProperties.setProperty("log.mining.strategy", "online_catalog"); dbzProperties.setProperty("snapshot.locking.mode", "none"); - dbzProperties.setProperty("database.history.store.only.captured.tables.ddl", "true"); + dbzProperties.setProperty("schema.history.internal.store.only.captured.tables.ddl", "true"); dbzProperties.setProperty("include.schema.changes", "true"); OracleSourceConfigFactory configFactory = new OracleSourceConfigFactory(); configFactory.username(CONNECTOR_USER); @@ -732,7 +733,7 @@ public void testInitialStartupMode() throws Exception { dbzProperties.setProperty("database.connection.adapter", "logminer"); dbzProperties.setProperty("log.mining.strategy", "online_catalog"); dbzProperties.setProperty("snapshot.locking.mode", "none"); - dbzProperties.setProperty("database.history.store.only.captured.tables.ddl", "true"); + dbzProperties.setProperty("schema.history.internal.store.only.captured.tables.ddl", "true"); dbzProperties.setProperty("include.schema.changes", "true"); OracleSourceConfigFactory configFactory = new OracleSourceConfigFactory(); configFactory @@ -1272,7 +1273,7 @@ public void testSnapshotOnlyMode() throws Exception { dbzProperties.setProperty("database.connection.adapter", "logminer"); dbzProperties.setProperty("log.mining.strategy", "online_catalog"); dbzProperties.setProperty("snapshot.locking.mode", "none"); - dbzProperties.setProperty("database.history.store.only.captured.tables.ddl", "true"); + dbzProperties.setProperty("schema.history.internal.store.only.captured.tables.ddl", "true"); dbzProperties.setProperty("include.schema.changes", "true"); OracleSourceConfigFactory configFactory = new OracleSourceConfigFactory(); configFactory.username(CONNECTOR_USER); @@ -1368,7 +1369,7 @@ private void restartFromSavePoint( dbzProperties.setProperty("database.connection.adapter", "logminer"); dbzProperties.setProperty("log.mining.strategy", "online_catalog"); dbzProperties.setProperty("snapshot.locking.mode", "none"); - dbzProperties.setProperty("database.history.store.only.captured.tables.ddl", "true"); + dbzProperties.setProperty("schema.history.internal.store.only.captured.tables.ddl", "true"); dbzProperties.setProperty("include.schema.changes", "true"); configFactory.username(CONNECTOR_USER); @@ -1432,15 +1433,22 @@ private void restartFromSavePoint( statement.execute(String.format("DELETE FROM %s.PRODUCTS WHERE ID = 111 ", "DEBEZIUM")); } - Thread.sleep(30_000); + // A snapshot-only source is BOUNDED, so the job terminates as soon as the snapshot is + // done. Sleeping a fixed 30s lets it reach FINISHED first, and stopWithSavepoint can + // never succeed on a terminated job. Wait only until the job is RUNNING so the savepoint + // is taken while the snapshot is still in flight. Unbounded modes keep running either + // way, so they retain the original settle time. + if ("snapshot".equals(options.get(SCAN_STARTUP_MODE.key()))) { + awaitJobRunning(jobClient); + } else { + Thread.sleep(30_000); + } // Trigger a savepoint and cancel the job LOG.info("Triggering savepoint"); finishedSavePointPath = triggerSavepointWithRetry(jobClient, savepointDirectory); LOG.info("Savepoint created at: {}", finishedSavePointPath); - if (!jobClient.getJobStatus().get().name().equals("FINISHED")) { - jobClient.cancel().get(); - } + cancelIfRunning(jobClient); iterator.close(); // Restore from savepoint @@ -1505,7 +1513,7 @@ private void restartFromSavePoint( restoreAfterEvents.add(restoredIterator.next()); } restoredIterator.close(); - restoredJobClient.cancel().get(); + cancelIfRunning(restoredJobClient); // Check if CreateTableEvent for new_products is present boolean hasCreateTableEvent = restoreAfterEvents.stream().anyMatch(event -> event instanceof CreateTableEvent); @@ -1518,6 +1526,39 @@ private void restartFromSavePoint( env.close(); } + /** + * Cancels the job unless it has already reached a terminal state. A snapshot-only source is + * bounded, so the job can finish on its own before the test reaches the cancel, and {@code + * cancel()} on a finished job throws {@code FlinkJobTerminatedWithoutCancellationException}. + */ + private void cancelIfRunning(JobClient jobClient) throws Exception { + if (!jobClient.getJobStatus().get().isTerminalState()) { + jobClient.cancel().get(); + } + } + + /** + * Waits until the job is RUNNING so that a savepoint can still be taken, failing fast with a + * meaningful message if it reaches a terminal state first. + */ + private void awaitJobRunning(JobClient jobClient) throws Exception { + final long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + JobStatus status = jobClient.getJobStatus().get(); + if (status == JobStatus.RUNNING) { + return; + } + if (status.isTerminalState()) { + throw new IllegalStateException( + "Job reached terminal state " + + status + + " before a savepoint could be taken"); + } + Thread.sleep(100); + } + throw new IllegalStateException("Job did not reach RUNNING within 60s"); + } + // Helper method to trigger a savepoint with retry mechanism private String triggerSavepointWithRetry(JobClient jobClient, String savepointDirectory) throws Exception { @@ -1529,6 +1570,13 @@ private String triggerSavepointWithRetry(JobClient jobClient, String savepointDi .stopWithSavepoint(true, savepointDirectory, SavepointFormatType.DEFAULT) .get(); } catch (Exception e) { + // A terminated job can never accept a savepoint, so retrying just burns the + // whole budget and then reports a misleading timeout instead of the real cause. + JobStatus status = jobClient.getJobStatus().get(); + if (status.isTerminalState()) { + throw new IllegalStateException( + "Cannot stop with savepoint: job already reached " + status, e); + } retryCount++; LOG.error( "Retry {}/{}: Failed to trigger savepoint: {}", diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres/src/main/java/org/apache/flink/cdc/connectors/postgres/utils/PostgresSchemaUtils.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres/src/main/java/org/apache/flink/cdc/connectors/postgres/utils/PostgresSchemaUtils.java index 93a7099ac39..e7f3c6fa740 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres/src/main/java/org/apache/flink/cdc/connectors/postgres/utils/PostgresSchemaUtils.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres/src/main/java/org/apache/flink/cdc/connectors/postgres/utils/PostgresSchemaUtils.java @@ -23,15 +23,15 @@ import org.apache.flink.cdc.connectors.postgres.source.PostgresDialect; import org.apache.flink.cdc.connectors.postgres.source.config.PostgresSourceConfig; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.postgresql.PostgresConnectorConfig; import io.debezium.connector.postgresql.PostgresObjectUtils; import io.debezium.connector.postgresql.PostgresSchema; -import io.debezium.connector.postgresql.PostgresTopicSelector; import io.debezium.connector.postgresql.TypeRegistry; import io.debezium.connector.postgresql.connection.PostgresConnection; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.Table; -import io.debezium.schema.TopicSelector; +import io.debezium.spi.topic.TopicNamingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -155,8 +155,10 @@ public static Schema getTableSchema( PostgresConnection jdbc) { try { // fetch table schemas - TopicSelector topicSelector = - PostgresTopicSelector.create(sourceConfig.getDbzConnectorConfig()); + TopicNamingStrategy topicSelector = + sourceConfig + .getDbzConnectorConfig() + .getTopicNamingStrategy(CommonConnectorConfig.TOPIC_NAMING_STRATEGY); PostgresConnection.PostgresValueConverterBuilder valueConverterBuilder = newPostgresValueConverterBuilder(sourceConfig.getDbzConnectorConfig()); PostgresSchema postgresSchema = diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-sqlserver/src/test/java/org/apache/flink/cdc/connectors/sqlserver/source/SqlServerEventDeserializerTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-sqlserver/src/test/java/org/apache/flink/cdc/connectors/sqlserver/source/SqlServerEventDeserializerTest.java index 1773de5be3b..1600993eaa8 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-sqlserver/src/test/java/org/apache/flink/cdc/connectors/sqlserver/source/SqlServerEventDeserializerTest.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-sqlserver/src/test/java/org/apache/flink/cdc/connectors/sqlserver/source/SqlServerEventDeserializerTest.java @@ -213,7 +213,7 @@ private static SourceRecord buildSchemaChangeRecord(TableChangeType type, List extends EventDispatc private final HistorizedDatabaseSchema historizedSchema; private final DataCollectionFilters.DataCollectionFilter filter; private final CommonConnectorConfig connectorConfig; - private final TopicSelector topicSelector; + private final TopicNamingStrategy topicNamingStrategy; private final Schema schemaChangeKeySchema; private final Schema schemaChangeValueSchema; private final String topic; @@ -82,7 +83,7 @@ public class JdbcSourceEventDispatcher

extends EventDispatc public JdbcSourceEventDispatcher( CommonConnectorConfig connectorConfig, - TopicSelector topicSelector, + TopicNamingStrategy topicNamingStrategy, DatabaseSchema schema, ChangeEventQueue queue, DataCollectionFilters.DataCollectionFilter filter, @@ -92,7 +93,7 @@ public JdbcSourceEventDispatcher( SchemaChangeEventHandler schemaChangeEventHandler) { super( connectorConfig, - topicSelector, + topicNamingStrategy, schema, queue, filter, @@ -106,8 +107,8 @@ public JdbcSourceEventDispatcher( this.filter = filter; this.queue = queue; this.connectorConfig = connectorConfig; - this.topicSelector = topicSelector; - this.topic = topicSelector.getPrimaryTopic(); + this.topicNamingStrategy = topicNamingStrategy; + this.topic = connectorConfig.getLogicalName(); this.schemaChangeKeySchema = SchemaBuilder.struct() .name( @@ -139,6 +140,7 @@ public ChangeEventQueue getQueue() { @Override public void dispatchSchemaChangeEvent( P partition, + OffsetContext offsetContext, TableId dataCollectionId, SchemaChangeEventEmitter schemaChangeEventEmitter) throws InterruptedException { @@ -152,7 +154,7 @@ public void dispatchSchemaChangeEvent( IncrementalSnapshotChangeEventSource incrementalEventSource = getIncrementalSnapshotChangeEventSource(); if (incrementalEventSource != null) { - incrementalEventSource.processSchemaChange(partition, dataCollectionId); + incrementalEventSource.processSchemaChange(partition, offsetContext, dataCollectionId); } } @@ -200,7 +202,8 @@ private Struct schemaChangeRecordValue(SchemaChangeEvent event) throws IOExcepti event.getDatabase(), event.getSchema(), event.getDdl(), - event.getTableChanges()); + event.getTableChanges(), + event.getTimestamp()); String historyStr = DOCUMENT_WRITER.write(historyRecord.document()); Struct value = new Struct(schemaChangeValueSchema); @@ -217,7 +220,7 @@ public void schemaChangeEvent(SchemaChangeEvent event) throws InterruptedExcepti historizedSchema.applySchemaChange(event); if (connectorConfig.isSchemaChangesHistoryEnabled()) { try { - final String topicName = topicSelector.getPrimaryTopic(); + final String topicName = topic; final Integer partition = 0; final Struct key = schemaChangeRecordKey(event); final Struct value = schemaChangeRecordValue(event); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/EmbeddedFlinkDatabaseHistory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/EmbeddedFlinkDatabaseHistory.java index 32e5d2198b1..aa992711eb7 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/EmbeddedFlinkDatabaseHistory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/EmbeddedFlinkDatabaseHistory.java @@ -23,14 +23,15 @@ import io.debezium.relational.TableId; import io.debezium.relational.Tables; import io.debezium.relational.ddl.DdlParser; -import io.debezium.relational.history.DatabaseHistory; -import io.debezium.relational.history.DatabaseHistoryException; -import io.debezium.relational.history.DatabaseHistoryListener; import io.debezium.relational.history.HistoryRecord; import io.debezium.relational.history.HistoryRecordComparator; +import io.debezium.relational.history.SchemaHistory; +import io.debezium.relational.history.SchemaHistoryException; +import io.debezium.relational.history.SchemaHistoryListener; import io.debezium.relational.history.TableChanges; import io.debezium.relational.history.TableChanges.TableChange; +import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -39,19 +40,20 @@ import java.util.concurrent.ConcurrentMap; /** - * A {@link DatabaseHistory} implementation which store the latest table schema in Flink state. + * A {@link SchemaHistory} implementation which store the latest table schema in Flink state. * *

It stores/recovers history using data offered by {@link SourceSplitState}. */ -public class EmbeddedFlinkDatabaseHistory implements DatabaseHistory { +public class EmbeddedFlinkDatabaseHistory implements SchemaHistory { - public static final String DATABASE_HISTORY_INSTANCE_NAME = "database.history.instance.name"; + public static final String DATABASE_HISTORY_INSTANCE_NAME = + "schema.history.internal.instance.name"; public static final ConcurrentMap> TABLE_SCHEMAS = new ConcurrentHashMap<>(); private Map tableSchemas; - private DatabaseHistoryListener listener; + private SchemaHistoryListener listener; private boolean storeOnlyMonitoredTablesDdl; private boolean skipUnparseableDDL; @@ -59,10 +61,10 @@ public class EmbeddedFlinkDatabaseHistory implements DatabaseHistory { public void configure( Configuration config, HistoryRecordComparator comparator, - DatabaseHistoryListener listener, + SchemaHistoryListener listener, boolean useCatalogBeforeSchema) { this.listener = listener; - this.storeOnlyMonitoredTablesDdl = config.getBoolean(STORE_ONLY_MONITORED_TABLES_DDL); + this.storeOnlyMonitoredTablesDdl = config.getBoolean(STORE_ONLY_CAPTURED_TABLES_DDL); this.skipUnparseableDDL = config.getBoolean(SKIP_UNPARSEABLE_DDL_STATEMENTS); // recover @@ -74,14 +76,19 @@ public void configure( } @Override - public void start() { - listener.started(); - } + // Debezium 2.0 wires SchemaHistoryMetrics in as the schema history listener. Its + // started() callback registers a JMX MBean whose name is built only from the connector + // context and the topic prefix, so every parallel subtask sharing a TaskManager JVM asks + // for the very same name. Debezium answers a name clash by sleeping 5 seconds and + // retrying, twelve times, so every reader but the first stalls for up to a minute each + // time it opens a split. Flink CDC publishes its own metrics and never reads these + // MBeans, and Debezium 1.9 did not register them either, so skip the registration. + public void start() {} @Override public void record( Map source, Map position, String databaseName, String ddl) - throws DatabaseHistoryException { + throws SchemaHistoryException { throw new UnsupportedOperationException("should not call here, error"); } @@ -92,10 +99,12 @@ public void record( String databaseName, String schemaName, String ddl, - TableChanges changes) - throws DatabaseHistoryException { + TableChanges changes, + Instant timestamp) + throws SchemaHistoryException { final HistoryRecord record = - new HistoryRecord(source, position, databaseName, schemaName, ddl, changes); + new HistoryRecord( + source, position, databaseName, schemaName, ddl, changes, timestamp); listener.onChangeApplied(record); } @@ -135,12 +144,12 @@ public void initializeStorage() { // do nothing } - @Override + // Debezium 2.2 moved storeOnlyCapturedTables()/skipUnparseableDdlStatements() from the + // SchemaHistory interface to HistorizedDatabaseSchema; kept here as plain helpers. public boolean storeOnlyCapturedTables() { return storeOnlyMonitoredTablesDdl; } - @Override public boolean skipUnparseableDdlStatements() { return skipUnparseableDDL; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/meta/wartermark/WatermarkEvent.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/meta/wartermark/WatermarkEvent.java index e57335ab420..b48ae7b9b5c 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/meta/wartermark/WatermarkEvent.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/meta/wartermark/WatermarkEvent.java @@ -19,7 +19,7 @@ import org.apache.flink.cdc.connectors.base.source.meta.offset.Offset; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/IncrementalSourceScanFetcher.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/IncrementalSourceScanFetcher.java index 3136bb97d25..c8d4900b385 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/IncrementalSourceScanFetcher.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/IncrementalSourceScanFetcher.java @@ -22,9 +22,11 @@ import org.apache.flink.cdc.connectors.base.source.meta.split.SourceRecords; import org.apache.flink.cdc.connectors.base.source.meta.split.SourceSplitBase; import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.TemporaryClassLoaderContext; import org.apache.flink.shaded.guava31.com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.base.ChangeEventQueue; import io.debezium.pipeline.DataChangeEvent; import org.apache.kafka.connect.data.Struct; @@ -89,7 +91,15 @@ public IncrementalSourceScanFetcher(FetchTask.Context taskContext, int subtaskId public void submitTask(FetchTask fetchTask) { this.snapshotSplitReadTask = fetchTask; this.currentSnapshotSplit = fetchTask.getSplit().asSnapshotSplit(); - taskContext.configure(currentSnapshotSplit); + // Debezium 2.0 changed io.debezium.config.Instantiator to resolve classes through the + // thread context class loader. On a Flink task thread that loader can be a user code class + // loader left over from a previous job attempt, which is already closed, so Debezium fails + // with "Trying to access closed classloader" while building its own objects (topic naming + // strategy, transaction metadata factory, ...). Pin it to the loader that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(CommonConnectorConfig.class.getClassLoader())) { + taskContext.configure(currentSnapshotSplit); + } this.queue = taskContext.getQueue(); this.hasNextElement.set(true); this.reachEnd.set(false); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/IncrementalSourceStreamFetcher.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/IncrementalSourceStreamFetcher.java index 937fefe67a1..d4d7c39bd12 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/IncrementalSourceStreamFetcher.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/IncrementalSourceStreamFetcher.java @@ -24,9 +24,11 @@ import org.apache.flink.cdc.connectors.base.source.meta.split.StreamSplit; import org.apache.flink.cdc.connectors.base.utils.SplitKeyUtils; import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.TemporaryClassLoaderContext; import org.apache.flink.shaded.guava31.com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.base.ChangeEventQueue; import io.debezium.pipeline.DataChangeEvent; import io.debezium.relational.TableId; @@ -89,7 +91,15 @@ public void submitTask(FetchTask fetchTask) { this.streamFetchTask = fetchTask; this.currentStreamSplit = fetchTask.getSplit().asStreamSplit(); configureFilter(); - taskContext.configure(currentStreamSplit); + // Debezium 2.0 changed io.debezium.config.Instantiator to resolve classes through the + // thread context class loader. On a Flink task thread that loader can be a user code class + // loader left over from a previous job attempt, which is already closed, so Debezium fails + // with "Trying to access closed classloader" while building its own objects (topic naming + // strategy, transaction metadata factory, ...). Pin it to the loader that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(CommonConnectorConfig.class.getClassLoader())) { + taskContext.configure(currentStreamSplit); + } this.queue = taskContext.getQueue(); startReadTask(); executorService.submit( diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/JdbcSourceFetchTaskContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/JdbcSourceFetchTaskContext.java index 9975331e469..c35e7851183 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/JdbcSourceFetchTaskContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/reader/external/JdbcSourceFetchTaskContext.java @@ -35,7 +35,7 @@ import io.debezium.relational.RelationalDatabaseSchema; import io.debezium.relational.Table; import io.debezium.relational.TableId; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/utils/SplitKeyUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/utils/SplitKeyUtils.java index 94be44aa653..7d73b164289 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/utils/SplitKeyUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/utils/SplitKeyUtils.java @@ -20,7 +20,7 @@ import org.apache.flink.cdc.connectors.base.source.meta.split.FinishedSnapshotSplitInfo; import org.apache.flink.table.types.logical.RowType; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/io/debezium/connector/db2/Db2Connection.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/io/debezium/connector/db2/Db2Connection.java index 63da894ec85..ac4b73100a3 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/io/debezium/connector/db2/Db2Connection.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/io/debezium/connector/db2/Db2Connection.java @@ -7,19 +7,26 @@ package io.debezium.connector.db2; import com.ibm.db2.jcc.DB2Driver; +import io.debezium.DebeziumException; +import io.debezium.config.CommonConnectorConfig; import io.debezium.config.Configuration; +import io.debezium.connector.db2.platform.Db2PlatformAdapter; import io.debezium.jdbc.JdbcConfiguration; import io.debezium.jdbc.JdbcConnection; +import io.debezium.pipeline.spi.OffsetContext; +import io.debezium.pipeline.spi.Partition; import io.debezium.relational.Column; import io.debezium.relational.ColumnEditor; import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.util.BoundedConcurrentHashMap; import io.debezium.util.Collect; +import org.apache.kafka.connect.errors.ConnectException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; @@ -28,14 +35,20 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +/** {@link JdbcConnection} extension to be used with IBM Db2 */ /** - * Copied from Debezium 1.9.8.Final. {@link JdbcConnection} extension to be used with IBM Db2 + * Copied from Debezium project(2.7.4.Final). * - * @author Horia Chiorean (hchiorea@redhat.com), Jiri Pechanec, Peter Urbanetz + *

Change 1: override {@code resolveCatalogName} to return the real database name. + * + *

Change 2: override {@code readPrimaryKeyNames} / {@code readTableUniqueIndices} / {@code + * readTableNames} to query the JDBC metadata with the schema rather than the catalog, which is how + * Db2 exposes them. */ public class Db2Connection extends JdbcConnection { @@ -44,48 +57,13 @@ public class Db2Connection extends JdbcConnection { private static Logger LOGGER = LoggerFactory.getLogger(Db2Connection.class); - private static final String CDC_SCHEMA = "ASNCDC"; - private static final String STATEMENTS_PLACEHOLDER = "#"; - private static final String GET_MAX_LSN = - "SELECT max(t.SYNCHPOINT) FROM ( SELECT CD_NEW_SYNCHPOINT AS SYNCHPOINT FROM " - + CDC_SCHEMA - + ".IBMSNAP_REGISTER UNION ALL SELECT SYNCHPOINT AS SYNCHPOINT FROM " - + CDC_SCHEMA - + ".IBMSNAP_REGISTER) t"; private static final String LOCK_TABLE = "SELECT * FROM # WITH CS"; // DB2 private static final String LSN_TO_TIMESTAMP = "SELECT CURRENT TIMEstamp FROM sysibm.sysdummy1 WHERE ? > X'00000000000000000000000000000000'"; - private static final String GET_ALL_CHANGES_FOR_TABLE = - "SELECT " - + "CASE " - + "WHEN IBMSNAP_OPERATION = 'D' AND (LEAD(cdc.IBMSNAP_OPERATION,1,'X') OVER (PARTITION BY cdc.IBMSNAP_COMMITSEQ ORDER BY cdc.IBMSNAP_INTENTSEQ)) ='I' THEN 3 " - + "WHEN IBMSNAP_OPERATION = 'I' AND (LAG(cdc.IBMSNAP_OPERATION,1,'X') OVER (PARTITION BY cdc.IBMSNAP_COMMITSEQ ORDER BY cdc.IBMSNAP_INTENTSEQ)) ='D' THEN 4 " - + "WHEN IBMSNAP_OPERATION = 'D' THEN 1 " - + "WHEN IBMSNAP_OPERATION = 'I' THEN 2 " - + "END " - + "OPCODE," - + "cdc.* " - + "FROM ASNCDC.# cdc WHERE IBMSNAP_COMMITSEQ >= ? AND IBMSNAP_COMMITSEQ <= ? " - + "order by IBMSNAP_COMMITSEQ, IBMSNAP_INTENTSEQ"; - - private static final String GET_LIST_OF_CDC_ENABLED_TABLES = - "select r.SOURCE_OWNER, r.SOURCE_TABLE, r.CD_OWNER, r.CD_TABLE, r.CD_NEW_SYNCHPOINT, r.CD_OLD_SYNCHPOINT, t.TBSPACEID, t.TABLEID , CAST((t.TBSPACEID * 65536 + t.TABLEID )AS INTEGER )from " - + CDC_SCHEMA - + ".IBMSNAP_REGISTER r left JOIN SYSCAT.TABLES t ON r.SOURCE_OWNER = t.TABSCHEMA AND r.SOURCE_TABLE = t.TABNAME WHERE r.SOURCE_OWNER <> ''"; - - // No new Tabels 1=0 - private static final String GET_LIST_OF_NEW_CDC_ENABLED_TABLES = - "select CAST((t.TBSPACEID * 65536 + t.TABLEID )AS INTEGER ) AS OBJECTID, " - + " CD_OWNER CONCAT '.' CONCAT CD_TABLE, " - + " CD_NEW_SYNCHPOINT, " - + " CD_OLD_SYNCHPOINT " - + "from ASNCDC.IBMSNAP_REGISTER r left JOIN SYSCAT.TABLES t ON r.SOURCE_OWNER = t.TABSCHEMA AND r.SOURCE_TABLE = t.TABNAME " - + "WHERE r.SOURCE_OWNER <> '' AND 1=0 AND CD_NEW_SYNCHPOINT > ? AND CD_OLD_SYNCHPOINT < ? "; - private static final String GET_LIST_OF_KEY_COLUMNS = "SELECT " + "CAST((t.TBSPACEID * 65536 + t.TABLEID )AS INTEGER ) as objectid, " @@ -123,15 +101,21 @@ public class Db2Connection extends JdbcConnection { private final BoundedConcurrentHashMap lsnToInstantCache; + private final Db2ConnectorConfig connectorConfig; + private final Db2PlatformAdapter platform; + /** * Creates a new connection using the supplied configuration. * * @param config {@link Configuration} instance, may not be null. */ - public Db2Connection(JdbcConfiguration config) { - super(config, FACTORY, QUOTED_CHARACTER, QUOTED_CHARACTER); + public Db2Connection(Db2ConnectorConfig config) { + super(config.getJdbcConfig(), FACTORY, QUOTED_CHARACTER, QUOTED_CHARACTER); + + connectorConfig = config; lsnToInstantCache = new BoundedConcurrentHashMap<>(100); realDatabaseName = retrieveRealDatabaseName(); + platform = connectorConfig.getDb2Platform().createAdapter(connectorConfig); } /** @@ -139,7 +123,7 @@ public Db2Connection(JdbcConfiguration config) { */ public Lsn getMaxLsn() throws SQLException { return queryAndMap( - GET_MAX_LSN, + platform.getMaxLsnQuery(), singleResultMapper( rs -> { final Lsn ret = Lsn.valueOf(rs.getBytes(1)); @@ -162,7 +146,8 @@ public void getChangesForTable( TableId tableId, Lsn fromLsn, Lsn toLsn, ResultSetConsumer consumer) throws SQLException { final String query = - GET_ALL_CHANGES_FOR_TABLE.replace(STATEMENTS_PLACEHOLDER, cdcNameForTable(tableId)); + platform.getAllChangesForTableQuery() + .replace(STATEMENTS_PLACEHOLDER, cdcNameForTable(tableId)); prepareQuery( query, statement -> { @@ -193,8 +178,8 @@ public void getChangesForTables( int idx = 0; for (Db2ChangeTable changeTable : changeTables) { final String query = - GET_ALL_CHANGES_FOR_TABLE.replace( - STATEMENTS_PLACEHOLDER, changeTable.getCaptureInstance()); + platform.getAllChangesForTableQuery() + .replace(STATEMENTS_PLACEHOLDER, changeTable.getCaptureInstance()); queries[idx] = query; // If the table was added in the middle of queried buffer we need // to adjust from to the first LSN available @@ -263,10 +248,10 @@ public Instant timestampOfLsn(Lsn lsn) throws SQLException { } @Override - public Optional getCurrentTimestamp() throws SQLException { + public Optional getCurrentTimestamp() throws SQLException { return queryAndMap( "SELECT CURRENT_TIMESTAMP result FROM sysibm.sysdummy1", - rs -> rs.next() ? Optional.of(rs.getTimestamp(1)) : Optional.empty()); + rs -> rs.next() ? Optional.of(rs.getTimestamp(1).toInstant()) : Optional.empty()); } /** @@ -309,10 +294,9 @@ public Lsn getFromLsn() { } public Set listOfChangeTables() throws SQLException { - final String query = GET_LIST_OF_CDC_ENABLED_TABLES; return queryAndMap( - query, + platform.getListOfCdcEnabledTablesQuery(), rs -> { final Set changeTables = new HashSet<>(); while (rs.next()) { @@ -325,22 +309,21 @@ public Set listOfChangeTables() throws SQLException { */ changeTables.add( new Db2ChangeTable( - new TableId( - realDatabaseName, rs.getString(1), rs.getString(2)), + new TableId("", rs.getString(1), rs.getString(2)), rs.getString(4), rs.getInt(9), Lsn.valueOf(rs.getBytes(5)), - Lsn.valueOf(rs.getBytes(6)))); + Lsn.valueOf(rs.getBytes(6)), + connectorConfig.getCdcChangeTablesSchema())); } return changeTables; }); } public Set listOfNewChangeTables(Lsn fromLsn, Lsn toLsn) throws SQLException { - final String query = GET_LIST_OF_NEW_CDC_ENABLED_TABLES; return prepareQueryAndMap( - query, + platform.getListOfNewCdcEnabledTablesQuery(), ps -> { ps.setBytes(1, fromLsn.getBinary()); ps.setBytes(2, toLsn.getBinary()); @@ -353,7 +336,8 @@ public Set listOfNewChangeTables(Lsn fromLsn, Lsn toLsn) throws rs.getString(2), rs.getInt(1), Lsn.valueOf(rs.getBytes(3)), - Lsn.valueOf(rs.getBytes(4)))); + Lsn.valueOf(rs.getBytes(4)), + connectorConfig.getCdcChangeTablesSchema())); } return changeTables; }); @@ -469,6 +453,13 @@ public String connectionString() { return connectionString(URL_PATTERN); } + @Override + public Optional nullsSortLast() { + // "The null value is higher than all other values" + // https://www.ibm.com/docs/en/db2/11.5?topic=subselect-order-by-clause + return Optional.of(true); + } + @Override public String quotedTableIdString(TableId tableId) { StringBuilder quoted = new StringBuilder(); @@ -479,6 +470,196 @@ public String quotedTableIdString(TableId tableId) { return quoted.toString(); } + @Override + public JdbcConnection prepareQuery( + String[] multiQuery, + StatementPreparer[] preparers, + BlockingMultiResultSetConsumer resultConsumer) + throws SQLException, InterruptedException { + final ResultSet[] resultSets = new ResultSet[multiQuery.length]; + final PreparedStatement[] preparedStatements = new PreparedStatement[multiQuery.length]; + + try { + for (int i = 0; i < multiQuery.length; i++) { + final String query = multiQuery[i]; + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("running '{}'", query); + } + // Purposely create the statement this way + final PreparedStatement statement = createPreparedStatement(query); + preparedStatements[i] = statement; + preparers[i].accept(statement); + resultSets[i] = statement.executeQuery(); + } + if (resultConsumer != null) { + resultConsumer.accept(resultSets); + } + } finally { + for (ResultSet rs : resultSets) { + if (rs != null) { + try { + rs.close(); + } catch (Exception ei) { + } + } + } + // Db2 requires closing prepared statements to avoid caching result-set column + // structures + for (PreparedStatement ps : preparedStatements) { + closePreparedStatement(ps); + } + } + return this; + } + + @Override + public JdbcConnection prepareQueryWithBlockingConsumer( + String preparedQueryString, + StatementPreparer preparer, + BlockingResultSetConsumer resultConsumer) + throws SQLException, InterruptedException { + // Db2 requires closing prepared statements to avoid caching result-set column structures + try (PreparedStatement statement = createPreparedStatement(preparedQueryString)) { + preparer.accept(statement); + try (ResultSet resultSet = statement.executeQuery(); ) { + if (resultConsumer != null) { + resultConsumer.accept(resultSet); + } + } + } + return this; + } + + @Override + public JdbcConnection prepareQuery(String preparedQueryString) throws SQLException { + // Db2 requires closing prepared statements to avoid caching result-set column structures + try (PreparedStatement statement = createPreparedStatement(preparedQueryString)) { + statement.executeQuery(); + } + return this; + } + + @Override + public JdbcConnection prepareQuery( + String preparedQueryString, + StatementPreparer preparer, + ResultSetConsumer resultConsumer) + throws SQLException { + // Db2 requires closing prepared statements to avoid caching result-set column structures + try (PreparedStatement statement = createPreparedStatement(preparedQueryString)) { + preparer.accept(statement); + try (ResultSet resultSet = statement.executeQuery(); ) { + if (resultConsumer != null) { + resultConsumer.accept(resultSet); + } + } + } + return this; + } + + @Override + public T prepareQueryAndMap( + String preparedQueryString, StatementPreparer preparer, ResultSetMapper mapper) + throws SQLException { + Objects.requireNonNull(mapper, "Mapper must be provided"); + // Db2 requires closing prepared statements to avoid caching result-set column structures + try (PreparedStatement statement = createPreparedStatement(preparedQueryString)) { + preparer.accept(statement); + try (ResultSet resultSet = statement.executeQuery(); ) { + return mapper.apply(resultSet); + } + } + } + + @Override + public JdbcConnection prepareUpdate(String stmt, StatementPreparer preparer) + throws SQLException { + // Db2 requires closing prepared statements to avoid caching result-set column structures + try (PreparedStatement statement = createPreparedStatement(stmt)) { + if (preparer != null) { + preparer.accept(statement); + } + LOGGER.trace("Executing statement '{}'", stmt); + statement.execute(); + } + return this; + } + + @Override + public JdbcConnection prepareQuery( + String preparedQueryString, + List parameters, + ParameterResultSetConsumer resultConsumer) + throws SQLException { + // Db2 requires closing prepared statements to avoid caching result-set column structures + try (PreparedStatement statement = createPreparedStatement(preparedQueryString)) { + int index = 1; + for (final Object parameter : parameters) { + statement.setObject(index++, parameter); + } + try (ResultSet resultSet = statement.executeQuery()) { + if (resultConsumer != null) { + resultConsumer.accept(parameters, resultSet); + } + } + } + return this; + } + + @Override + public TableId createTableId(String databaseName, String schemaName, String tableName) { + return new TableId(null, schemaName, tableName); + } + + public boolean validateLogPosition( + Partition partition, OffsetContext offset, CommonConnectorConfig config) { + + final Lsn storedLsn = ((Db2OffsetContext) offset).getChangePosition().getCommitLsn(); + + String oldestFirstChangeQuery = + String.format( + "SELECT min(RESTART_SEQ) FROM %s.IBMSNAP_CAPMON;", + connectorConfig.getCdcControlSchema()); + + try { + final String oldestScn = + singleOptionalValue(oldestFirstChangeQuery, rs -> rs.getString(1)); + + if (oldestScn == null) { + return false; + } + + LOGGER.trace("Oldest SCN in logs is '{}'", oldestScn); + return storedLsn == null || Lsn.valueOf(oldestScn).compareTo(storedLsn) < 0; + } catch (SQLException e) { + throw new DebeziumException("Unable to get last available log position", e); + } + } + + public T singleOptionalValue(String query, ResultSetExtractor extractor) + throws SQLException { + return queryAndMap(query, rs -> rs.next() ? extractor.apply(rs) : null); + } + + private PreparedStatement createPreparedStatement(String query) { + try { + LOGGER.trace("Creating prepared statement '{}'", query); + return connection().prepareStatement(query); + } catch (SQLException e) { + throw new ConnectException(e); + } + } + + private void closePreparedStatement(PreparedStatement statement) { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + // ignored + } + } + } + protected String resolveCatalogName(String catalogName) { return realDatabaseName; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/io/debezium/connector/db2/Db2StreamingChangeEventSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/io/debezium/connector/db2/Db2StreamingChangeEventSource.java index 261a59fb2f2..fd75715d706 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/io/debezium/connector/db2/Db2StreamingChangeEventSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/io/debezium/connector/db2/Db2StreamingChangeEventSource.java @@ -10,9 +10,11 @@ import io.debezium.pipeline.EventDispatcher; import io.debezium.pipeline.source.spi.ChangeTableResultSet; import io.debezium.pipeline.source.spi.StreamingChangeEventSource; +import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.schema.DatabaseSchema; import io.debezium.schema.SchemaChangeEvent.SchemaChangeEventType; +import io.debezium.snapshot.SnapshotterService; import io.debezium.util.Clock; import io.debezium.util.Metronome; import org.slf4j.Logger; @@ -35,27 +37,24 @@ import java.util.stream.Collectors; /** - * Copied from Debezium project(1.9.8.final) + * Copied from Debezium project(2.7.4.Final).. * - *

A {@link StreamingChangeEventSource} based on DB2 change data capture functionality. A main - * loop polls database DDL change and change data tables and turns them into change events. + *

Change 1: the effective offset context is initialised at the top of {@code execute} — + * Debezium's coordinator calls {@code init(offsetContext)} before {@code execute}, but Flink CDC + * drives this source directly and never calls {@code init}. * - *

The connector uses CDC functionality of DB2 that is implemented as as a process that monitors - * source table and write changes from the table into the change table. + *

Change 2: add the {@code afterHandleLsn} hook, called after each iteration commits its + * position, so Flink CDC's bounded stream task can stop at the high watermark. * - *

The main loop keeps a pointer to the LSN of changes that were already processed. It queries - * all change tables and get result set of changes. It always finds the smallest LSN across all - * tables and the change is converted into the event message and sent downstream. The process - * repeats until all result sets are empty. The LSN is marked and the procedure repeats. - * - *

The schema changes detection follows the procedure recommended by DB2 CDC documentation. The - * database operator should create one more capture process (and table) when a table schema is - * updated. The code detects presence of two change tables for a single source table. It decides - * which table is the new one depending on LSNs stored in them. The loop streams changes from the - * older table till there are events in new table with the LSN larger than in the old one. Then the - * change table is switched and streaming is executed from the new one. - * - * @author Jiri Pechanec, Peter Urbanetz + *

Change 3: keep the pre-2.7 exclusive upper bound when deciding which capture instances + * represent a schema change. Upstream 2.7 widened this to {@code currentMaxLsn.increment()}, making + * the window inclusive. Flink CDC reads a change table's {@code startLsn} from a column that tracks + * the capture progress rather than a fixed creation LSN, so {@code startLsn} equals {@code + * currentMaxLsn} on virtually every poll. With the inclusive bound the same capture instance is + * queued as a schema change over and over, and {@code migrateTable} re-reads the live source table + * schema each time. After an {@code ALTER TABLE ... ADD COLUMN} that re-read installs the new, + * wider schema while the still-open change-table result set only yields the old column count, + * producing "Data row is smaller than a column index". */ public class Db2StreamingChangeEventSource implements StreamingChangeEventSource { @@ -86,6 +85,9 @@ public class Db2StreamingChangeEventSource private final Db2DatabaseSchema schema; private final Duration pollInterval; private final Db2ConnectorConfig connectorConfig; + private Db2OffsetContext effectiveOffsetContext; + + private final SnapshotterService snapshotterService; public Db2StreamingChangeEventSource( Db2ConnectorConfig connectorConfig, @@ -94,7 +96,8 @@ public Db2StreamingChangeEventSource( EventDispatcher dispatcher, ErrorHandler errorHandler, Clock clock, - Db2DatabaseSchema schema) { + Db2DatabaseSchema schema, + SnapshotterService snapshotterService) { this.connectorConfig = connectorConfig; this.dataConnection = dataConnection; this.metadataConnection = metadataConnection; @@ -103,6 +106,15 @@ public Db2StreamingChangeEventSource( this.clock = clock; this.schema = schema; this.pollInterval = connectorConfig.getPollInterval(); + this.snapshotterService = snapshotterService; + } + + public void init(Db2OffsetContext offsetContext) { + + this.effectiveOffsetContext = + offsetContext != null + ? offsetContext + : new Db2OffsetContext(connectorConfig, TxLogPosition.NULL, false, false); } @Override @@ -111,9 +123,11 @@ public void execute( Db2Partition partition, Db2OffsetContext offsetContext) throws InterruptedException { - if (!connectorConfig.getSnapshotMode().shouldStream()) { - LOGGER.info("Streaming is not enabled in current configuration"); - return; + // Debezium's ChangeEventSourceCoordinator calls init(offsetContext) before execute(...) to + // set the effective offset context. Flink CDC drives this source directly and never calls + // init, so fall back to the offset context it passes in. + if (getOffsetContext() == null) { + init(offsetContext); } final Metronome metronome = Metronome.sleeper(pollInterval, clock); @@ -142,7 +156,7 @@ public void execute( // situation if (!currentMaxLsn.isAvailable()) { LOGGER.warn( - "No maximum LSN recorded in the database; please ensure that the DB2 Agent is running"); + "No maximum LSN in the database; please ensure that the DB2 Agent is running"); metronome.pause(); continue; } @@ -338,7 +352,8 @@ public void execute( operation, data, dataNext, - clock)); + clock, + connectorConfig)); tableWithSmallestLsn.next(); } }); @@ -350,12 +365,24 @@ public void execute( } catch (SQLException e) { tablesSlot.set(processErrorFromChangeTableQuery(e, tablesSlot.get())); } + + if (context.isPaused()) { + LOGGER.info("Streaming will now pause"); + context.streamingPaused(); + context.waitSnapshotCompletion(); + LOGGER.info("Streaming resumed"); + } } } catch (Exception e) { errorHandler.setProducerThrowable(e); } } + @Override + public Db2OffsetContext getOffsetContext() { + return effectiveOffsetContext; + } + private void migrateTable( Db2Partition partition, Db2OffsetContext offsetContext, @@ -363,15 +390,21 @@ private void migrateTable( throws InterruptedException, SQLException { final Db2ChangeTable newTable = schemaChangeCheckpoints.poll(); LOGGER.info("Migrating schema to {}", newTable); + Table tableSchema = metadataConnection.getTableSchemaFromTable(newTable); + offsetContext.event(newTable.getSourceTableId(), Instant.now()); dispatcher.dispatchSchemaChangeEvent( partition, + offsetContext, newTable.getSourceTableId(), new Db2SchemaChangeEventEmitter( partition, offsetContext, newTable, - metadataConnection.getTableSchemaFromTable(newTable), + tableSchema, + schema, SchemaChangeEventType.ALTER)); + + newTable.setSourceTable(tableSchema); } private Db2ChangeTable[] processErrorFromChangeTableQuery( @@ -450,12 +483,14 @@ private Db2ChangeTable[] getCdcTablesToQuery( // obtained from change table dispatcher.dispatchSchemaChangeEvent( partition, + offsetContext, currentTable.getSourceTableId(), new Db2SchemaChangeEventEmitter( partition, offsetContext, currentTable, dataConnection.getTableSchemaFromTable(currentTable), + schema, SchemaChangeEventType.CREATE)); } tables.add(currentTable); @@ -470,13 +505,11 @@ private Db2ChangeTable[] getCdcTablesToQuery( * changes across all tables.
* This class represents an open database cursor over the change table that is able to move the * cursor forward and report the LSN for the change to which the cursor now points. - * - * @author Jiri Pechanec */ private static class ChangeTablePointer extends ChangeTableResultSet { - public ChangeTablePointer(Db2ChangeTable changeTable, ResultSet resultSet) { + ChangeTablePointer(Db2ChangeTable changeTable, ResultSet resultSet) { super(changeTable, resultSet, COL_DATA); } @@ -495,7 +528,7 @@ protected TxLogPosition getNextChangePosition(ResultSet resultSet) throws SQLExc } } - /** expose control to the user to stop the connector. */ + /** Expose control to the subclass to stop the connector. */ protected void afterHandleLsn(Db2Partition partition, Lsn toLsn) { // do nothing } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/Db2Source.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/Db2Source.java index 2b704562d45..1a2b0c9e174 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/Db2Source.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/Db2Source.java @@ -61,8 +61,9 @@ public DebeziumSourceFunction build() { props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); props.setProperty("database.dbname", checkNotNull(database)); - props.setProperty("database.server.name", DB2_DATABASE_SERVER_NAME); // Hard-coded here - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", DB2_DATABASE_SERVER_NAME); // Hard-coded here + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); if (tableList != null) { props.setProperty("table.include.list", String.join(",", tableList)); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfig.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfig.java index ab35aa409ea..ee3b1018545 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfig.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfig.java @@ -19,6 +19,7 @@ import org.apache.flink.cdc.connectors.base.config.JdbcSourceConfig; import org.apache.flink.cdc.connectors.base.options.StartupOptions; +import org.apache.flink.util.TemporaryClassLoaderContext; import io.debezium.config.Configuration; import io.debezium.connector.db2.Db2ConnectorConfig; @@ -89,6 +90,13 @@ public Db2SourceConfig( @Override public Db2ConnectorConfig getDbzConnectorConfig() { - return new Db2ConnectorConfig(getDbzConfiguration()); + // Debezium 2.0 resolves classes through the thread context class loader + // (io.debezium.config.Instantiator, and the ServiceLoader based SPIs). On a Flink + // thread that loader may be a user code class loader from a previous job attempt, + // which is already closed. Pin it to the loader that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(Db2ConnectorConfig.class.getClassLoader())) { + return new Db2ConnectorConfig(getDbzConfiguration()); + } } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfigFactory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfigFactory.java index 365e7871099..82365e84075 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfigFactory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfigFactory.java @@ -43,8 +43,9 @@ public Db2SourceConfig create(int subtask) { // set database history impl to flink database history props.setProperty( - "database.history", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); - props.setProperty("database.history.instance.name", UUID.randomUUID() + "_" + subtask); + "schema.history.internal", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); + props.setProperty( + "schema.history.internal.instance.name", UUID.randomUUID() + "_" + subtask); // hard code server name, because we don't need to distinguish it, docs: // Logical name that identifies and provides a namespace for the SQL Server database @@ -52,12 +53,13 @@ public Db2SourceConfig create(int subtask) { // all other connectors, since it is used as a prefix for all Kafka topic names // emanating from this connector. Only alphanumeric characters and underscores should be // used. - props.setProperty("database.server.name", DATABASE_SERVER_NAME); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", DATABASE_SERVER_NAME); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); props.setProperty("database.port", String.valueOf(port)); - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); props.setProperty("database.dbname", checkNotNull(databaseList.get(0))); if (tableList != null) { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2ScanFetchTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2ScanFetchTask.java index d6baf843a5c..49233040779 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2ScanFetchTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2ScanFetchTask.java @@ -21,17 +21,22 @@ import org.apache.flink.cdc.connectors.base.source.meta.split.StreamSplit; import org.apache.flink.cdc.connectors.base.source.reader.external.AbstractScanFetchTask; import org.apache.flink.cdc.connectors.db2.source.fetch.Db2StreamFetchTask.StreamSplitReadTask; +import org.apache.flink.cdc.debezium.internal.SnapshotterServiceFactory; import io.debezium.DebeziumException; import io.debezium.config.Configuration; import io.debezium.connector.db2.Db2Connection; +import io.debezium.connector.db2.Db2Connector; import io.debezium.connector.db2.Db2ConnectorConfig; import io.debezium.connector.db2.Db2DatabaseSchema; import io.debezium.connector.db2.Db2OffsetContext; import io.debezium.connector.db2.Db2Partition; import io.debezium.heartbeat.Heartbeat; import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.notification.NotificationService; +import io.debezium.pipeline.signal.actions.snapshotting.SnapshotConfiguration; import io.debezium.pipeline.source.AbstractSnapshotChangeEventSource; +import io.debezium.pipeline.source.SnapshottingTask; import io.debezium.pipeline.source.spi.ChangeEventSource; import io.debezium.pipeline.source.spi.SnapshotProgressListener; import io.debezium.pipeline.spi.ChangeRecordEmitter; @@ -40,6 +45,7 @@ import io.debezium.relational.SnapshotChangeRecordEmitter; import io.debezium.relational.Table; import io.debezium.relational.TableId; +import io.debezium.schema.SchemaFactory; import io.debezium.util.Clock; import io.debezium.util.ColumnUtils; import io.debezium.util.Strings; @@ -52,6 +58,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Duration; +import java.util.Collections; import static org.apache.flink.cdc.connectors.db2.source.utils.Db2Utils.buildSplitScanQuery; import static org.apache.flink.cdc.connectors.db2.source.utils.Db2Utils.readTableSplitDataStatement; @@ -82,7 +89,10 @@ protected void executeDataSnapshot(Context context) throws Exception { snapshotSplitReadTask.execute( changeEventSourceContext, sourceFetchContext.getPartition(), - sourceFetchContext.getOffsetContext()); + sourceFetchContext.getOffsetContext(), + snapshotSplitReadTask.getSnapshottingTask( + sourceFetchContext.getPartition(), + sourceFetchContext.getOffsetContext())); // execute stream read task if (!snapshotResult.isCompletedOrSkipped()) { taskRunning = false; @@ -127,14 +137,16 @@ private StreamSplitReadTask createBackFillLsnSplitReadTask( .with(Heartbeat.HEARTBEAT_INTERVAL, 0) .build(); // task to read wal and backfill for current split + Db2ConnectorConfig backfillConnectorConfig = new Db2ConnectorConfig(dezConf); return new StreamSplitReadTask( - new Db2ConnectorConfig(dezConf), + backfillConnectorConfig, context.getConnection(), context.getMetaDataConnection(), context.getEventDispatcher(), context.getWaterMarkDispatcher(), context.getErrorHandler(), context.getDatabaseSchema(), + SnapshotterServiceFactory.create(backfillConnectorConfig, Db2Connector.class), backfillBinlogSplit); } @@ -166,7 +178,14 @@ public Db2SnapshotSplitReadTask( EventDispatcher dispatcher, EventDispatcher.SnapshotReceiver snapshotReceiver, SnapshotSplit snapshotSplit) { - super(connectorConfig, snapshotProgressListener); + super( + connectorConfig, + snapshotProgressListener, + new NotificationService<>( + Collections.emptyList(), + connectorConfig, + SchemaFactory.get(), + notification -> {})); this.offsetContext = previousOffset; this.connectorConfig = connectorConfig; this.databaseSchema = databaseSchema; @@ -182,12 +201,12 @@ public Db2SnapshotSplitReadTask( public SnapshotResult execute( ChangeEventSourceContext context, Db2Partition partition, - Db2OffsetContext previousOffset) + Db2OffsetContext previousOffset, + SnapshottingTask snapshottingTask) throws InterruptedException { - SnapshottingTask snapshottingTask = getSnapshottingTask(partition, previousOffset); final Db2SnapshotContext ctx; try { - ctx = prepare(partition); + ctx = prepare(partition, false); } catch (Exception e) { LOG.error("Failed to initialize snapshot context.", e); throw new RuntimeException(e); @@ -218,14 +237,26 @@ protected SnapshotResult doExecute( } @Override - protected SnapshottingTask getSnapshottingTask( + public SnapshottingTask getSnapshottingTask( Db2Partition partition, Db2OffsetContext previousOffset) { - return new SnapshottingTask(false, true); + return new SnapshottingTask( + false, true, Collections.emptyList(), Collections.emptyMap(), false); } @Override - protected Db2SnapshotContext prepare(Db2Partition partition) throws Exception { - return new Db2SnapshotContext(partition); + public SnapshottingTask getBlockingSnapshottingTask( + Db2Partition partition, + Db2OffsetContext previousOffset, + SnapshotConfiguration snapshotConfiguration) { + // Debezium 2.6 made this abstract. Flink CDC drives its own split snapshot and never + // runs Debezium's signal-based blocking snapshot, so it behaves like the regular one. + return getSnapshottingTask(partition, previousOffset); + } + + @Override + protected Db2SnapshotContext prepare(Db2Partition partition, boolean onDemand) + throws Exception { + return new Db2SnapshotContext(partition, onDemand); } private void createDataEvents(Db2SnapshotContext snapshotContext, TableId tableId) @@ -279,8 +310,7 @@ private void createDataEventsForTable( while (rs.next()) { rows++; - final Object[] row = - jdbcConnection.rowToArray(table, databaseSchema, rs, columnArray); + final Object[] row = jdbcConnection.rowToArray(table, rs, columnArray); if (logTimer.expired()) { long stop = clock.currentTimeInMillis(); LOG.info( @@ -312,7 +342,7 @@ protected ChangeRecordEmitter getChangeRecordEmitter( Db2SnapshotContext snapshotContext, TableId tableId, Object[] row) { snapshotContext.offset.event(tableId, clock.currentTime()); return new SnapshotChangeRecordEmitter<>( - snapshotContext.partition, snapshotContext.offset, row, clock); + snapshotContext.partition, snapshotContext.offset, row, clock, connectorConfig); } private Threads.Timer getTableScanLogTimer() { @@ -323,8 +353,9 @@ private static class Db2SnapshotContext extends RelationalSnapshotChangeEventSource.RelationalSnapshotContext< Db2Partition, Db2OffsetContext> { - public Db2SnapshotContext(Db2Partition partition) throws SQLException { - super(partition, ""); + public Db2SnapshotContext(Db2Partition partition, boolean onDemand) + throws SQLException { + super(partition, "", onDemand); } } } @@ -344,5 +375,25 @@ public void finished() { public boolean isRunning() { return taskRunning; } + + // The following methods are only used by Debezium's signal-based blocking snapshot, + // which Flink CDC does not use, so they are no-ops here. + + @Override + public boolean isPaused() { + return false; + } + + @Override + public void resumeStreaming() throws InterruptedException {} + + @Override + public void waitSnapshotCompletion() throws InterruptedException {} + + @Override + public void streamingPaused() {} + + @Override + public void waitStreamingPaused() throws InterruptedException {} } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2SourceFetchTaskContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2SourceFetchTaskContext.java index 64e99c32fd2..a4b3bf4122a 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2SourceFetchTaskContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2SourceFetchTaskContext.java @@ -31,6 +31,7 @@ import org.apache.flink.cdc.connectors.db2.source.utils.Db2Utils; import org.apache.flink.table.types.logical.RowType; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.base.ChangeEventQueue; import io.debezium.connector.base.ChangeEventQueue.Builder; import io.debezium.connector.db2.Db2Connection; @@ -41,7 +42,6 @@ import io.debezium.connector.db2.Db2OffsetContext.Loader; import io.debezium.connector.db2.Db2Partition; import io.debezium.connector.db2.Db2TaskContext; -import io.debezium.connector.db2.Db2TopicSelector; import io.debezium.connector.db2.SourceInfo; import io.debezium.data.Envelope.FieldName; import io.debezium.pipeline.DataChangeEvent; @@ -56,10 +56,10 @@ import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.relational.Tables.TableFilter; -import io.debezium.schema.DataCollectionId; -import io.debezium.schema.TopicSelector; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.schema.DataCollectionId; +import io.debezium.spi.topic.TopicNamingStrategy; import io.debezium.util.Collect; -import io.debezium.util.SchemaNameAdjuster; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; @@ -82,7 +82,7 @@ public class Db2SourceFetchTaskContext extends JdbcSourceFetchTaskContext { private ErrorHandler errorHandler; private ChangeEventQueue queue; private Db2TaskContext taskContext; - private TopicSelector topicSelector; + private TopicNamingStrategy topicSelector; private EventDispatcher.SnapshotReceiver snapshotReceiver; private SnapshotChangeEventSourceMetrics snapshotChangeEventSourceMetrics; private StreamingChangeEventSourceMetrics streamingChangeEventSourceMetrics; @@ -102,7 +102,8 @@ public Db2SourceFetchTaskContext( public void configure(SourceSplitBase sourceSplitBase) { // initial stateful objects final Db2ConnectorConfig connectorConfig = getDbzConnectorConfig(); - this.topicSelector = Db2TopicSelector.defaultSelector(connectorConfig); + this.topicSelector = + connectorConfig.getTopicNamingStrategy(CommonConnectorConfig.TOPIC_NAMING_STRATEGY); EmbeddedFlinkDatabaseHistory.registerHistory( sourceConfig .getDbzConfiguration() @@ -156,7 +157,7 @@ public void configure(SourceSplitBase sourceSplitBase) { this.streamingChangeEventSourceMetrics = changeEventSourceMetricsFactory.getStreamingMetrics( taskContext, queue, metadataProvider); - this.errorHandler = new ErrorHandler(Db2Connector.class, connectorConfig, queue); + this.errorHandler = new ErrorHandler(Db2Connector.class, connectorConfig, queue, null); } /** Loads the connector's persistent offset (if present) via the given loader. */ diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2StreamFetchTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2StreamFetchTask.java index 43581c54917..aa9b7c6b6a9 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2StreamFetchTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/fetch/Db2StreamFetchTask.java @@ -24,9 +24,11 @@ import org.apache.flink.cdc.connectors.base.source.meta.wartermark.WatermarkKind; import org.apache.flink.cdc.connectors.base.source.reader.external.FetchTask; import org.apache.flink.cdc.connectors.db2.source.offset.LsnOffset; +import org.apache.flink.cdc.debezium.internal.SnapshotterServiceFactory; import io.debezium.DebeziumException; import io.debezium.connector.db2.Db2Connection; +import io.debezium.connector.db2.Db2Connector; import io.debezium.connector.db2.Db2ConnectorConfig; import io.debezium.connector.db2.Db2DatabaseSchema; import io.debezium.connector.db2.Db2OffsetContext; @@ -37,6 +39,7 @@ import io.debezium.pipeline.EventDispatcher; import io.debezium.pipeline.source.spi.ChangeEventSource.ChangeEventSourceContext; import io.debezium.relational.TableId; +import io.debezium.snapshot.SnapshotterService; import io.debezium.util.Clock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +70,8 @@ public void execute(Context context) throws Exception { sourceFetchContext.getWaterMarkDispatcher(), sourceFetchContext.getErrorHandler(), sourceFetchContext.getDatabaseSchema(), + SnapshotterServiceFactory.create( + sourceFetchContext.getDbzConnectorConfig(), Db2Connector.class), split); RedoLogSplitChangeEventSourceContext changeEventSourceContext = new RedoLogSplitChangeEventSourceContext(); @@ -111,6 +116,7 @@ public StreamSplitReadTask( WatermarkDispatcher watermarkDispatcher, ErrorHandler errorHandler, Db2DatabaseSchema schema, + SnapshotterService snapshotterService, StreamSplit lsnSplit) { super( connectorConfig, @@ -119,7 +125,8 @@ public StreamSplitReadTask( eventDispatcher, errorHandler, Clock.system(), - schema); + schema, + snapshotterService); this.lsnSplit = lsnSplit; this.watermarkDispatcher = watermarkDispatcher; this.errorHandler = errorHandler; @@ -173,5 +180,25 @@ private class RedoLogSplitChangeEventSourceContext implements ChangeEventSourceC public boolean isRunning() { return taskRunning; } + + // The following methods are only used by Debezium's signal-based blocking snapshot, + // which Flink CDC does not use, so they are no-ops here. + + @Override + public boolean isPaused() { + return false; + } + + @Override + public void resumeStreaming() throws InterruptedException {} + + @Override + public void waitSnapshotCompletion() throws InterruptedException {} + + @Override + public void streamingPaused() {} + + @Override + public void waitStreamingPaused() throws InterruptedException {} } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/utils/Db2ConnectionUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/utils/Db2ConnectionUtils.java index 58bffa63c8c..d931a65b5af 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/utils/Db2ConnectionUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/utils/Db2ConnectionUtils.java @@ -17,9 +17,8 @@ package org.apache.flink.cdc.connectors.db2.source.utils; -import io.debezium.config.Configuration; import io.debezium.connector.db2.Db2Connection; -import io.debezium.jdbc.JdbcConfiguration; +import io.debezium.connector.db2.Db2ConnectorConfig; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.RelationalDatabaseConnectorConfig; import io.debezium.relational.RelationalTableFilters; @@ -40,8 +39,9 @@ public class Db2ConnectionUtils { public static Db2Connection createDb2Connection( RelationalDatabaseConnectorConfig connectorConfig) { - Configuration dbzConnectorConfig = connectorConfig.getJdbcConfig(); - return new Db2Connection(JdbcConfiguration.adapt(dbzConnectorConfig)); + // Debezium 2.7 takes the Db2ConnectorConfig itself rather than a JdbcConfiguration; the + // connection needs the config to resolve the CDC schema and the Db2 platform adapter. + return new Db2Connection((Db2ConnectorConfig) connectorConfig); } public static List listTables(JdbcConnection jdbc, RelationalTableFilters tableFilters) diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/utils/Db2Utils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/utils/Db2Utils.java index 34e328dadca..2266ff1e433 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/utils/Db2Utils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/main/java/org/apache/flink/cdc/connectors/db2/source/utils/Db2Utils.java @@ -22,19 +22,21 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.TemporaryClassLoaderContext; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.db2.Db2Connection; import io.debezium.connector.db2.Db2ConnectorConfig; import io.debezium.connector.db2.Db2DatabaseSchema; -import io.debezium.connector.db2.Db2TopicSelector; +import io.debezium.connector.db2.Db2ValueConverters; import io.debezium.connector.db2.Lsn; import io.debezium.connector.db2.SourceInfo; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.Column; import io.debezium.relational.Table; import io.debezium.relational.TableId; -import io.debezium.schema.TopicSelector; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.topic.TopicNamingStrategy; import org.apache.kafka.connect.source.SourceRecord; import javax.annotation.Nullable; @@ -229,11 +231,24 @@ public static PreparedStatement readTableSplitDataStatement( public static Db2DatabaseSchema createDb2DatabaseSchema( Db2ConnectorConfig connectorConfig, Db2Connection connection) { - TopicSelector topicSelector = Db2TopicSelector.defaultSelector(connectorConfig); + TopicNamingStrategy topicSelector; + // Debezium 2.0 resolves classes through the thread context class loader; pin it to the + // loader that loaded Debezium so a stale/closed Flink user class loader cannot break it. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(CommonConnectorConfig.class.getClassLoader())) { + topicSelector = + connectorConfig.getTopicNamingStrategy( + CommonConnectorConfig.TOPIC_NAMING_STRATEGY); + } SchemaNameAdjuster schemaNameAdjuster = SchemaNameAdjuster.create(); + // Debezium 2.0 Db2DatabaseSchema requires the value converters explicitly. + Db2ValueConverters valueConverters = + new Db2ValueConverters( + connectorConfig.getDecimalMode(), + connectorConfig.getTemporalPrecisionMode()); return new Db2DatabaseSchema( - connectorConfig, schemaNameAdjuster, topicSelector, connection); + connectorConfig, valueConverters, schemaNameAdjuster, topicSelector, connection); } // --------------------------private method------------------------------- diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/test/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfigFactoryTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/test/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfigFactoryTest.java new file mode 100644 index 00000000000..278c929fd04 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc/src/test/java/org/apache/flink/cdc/connectors/db2/source/config/Db2SourceConfigFactoryTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.db2.source.config; + +import org.apache.flink.cdc.connectors.base.options.StartupOptions; + +import io.debezium.config.CommonConnectorConfig; +import io.debezium.connector.db2.Db2ConnectorConfig; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link Db2SourceConfigFactory}. */ +class Db2SourceConfigFactoryTest { + + /** + * Debezium 2.0 renamed {@code database.server.name} to {@code topic.prefix}. Without it the + * connector configuration does not validate and the topic naming strategy cannot be built. + */ + @Test + void testTopicPrefixIsConfigured() { + Db2SourceConfigFactory factory = new Db2SourceConfigFactory(); + factory.hostname("localhost"); + factory.port(50000); + factory.username("db2inst1"); + factory.password("flinkpw"); + factory.databaseList("testdb"); + factory.tableList("testdb.DB2INST1.PRODUCTS"); + factory.startupOptions(StartupOptions.initial()); + + Db2ConnectorConfig connectorConfig = + new Db2ConnectorConfig(factory.create(0).getDbzConfiguration()); + + assertThat(connectorConfig.getLogicalName()).isEqualTo("Db2_transaction_log_source"); + assertThat( + connectorConfig.getTopicNamingStrategy( + CommonConnectorConfig.TOPIC_NAMING_STRATEGY)) + .isNotNull(); + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/embedded/EmbeddedEngineChangeEvent.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/embedded/EmbeddedEngineChangeEvent.java index cf39b79f4a3..731a3ff61f7 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/embedded/EmbeddedEngineChangeEvent.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/embedded/EmbeddedEngineChangeEvent.java @@ -6,24 +6,31 @@ package io.debezium.embedded; -import org.apache.flink.cdc.debezium.internal.DebeziumChangeFetcher; - import io.debezium.engine.ChangeEvent; +import io.debezium.engine.Header; import io.debezium.engine.RecordChangeEvent; import org.apache.kafka.connect.source.SourceRecord; +import java.util.List; + /** - * Copied from Debezium project. Make it public to be accessible from {@link DebeziumChangeFetcher}. + * Copied from Debezium project(2.7.4.Final).. + * + *

Change 1: the constructor is public so {@code DebeziumChangeConsumer} can construct the event + * when handing records to the legacy DataStream source (upstream keeps it package-private). */ -public class EmbeddedEngineChangeEvent implements ChangeEvent, RecordChangeEvent { +public class EmbeddedEngineChangeEvent implements ChangeEvent, RecordChangeEvent { private final K key; private final V value; + private final List> headers; private final SourceRecord sourceRecord; - public EmbeddedEngineChangeEvent(K key, V value, SourceRecord sourceRecord) { + public EmbeddedEngineChangeEvent( + K key, V value, List> headers, SourceRecord sourceRecord) { this.key = key; this.value = value; + this.headers = headers; this.sourceRecord = sourceRecord; } @@ -37,6 +44,12 @@ public V value() { return value; } + @SuppressWarnings("unchecked") + @Override + public List> headers() { + return headers; + } + @Override public V record() { return value; @@ -47,6 +60,11 @@ public String destination() { return sourceRecord.topic(); } + @Override + public Integer partition() { + return sourceRecord.kafkaPartition(); + } + public SourceRecord sourceRecord() { return sourceRecord; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/CachedTableFilter.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/CachedTableFilter.java index 18e9ef93ac1..33645185cd9 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/CachedTableFilter.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/CachedTableFilter.java @@ -25,7 +25,12 @@ import static org.apache.flink.util.Preconditions.checkNotNull; -/** A bounded cache for table filter results. */ +/** + * A bounded cache for table filter results. * + * + *

This is a Flink CDC owned class (not a Debezium fork), placed in the {@code io.debezium} + * package for access to package-private Debezium members. + */ public class CachedTableFilter implements TableFilter { private static final long TABLE_FILTER_CACHE_MAXIMUM_SIZE = 32 * 1024; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/HistorizedRelationalDatabaseConnectorConfig.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/HistorizedRelationalDatabaseConnectorConfig.java deleted file mode 100644 index 98d25b74f11..00000000000 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/HistorizedRelationalDatabaseConnectorConfig.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright Debezium Authors. - * - * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 - */ - -package io.debezium.relational; - -import io.debezium.config.ConfigDefinition; -import io.debezium.config.Configuration; -import io.debezium.config.Field; -import io.debezium.relational.Selectors.TableIdToStringMapper; -import io.debezium.relational.Tables.TableFilter; -import io.debezium.relational.history.DatabaseHistory; -import io.debezium.relational.history.DatabaseHistoryListener; -import io.debezium.relational.history.DatabaseHistoryMetrics; -import io.debezium.relational.history.HistoryRecordComparator; -import io.debezium.relational.history.KafkaDatabaseHistory; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; -import org.apache.kafka.common.config.ConfigDef.Width; -import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.source.SourceConnector; - -/** - * Copied from Debezium project. Configuration options shared across the relational CDC connectors - * which use a persistent database schema history. - * - *

Added JMX_METRICS_ENABLED option. - */ -public abstract class HistorizedRelationalDatabaseConnectorConfig - extends RelationalDatabaseConnectorConfig { - - protected static final int DEFAULT_SNAPSHOT_FETCH_SIZE = 2_000; - - private boolean useCatalogBeforeSchema; - private final String logicalName; - private final Class connectorClass; - private final boolean multiPartitionMode; - - /** - * The database history class is hidden in the {@link #configDef()} since that is designed to - * work with a user interface, and in these situations using Kafka is the only way to go. - */ - public static final Field DATABASE_HISTORY = - Field.create("database.history") - .withDisplayName("Database history class") - .withType(Type.CLASS) - .withWidth(Width.LONG) - .withImportance(Importance.LOW) - .withInvisibleRecommender() - .withDescription( - "The name of the DatabaseHistory class that should be used to store and recover database schema changes. " - + "The configuration properties for the history are prefixed with the '" - + DatabaseHistory.CONFIGURATION_FIELD_PREFIX_STRING - + "' string.") - .withDefault(KafkaDatabaseHistory.class.getName()); - - public static final Field JMX_METRICS_ENABLED = - Field.create(DatabaseHistory.CONFIGURATION_FIELD_PREFIX_STRING + "metrics.enabled") - .withDisplayName("Skip DDL statements that cannot be parsed") - .withType(Type.BOOLEAN) - .withImportance(Importance.LOW) - .withDescription("Whether to enable JMX history metrics") - .withDefault(false); - - protected static final ConfigDefinition CONFIG_DEFINITION = - RelationalDatabaseConnectorConfig.CONFIG_DEFINITION - .edit() - .history( - DATABASE_HISTORY, - DatabaseHistory.SKIP_UNPARSEABLE_DDL_STATEMENTS, - DatabaseHistory.STORE_ONLY_MONITORED_TABLES_DDL, - DatabaseHistory.STORE_ONLY_CAPTURED_TABLES_DDL, - KafkaDatabaseHistory.BOOTSTRAP_SERVERS, - KafkaDatabaseHistory.TOPIC, - KafkaDatabaseHistory.RECOVERY_POLL_ATTEMPTS, - KafkaDatabaseHistory.RECOVERY_POLL_INTERVAL_MS, - KafkaDatabaseHistory.KAFKA_QUERY_TIMEOUT_MS) - .create(); - - protected HistorizedRelationalDatabaseConnectorConfig( - Class connectorClass, - Configuration config, - String logicalName, - TableFilter systemTablesFilter, - boolean useCatalogBeforeSchema, - int defaultSnapshotFetchSize, - ColumnFilterMode columnFilterMode, - boolean multiPartitionMode) { - super( - config, - logicalName, - systemTablesFilter, - TableId::toString, - defaultSnapshotFetchSize, - columnFilterMode); - this.useCatalogBeforeSchema = useCatalogBeforeSchema; - this.logicalName = logicalName; - this.connectorClass = connectorClass; - this.multiPartitionMode = multiPartitionMode; - } - - protected HistorizedRelationalDatabaseConnectorConfig( - Class connectorClass, - Configuration config, - String logicalName, - TableFilter systemTablesFilter, - TableIdToStringMapper tableIdMapper, - boolean useCatalogBeforeSchema, - ColumnFilterMode columnFilterMode, - boolean multiPartitionMode) { - super( - config, - logicalName, - systemTablesFilter, - tableIdMapper, - DEFAULT_SNAPSHOT_FETCH_SIZE, - columnFilterMode); - this.useCatalogBeforeSchema = useCatalogBeforeSchema; - this.logicalName = logicalName; - this.connectorClass = connectorClass; - this.multiPartitionMode = multiPartitionMode; - } - - /** Returns a configured (but not yet started) instance of the database history. */ - public DatabaseHistory getDatabaseHistory() { - Configuration config = getConfig(); - - DatabaseHistory databaseHistory = - config.getInstance( - HistorizedRelationalDatabaseConnectorConfig.DATABASE_HISTORY, - DatabaseHistory.class); - if (databaseHistory == null) { - throw new ConnectException( - "Unable to instantiate the database history class " - + config.getString( - HistorizedRelationalDatabaseConnectorConfig.DATABASE_HISTORY)); - } - - // Do not remove the prefix from the subset of config properties ... - Configuration dbHistoryConfig = - config.subset(DatabaseHistory.CONFIGURATION_FIELD_PREFIX_STRING, false) - .edit() - .withDefault(DatabaseHistory.NAME, getLogicalName() + "-dbhistory") - .withDefault( - KafkaDatabaseHistory.INTERNAL_CONNECTOR_CLASS, - connectorClass.getName()) - .withDefault(KafkaDatabaseHistory.INTERNAL_CONNECTOR_ID, logicalName) - .build(); - - DatabaseHistoryListener listener = - config.getBoolean(JMX_METRICS_ENABLED) - ? new DatabaseHistoryMetrics(this, multiPartitionMode) - : DatabaseHistoryListener.NOOP; - - HistoryRecordComparator historyComparator = getHistoryRecordComparator(); - databaseHistory.configure( - dbHistoryConfig, historyComparator, listener, useCatalogBeforeSchema); // validates - - return databaseHistory; - } - - public boolean useCatalogBeforeSchema() { - return useCatalogBeforeSchema; - } - - /** - * Returns a comparator to be used when recovering records from the schema history, making sure - * no history entries newer than the offset we resume from are recovered (which could happen - * when restarting a connector after history records have been persisted but no new offset has - * been committed yet). - */ - protected abstract HistoryRecordComparator getHistoryRecordComparator(); -} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/RelationalChangeRecordEmitter.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/RelationalChangeRecordEmitter.java index 2b0688a5dc4..4fdc3630442 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/RelationalChangeRecordEmitter.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/RelationalChangeRecordEmitter.java @@ -3,12 +3,10 @@ * * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 */ - package io.debezium.relational; import io.debezium.data.Envelope.Operation; import io.debezium.pipeline.AbstractChangeRecordEmitter; -import io.debezium.pipeline.spi.ChangeRecordEmitter; import io.debezium.pipeline.spi.OffsetContext; import io.debezium.pipeline.spi.Partition; import io.debezium.schema.DataCollectionSchema; @@ -22,13 +20,14 @@ import java.util.Optional; /** - * Copied from Debezium 1.9.8.Final. - * - *

Base class for {@link ChangeRecordEmitter} implementations based on a relational database. + * Base class for {@link io.debezium.pipeline.spi.ChangeRecordEmitter} implementations based on a + * relational database. * - *

This class overrides the emit methods to put some values in the header. + *

Copied from Debezium project(2.7.4.Final).. * - *

Line 59 ~ 257: add other headers and emit. + *

Change 1: add the {@code getEmitConnectHeaders()} hook and pass its value as the change record + * headers in every emit method, so subclasses (e.g. the Oracle LogMiner emitter's ROWID header) can + * attach Connect headers to emitted records. */ public abstract class RelationalChangeRecordEmitter

extends AbstractChangeRecordEmitter { @@ -39,8 +38,12 @@ public abstract class RelationalChangeRecordEmitter

public static final String PK_UPDATE_OLDKEY_FIELD = "__debezium.oldkey"; public static final String PK_UPDATE_NEWKEY_FIELD = "__debezium.newkey"; - public RelationalChangeRecordEmitter(P partition, OffsetContext offsetContext, Clock clock) { - super(partition, offsetContext, clock); + public RelationalChangeRecordEmitter( + P partition, + OffsetContext offsetContext, + Clock clock, + RelationalDatabaseConnectorConfig connectorConfig) { + super(partition, offsetContext, clock, connectorConfig); } @Override @@ -87,7 +90,7 @@ protected void emitCreateRecord(Receiver

receiver, TableSchema tableSchema) if (skipEmptyMessages() && (newColumnValues == null || newColumnValues.length == 0)) { // This case can be hit on UPDATE / DELETE when there's no primary key defined while // using certain decoders - LOGGER.warn( + LOGGER.debug( "no new values found for table '{}' from create message at '{}'; skipping record", tableSchema, getOffset().getSourceInfo()); @@ -140,12 +143,25 @@ protected void emitUpdateRecord(Receiver

receiver, TableSchema tableSchema) Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); if (skipEmptyMessages() && (newColumnValues == null || newColumnValues.length == 0)) { - LOGGER.warn( + LOGGER.debug( "no new values found for table '{}' from update message at '{}'; skipping record", tableSchema, getOffset().getSourceInfo()); return; } + + /* + * If skip.messages.without.change is configured true, + * Skip Publishing the message in case there is no change in monitored columns + * (Postgres) Only works if REPLICA IDENTITY is set to FULL - as oldValues won't be available + */ + if (skipMessagesWithoutChange() && Objects.nonNull(newValue) && newValue.equals(oldValue)) { + LOGGER.debug( + "No new values found for table '{}' in included columns from update message at '{}'; skipping record", + tableSchema, + getOffset().getSourceInfo()); + return; + } // some configurations does not provide old values in case of updates // in this case we handle all updates as regular ones if (oldKey == null || Objects.equals(oldKey, newKey)) { @@ -181,7 +197,7 @@ protected void emitDeleteRecord(Receiver

receiver, TableSchema tableSchema) Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); if (skipEmptyMessages() && (oldColumnValues == null || oldColumnValues.length == 0)) { - LOGGER.warn( + LOGGER.debug( "no old values found for table '{}' from delete message at '{}'; skipping record", tableSchema, getOffset().getSourceInfo()); @@ -219,8 +235,8 @@ protected void emitTruncateRecord(Receiver

receiver, TableSchema schema) /** * Whether empty data messages should be ignored. * - * @return true if empty data messages coming from data source should be ignored. Typical use - * case are PostgreSQL changes without FULL replica identity. + * @return true if empty data messages coming from data source should be ignored.
Typical + * use case are PostgreSQL changes without FULL replica identity. */ protected boolean skipEmptyMessages() { return false; @@ -234,45 +250,34 @@ protected void emitUpdateAsPrimaryKeyChangeRecord( Struct oldValue, Struct newValue) throws InterruptedException { - ConnectHeaders headers = getEmitConnectHeaders().orElse(new ConnectHeaders()); + final OffsetContext offset = getOffset(); + final Struct sourceInfo = offset.getSourceInfo(); + + ConnectHeaders headers = new ConnectHeaders(); headers.add(PK_UPDATE_NEWKEY_FIELD, newKey, tableSchema.keySchema()); Struct envelope = tableSchema .getEnvelopeSchema() - .delete( - oldValue, - getOffset().getSourceInfo(), - getClock().currentTimeAsInstant()); + .delete(oldValue, sourceInfo, getClock().currentTimeAsInstant()); receiver.changeRecord( - getPartition(), - tableSchema, - Operation.DELETE, - oldKey, - envelope, - getOffset(), - headers); + getPartition(), tableSchema, Operation.DELETE, oldKey, envelope, offset, headers); - headers = getEmitConnectHeaders().orElse(new ConnectHeaders()); + headers = new ConnectHeaders(); headers.add(PK_UPDATE_OLDKEY_FIELD, oldKey, tableSchema.keySchema()); envelope = tableSchema .getEnvelopeSchema() - .create( - newValue, - getOffset().getSourceInfo(), - getClock().currentTimeAsInstant()); + .create(newValue, sourceInfo, getClock().currentTimeAsInstant()); receiver.changeRecord( - getPartition(), - tableSchema, - Operation.CREATE, - newKey, - envelope, - getOffset(), - headers); + getPartition(), tableSchema, Operation.CREATE, newKey, envelope, offset, headers); } + /** + * Returns the Connect headers to attach to emitted change records; {@link Optional#empty()} by + * default. Flink CDC addition — see the class javadoc. + */ protected Optional getEmitConnectHeaders() { return Optional.empty(); } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/RelationalTableFilters.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/RelationalTableFilters.java index 4d91ff83c2e..98e0aeb008b 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/RelationalTableFilters.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/io/debezium/relational/RelationalTableFilters.java @@ -9,30 +9,32 @@ import io.debezium.relational.Selectors.TableIdToStringMapper; import io.debezium.relational.Selectors.TableSelectionPredicateBuilder; import io.debezium.relational.Tables.TableFilter; -import io.debezium.relational.history.DatabaseHistory; +import io.debezium.relational.history.SchemaHistory; import io.debezium.schema.DataCollectionFilters; import java.util.function.Predicate; -import static io.debezium.relational.RelationalDatabaseConnectorConfig.COLUMN_BLACKLIST; import static io.debezium.relational.RelationalDatabaseConnectorConfig.COLUMN_EXCLUDE_LIST; /** - * Copied from Debezium 1.9.8.Final. + * Copied from Debezium project(2.7.4.Final).. * - *

Line 98: cache table filter results. + *

Change 1: wrap the table filter in {@link CachedTableFilter} so repeated filter evaluations + * (once per change record) hit a cache instead of re-running the predicate chain. * - *

Line 148: add a method to update the tableFilter variable. + *

Change 2: add {@code setDataCollectionFilters} so Flink CDC can replace the table filter after + * construction (used by the MySQL source config when the captured-table list is refined). */ public class RelationalTableFilters implements DataCollectionFilters { - // Filter that filters tables based only on datbase/schema/system table filters but not table + // Filter that filters tables based only on database/schema/system table filters but not table // filters // Represents the list of tables whose schema needs to be captured private final TableFilter eligibleTableFilter; // Filter that filters tables based on table filters private TableFilter tableFilter; private final Predicate databaseFilter; + private final Predicate schemaFilter; private final String excludeColumns; /** @@ -45,26 +47,23 @@ public class RelationalTableFilters implements DataCollectionFilters { public RelationalTableFilters( Configuration config, TableFilter systemTablesFilter, - TableIdToStringMapper tableIdMapper) { + TableIdToStringMapper tableIdMapper, + boolean useCatalogBeforeSchema) { // Define the filter that provides the list of tables that could be captured if configured final TableSelectionPredicateBuilder eligibleTables = Selectors.tableSelector() .includeDatabases( - config.getFallbackStringProperty( - RelationalDatabaseConnectorConfig.DATABASE_INCLUDE_LIST, - RelationalDatabaseConnectorConfig.DATABASE_WHITELIST)) + config.getString( + RelationalDatabaseConnectorConfig.DATABASE_INCLUDE_LIST)) .excludeDatabases( - config.getFallbackStringProperty( - RelationalDatabaseConnectorConfig.DATABASE_EXCLUDE_LIST, - RelationalDatabaseConnectorConfig.DATABASE_BLACKLIST)) + config.getString( + RelationalDatabaseConnectorConfig.DATABASE_EXCLUDE_LIST)) .includeSchemas( - config.getFallbackStringProperty( - RelationalDatabaseConnectorConfig.SCHEMA_INCLUDE_LIST, - RelationalDatabaseConnectorConfig.SCHEMA_WHITELIST)) + config.getString( + RelationalDatabaseConnectorConfig.SCHEMA_INCLUDE_LIST)) .excludeSchemas( - config.getFallbackStringProperty( - RelationalDatabaseConnectorConfig.SCHEMA_EXCLUDE_LIST, - RelationalDatabaseConnectorConfig.SCHEMA_BLACKLIST)); + config.getString( + RelationalDatabaseConnectorConfig.SCHEMA_EXCLUDE_LIST)); final Predicate eligibleTablePredicate = eligibleTables.build(); Predicate finalEligibleTablePredicate = @@ -78,14 +77,12 @@ public RelationalTableFilters( Predicate tablePredicate = eligibleTables .includeTables( - config.getFallbackStringProperty( - RelationalDatabaseConnectorConfig.TABLE_INCLUDE_LIST, - RelationalDatabaseConnectorConfig.TABLE_WHITELIST), + config.getString( + RelationalDatabaseConnectorConfig.TABLE_INCLUDE_LIST), tableIdMapper) .excludeTables( - config.getFallbackStringProperty( - RelationalDatabaseConnectorConfig.TABLE_EXCLUDE_LIST, - RelationalDatabaseConnectorConfig.TABLE_BLACKLIST), + config.getString( + RelationalDatabaseConnectorConfig.TABLE_EXCLUDE_LIST), tableIdMapper) .build(); @@ -93,21 +90,41 @@ public RelationalTableFilters( config.getBoolean(RelationalDatabaseConnectorConfig.TABLE_IGNORE_BUILTIN) ? tablePredicate.and(systemTablesFilter::isIncluded) : tablePredicate; - - TableFilter initialTableFilter = finalTablePredicate::test; - this.tableFilter = CachedTableFilter.from(initialTableFilter); + String signalDataCollection = + config.getString(RelationalDatabaseConnectorConfig.SIGNAL_DATA_COLLECTION); + if (signalDataCollection != null) { + TableId signalDataCollectionTableId = + TableId.parse(signalDataCollection, useCatalogBeforeSchema); + if (!finalTablePredicate.test(signalDataCollectionTableId)) { + final Predicate signalDataCollectionPredicate = + Selectors.tableSelector() + .includeTables( + tableIdMapper.toString(signalDataCollectionTableId), + tableIdMapper) + .build(); + finalTablePredicate = finalTablePredicate.or(signalDataCollectionPredicate); + } + } + this.tableFilter = CachedTableFilter.from(finalTablePredicate::test); // Define the database filter using the include and exclude lists for database names ... this.databaseFilter = Selectors.databaseSelector() .includeDatabases( - config.getFallbackStringProperty( - RelationalDatabaseConnectorConfig.DATABASE_INCLUDE_LIST, - RelationalDatabaseConnectorConfig.DATABASE_WHITELIST)) + config.getString( + RelationalDatabaseConnectorConfig.DATABASE_INCLUDE_LIST)) .excludeDatabases( - config.getFallbackStringProperty( - RelationalDatabaseConnectorConfig.DATABASE_EXCLUDE_LIST, - RelationalDatabaseConnectorConfig.DATABASE_BLACKLIST)) + config.getString( + RelationalDatabaseConnectorConfig.DATABASE_EXCLUDE_LIST)) + .build(); + this.schemaFilter = + Selectors.databaseSelector() + .includeDatabases( + config.getString( + RelationalDatabaseConnectorConfig.SCHEMA_INCLUDE_LIST)) + .excludeDatabases( + config.getString( + RelationalDatabaseConnectorConfig.SCHEMA_EXCLUDE_LIST)) .build(); Predicate eligibleSchemaPredicate = @@ -115,13 +132,15 @@ public RelationalTableFilters( ? systemTablesFilter::isIncluded : x -> true; - this.schemaSnapshotFilter = - config.getBoolean(DatabaseHistory.STORE_ONLY_CAPTURED_TABLES_DDL) - ? eligibleSchemaPredicate.and(initialTableFilter::isIncluded)::test - : eligibleSchemaPredicate::test; + if (config.getBoolean(SchemaHistory.STORE_ONLY_CAPTURED_TABLES_DDL)) { + this.schemaSnapshotFilter = eligibleSchemaPredicate.and(tableFilter::isIncluded)::test; + } else if (config.getBoolean(SchemaHistory.STORE_ONLY_CAPTURED_DATABASES_DDL)) { + this.schemaSnapshotFilter = finalEligibleTablePredicate::test; + } else { + this.schemaSnapshotFilter = eligibleSchemaPredicate::test; + } - this.excludeColumns = - config.getFallbackStringProperty(COLUMN_EXCLUDE_LIST, COLUMN_BLACKLIST); + this.excludeColumns = config.getString(COLUMN_EXCLUDE_LIST); } @Override @@ -141,10 +160,15 @@ public Predicate databaseFilter() { return databaseFilter; } + public Predicate schemaFilter() { + return schemaFilter; + } + public String getExcludeColumns() { return excludeColumns; } + /** Replaces the table filter. Flink CDC addition — see the class javadoc. */ public void setDataCollectionFilters(TableFilter tableFilter) { this.tableFilter = tableFilter; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/DebeziumSourceFunction.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/DebeziumSourceFunction.java index 21d822a06ee..2302242bb58 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/DebeziumSourceFunction.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/DebeziumSourceFunction.java @@ -49,6 +49,7 @@ import org.apache.flink.shaded.guava31.com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.debezium.DebeziumException; import io.debezium.document.DocumentReader; import io.debezium.document.DocumentWriter; import io.debezium.embedded.Connect; @@ -370,6 +371,16 @@ private void snapshotHistoryRecordsState() throws Exception { @Override public void run(SourceContext sourceContext) throws Exception { properties.putIfAbsent("name", "engine"); + // Debezium 2.x tags the JMX ObjectName of a multi-partition connector's metrics with + // "task.id" in addition to the topic prefix. Kafka Connect sets that property; Flink + // never has, so Metrics#metricName hands null to Sanitizer.jmxSanitize and the task + // fails with a NullPointerException before producing anything. SQL Server is the only + // multi-partition connector today, but the property is connector-agnostic, so set it + // here for every embedded engine. The subtask index is the natural analogue of a + // Connect task id and keeps the name unique within the TaskManager JVM. + properties.putIfAbsent( + "task.id", + String.valueOf(getRuntimeContext().getTaskInfo().getIndexOfThisSubtask())); properties.setProperty("offset.storage", FlinkOffsetBackingStore.class.getCanonicalName()); if (restoredOffsetState != null) { // restored from state @@ -381,6 +392,13 @@ public void run(SourceContext sourceContext) throws Exception { properties.putIfAbsent("offset.flush.interval.ms", String.valueOf(Long.MAX_VALUE)); // disable tombstones properties.setProperty("tombstones.on.delete", "false"); + // Debezium 2.4 made the embedded engine restart the connector itself on a retriable + // error, retrying forever by default ("errors.max.retries" = -1). Debezium 2.3 and older + // had no such loop: the error reached the completion callback, the Flink job failed and + // Flink's restart strategy recovered it from the last checkpoint. Keep that behavior - an + // engine silently retrying forever looks like a source that simply stopped producing. + // Users who want the Debezium-side retries can still set "debezium.errors.max.retries". + properties.putIfAbsent("errors.max.retries", "0"); if (engineInstanceName == null) { // not restore from recovery engineInstanceName = UUID.randomUUID().toString(); @@ -393,7 +411,7 @@ public void run(SourceContext sourceContext) throws Exception { // see // https://stackoverflow.com/questions/57147584/debezium-error-schema-isnt-know-to-this-connector // and https://debezium.io/blog/2018/03/16/note-on-database-history-topic-configuration/ - properties.setProperty("database.history", determineDatabase().getCanonicalName()); + properties.setProperty("schema.history.internal", determineDatabase().getCanonicalName()); // we have to filter out the heartbeat events, otherwise the deserializer will fail String dbzHeartbeatPrefix = @@ -420,7 +438,16 @@ public void run(SourceContext sourceContext) throws Exception { // Close the handover and prepare to exit. handover.close(); } else { - handover.reportError(error); + // Debezium may report a failure without a throwable, e.g. + // when the connector configuration does not validate. In + // that case the message is the only diagnostic we have. + handover.reportError( + error != null + ? error + : new DebeziumException( + message == null + ? "The Debezium engine has stopped unexpectedly." + : message)); } }) .build(); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/history/FlinkJsonTableChangeSerializer.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/history/FlinkJsonTableChangeSerializer.java index 234673c3d5c..45e5136c646 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/history/FlinkJsonTableChangeSerializer.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/history/FlinkJsonTableChangeSerializer.java @@ -58,8 +58,10 @@ public Document toDocument(TableChange tableChange) { document.setString("type", tableChange.getType().name()); document.setString("id", tableChange.getId().toDoubleQuotedString()); - document.setDocument("table", toDocument(tableChange.getTable())); - document.setString("comment", tableChange.getTable().comment()); + if (tableChange.getTable() != null) { + document.setDocument("table", toDocument(tableChange.getTable())); + document.setString("comment", tableChange.getTable().comment()); + } return document; } @@ -127,7 +129,7 @@ public TableChanges deserialize(Array array, boolean useCatalogBeforeSchema) { } else if (change.getType() == TableChangeType.ALTER) { tableChanges.alter(change.getTable()); } else if (change.getType() == TableChangeType.DROP) { - tableChanges.drop(change.getTable()); + tableChanges.drop(change.getId()); } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/DebeziumChangeConsumer.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/DebeziumChangeConsumer.java index 03aae4b3fe8..25ffbb51ff0 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/DebeziumChangeConsumer.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/DebeziumChangeConsumer.java @@ -28,6 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -78,8 +79,9 @@ public void commitOffset(DebeziumOffset offset) throws InterruptedException { "DUMMY", Schema.BOOLEAN_SCHEMA, true); - EmbeddedEngineChangeEvent changeEvent = - new EmbeddedEngineChangeEvent<>(null, recordWrapper, recordWrapper); + EmbeddedEngineChangeEvent changeEvent = + new EmbeddedEngineChangeEvent<>( + null, recordWrapper, Collections.emptyList(), recordWrapper); currentCommitter.markProcessed(changeEvent); currentCommitter.markBatchFinished(); } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkDatabaseHistory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkDatabaseHistory.java index f4f880b9f39..983887966c1 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkDatabaseHistory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkDatabaseHistory.java @@ -18,11 +18,11 @@ package org.apache.flink.cdc.debezium.internal; import io.debezium.config.Configuration; -import io.debezium.relational.history.AbstractDatabaseHistory; -import io.debezium.relational.history.DatabaseHistoryException; -import io.debezium.relational.history.DatabaseHistoryListener; +import io.debezium.relational.history.AbstractSchemaHistory; import io.debezium.relational.history.HistoryRecord; import io.debezium.relational.history.HistoryRecordComparator; +import io.debezium.relational.history.SchemaHistoryException; +import io.debezium.relational.history.SchemaHistoryListener; import java.util.Collection; import java.util.concurrent.ConcurrentLinkedQueue; @@ -40,9 +40,10 @@ * records will be stored in state (grow infinitely). We may need to come up with a * FileSystemDatabaseHistory in the future to store history in HDFS. */ -public class FlinkDatabaseHistory extends AbstractDatabaseHistory { +public class FlinkDatabaseHistory extends AbstractSchemaHistory { - public static final String DATABASE_HISTORY_INSTANCE_NAME = "database.history.instance.name"; + public static final String DATABASE_HISTORY_INSTANCE_NAME = + "schema.history.internal.instance.name"; private ConcurrentLinkedQueue schemaRecords; private String instanceName; @@ -57,7 +58,7 @@ private ConcurrentLinkedQueue getRegisteredHistoryRecord(String in public void configure( Configuration config, HistoryRecordComparator comparator, - DatabaseHistoryListener listener, + SchemaHistoryListener listener, boolean useCatalogBeforeSchema) { super.configure(config, comparator, listener, useCatalogBeforeSchema); this.instanceName = config.getString(DATABASE_HISTORY_INSTANCE_NAME); @@ -68,6 +69,16 @@ public void configure( registerHistory(instanceName, schemaRecords); } + // Debezium 2.0 wires SchemaHistoryMetrics in as the schema history listener, and + // AbstractSchemaHistory#start does nothing but hand it a started() callback. That callback + // registers a JMX MBean named only after the connector context and the topic prefix, so + // every parallel subtask sharing a TaskManager JVM asks for the very same name. Debezium + // answers a name clash by sleeping 5 seconds and retrying, twelve times. Flink CDC + // publishes its own metrics and never reads these MBeans, and Debezium 1.9 did not + // register them either, so skip the registration. + @Override + public void start() {} + @Override public void stop() { super.stop(); @@ -75,7 +86,7 @@ public void stop() { } @Override - protected void storeRecord(HistoryRecord record) throws DatabaseHistoryException { + protected void storeRecord(HistoryRecord record) throws SchemaHistoryException { this.schemaRecords.add(new SchemaRecord(record)); } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkDatabaseSchemaHistory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkDatabaseSchemaHistory.java index b9fc8fd7795..d509d60029f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkDatabaseSchemaHistory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkDatabaseSchemaHistory.java @@ -23,14 +23,15 @@ import io.debezium.relational.TableId; import io.debezium.relational.Tables; import io.debezium.relational.ddl.DdlParser; -import io.debezium.relational.history.DatabaseHistory; -import io.debezium.relational.history.DatabaseHistoryException; -import io.debezium.relational.history.DatabaseHistoryListener; import io.debezium.relational.history.HistoryRecord; import io.debezium.relational.history.HistoryRecordComparator; +import io.debezium.relational.history.SchemaHistory; +import io.debezium.relational.history.SchemaHistoryException; +import io.debezium.relational.history.SchemaHistoryListener; import io.debezium.relational.history.TableChanges; import io.debezium.schema.DatabaseSchema; +import java.time.Instant; import java.util.Collection; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -51,16 +52,17 @@ * FlinkDatabaseHistory}. Because it only maintains the latest schema of the table rather than all * history DDLs, it's useful to prevent OOM when meet massive history DDLs. */ -public class FlinkDatabaseSchemaHistory implements DatabaseHistory { +public class FlinkDatabaseSchemaHistory implements SchemaHistory { - public static final String DATABASE_HISTORY_INSTANCE_NAME = "database.history.instance.name"; + public static final String DATABASE_HISTORY_INSTANCE_NAME = + "schema.history.internal.instance.name"; private final FlinkJsonTableChangeSerializer tableChangesSerializer = new FlinkJsonTableChangeSerializer(); private ConcurrentMap latestTables; private String instanceName; - private DatabaseHistoryListener listener; + private SchemaHistoryListener listener; private boolean storeOnlyMonitoredTablesDdl; private boolean skipUnparseableDDL; private boolean useCatalogBeforeSchema; @@ -69,11 +71,11 @@ public class FlinkDatabaseSchemaHistory implements DatabaseHistory { public void configure( Configuration config, HistoryRecordComparator comparator, - DatabaseHistoryListener listener, + SchemaHistoryListener listener, boolean useCatalogBeforeSchema) { this.instanceName = config.getString(DATABASE_HISTORY_INSTANCE_NAME); this.listener = listener; - this.storeOnlyMonitoredTablesDdl = config.getBoolean(STORE_ONLY_MONITORED_TABLES_DDL); + this.storeOnlyMonitoredTablesDdl = config.getBoolean(STORE_ONLY_CAPTURED_TABLES_DDL); this.skipUnparseableDDL = config.getBoolean(SKIP_UNPARSEABLE_DDL_STATEMENTS); this.useCatalogBeforeSchema = useCatalogBeforeSchema; @@ -91,14 +93,19 @@ public void configure( } @Override - public void start() { - listener.started(); - } + // Debezium 2.0 wires SchemaHistoryMetrics in as the schema history listener. Its + // started() callback registers a JMX MBean whose name is built only from the connector + // context and the topic prefix, so every parallel subtask sharing a TaskManager JVM asks + // for the very same name. Debezium answers a name clash by sleeping 5 seconds and + // retrying, twelve times, so every reader but the first stalls for up to a minute each + // time it opens a split. Flink CDC publishes its own metrics and never reads these + // MBeans, and Debezium 1.9 did not register them either, so skip the registration. + public void start() {} @Override public void record( Map source, Map position, String databaseName, String ddl) - throws DatabaseHistoryException { + throws SchemaHistoryException { throw new UnsupportedOperationException( String.format( "The %s cannot work with 'debezium.internal.implementation' = 'legacy'," @@ -114,8 +121,9 @@ public void record( String databaseName, String schemaName, String ddl, - TableChanges changes) - throws DatabaseHistoryException { + TableChanges changes, + Instant timestamp) + throws SchemaHistoryException { for (TableChanges.TableChange change : changes) { switch (change.getType()) { case CREATE: @@ -134,7 +142,8 @@ public void record( } } listener.onChangeApplied( - new HistoryRecord(source, position, databaseName, schemaName, ddl, changes)); + new HistoryRecord( + source, position, databaseName, schemaName, ddl, changes, timestamp)); } @Override @@ -186,12 +195,12 @@ public void initializeStorage() { // do nothing } - @Override + // Debezium 2.2 moved storeOnlyCapturedTables()/skipUnparseableDdlStatements() from the + // SchemaHistory interface to HistorizedDatabaseSchema; kept here as plain helpers. public boolean storeOnlyCapturedTables() { return storeOnlyMonitoredTablesDdl; } - @Override public boolean skipUnparseableDdlStatements() { return skipUnparseableDDL; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkOffsetBackingStore.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkOffsetBackingStore.java index dbce64eb75f..7281b9f3754 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkOffsetBackingStore.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/FlinkOffsetBackingStore.java @@ -36,8 +36,10 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -90,10 +92,14 @@ public void configure(WorkerConfig config) { String engineName = (String) conf.get(EmbeddedEngine.ENGINE_NAME.name()); Converter keyConverter = new JsonConverter(); Converter valueConverter = new JsonConverter(); - keyConverter.configure(config.originals(), true); - Map valueConfigs = new HashMap<>(conf); - valueConfigs.put("schemas.enable", false); - valueConverter.configure(valueConfigs, true); + // Debezium 2.0's embedded engine serializes offset keys/values as raw JSON without the + // {"schema":...,"payload":...} envelope. Disable schemas on BOTH converters so the key we + // seed here matches the key the engine looks up on recovery (otherwise the restored offset + // is never found and the connector falls back to a schema-only-recovery snapshot). + Map converterConfigs = new HashMap<>(conf); + converterConfigs.put("schemas.enable", false); + keyConverter.configure(converterConfigs, true); + valueConverter.configure(converterConfigs, false); OffsetStorageWriter offsetWriter = new OffsetStorageWriter( this, @@ -200,4 +206,14 @@ public Future set( return null; }); } + + /** + * Added in Kafka Connect 3.5 (KAFKA-14304) for exactly-once source support and the offset-reset + * REST endpoint. Neither code path is exercised by the Flink embedded engine, which drives + * offsets through Flink state, so an empty set is a safe no-op. + */ + @Override + public Set> connectorPartitions(String connectorName) { + return Collections.emptySet(); + } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/SnapshotterServiceFactory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/SnapshotterServiceFactory.java new file mode 100644 index 00000000000..77a015b0c4c --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/internal/SnapshotterServiceFactory.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.debezium.internal; + +import org.apache.flink.util.TemporaryClassLoaderContext; + +import io.debezium.bean.DefaultBeanRegistry; +import io.debezium.bean.StandardBeanNames; +import io.debezium.bean.spi.BeanRegistry; +import io.debezium.config.CommonConnectorConfig; +import io.debezium.config.Configuration; +import io.debezium.service.DefaultServiceRegistry; +import io.debezium.snapshot.SnapshotLockProvider; +import io.debezium.snapshot.SnapshotQueryProvider; +import io.debezium.snapshot.SnapshotterService; +import io.debezium.snapshot.SnapshotterServiceProvider; + +/** + * Builds the {@link SnapshotterService} that Debezium 2.6 threads through every snapshot and + * streaming change event source. + * + *

Debezium wires this up in {@code BaseSourceTask.registerServiceProviders}, which is part of + * the Kafka Connect task lifecycle that Flink CDC does not run — Flink CDC drives the change event + * sources directly. This builds an equivalent registry instead. + * + *

The registry is created here rather than reused from {@link + * CommonConnectorConfig#getServiceRegistry()} because the snapshot providers resolve the + * connector-specific {@code SnapshotQuery} and {@code SnapshotLock} implementations by calling + * {@code Class.forName(config.getString("connector.class"))}. Flink CDC builds its Debezium + * configuration itself and never sets that property, so the lookup would throw a {@link + * NullPointerException}; the connector class is supplied explicitly instead. + */ +public class SnapshotterServiceFactory { + + private static final String CONNECTOR_CLASS_PROPERTY = "connector.class"; + + private SnapshotterServiceFactory() {} + + /** + * Creates the snapshotter service for the given connector configuration. + * + * @param connectorConfig the Debezium connector configuration; the snapshotter is resolved from + * its {@code snapshot.mode} + * @param connectorClass the Debezium connector class, used to select the connector-specific + * snapshot query and lock implementations + * @return the snapshotter service, never null + */ + public static SnapshotterService create( + CommonConnectorConfig connectorConfig, Class connectorClass) { + // Debezium 2.6 resolves the Snapshotter SPI through ServiceLoader, which uses the + // thread context class loader. On a Flink task thread that may be a user code class + // loader from a previous job attempt, which is already closed. Pin it to the loader + // that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(SnapshotterServiceProvider.class.getClassLoader())) { + final Configuration configuration = + connectorConfig + .getConfig() + .edit() + .with(CONNECTOR_CLASS_PROPERTY, connectorClass.getName()) + .build(); + + final BeanRegistry beanRegistry = new DefaultBeanRegistry(); + beanRegistry.add(StandardBeanNames.CONNECTOR_CONFIG, connectorConfig); + + final DefaultServiceRegistry serviceRegistry = + new DefaultServiceRegistry(configuration, beanRegistry); + serviceRegistry.registerServiceProvider(new SnapshotLockProvider()); + serviceRegistry.registerServiceProvider(new SnapshotQueryProvider()); + serviceRegistry.registerServiceProvider(new SnapshotterServiceProvider()); + + return serviceRegistry.getService(SnapshotterService.class); + } + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/utils/DatabaseHistoryUtil.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/utils/DatabaseHistoryUtil.java index 70b86cf0c1b..8ed8bca6a81 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/utils/DatabaseHistoryUtil.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/utils/DatabaseHistoryUtil.java @@ -20,7 +20,7 @@ import org.apache.flink.cdc.debezium.DebeziumSourceFunction; import org.apache.flink.cdc.debezium.internal.SchemaRecord; -import io.debezium.relational.history.DatabaseHistory; +import io.debezium.relational.history.SchemaHistory; import java.util.Collection; import java.util.Collections; @@ -28,7 +28,7 @@ import java.util.Map; /** - * Util to safely visit schema history between {@link DatabaseHistory} and {@link + * Util to safely visit schema history between {@link SchemaHistory} and {@link * DebeziumSourceFunction}. */ public class DatabaseHistoryUtil { @@ -39,13 +39,13 @@ private DatabaseHistoryUtil() { /** * Structure to maintain the current schema history. The content in {@link SchemaRecord} is up - * to the implementation of the {@link DatabaseHistory}. + * to the implementation of the {@link SchemaHistory}. */ private static final Map> HISTORY = new HashMap<>(); /** - * The schema history will be clean up once {@link DatabaseHistory#stop()}, the checkpoint - * should fail when this happens. + * The schema history will be clean up once {@link SchemaHistory#stop()}, the checkpoint should + * fail when this happens. */ private static final Map HISTORY_CLEANUP_STATUS = new HashMap<>(); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/test/java/io/debezium/relational/RelationalTableFiltersTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/test/java/io/debezium/relational/RelationalTableFiltersTest.java index 836bc91223b..868a3ec8ef1 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/test/java/io/debezium/relational/RelationalTableFiltersTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/test/java/io/debezium/relational/RelationalTableFiltersTest.java @@ -44,6 +44,6 @@ void testSetDataCollectionFiltersRetainsReplacement() { private static RelationalTableFilters createTableFilters() { return new RelationalTableFilters( - Configuration.empty(), tableId -> true, TableId::toString); + Configuration.empty(), tableId -> true, TableId::toString, false); } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml index 026181af508..e4dfe51aa38 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml @@ -48,6 +48,13 @@ limitations under the License. + + + io.debezium + debezium-connector-binlog + ${debezium.version} + + io.debezium debezium-connector-mysql diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/binlog/BinlogSnapshotChangeEventSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/binlog/BinlogSnapshotChangeEventSource.java new file mode 100644 index 00000000000..504d0898178 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/binlog/BinlogSnapshotChangeEventSource.java @@ -0,0 +1,829 @@ +/* + * Copyright Debezium Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ +package io.debezium.connector.binlog; + +import io.debezium.DebeziumException; +import io.debezium.connector.SnapshotRecord; +import io.debezium.connector.binlog.jdbc.BinlogConnectorConnection; +import io.debezium.connector.binlog.jdbc.BinlogConnectorConnection.DatabaseLocales; +import io.debezium.connector.binlog.metrics.BinlogSnapshotChangeEventSourceMetrics; +import io.debezium.data.Envelope; +import io.debezium.function.BlockingConsumer; +import io.debezium.jdbc.JdbcConnection; +import io.debezium.jdbc.MainConnectionProvidingConnectionFactory; +import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.notification.NotificationService; +import io.debezium.pipeline.source.SnapshottingTask; +import io.debezium.relational.RelationalSnapshotChangeEventSource; +import io.debezium.relational.RelationalTableFilters; +import io.debezium.relational.Table; +import io.debezium.relational.TableId; +import io.debezium.schema.SchemaChangeEvent; +import io.debezium.snapshot.SnapshotterService; +import io.debezium.util.Clock; +import io.debezium.util.Collect; +import io.debezium.util.Strings; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** An abstract implementation of {@link SnapshotChangeEventSource} for binlog-based connectors. */ +/** + * Copied from Debezium project(2.7.4.Final). + * + *

Flink CDC drives the snapshot itself, so the lifecycle hooks Debezium uses to mark snapshot + * boundaries are neutralised: + * + *

Change 1: {@code connectionCreated} and {@code completed} are no-ops. + * + *

Change 2: {@code lastSnapshotRecord} only delegates once the delayed schema snapshot is done, + * and the schema-event loops iterate with an explicit {@code Iterator} so the last event can set + * the last-snapshot-record flag before being dispatched. + * + *

Change 3: {@code rowCountForTable} always returns the estimated table size. + * + *

Debezium 2.7 split the shared binlog engine out of the MySQL connector, so these patches moved + * here from {@code io.debezium.connector.mysql.MySqlSnapshotChangeEventSource}. + */ +public abstract class BinlogSnapshotChangeEventSource< + P extends BinlogPartition, O extends BinlogOffsetContext> + extends RelationalSnapshotChangeEventSource { + + private static final Logger LOGGER = + LoggerFactory.getLogger(BinlogSnapshotChangeEventSource.class); + private static final Logger ROW_ESTIMATE_LOGGER = + LoggerFactory.getLogger( + BinlogSnapshotChangeEventSource.class.getName() + ".RowEstimate"); + + private final BinlogConnectorConfig connectorConfig; + private final BinlogConnectorConnection connection; + private final RelationalTableFilters filters; + private final BinlogSnapshotChangeEventSourceMetrics

metrics; + private final BinlogDatabaseSchema databaseSchema; + private final Set schemaEvents = new LinkedHashSet<>(); + private final BlockingConsumer> lastEventProcessor; + private final Runnable preSnapshotAction; + private Set delayedSchemaSnapshotTables = Collections.emptySet(); + private long globalLockAcquiredAt = -1; + private long tableLockAcquiredAt = -1; + + public BinlogSnapshotChangeEventSource( + BinlogConnectorConfig connectorConfig, + MainConnectionProvidingConnectionFactory connectionFactory, + BinlogDatabaseSchema schema, + EventDispatcher dispatcher, + Clock clock, + BinlogSnapshotChangeEventSourceMetrics

metrics, + BlockingConsumer> lastEventProcessor, + Runnable preSnapshotAction, + NotificationService notificationService, + SnapshotterService snapshotterService) { + super( + connectorConfig, + connectionFactory, + schema, + dispatcher, + clock, + metrics, + notificationService, + snapshotterService); + this.connectorConfig = connectorConfig; + this.connection = connectionFactory.mainConnection(); + this.filters = connectorConfig.getTableFilters(); + this.metrics = metrics; + this.databaseSchema = schema; + this.lastEventProcessor = lastEventProcessor; + this.preSnapshotAction = preSnapshotAction; + } + + @Override + protected void connectionCreated(RelationalSnapshotContext snapshotContext) + throws Exception {} + + @Override + protected SnapshotContext prepare(P partition, boolean onDemand) { + return new BinlogSnapshotContext<>(partition, onDemand); + } + + @Override + protected Set getAllTableIds(RelationalSnapshotContext ctx) throws Exception { + // ------------------- + // READ DATABASE NAMES + // ------------------- + // Get the list of databases ... + LOGGER.info("Read list of available databases"); + final List databaseNames = connection.availableDatabases(); + LOGGER.info("\t list of available databases is: {}", databaseNames); + + // ---------------- + // READ TABLE NAMES + // ---------------- + // Get the list of table IDs for each database. We can't use a prepared statement with + // MySQL, so we have to + // build the SQL statement each time. Although in other cases this might lead to SQL + // injection, in our case + // we are reading the database names from the database and not taking them from the user ... + LOGGER.info("Read list of available tables in each database"); + final Set tableIds = new HashSet<>(); + final Set readableDatabaseNames = new HashSet<>(); + for (String dbName : databaseNames) { + try { + // MySQL sometimes considers some local files as databases (see DBZ-164), + // so we will simply try each one and ignore the problematic ones ... + connection.query( + "SHOW FULL TABLES IN " + quote(dbName) + " where Table_Type = 'BASE TABLE'", + rs -> { + while (rs.next()) { + TableId id = new TableId(dbName, null, rs.getString(1)); + tableIds.add(id); + } + }); + readableDatabaseNames.add(dbName); + } catch (SQLException e) { + // We were unable to execute the query or process the results, so skip this ... + LOGGER.warn( + "\t skipping database '{}' due to error reading tables: {}", + dbName, + e.getMessage()); + } + } + final Set includedDatabaseNames = + readableDatabaseNames.stream() + .filter(filters.databaseFilter()) + .collect(Collectors.toSet()); + LOGGER.info("\tsnapshot continuing with database(s): {}", includedDatabaseNames); + return tableIds; + } + + @Override + protected void lockTablesForSchemaSnapshot( + ChangeEventSourceContext sourceContext, RelationalSnapshotContext snapshotContext) + throws SQLException { + // Set the transaction isolation level to REPEATABLE READ. This is the default, but the + // default can be changed + // which is why we explicitly set it here. + // + // With REPEATABLE READ, all SELECT queries within the scope of a transaction (which we + // don't yet have) will read + // from the same MVCC snapshot. Thus each plain (non-locking) SELECT statements within the + // same transaction are + // consistent also with respect to each other. + // + // See: https://dev.mysql.com/doc/refman/8.2/en/set-transaction.html + // See: https://dev.mysql.com/doc/refman/8.2/en/innodb-transaction-isolation-levels.html + // See: https://dev.mysql.com/doc/refman/8.2/en/innodb-consistent-read.html + connection.connection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ); + connection.executeWithoutCommitting( + "SET SESSION lock_wait_timeout=" + + connectorConfig.snapshotLockTimeout().getSeconds()); + try { + connection.executeWithoutCommitting( + "SET SESSION innodb_lock_wait_timeout=" + + connectorConfig.snapshotLockTimeout().getSeconds()); + } catch (SQLException e) { + LOGGER.warn("Unable to set innodb_lock_wait_timeout", e); + } + + // ------------------------------------ + // LOCK TABLES + // ------------------------------------ + // Obtain read lock on all tables. This statement closes all open tables and locks all + // tables + // for all databases with a global read lock, and it prevents ALL updates while we have this + // lock. + // It also ensures that everything we do while we have this lock will be consistent. + if (connectorConfig.getSnapshotLockingStrategy().isLockingEnabled() + && connectorConfig.isGlobalLockUseRequested()) { + try { + globalLock(); + metrics.setGlobalLockAcquired(); + } catch (SQLException e) { + LOGGER.info( + "Unable to flush and acquire global read lock, will use table read locks after reading table names"); + // Continue anyway, since RDS (among others) don't allow setting a global lock + assert !isGloballyLocked(); + } + if (connectorConfig.getSnapshotLockingStrategy().isIsolationLevelResetOnFlush()) { + // FLUSH TABLES resets TX and isolation level + connection.executeWithoutCommitting( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + } + } + } + + @Override + protected void releaseSchemaSnapshotLocks(RelationalSnapshotContext snapshotContext) + throws SQLException { + if (connectorConfig.getSnapshotLockingStrategy().isMinimalLockingEnabled()) { + if (isGloballyLocked()) { + globalUnlock(); + } + if (isTablesLocked()) { + // We could not acquire a global read lock and instead had to obtain individual + // table-level read locks + // using 'FLUSH TABLE WITH READ LOCK'. However, if we were to do this, + // the 'UNLOCK TABLES' + // would implicitly commit our active transaction, and this would break our + // consistent snapshot logic. + // Therefore, we cannot unlock the tables here! + // https://dev.mysql.com/doc/refman/8.2/en/flush.html + LOGGER.warn( + "Tables were locked explicitly, but to get a consistent snapshot we cannot release the locks until we've read all tables."); + } + } + } + + @Override + protected void releaseDataSnapshotLocks(RelationalSnapshotContext snapshotContext) + throws Exception { + if (isGloballyLocked()) { + globalUnlock(); + } + if (isTablesLocked()) { + tableUnlock(); + if (!delayedSchemaSnapshotTables.isEmpty()) { + schemaEvents.clear(); + if (connectorConfig.getSnapshotLockingStrategy().isLockingEnabled()) { + createSchemaEventsForTables( + snapshotContext, delayedSchemaSnapshotTables, false); + } else { + int snapshotMaxThreads = connectionPool.size(); + LOGGER.info( + "Creating delayed schema snapshot worker pool with {} worker thread(s)", + snapshotMaxThreads); + ExecutorService executorService = + Executors.newFixedThreadPool(snapshotMaxThreads); + try { + createSchemaEventsForTables( + snapshotContext, + delayedSchemaSnapshotTables, + false, + executorService); + } finally { + executorService.shutdownNow(); + } + } + + for (Iterator i = schemaEvents.iterator(); i.hasNext(); ) { + final SchemaChangeEvent event = i.next(); + + if (databaseSchema.storeOnlyCapturedTables() + && event.getDatabase() != null + && !event.getDatabase().isEmpty() + && !connectorConfig + .getTableFilters() + .databaseFilter() + .test(event.getDatabase())) { + LOGGER.debug( + "Skipping schema event as it belongs to a non-captured database: '{}'", + event); + continue; + } + + LOGGER.debug("Processing schema event {}", event); + + final TableId tableId = + event.getTables().isEmpty() + ? null + : event.getTables().iterator().next().id(); + snapshotContext.offset.event(tableId, getClock().currentTime()); + + if (!i.hasNext()) { + super.lastSnapshotRecord(snapshotContext); + } + + dispatcher.dispatchSchemaChangeEvent( + snapshotContext.partition, + snapshotContext.offset, + tableId, + (receiver) -> receiver.schemaChangeEvent(event)); + } + + // Make schema available for snapshot source + databaseSchema + .tableIds() + .forEach( + x -> + snapshotContext.tables.overwriteTable( + databaseSchema.tableFor(x))); + } + } + } + + @Override + protected void determineSnapshotOffset(RelationalSnapshotContext ctx, O previousOffset) + throws Exception { + if (!isGloballyLocked() + && !isTablesLocked() + && connectorConfig.getSnapshotLockingStrategy().isLockingEnabled()) { + return; + } + + if (previousOffset != null) { + ctx.offset = previousOffset; + tryStartingSnapshot(ctx); + return; + } + + final O offsetContext = getInitialOffsetContext(connectorConfig); + ctx.offset = offsetContext; + + setOffsetContextBinlogPositionAndGtidDetailsForSnapshot( + offsetContext, connection, snapshotterService); + tryStartingSnapshot(ctx); + } + + protected abstract O getInitialOffsetContext(BinlogConnectorConfig connectorConfig); + + protected abstract void setOffsetContextBinlogPositionAndGtidDetailsForSnapshot( + O offsetContext, + BinlogConnectorConnection connection, + SnapshotterService snapshotterService) + throws Exception; + + private void addSchemaEvent( + RelationalSnapshotContext snapshotContext, String database, String ddl) { + List schemaChangeEvents = + databaseSchema.parseSnapshotDdl( + snapshotContext.partition, + ddl, + database, + snapshotContext.offset, + clock.currentTimeAsInstant()); + schemaEvents.addAll(new LinkedHashSet<>(schemaChangeEvents)); + } + + @Override + protected void readTableStructure( + ChangeEventSourceContext sourceContext, + RelationalSnapshotContext snapshotContext, + O offsetContext, + SnapshottingTask snapshottingTask) + throws Exception { + Set capturedSchemaTables; + if (twoPhaseSchemaSnapshot()) { + // Capture schema of captured tables after they are locked + tableLock(snapshotContext); + determineSnapshotOffset(snapshotContext, offsetContext); + capturedSchemaTables = snapshotContext.capturedTables; + LOGGER.info( + "Table level locking is in place, the schema will be capture in two phases, now capturing: {}", + capturedSchemaTables); + delayedSchemaSnapshotTables = + Collect.minus( + snapshotContext.capturedSchemaTables, snapshotContext.capturedTables); + LOGGER.info("Tables for delayed schema capture: {}", delayedSchemaSnapshotTables); + } + if (databaseSchema.storeOnlyCapturedTables()) { + capturedSchemaTables = snapshotContext.capturedTables; + LOGGER.info( + "Only captured tables schema should be captured, capturing: {}", + capturedSchemaTables); + } else { + capturedSchemaTables = snapshotContext.capturedSchemaTables; + LOGGER.info( + "All eligible tables schema should be captured, capturing: {}", + capturedSchemaTables); + } + final Map> tablesToRead = + capturedSchemaTables.stream() + .collect( + Collectors.groupingBy( + TableId::catalog, LinkedHashMap::new, Collectors.toList())); + final Set databases = tablesToRead.keySet(); + + if (!snapshottingTask.isOnDemand()) { + // Record default charset + addSchemaEvent( + snapshotContext, + "", + connection.setStatementFor(connection.readCharsetSystemVariables())); + } + + for (TableId tableId : capturedSchemaTables) { + if (!sourceContext.isRunning()) { + throw new InterruptedException( + "Interrupted while emitting initial DROP TABLE events"); + } + addSchemaEvent( + snapshotContext, tableId.catalog(), "DROP TABLE IF EXISTS " + quote(tableId)); + } + + final Map databaseCharsets = connection.readDatabaseCollations(); + + ExecutorService executorService = null; + if (!connectorConfig.getSnapshotLockingStrategy().isLockingEnabled()) { + int snapshotMaxThreads = connectionPool.size(); + LOGGER.info( + "Creating schema snapshot worker pool with {} worker thread(s)", + snapshotMaxThreads); + executorService = Executors.newFixedThreadPool(snapshotMaxThreads); + } + try { + for (String database : databases) { + if (!sourceContext.isRunning()) { + throw new InterruptedException( + "Interrupted while reading structure of schema " + databases); + } + + if (!snapshottingTask.isOnDemand()) { + // in case of blocking snapshot we want to read structures only for collections + // specified in the signal + LOGGER.info("Reading structure of database '{}'", database); + addSchemaEvent( + snapshotContext, + database, + "DROP DATABASE IF EXISTS " + quote(database)); + final StringBuilder createDatabaseDdl = + new StringBuilder("CREATE DATABASE " + quote(database)); + final DatabaseLocales defaultDatabaseLocales = databaseCharsets.get(database); + if (defaultDatabaseLocales != null) { + defaultDatabaseLocales.appendToDdlStatement(database, createDatabaseDdl); + } + addSchemaEvent(snapshotContext, database, createDatabaseDdl.toString()); + addSchemaEvent(snapshotContext, database, "USE " + quote(database)); + } + + if (connectorConfig.getSnapshotLockingStrategy().isLockingEnabled()) { + createSchemaEventsForTables(snapshotContext, tablesToRead.get(database), true); + } else { + assert executorService != null; + createSchemaEventsForTables( + snapshotContext, tablesToRead.get(database), true, executorService); + } + } + } finally { + if (executorService != null) { + executorService.shutdownNow(); + } + } + } + + private void createSchemaEventsForTables( + RelationalSnapshotContext snapshotContext, + Collection tablesToRead, + boolean firstPhase) + throws Exception { + List realTablesToRead = new ArrayList<>(tablesToRead); + if (firstPhase) { + realTablesToRead = + realTablesToRead.stream() + .filter(id -> !delayedSchemaSnapshotTables.contains(id)) + .collect(Collectors.toList()); + } + for (TableId tableId : realTablesToRead) { + connection.query( + "SHOW CREATE TABLE " + quote(tableId), + rs -> { + if (rs.next()) { + addSchemaEvent(snapshotContext, tableId.catalog(), rs.getString(2)); + } + }); + } + } + + private void createSchemaEventsForTables( + RelationalSnapshotContext snapshotContext, + Collection tablesToRead, + boolean firstPhase, + ExecutorService executorService) + throws Exception { + List realTablesToRead = new ArrayList<>(tablesToRead); + if (firstPhase) { + realTablesToRead = + realTablesToRead.stream() + .filter(id -> !delayedSchemaSnapshotTables.contains(id)) + .collect(Collectors.toList()); + } + if (!realTablesToRead.isEmpty()) { + CompletionService> completionService = + new ExecutorCompletionService<>(executorService); + for (TableId tableId : realTablesToRead) { + completionService.submit(createDdlForTableCallable(tableId, connectionPool)); + } + Map ddls = new HashMap<>(); + for (int i = 0; i < realTablesToRead.size(); i++) { + Map ddl = completionService.take().get(); + if (ddl != null) { + ddls.putAll(ddl); + } + } + ddls.forEach((key, value) -> addSchemaEvent(snapshotContext, key.catalog(), value)); + } + } + + private Callable> createDdlForTableCallable( + TableId tableId, Queue connectionPool) { + return () -> { + JdbcConnection connection = connectionPool.poll(); + assert connection != null; + try { + Map result = new HashMap<>(); + connection.query( + "SHOW CREATE TABLE " + quote(tableId), + rs -> { + if (rs.next()) { + result.put(tableId, rs.getString(2)); + } + }); + return result; + } finally { + connectionPool.add(connection); + } + }; + } + + private boolean twoPhaseSchemaSnapshot() { + return connectorConfig.getSnapshotLockingStrategy().isLockingEnabled() + && !isGloballyLocked(); + } + + @Override + protected SchemaChangeEvent getCreateTableEvent( + RelationalSnapshotContext snapshotContext, Table table) { + return SchemaChangeEvent.ofSnapshotCreate( + snapshotContext.partition, + snapshotContext.offset, + snapshotContext.catalogName, + table); + } + + /** + * Generate a valid MySQL query string for the specified table and columns + * + * @param tableId the table to generate a query for + * @return a valid query string + */ + @Override + protected Optional getSnapshotSelect( + RelationalSnapshotContext snapshotContext, + TableId tableId, + List columns) { + return getSnapshotSelect(tableId, columns); + } + + private Optional getSnapshotSelect(TableId tableId, List columns) { + return snapshotterService + .getSnapshotQuery() + .snapshotQuery(tableId.toQuotedString('`'), columns); + } + + @Override + protected Optional getSnapshotConnectionFirstSelect( + RelationalSnapshotContext snapshotContext, TableId tableId) { + if (getSnapshotSelect(tableId, List.of("*")).isEmpty()) { + return Optional.empty(); + } + return Optional.of(getSnapshotSelect(tableId, List.of("*")).get() + " LIMIT 1"); + } + + private boolean isGloballyLocked() { + return globalLockAcquiredAt != -1; + } + + private boolean isTablesLocked() { + return tableLockAcquiredAt != -1; + } + + private void globalLock() throws SQLException { + LOGGER.info("Flush and obtain global read lock to prevent writes to database"); + Optional lockingStatement = + snapshotterService.getSnapshotLock().tableLockingStatement(null, null); + if (lockingStatement.isPresent()) { + connection.executeWithoutCommitting(lockingStatement.get()); + globalLockAcquiredAt = clock.currentTimeInMillis(); + } + } + + private void globalUnlock() throws SQLException { + LOGGER.info("Releasing global read lock to enable MySQL writes"); + connection.executeWithoutCommitting("UNLOCK TABLES"); + long lockReleased = clock.currentTimeInMillis(); + metrics.setGlobalLockReleased(); + LOGGER.info( + "Writes to MySQL tables prevented for a total of {}", + Strings.duration(lockReleased - globalLockAcquiredAt)); + globalLockAcquiredAt = -1; + } + + private void tableLock(RelationalSnapshotContext snapshotContext) throws SQLException { + // ------------------------------------ + // LOCK TABLES and READ BINLOG POSITION + // ------------------------------------ + // We were not able to acquire the global read lock, so instead we have to obtain a read + // lock on each table. + // This requires different privileges than normal, and also means we can't unlock the tables + // without + // implicitly committing our transaction ... + if (!connection.userHasPrivileges("LOCK TABLES")) { + // We don't have the right privileges + throw new DebeziumException( + "User does not have the 'LOCK TABLES' privilege required to obtain a " + + "consistent snapshot by preventing concurrent writes to tables."); + } + // We have the required privileges, so try to lock all of the tables we're interested in ... + LOGGER.info( + "Flush and obtain read lock for {} tables (preventing writes)", + snapshotContext.capturedTables); + if (!snapshotContext.capturedTables.isEmpty()) { + final String tableList = + snapshotContext.capturedTables.stream() + .map(this::quote) + .collect(Collectors.joining(",")); + connection.executeWithoutCommitting("FLUSH TABLES " + tableList + " WITH READ LOCK"); + } + tableLockAcquiredAt = clock.currentTimeInMillis(); + metrics.setGlobalLockAcquired(); + } + + private void tableUnlock() throws SQLException { + LOGGER.info("Releasing table read lock to enable MySQL writes"); + connection.executeWithoutCommitting("UNLOCK TABLES"); + long lockReleased = clock.currentTimeInMillis(); + metrics.setGlobalLockReleased(); + LOGGER.info( + "Writes to MySQL tables prevented for a total of {}", + Strings.duration(lockReleased - tableLockAcquiredAt)); + tableLockAcquiredAt = -1; + } + + private String quote(String dbOrTableName) { + return "`" + dbOrTableName + "`"; + } + + private String quote(TableId id) { + return quote(id.catalog()) + "." + quote(id.table()); + } + + @Override + protected OptionalLong rowCountForTable(TableId tableId) { + return connection.getEstimatedTableSize(tableId); + } + + @Override + protected Statement readTableStatement(JdbcConnection jdbcConnection, OptionalLong rowCount) + throws SQLException { + BinlogConnectorConnection connection = (BinlogConnectorConnection) jdbcConnection; + final long largeTableRowCount = connectorConfig.getRowCountForLargeTable(); + if (rowCount.isEmpty() + || largeTableRowCount == 0 + || rowCount.getAsLong() <= largeTableRowCount) { + return super.readTableStatement(connection, rowCount); + } + return createStatementWithLargeResultSet(connection); + } + + /** + * Create a JDBC statement that can be used for large result sets. + * + *

By default, the MySQL Connector/J driver retrieves all rows for ResultSets and stores them + * in memory. In most cases this is the most efficient way to operate and, due to the design of + * the MySQL network protocol, is easier to implement. However, when ResultSets that have a + * large number of rows or large values, the driver may not be able to allocate heap space in + * the JVM and may result in an {@link OutOfMemoryError}. See DBZ-94 for details. + * + *

This method handles such cases using the recommended + * technique for MySQL by creating the JDBC {@link Statement} with {@link + * ResultSet#TYPE_FORWARD_ONLY forward-only} cursor and {@link ResultSet#CONCUR_READ_ONLY + * read-only concurrency} flags, and with a {@link Integer#MIN_VALUE minimum value} {@link + * Statement#setFetchSize(int) fetch size hint}. + * + * @return the statement; never null + * @throws SQLException if there is a problem creating the statement + */ + private Statement createStatementWithLargeResultSet(BinlogConnectorConnection connection) + throws SQLException { + int fetchSize = connectorConfig.getSnapshotFetchSize(); + Statement stmt = + connection + .connection() + .createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + stmt.setFetchSize(fetchSize); + return stmt; + } + + /** Mutable context which is populated in the course of snapshotting. */ + private static class BinlogSnapshotContext< + P extends BinlogPartition, O extends BinlogOffsetContext> + extends RelationalSnapshotContext { + BinlogSnapshotContext(P partition, boolean onDemand) { + super(partition, "", onDemand); + } + } + + @Override + protected void createSchemaChangeEventsForTables( + ChangeEventSourceContext sourceContext, + RelationalSnapshotContext snapshotContext, + SnapshottingTask snapshottingTask) + throws Exception { + tryStartingSnapshot(snapshotContext); + + for (Iterator i = schemaEvents.iterator(); i.hasNext(); ) { + final SchemaChangeEvent event = i.next(); + if (!sourceContext.isRunning()) { + throw new InterruptedException("Interrupted while processing event " + event); + } + + if (databaseSchema.skipSchemaChangeEvent(event)) { + continue; + } + + LOGGER.debug("Processing schema event {}", event); + + final TableId tableId = + event.getTables().isEmpty() ? null : event.getTables().iterator().next().id(); + if (snapshottingTask.isOnDemand() + && !snapshotContext.capturedTables.contains(tableId)) { + LOGGER.debug( + "Event {} will be skipped since it's not related to blocking snapshot captured table {}", + event, + snapshotContext.capturedTables); + continue; + } + snapshotContext.offset.event(tableId, getClock().currentTime()); + + // If data are not snapshotted then the last schema change must set last snapshot flag + if (!snapshottingTask.snapshotData() && !i.hasNext()) { + lastSnapshotRecord(snapshotContext); + } + dispatcher.dispatchSchemaChangeEvent( + snapshotContext.partition, + snapshotContext.offset, + tableId, + (receiver) -> receiver.schemaChangeEvent(event)); + } + + // Make schema available for snapshot source + databaseSchema + .tableIds() + .forEach(x -> snapshotContext.tables.overwriteTable(databaseSchema.tableFor(x))); + } + + @Override + protected void completed(SnapshotContext snapshotContext) {} + + @Override + protected void lastSnapshotRecord(RelationalSnapshotContext snapshotContext) { + if (delayedSchemaSnapshotTables.isEmpty()) { + super.lastSnapshotRecord(snapshotContext); + } + } + + @Override + protected void postSnapshot() throws InterruptedException { + // We cannot be sure that the last event as the last one + // - last table could be empty + // - data snapshot was not executed + // - the last table schema snaphsotted is not monitored and storing of monitored is disabled + lastEventProcessor.accept( + record -> { + record.sourceOffset().remove(BinlogSourceInfo.SNAPSHOT_KEY); + ((Struct) record.value()) + .getStruct(Envelope.FieldName.SOURCE) + .put( + BinlogSourceInfo.SNAPSHOT_KEY, + SnapshotRecord.LAST.toString().toLowerCase()); + return record; + }); + super.postSnapshot(); + } + + @Override + protected void preSnapshot() throws InterruptedException { + preSnapshotAction.run(); + super.preSnapshot(); + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/binlog/BinlogStreamingChangeEventSource.java similarity index 65% rename from flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java rename to flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/binlog/BinlogStreamingChangeEventSource.java index 205582ca83b..f249a018bd0 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/binlog/BinlogStreamingChangeEventSource.java @@ -3,22 +3,21 @@ * * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 */ - -package io.debezium.connector.mysql; +package io.debezium.connector.binlog; import com.github.shyiko.mysql.binlog.BinaryLogClient; -import com.github.shyiko.mysql.binlog.BinaryLogClient.LifecycleListener; +import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData; import com.github.shyiko.mysql.binlog.event.DeleteRowsEventData; import com.github.shyiko.mysql.binlog.event.Event; import com.github.shyiko.mysql.binlog.event.EventData; import com.github.shyiko.mysql.binlog.event.EventHeader; import com.github.shyiko.mysql.binlog.event.EventHeaderV4; import com.github.shyiko.mysql.binlog.event.EventType; -import com.github.shyiko.mysql.binlog.event.GtidEventData; import com.github.shyiko.mysql.binlog.event.QueryEventData; import com.github.shyiko.mysql.binlog.event.RotateEventData; import com.github.shyiko.mysql.binlog.event.RowsQueryEventData; import com.github.shyiko.mysql.binlog.event.TableMapEventData; +import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData; import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData; import com.github.shyiko.mysql.binlog.event.WriteRowsEventData; import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException; @@ -34,20 +33,31 @@ import io.debezium.annotation.SingleThreadAccess; import io.debezium.config.CommonConnectorConfig.EventProcessingFailureHandlingMode; import io.debezium.config.Configuration; -import io.debezium.connector.mysql.MySqlConnectorConfig.GtidNewChannelPosition; -import io.debezium.connector.mysql.MySqlConnectorConfig.SecureConnectionMode; +import io.debezium.connector.binlog.BinlogConnectorConfig.SecureConnectionMode; +import io.debezium.connector.binlog.event.EventDataDeserializationExceptionData; +import io.debezium.connector.binlog.event.RowDeserializers; +import io.debezium.connector.binlog.event.StopEventDataDeserializer; +import io.debezium.connector.binlog.event.TransactionPayloadDeserializer; +import io.debezium.connector.binlog.gtid.GtidSet; +import io.debezium.connector.binlog.jdbc.BinlogConnectorConnection; +import io.debezium.connector.binlog.metrics.BinlogStreamingChangeEventSourceMetrics; import io.debezium.connector.mysql.util.ErrorMessageUtils; -import io.debezium.data.Envelope.Operation; +import io.debezium.data.Envelope; import io.debezium.function.BlockingConsumer; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.EventDispatcher; import io.debezium.pipeline.source.spi.StreamingChangeEventSource; +import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.schema.SchemaChangeEvent; +import io.debezium.snapshot.SnapshotterService; +import io.debezium.snapshot.mode.NeverSnapshotter; +import io.debezium.time.Conversions; import io.debezium.util.Clock; import io.debezium.util.Metronome; import io.debezium.util.Strings; import io.debezium.util.Threads; +import org.apache.kafka.connect.source.SourceConnector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; @@ -84,180 +94,435 @@ import static io.debezium.util.Strings.isNullOrEmpty; /** - * Copied from Debezium project(1.9.8.Final) to fix - * https://github.com/ververica/flink-cdc-connectors/issues/1944. + * An abstract common implementation of {@link StreamingChangeEventSource} for binlog-based + * connectors. + */ +/** + * Copied from Debezium project(2.7.4.Final). + * + *

Change 1: the effective offset context is initialised at the top of {@code execute} — + * Debezium's coordinator calls {@code init(offsetContext)} before {@code execute}, but Flink CDC + * drives this source directly and never calls {@code init}. * - *

Line 1432-1443 : Adjust GTID merging logic to support recovering from job which previously - * specifying starting offset on start. Uses {@link GtidUtils#fixOldChannelsGtidSet} for shared - * EARLIEST/LATEST logic. + *

Change 2: {@code handleChange} iterates the row list instead of indexing it — the list may be + * a linked list, in which case {@code get(i)} turns the loop into O(n^2). * - *

Line 1444-1452 : Fix LATEST mode GTID merging to avoid replaying pre-checkpoint transactions - * when checkpoint GTID has non-contiguous ranges. Delegates to {@link - * GtidUtils#computeLatestModeGtidSet}. See FLINK-39149. + *

Change 3: {@code wrap()} runs the message through {@code ErrorMessageUtils} to keep the error + * text bounded. * - *

Line 1490 : Add more error details for some exceptions. + *

Change 4: when the offset carries no GTID set but does pin a binlog file/position, keep the + * binlog client in file/position mode instead of switching it to GTID mode. Debezium always + * switches to GTID mode here because upstream only reaches this branch with an empty binlog + * filename; Flink CDC reaches it whenever a job starts from a specific binlog offset, and there + * {@code BinaryLogClient} would stream from the GTID set (ignoring the requested position) or, with + * an empty GTID set, fail with an NPE in {@code DumpBinaryLogGtidCommand} — it only materialises + * its internal GtidSet when the binlog filename is empty. * - *

Line 951-963 : Use iterator instead of index-based loop to avoid O(n²) complexity when - * processing LinkedList rows in handleChange method. See FLINK-38846. + *

Debezium 2.7 split the shared binlog engine out of the MySQL connector, so these patches moved + * here from {@code io.debezium.connector.mysql.MySqlStreamingChangeEventSource}, which is now a + * thin subclass and no longer forked. */ -public class MySqlStreamingChangeEventSource - implements StreamingChangeEventSource { +public abstract class BinlogStreamingChangeEventSource< + P extends BinlogPartition, O extends BinlogOffsetContext> + implements StreamingChangeEventSource { private static final Logger LOGGER = - LoggerFactory.getLogger(MySqlStreamingChangeEventSource.class); + LoggerFactory.getLogger(BinlogStreamingChangeEventSource.class); private static final String KEEPALIVE_THREAD_NAME = "blc-keepalive"; + private static final String SET_STATEMENT_REGEX = "SET STATEMENT .* FOR"; - private final EnumMap> eventHandlers = - new EnumMap<>(EventType.class); private final BinaryLogClient client; - private final MySqlStreamingChangeEventSourceMetrics metrics; + private final BinlogStreamingChangeEventSourceMetrics metrics; + // todo: can we go back to accessing schema via task context? issue is all the generics :/ + private final BinlogTaskContext taskContext; + private final BinlogDatabaseSchema schema; + private final BinlogConnectorConfig connectorConfig; + private final BinlogConnectorConnection connection; + private final EventDispatcher eventDispatcher; private final Clock clock; + private final ErrorHandler errorHandler; private final EventProcessingFailureHandlingMode eventDeserializationFailureHandlingMode; private final EventProcessingFailureHandlingMode inconsistentSchemaHandlingMode; + private final SnapshotterService snapshotterService; + private final Predicate gtidDmlSourceFilter; + private final boolean isGtidModeEnabled; + private final AtomicLong totalRecordCounter = new AtomicLong(); + private final Map binaryLogClientThreads = new ConcurrentHashMap<>(4); + private final EnumMap> eventHandlers = + new EnumMap<>(EventType.class); + private final float heartbeatIntervalFactor = 0.8f; private int startingRowNumber = 0; private long initialEventsToSkip = 0L; private boolean skipEvent = false; private boolean ignoreDmlEventByGtidSource = false; - private final Predicate gtidDmlSourceFilter; - private final AtomicLong totalRecordCounter = new AtomicLong(); private volatile Map lastOffset = null; - private com.github.shyiko.mysql.binlog.GtidSet gtidSet; - private final float heartbeatIntervalFactor = 0.8f; - private final Map binaryLogClientThreads = new ConcurrentHashMap<>(4); - private final MySqlTaskContext taskContext; - private final MySqlConnectorConfig connectorConfig; - private final MySqlConnection connection; - private final EventDispatcher eventDispatcher; - private final ErrorHandler errorHandler; + private O effectiveOffsetContext; @SingleThreadAccess("binlog client thread") - private Instant eventTimestamp; + protected Instant eventTimestamp; - /** Describe binlog position. */ - public static class BinlogPosition { - final String filename; - final long position; + public BinlogStreamingChangeEventSource( + BinlogConnectorConfig connectorConfig, + BinlogConnectorConnection connection, + EventDispatcher dispatcher, + ErrorHandler errorHandler, + Clock clock, + BinlogTaskContext taskContext, + BinlogDatabaseSchema schema, + BinlogStreamingChangeEventSourceMetrics metrics, + SnapshotterService snapshotterService) { + this.taskContext = taskContext; + this.connectorConfig = connectorConfig; + this.schema = schema; + this.connection = connection; + this.clock = clock; + this.eventDispatcher = dispatcher; + this.errorHandler = errorHandler; + this.metrics = metrics; + this.eventDeserializationFailureHandlingMode = + connectorConfig.getEventProcessingFailureHandlingMode(); + this.inconsistentSchemaHandlingMode = + connectorConfig.getInconsistentSchemaFailureHandlingMode(); + this.snapshotterService = snapshotterService; + this.client = + createBinaryLogClient( + taskContext, connectorConfig, binaryLogClientThreads, connection); + this.gtidDmlSourceFilter = getGtidDmlSourceFilter(); + this.isGtidModeEnabled = connection.isGtidModeEnabled(); + } - public BinlogPosition(String filename, long position) { - assert filename != null; + @Override + public void execute(ChangeEventSourceContext context, P partition, O offsetContext) + throws InterruptedException { + // Debezium's ChangeEventSourceCoordinator calls init(offsetContext) before execute(...) to + // set the effective offset context. Flink CDC drives this source directly and never calls + // init, so fall back to the offset context it passes in. + if (getOffsetContext() == null) { + setEffectiveOffsetContext(offsetContext); + } + if (!(snapshotterService.getSnapshotter() instanceof NeverSnapshotter)) { + taskContext.getSchema().assureNonEmptySchema(); + } + final Set skippedOperations = connectorConfig.getSkippedOperations(); - this.filename = filename; - this.position = position; + // Register our event handlers ... + eventHandlers.put( + EventType.STOP, (event) -> handleServerStop(effectiveOffsetContext, event)); + eventHandlers.put( + EventType.HEARTBEAT, + (event) -> handleServerHeartbeat(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.INCIDENT, + (event) -> handleServerIncident(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.ROTATE, (event) -> handleRotateLogsEvent(effectiveOffsetContext, event)); + eventHandlers.put( + EventType.TABLE_MAP, + (event) -> handleUpdateTableMetadata(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.QUERY, + (event) -> handleQueryEvent(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.TRANSACTION_PAYLOAD, + (event) -> handleTransactionPayload(partition, effectiveOffsetContext, event)); + + if (!skippedOperations.contains(Envelope.Operation.CREATE)) { + eventHandlers.put( + EventType.WRITE_ROWS, + (event) -> handleInsert(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.EXT_WRITE_ROWS, + (event) -> handleInsert(partition, effectiveOffsetContext, event)); } - public String getFilename() { - return filename; + if (!skippedOperations.contains(Envelope.Operation.UPDATE)) { + eventHandlers.put( + EventType.UPDATE_ROWS, + (event) -> handleUpdate(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.EXT_UPDATE_ROWS, + (event) -> handleUpdate(partition, effectiveOffsetContext, event)); } - public long getPosition() { - return position; + if (!skippedOperations.contains(Envelope.Operation.DELETE)) { + eventHandlers.put( + EventType.DELETE_ROWS, + (event) -> handleDelete(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.EXT_DELETE_ROWS, + (event) -> handleDelete(partition, effectiveOffsetContext, event)); } - @Override - public String toString() { - return filename + "/" + position; + eventHandlers.put( + EventType.VIEW_CHANGE, (event) -> viewChange(effectiveOffsetContext, event)); + eventHandlers.put( + EventType.XA_PREPARE, (event) -> prepareTransaction(effectiveOffsetContext, event)); + eventHandlers.put( + EventType.XID, + (event) -> handleTransactionCompletion(partition, effectiveOffsetContext, event)); + + // Conditionally register ROWS_QUERY handler to parse SQL statements. + if (connectorConfig.isSqlQueryIncluded()) { + final EventType eventType = getIncludeQueryEventType(); + eventHandlers.put( + eventType, (event) -> handleRecordingQuery(effectiveOffsetContext, event)); } - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + filename.hashCode(); - result = prime * result + (int) (position ^ (position >>> 32)); - return result; + BinaryLogClient.EventListener listener; + if (connectorConfig.getBufferSizeForStreamingChangeEventSource() == 0) { + listener = (event) -> handleEvent(partition, effectiveOffsetContext, context, event); + } else { + EventBuffer buffer = + new EventBuffer<>( + connectorConfig.getBufferSizeForStreamingChangeEventSource(), + this, + context); + listener = (event) -> buffer.add(partition, effectiveOffsetContext, event); } + client.registerEventListener(listener); - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; + client.registerLifecycleListener(new ReaderThreadLifecycleListener(effectiveOffsetContext)); + client.registerEventListener((event) -> onEvent(effectiveOffsetContext, event)); + if (LOGGER.isDebugEnabled()) { + client.registerEventListener((event) -> logEvent(effectiveOffsetContext, event)); + } + + metrics.setIsGtidModeEnabled(isGtidModeEnabled); + + // Get the current GtidSet from MySQL so we can get a filtered/merged GtidSet based off of + // the last Debezium checkpoint. + if (isGtidModeEnabled) { + // The server is using GTIDs, so enable the handler ... + eventHandlers.put( + getGtidEventType(), + (event) -> + handleGtidEvent( + partition, effectiveOffsetContext, event, gtidDmlSourceFilter)); + + // Now look at the GTID set from the server and what we've previously seen ... + GtidSet availableServerGtidSet = connection.knownGtidSet(); + + // also take into account purged GTID logs + GtidSet purgedServerGtidSet = connection.purgedGtidSet(); + LOGGER.info("GTID set purged on server: '{}'", purgedServerGtidSet); + + final GtidSet filteredGtidSet = + connection.filterGtidSet( + connectorConfig.getGtidSourceFilter(), + effectiveOffsetContext.gtidSet(), + availableServerGtidSet, + purgedServerGtidSet); + + if (filteredGtidSet != null) { + // We've seen at least some GTIDs, so start reading from the filtered GTID set ... + LOGGER.info("Registering binlog reader with GTID set: '{}'", filteredGtidSet); + String filteredGtidSetStr = filteredGtidSet.toString(); + client.setGtidSet(filteredGtidSetStr); + effectiveOffsetContext.setCompletedGtidSet(filteredGtidSetStr); + initializeGtidSet(filteredGtidSetStr); + } else { + // We've not yet seen any GTIDs, so that means we have to start reading the binlog + // from the beginning ... + final String startBinlogFilename = + effectiveOffsetContext.getSource().binlogFilename(); + client.setBinlogFilename(startBinlogFilename); + client.setBinlogPosition(effectiveOffsetContext.getSource().binlogPosition()); + if (!Strings.isNullOrEmpty(startBinlogFilename)) { + LOGGER.info( + "No GTID stored in the offset, registering binlog reader with binlog file '{}' and position {}.", + startBinlogFilename, + effectiveOffsetContext.getSource().binlogPosition()); + initializeGtidSet(""); + } else if (purgedServerGtidSet == null || purgedServerGtidSet.isEmpty()) { + LOGGER.info( + "No GTID stored in the offset, registering binlog reader with empty GTID set."); + client.setGtidSet(""); + initializeGtidSet(""); + } else { + LOGGER.info( + "No GTID stored in the offset, but there is non-empty purged GTID set. Registering binlog reader with purged GTID set: '{}'", + purgedServerGtidSet.toString()); + client.setGtidSet(purgedServerGtidSet.toString()); + // We don't have stored any GTID in the offset, so start from empty GTID set. + initializeGtidSet(""); + } } - if (getClass() != obj.getClass()) { - return false; + } else { + // The server is not using GTIDs, so start reading the binlog based upon where we last + // left off ... + client.setBinlogFilename(effectiveOffsetContext.getSource().binlogFilename()); + client.setBinlogPosition(effectiveOffsetContext.getSource().binlogPosition()); + } + + // We may be restarting in the middle of a transaction, so see how far into the transaction + // we have already processed... + initialEventsToSkip = effectiveOffsetContext.eventsToSkipUponRestart(); + LOGGER.info("Skip {} events on streaming start", initialEventsToSkip); + + // Set the starting row number, which is the next row number to be read ... + startingRowNumber = effectiveOffsetContext.rowsToSkipUponRestart(); + LOGGER.info("Skip {} rows on streaming start", startingRowNumber); + + // Only when we reach the first BEGIN event will we start to skip events ... + skipEvent = false; + + try { + // Start the log reader, which starts background threads ... + if (context.isRunning()) { + long timeout = connectorConfig.getConnectionTimeout().toMillis(); + long started = clock.currentTimeInMillis(); + try { + LOGGER.debug( + "Attempting to establish binlog reader connection with timeout of {} ms", + timeout); + client.connect(timeout); + // Need to wait for keepalive thread to be running, otherwise it can be left + // orphaned + // The problem is with timing. When the close is called too early after connect + // then + // the keepalive thread is not terminated + if (client.isKeepAlive()) { + LOGGER.info("Waiting for keepalive thread to start"); + final Metronome metronome = Metronome.parker(Duration.ofMillis(100), clock); + int waitAttempts = 50; + boolean keepAliveThreadRunning = false; + while (!keepAliveThreadRunning && waitAttempts-- > 0) { + for (Thread t : binaryLogClientThreads.values()) { + if (t.getName().startsWith(KEEPALIVE_THREAD_NAME) && t.isAlive()) { + LOGGER.info("Keepalive thread is running"); + keepAliveThreadRunning = true; + } + } + metronome.pause(); + } + } + } catch (TimeoutException e) { + // If the client thread is interrupted *before* the client could connect, the + // client throws a timeout exception + // The only way we can distinguish this is if we get the timeout exception + // before the specified timeout has + // elapsed, so we simply check this (within 10%) ... + long duration = clock.currentTimeInMillis() - started; + if (duration > (0.9 * timeout)) { + double actualSeconds = TimeUnit.MILLISECONDS.toSeconds(duration); + throw new DebeziumException( + String.format( + "Timed out after %f seconds while waiting to connect to database at %s:%d with user '%s'", + actualSeconds, + connectorConfig.getHostName(), + connectorConfig.getPort(), + connectorConfig.getUserName()), + e); + } + // Otherwise, we were told to shutdown, so we don't care about the timeout + // exception + } catch (AuthenticationException e) { + throw new DebeziumException( + String.format( + "Failed to authenticate to the database at %s:%d with user '%s'", + connectorConfig.getHostName(), + connectorConfig.getPort(), + connectorConfig.getUserName()), + e); + } catch (Throwable e) { + throw new DebeziumException( + String.format( + "Unable to connect to the database at %s:%d with user '%s': %s", + connectorConfig.getHostName(), + connectorConfig.getPort(), + connectorConfig.getUserName(), + e.getMessage()), + e); + } } - BinlogPosition other = (BinlogPosition) obj; - if (!filename.equals(other.filename)) { - return false; + while (context.isRunning()) { + Thread.sleep(100); + waitWhenStreamingPaused(context); } - if (position != other.position) { - return false; + } finally { + try { + client.disconnect(); + } catch (Exception e) { + LOGGER.info("Exception while stopping binary log client", e); } - return true; } } - @FunctionalInterface - private interface BinlogChangeEmitter { - void emit(TableId tableId, T data) throws InterruptedException; + @Override + public O getOffsetContext() { + return effectiveOffsetContext; } - public MySqlStreamingChangeEventSource( - MySqlConnectorConfig connectorConfig, - MySqlConnection connection, - EventDispatcher dispatcher, - ErrorHandler errorHandler, - Clock clock, - MySqlTaskContext taskContext, - MySqlStreamingChangeEventSourceMetrics metrics) { + protected void setEffectiveOffsetContext(O offsetContext) { + this.effectiveOffsetContext = offsetContext; + } - this.taskContext = taskContext; - this.connectorConfig = connectorConfig; - this.connection = connection; - this.clock = clock; - this.eventDispatcher = dispatcher; - this.errorHandler = errorHandler; - this.metrics = metrics; + protected void setIgnoreDmlEventByGtidSource(boolean ignoreDmlEventByGtidSource) { + this.ignoreDmlEventByGtidSource = ignoreDmlEventByGtidSource; + } - eventDeserializationFailureHandlingMode = - connectorConfig.getEventProcessingFailureHandlingMode(); - inconsistentSchemaHandlingMode = connectorConfig.inconsistentSchemaFailureHandlingMode(); + protected void setGtidChanged(String gtid) { + metrics.onGtidChange(gtid); + } + + protected boolean isGtidModeEnabled() { + return isGtidModeEnabled; + } - // Set up the log reader ... - client = taskContext.getBinaryLogClient(); - // BinaryLogClient will overwrite thread names later + // todo: perhaps refactor back out to a binary log configurator instance? + protected BinaryLogClient createBinaryLogClient( + BinlogTaskContext taskContext, + BinlogConnectorConfig connectorConfig, + Map clientThreads, + BinlogConnectorConnection connection) { + final BinaryLogClient client = taskContext.getBinaryLogClient(); client.setThreadFactory( Threads.threadFactory( - MySqlConnector.class, + getConnectorClass(), connectorConfig.getLogicalName(), "binlog-client", false, false, - x -> binaryLogClientThreads.put(x.getName(), x))); - client.setServerId(connectorConfig.serverId()); - client.setSSLMode(sslModeFor(connectorConfig.sslMode())); - if (connectorConfig.sslModeEnabled()) { + x -> clientThreads.put(x.getName(), x))); + client.setServerId(connectorConfig.getServerId()); + + client.setSSLMode(sslModeFor(connectorConfig.getSslMode())); + if (connectorConfig.isSslModeEnabled()) { SSLSocketFactory sslSocketFactory = getBinlogSslSocketFactory(connectorConfig, connection); if (sslSocketFactory != null) { client.setSslSocketFactory(sslSocketFactory); } } + client.setUseNonGracefulDisconnect(connectorConfig.usesNonGracefulDisconnect()); + + configureReplicaCompatibility(client); + Configuration configuration = connectorConfig.getConfig(); - client.setKeepAlive(configuration.getBoolean(MySqlConnectorConfig.KEEP_ALIVE)); final long keepAliveInterval = - configuration.getLong(MySqlConnectorConfig.KEEP_ALIVE_INTERVAL_MS); + configuration.getLong(BinlogConnectorConfig.KEEP_ALIVE_INTERVAL_MS); + + client.setKeepAlive(configuration.getBoolean(BinlogConnectorConfig.KEEP_ALIVE)); client.setKeepAliveInterval(keepAliveInterval); - // Considering heartbeatInterval should be less than keepAliveInterval, we use the + + // Considering heartbeatInterval should be less than keepAliveInterval, use the // heartbeatIntervalFactor - // multiply by keepAliveInterval and set the result value to heartbeatInterval.The default - // value of heartbeatIntervalFactor - // is 0.8, and we believe the left time (0.2 * keepAliveInterval) is enough to process the - // packet received from the MySQL server. + // multiply by keepAliveInterval and set the result value to heartbeatInterval. The default + // value of + // heartbeatIntervalFactor is 0.0, and we believe the left time (0.2 * keepAliveInterval) is + // enough + // to process the packet received from the database server. client.setHeartbeatInterval((long) (keepAliveInterval * heartbeatIntervalFactor)); + client.setEventDeserializer(createEventDeserializer()); + return client; + } - boolean filterDmlEventsByGtidSource = - configuration.getBoolean(MySqlConnectorConfig.GTID_SOURCE_FILTER_DML_EVENTS); - gtidDmlSourceFilter = - filterDmlEventsByGtidSource ? connectorConfig.gtidSourceFilter() : null; + protected void configureReplicaCompatibility(BinaryLogClient client) {} - // Set up the event deserializer with additional type(s) ... - final Map tableMapEventByTableId = - new HashMap(); + protected EventDeserializer createEventDeserializer() { + final Map tableMapEventByTableId = new HashMap<>(); EventDeserializer eventDeserializer = new EventDeserializer() { @Override @@ -274,6 +539,28 @@ public Event nextEvent(ByteArrayInputStream inputStream) throws IOException { tableMapEvent.getTableId(), tableMapEvent); } + // DBZ-2663 Handle for transaction payload and capture the table map + // event and add it to the map + if (event.getHeader().getEventType() == EventType.TRANSACTION_PAYLOAD) { + TransactionPayloadEventData transactionPayloadEventData = + event.getData(); + /** + * Loop over the uncompressed events in the transaction payload + * event and add the table map event in the map of table events + */ + for (Event uncompressedEvent : + transactionPayloadEventData.getUncompressedEvents()) { + if (uncompressedEvent.getHeader().getEventType() + == EventType.TABLE_MAP + && uncompressedEvent.getData() != null) { + TableMapEventData tableMapEvent = + uncompressedEvent.getData(); + tableMapEventByTableId.put( + tableMapEvent.getTableId(), tableMapEvent); + } + } + } + // DBZ-5126 Clean cache on rotate event to prevent it from growing // indefinitely. if (event.getHeader().getEventType() == EventType.ROTATE @@ -316,35 +603,46 @@ public Event nextEvent(ByteArrayInputStream inputStream) throws IOException { } }; - // Add our custom deserializers ... eventDeserializer.setEventDataDeserializer(EventType.STOP, new StopEventDataDeserializer()); eventDeserializer.setEventDataDeserializer(EventType.GTID, new GtidEventDataDeserializer()); eventDeserializer.setEventDataDeserializer( EventType.WRITE_ROWS, - new RowDeserializers.WriteRowsDeserializer(tableMapEventByTableId)); + new RowDeserializers.WriteRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode)); eventDeserializer.setEventDataDeserializer( EventType.UPDATE_ROWS, - new RowDeserializers.UpdateRowsDeserializer(tableMapEventByTableId)); + new RowDeserializers.UpdateRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode)); eventDeserializer.setEventDataDeserializer( EventType.DELETE_ROWS, - new RowDeserializers.DeleteRowsDeserializer(tableMapEventByTableId)); + new RowDeserializers.DeleteRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode)); eventDeserializer.setEventDataDeserializer( EventType.EXT_WRITE_ROWS, - new RowDeserializers.WriteRowsDeserializer(tableMapEventByTableId) + new RowDeserializers.WriteRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode) .setMayContainExtraInformation(true)); eventDeserializer.setEventDataDeserializer( EventType.EXT_UPDATE_ROWS, - new RowDeserializers.UpdateRowsDeserializer(tableMapEventByTableId) + new RowDeserializers.UpdateRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode) .setMayContainExtraInformation(true)); eventDeserializer.setEventDataDeserializer( EventType.EXT_DELETE_ROWS, - new RowDeserializers.DeleteRowsDeserializer(tableMapEventByTableId) + new RowDeserializers.DeleteRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode) .setMayContainExtraInformation(true)); - client.setEventDeserializer(eventDeserializer); + eventDeserializer.setEventDataDeserializer( + EventType.TRANSACTION_PAYLOAD, + new TransactionPayloadDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode)); + + return eventDeserializer; } - protected void onEvent(MySqlOffsetContext offsetContext, Event event) { + protected void onEvent(O offsetContext, Event event) { long ts = 0; + totalRecordCounter.incrementAndGet(); if (event.getHeader().getEventType() == EventType.HEARTBEAT) { // HEARTBEAT events have no timestamp but are fired only when @@ -363,31 +661,26 @@ protected void onEvent(MySqlOffsetContext offsetContext, Event event) { return; } - ts = clock.currentTimeInMillis() - eventTs; + setEventTimestamp(event, eventTs); + + ts = clock.currentTimeInMillis() - eventTimestamp.toEpochMilli(); LOGGER.trace("Current milliseconds behind source: {} ms", ts); metrics.setMilliSecondsBehindSource(ts); } - protected void ignoreEvent(MySqlOffsetContext offsetContext, Event event) { + protected abstract void setEventTimestamp(Event event, long eventTs); + + protected void ignoreEvent(O offsetContext, Event event) { LOGGER.trace("Ignoring event due to missing handler: {}", event); } protected void handleEvent( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) { + P partition, O offsetContext, ChangeEventSourceContext context, Event event) { if (event == null) { return; } final EventHeader eventHeader = event.getHeader(); - // Update the source offset info. Note that the client returns the value in *milliseconds*, - // even though the binlog - // contains only *seconds* precision ... - // HEARTBEAT events have no timestamp; only set the timestamp if the event is not a - // HEARTBEAT - eventTimestamp = - !eventHeader.getEventType().equals(EventType.HEARTBEAT) - ? Instant.ofEpochMilli(eventHeader.getTimestamp()) - : null; offsetContext.setBinlogServerId(eventHeader.getServerId()); final EventType eventType = eventHeader.getEventType(); @@ -411,6 +704,8 @@ protected void handleEvent( // If there is a handler for this event, forward the event to it ... try { + waitWhenStreamingPaused(context); + // Forward the event to the handler ... eventHandlers .getOrDefault(eventType, (e) -> ignoreEvent(offsetContext, e)) @@ -461,11 +756,11 @@ protected T unwrapData(Event event) { } /** - * Handle the supplied event that signals that mysqld has stopped. + * Handle the supplied event that signals that the database has stopped. * * @param event the server stopped event to be processed; may not be null */ - protected void handleServerStop(MySqlOffsetContext offsetContext, Event event) { + protected void handleServerStop(O offsetContext, Event event) { LOGGER.debug("Server stopped: {}", event); } @@ -475,8 +770,7 @@ protected void handleServerStop(MySqlOffsetContext offsetContext, Event event) { * * @param event the server stopped event to be processed; may not be null */ - protected void handleServerHeartbeat( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleServerHeartbeat(P partition, O offsetContext, Event event) throws InterruptedException { LOGGER.trace("Server heartbeat: {}", event); eventDispatcher.dispatchServerHeartbeatEvent(partition, offsetContext); @@ -489,8 +783,7 @@ protected void handleServerHeartbeat( * * @param event the server stopped event to be processed; may not be null */ - protected void handleServerIncident( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) { + protected void handleServerIncident(P partition, O offsetContext, Event event) { if (event.getData() instanceof EventDataDeserializationExceptionData) { metrics.onErroneousEvent(partition, "source = " + event); EventDataDeserializationExceptionData data = event.getData(); @@ -536,91 +829,34 @@ protected void handleServerIncident( /** * Handle the supplied event with a {@link RotateEventData} that signals the logs are being * rotated. This means that either the server was restarted, or the binlog has transitioned to a - * new file. In either case, subsequent table numbers will be different than those seen to this + * new file. In either case, subsequent table numbers will be different from those seen to this * point. * * @param event the database change data event to be processed; may not be null */ - protected void handleRotateLogsEvent(MySqlOffsetContext offsetContext, Event event) { + protected void handleRotateLogsEvent(O offsetContext, Event event) { LOGGER.debug("Rotating logs: {}", event); RotateEventData command = unwrapData(event); assert command != null; - taskContext.getSchema().clearTableMappings(); - } - - /** - * Handle the supplied event with a {@link GtidEventData} that signals the beginning of a GTID - * transaction. We don't yet know whether this transaction contains any events we're interested - * in, but we have to record it so that we know the position of this event and know we've - * processed the binlog to this point. - * - *

Note that this captures the current GTID and complete GTID set, regardless of whether the - * connector is {@link MySqlTaskContext#gtidSourceFilter() filtering} the GTID set upon - * connection. We do this because we actually want to capture all GTID set values found in the - * binlog, whether or not we process them. However, only when we connect do we actually want to - * pass to MySQL only those GTID ranges that are applicable per the configuration. - * - * @param event the GTID event to be processed; may not be null - */ - protected void handleGtidEvent(MySqlOffsetContext offsetContext, Event event) { - LOGGER.debug("GTID transaction: {}", event); - GtidEventData gtidEvent = unwrapData(event); - String gtid = gtidEvent.getGtid(); - gtidSet.add(gtid); - offsetContext.startGtid(gtid, gtidSet.toString()); // rather than use the client's GTID set - ignoreDmlEventByGtidSource = false; - if (gtidDmlSourceFilter != null && gtid != null) { - String uuid = gtid.trim().substring(0, gtid.indexOf(":")); - if (!gtidDmlSourceFilter.test(uuid)) { - ignoreDmlEventByGtidSource = true; - } - } - metrics.onGtidChange(gtid); - } - - /** - * Handle the supplied event with an {@link RowsQueryEventData} by recording the original SQL - * query that generated the event. - * - * @param event the database change data event to be processed; may not be null - */ - protected void handleRowsQuery(MySqlOffsetContext offsetContext, Event event) { - // Unwrap the RowsQueryEvent - final RowsQueryEventData lastRowsQueryEventData = unwrapData(event); - - // Set the query on the source - offsetContext.setQuery(lastRowsQueryEventData.getQuery()); + schema.clearTableMappings(); } /** * Handle the supplied event with an {@link QueryEventData} by possibly recording the DDL - * statements as changes in the MySQL schemas. + * statements as changes in the database schemas. * * @param partition the partition in which the even occurred * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while recording the DDL statements */ - protected void handleQueryEvent( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleQueryEvent(P partition, O offsetContext, Event event) throws InterruptedException { + Instant eventTime = Conversions.toInstantFromMillis(eventTimestamp.toEpochMilli()); QueryEventData command = unwrapData(event); LOGGER.debug("Received query command: {}", event); String sql = command.getSql().trim(); if (sql.equalsIgnoreCase("BEGIN")) { - // We are starting a new transaction ... - offsetContext.startNextTransaction(); - eventDispatcher.dispatchTransactionStartedEvent( - partition, offsetContext.getTransactionId(), offsetContext); - offsetContext.setBinlogThread(command.getThreadId()); - if (initialEventsToSkip != 0) { - LOGGER.debug( - "Restarting partially-processed transaction; change events will not be created for the first {} events plus {} more rows in the next event", - initialEventsToSkip, - startingRowNumber); - // We are restarting, so we need to skip the events in this transaction that we - // processed previously... - skipEvent = true; - } + handleTransactionBegin(partition, offsetContext, event, command.getThreadId()); return; } if (sql.equalsIgnoreCase("COMMIT")) { @@ -628,13 +864,14 @@ protected void handleQueryEvent( return; } - String upperCasedStatementBegin = Strings.getBegin(sql, 7).toUpperCase(); + String upperCasedStatementBegin = + Strings.getBegin(removeSetStatement(sql), 7).toUpperCase(); if (upperCasedStatementBegin.startsWith("XA ")) { // This is an XA transaction, and we currently ignore these and do nothing ... return; } - if (connectorConfig.getDdlFilter().test(sql)) { + if (schema.ddlFilter().test(sql)) { LOGGER.debug("DDL '{}' was filtered out of processing", sql); return; } @@ -651,39 +888,15 @@ protected void handleQueryEvent( // We have hit a ROLLBACK which is not supported LOGGER.warn( "Rollback statements cannot be handled without binlog buffering, the connector will fail. Please check '{}' to see how to enable buffering", - MySqlConnectorConfig.BUFFER_SIZE_FOR_BINLOG_READER.name()); - } else { - // ROLLBACK is not a DDL event and already has a dedicated warning above. - EventHeader header = event.getHeader(); - if (header instanceof EventHeaderV4) { - EventHeaderV4 eventHeader = (EventHeaderV4) header; - LOGGER.info( - "Received MySQL DDL event at {} [{}-{}], database={}, ddl={}", - offsetContext.getSource().binlogFilename(), - eventHeader.getPosition(), - eventHeader.getNextPosition(), - command.getDatabase(), - sql); - } else { - LOGGER.info( - "Received MySQL DDL event, database={}, ddl={}", - command.getDatabase(), - sql); - } + BinlogConnectorConfig.BUFFER_SIZE_FOR_BINLOG_READER.name()); } final List schemaChangeEvents = - taskContext - .getSchema() - .parseStreamingDdl( - partition, - sql, - command.getDatabase(), - offsetContext, - clock.currentTimeAsInstant()); + schema.parseStreamingDdl( + partition, sql, command.getDatabase(), offsetContext, eventTime); try { for (SchemaChangeEvent schemaChangeEvent : schemaChangeEvents) { - if (taskContext.getSchema().skipSchemaChangeEvent(schemaChangeEvent)) { + if (schema.skipSchemaChangeEvent(schemaChangeEvent)) { continue; } @@ -691,8 +904,28 @@ protected void handleQueryEvent( schemaChangeEvent.getTables().isEmpty() ? null : schemaChangeEvent.getTables().iterator().next().id(); + if (tableId != null + && !connectorConfig + .getSkippedOperations() + .contains(Envelope.Operation.TRUNCATE) + && schemaChangeEvent + .getType() + .equals(SchemaChangeEvent.SchemaChangeEventType.TRUNCATE)) { + eventDispatcher.dispatchDataChangeEvent( + partition, + tableId, + new BinlogChangeRecordEmitter<>( + partition, + offsetContext, + clock, + Envelope.Operation.TRUNCATE, + null, + null, + connectorConfig)); + } eventDispatcher.dispatchSchemaChangeEvent( partition, + offsetContext, tableId, (receiver) -> { try { @@ -707,15 +940,8 @@ protected void handleQueryEvent( } } - private void handleTransactionCompletion( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) - throws InterruptedException { - // We are completing the transaction ... - eventDispatcher.dispatchTransactionCommittedEvent(partition, offsetContext); - offsetContext.commitTransaction(); - offsetContext.setBinlogThread(-1L); - skipEvent = false; - ignoreDmlEventByGtidSource = false; + private String removeSetStatement(String sql) { + return sql.replaceAll(SET_STATEMENT_REGEX, "").trim(); } /** @@ -731,17 +957,18 @@ private void handleTransactionCompletion( *

  • MySQL rotates to a new binary log file, even if the table structure does not change. * * + * @param partition the partition; never null + * @param offsetContext the offset context; never null * @param event the update event; never null */ - protected void handleUpdateTableMetadata( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleUpdateTableMetadata(P partition, O offsetContext, Event event) throws InterruptedException { TableMapEventData metadata = unwrapData(event); long tableNumber = metadata.getTableId(); String databaseName = metadata.getDatabase(); String tableName = metadata.getTable(); TableId tableId = new TableId(databaseName, null, tableName); - if (taskContext.getSchema().assignTableNumber(tableNumber, tableId)) { + if (schema.assignTableNumber(tableNumber, tableId)) { LOGGER.debug("Received update table metadata event: {}", event); } else { informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId); @@ -749,103 +976,31 @@ protected void handleUpdateTableMetadata( } /** - * If we receive an event for a table that is monitored but whose metadata we don't know, either - * ignore that event or raise a warning or error as per the {@link - * MySqlConnectorConfig#INCONSISTENT_SCHEMA_HANDLING_MODE} configuration. + * Handle an event of type TRANSACTION_PAYLOAD_EVENT + * + *

    This method should be called whenever a transaction payload event is encountered by the + * mysql binlog connector. A Transaction payload event is propagated from the binlog connector + * when compression is turned on over binlog. This method loops over the individual events in + * the compressed binlog and calls the respective atomic event handlers. + * + * @param partition the partition; never null + * @param offsetContext the offset context; never null + * @param event the transaction payload event; never null */ - private void informAboutUnknownTableIfRequired( - MySqlPartition partition, - MySqlOffsetContext offsetContext, - Event event, - TableId tableId, - Operation operation) + protected void handleTransactionPayload(P partition, O offsetContext, Event event) throws InterruptedException { - if (tableId != null - && connectorConfig.getTableFilters().dataCollectionFilter().isIncluded(tableId)) { - metrics.onErroneousEvent( - partition, "source = " + tableId + ", event " + event, operation); - EventHeaderV4 eventHeader = event.getHeader(); - - if (inconsistentSchemaHandlingMode == EventProcessingFailureHandlingMode.FAIL) { - LOGGER.error( - "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database history topic. Take a new snapshot in this case.{}" - + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", - event, - offsetContext.getOffset(), - tableId, - System.lineSeparator(), - eventHeader.getPosition(), - eventHeader.getNextPosition(), - offsetContext.getSource().binlogFilename()); - throw new DebeziumException( - "Encountered change event for table " - + tableId - + " whose schema isn't known to this connector"); - } else if (inconsistentSchemaHandlingMode == EventProcessingFailureHandlingMode.WARN) { - LOGGER.warn( - "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database history topic. Take a new snapshot in this case.{}" - + "The event will be ignored.{}" - + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", - event, - offsetContext.getOffset(), - tableId, - System.lineSeparator(), - System.lineSeparator(), - eventHeader.getPosition(), - eventHeader.getNextPosition(), - offsetContext.getSource().binlogFilename()); - } else { - LOGGER.debug( - "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database history topic. Take a new snapshot in this case.{}" - + "The event will be ignored.{}" - + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", - event, - offsetContext.getOffset(), - tableId, - System.lineSeparator(), - System.lineSeparator(), - eventHeader.getPosition(), - eventHeader.getNextPosition(), - offsetContext.getSource().binlogFilename()); - } - } else { - if (tableId == null) { - EventData eventData = unwrapData(event); - if (eventData instanceof WriteRowsEventData) { - tableId = - taskContext - .getSchema() - .getExcludeTableId( - ((WriteRowsEventData) eventData).getTableId()); - } else if (eventData instanceof UpdateRowsEventData) { - tableId = - taskContext - .getSchema() - .getExcludeTableId( - ((UpdateRowsEventData) eventData).getTableId()); - } else if (eventData instanceof DeleteRowsEventData) { - tableId = - taskContext - .getSchema() - .getExcludeTableId( - ((DeleteRowsEventData) eventData).getTableId()); - } - } - LOGGER.trace("Filtered {} event for {}", event.getHeader().getEventType(), tableId); - metrics.onFilteredEvent(partition, "source = " + tableId, operation); - eventDispatcher.dispatchFilteredEvent(partition, offsetContext); + TransactionPayloadEventData transactionPayloadEventData = event.getData(); + // Loop over the uncompressed events in the transaction payload event and add the table map + // event in the map of table events + for (Event uncompressedEvent : transactionPayloadEventData.getUncompressedEvents()) { + eventHandlers + .getOrDefault( + uncompressedEvent.getHeader().getEventType(), + (e) -> ignoreEvent(offsetContext, uncompressedEvent)) + .accept(uncompressedEvent); } } - private void informAboutUnknownTableIfRequired( - MySqlPartition partition, - MySqlOffsetContext offsetContext, - Event event, - TableId tableId) - throws InterruptedException { - informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId, null); - } - /** * Generate source records for the supplied event with an {@link WriteRowsEventData}. * @@ -853,28 +1008,31 @@ private void informAboutUnknownTableIfRequired( * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void handleInsert( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleInsert(P partition, O offsetContext, Event event) throws InterruptedException { handleChange( partition, offsetContext, event, - Operation.CREATE, + Envelope.Operation.CREATE, WriteRowsEventData.class, - x -> taskContext.getSchema().getTableId(x.getTableId()), + x -> schema.getTableId(x.getTableId()), WriteRowsEventData::getRows, (tableId, row) -> eventDispatcher.dispatchDataChangeEvent( partition, tableId, - new MySqlChangeRecordEmitter( + new BinlogChangeRecordEmitter<>( partition, offsetContext, clock, - Operation.CREATE, + Envelope.Operation.CREATE, null, - row))); + row, + connectorConfig)), + (tableId, row) -> + validateChangeEventWithTable( + taskContext.getSchema().tableFor(tableId), null, row)); } /** @@ -884,28 +1042,33 @@ protected void handleInsert( * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void handleUpdate( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleUpdate(P partition, O offsetContext, Event event) throws InterruptedException { handleChange( partition, offsetContext, event, - Operation.UPDATE, + Envelope.Operation.UPDATE, UpdateRowsEventData.class, - x -> taskContext.getSchema().getTableId(x.getTableId()), + x -> schema.getTableId(x.getTableId()), UpdateRowsEventData::getRows, (tableId, row) -> eventDispatcher.dispatchDataChangeEvent( partition, tableId, - new MySqlChangeRecordEmitter( + new BinlogChangeRecordEmitter<>( partition, offsetContext, clock, - Operation.UPDATE, + Envelope.Operation.UPDATE, row.getKey(), - row.getValue()))); + row.getValue(), + connectorConfig)), + (tableId, row) -> + validateChangeEventWithTable( + taskContext.getSchema().tableFor(tableId), + row.getKey(), + row.getValue())); } /** @@ -915,93 +1078,31 @@ protected void handleUpdate( * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void handleDelete( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleDelete(P partition, O offsetContext, Event event) throws InterruptedException { handleChange( partition, offsetContext, event, - Operation.DELETE, + Envelope.Operation.DELETE, DeleteRowsEventData.class, - x -> taskContext.getSchema().getTableId(x.getTableId()), + x -> schema.getTableId(x.getTableId()), DeleteRowsEventData::getRows, (tableId, row) -> eventDispatcher.dispatchDataChangeEvent( partition, tableId, - new MySqlChangeRecordEmitter( + new BinlogChangeRecordEmitter<>( partition, offsetContext, clock, - Operation.DELETE, + Envelope.Operation.DELETE, row, - null))); - } - - private void handleChange( - MySqlPartition partition, - MySqlOffsetContext offsetContext, - Event event, - Operation operation, - Class eventDataClass, - TableIdProvider tableIdProvider, - RowsProvider rowsProvider, - BinlogChangeEmitter changeEmitter) - throws InterruptedException { - if (skipEvent) { - // We can skip this because we should already be at least this far ... - LOGGER.info("Skipping previously processed row event: {}", event); - return; - } - if (ignoreDmlEventByGtidSource) { - LOGGER.debug("Skipping DML event because this GTID source is filtered: {}", event); - return; - } - final T data = unwrapData(event); - final TableId tableId = tableIdProvider.getTableId(data); - final List rows = rowsProvider.getRows(data); - String changeType = operation.name(); - - if (tableId != null && taskContext.getSchema().schemaFor(tableId) != null) { - int count = 0; - int numRows = rows.size(); - if (startingRowNumber < numRows) { - // Use iterator to avoid O(n²) complexity when rows is a LinkedList - // (mysql-binlog-connector-java uses LinkedList in WriteRowsEventDataDeserializer - // and DeleteRowsEventDataDeserializer) - int rowIndex = 0; - for (U rowData : rows) { - if (rowIndex >= startingRowNumber) { - offsetContext.setRowNumber(rowIndex, numRows); - offsetContext.event(tableId, eventTimestamp); - changeEmitter.emit(tableId, rowData); - count++; - } - rowIndex++; - } - if (LOGGER.isDebugEnabled()) { - if (startingRowNumber != 0) { - LOGGER.debug( - "Emitted {} {} record(s) for last {} row(s) in event: {}", - count, - changeType, - numRows - startingRowNumber, - event); - } else { - LOGGER.debug( - "Emitted {} {} record(s) for event: {}", count, changeType, event); - } - } - offsetContext.changeEventCompleted(); - } else { - // All rows were previously processed ... - LOGGER.debug("Skipping previously processed {} event: {}", changeType, event); - } - } else { - informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId, operation); - } - startingRowNumber = 0; + null, + connectorConfig)), + (tableId, row) -> + validateChangeEventWithTable( + taskContext.getSchema().tableFor(tableId), row, null)); } /** @@ -1010,8 +1111,7 @@ private void handleChange( * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void viewChange(MySqlOffsetContext offsetContext, Event event) - throws InterruptedException { + protected void viewChange(O offsetContext, Event event) throws InterruptedException { LOGGER.debug("View Change event: {}", event); // do nothing } @@ -1022,357 +1122,319 @@ protected void viewChange(MySqlOffsetContext offsetContext, Event event) * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void prepareTransaction(MySqlOffsetContext offsetContext, Event event) - throws InterruptedException { + protected void prepareTransaction(O offsetContext, Event event) throws InterruptedException { LOGGER.debug("XA Prepare event: {}", event); // do nothing } - private SSLMode sslModeFor(SecureConnectionMode mode) { - switch (mode) { - case DISABLED: - return SSLMode.DISABLED; - case PREFERRED: - return SSLMode.PREFERRED; - case REQUIRED: - return SSLMode.REQUIRED; - case VERIFY_CA: - return SSLMode.VERIFY_CA; - case VERIFY_IDENTITY: - return SSLMode.VERIFY_IDENTITY; - } - return null; - } - - @Override - public void execute( - ChangeEventSourceContext context, - MySqlPartition partition, - MySqlOffsetContext offsetContext) + protected void handleTransactionBegin(P partition, O offsetContext, Event event, Long threadId) throws InterruptedException { - if (!connectorConfig.getSnapshotMode().shouldStream()) { - LOGGER.info( - "Streaming is disabled for snapshot mode {}", - connectorConfig.getSnapshotMode()); - return; + Instant eventTime = Conversions.toInstantFromMillis(eventTimestamp.toEpochMilli()); + // We are starting a new transaction ... + offsetContext.startNextTransaction(); + eventDispatcher.dispatchTransactionStartedEvent( + partition, offsetContext.getTransactionId(), offsetContext, eventTime); + if (threadId != null) { + offsetContext.setBinlogThread(threadId); } - if (connectorConfig.getSnapshotMode() != MySqlConnectorConfig.SnapshotMode.NEVER) { - taskContext.getSchema().assureNonEmptySchema(); + if (initialEventsToSkip != 0) { + LOGGER.debug( + "Restarting partially-processed transaction; change events will not be created for the first {} events plus {} more rows in the next event", + initialEventsToSkip, + startingRowNumber); + // We are restarting, so we need to skip the events in this transaction that we + // processed previously... + skipEvent = true; } - final Set skippedOperations = connectorConfig.getSkippedOperations(); - - final MySqlOffsetContext effectiveOffsetContext = - offsetContext != null ? offsetContext : MySqlOffsetContext.initial(connectorConfig); - - // Register our event handlers ... - eventHandlers.put( - EventType.STOP, (event) -> handleServerStop(effectiveOffsetContext, event)); - eventHandlers.put( - EventType.HEARTBEAT, - (event) -> handleServerHeartbeat(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.INCIDENT, - (event) -> handleServerIncident(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.ROTATE, (event) -> handleRotateLogsEvent(effectiveOffsetContext, event)); - eventHandlers.put( - EventType.TABLE_MAP, - (event) -> handleUpdateTableMetadata(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.QUERY, - (event) -> handleQueryEvent(partition, effectiveOffsetContext, event)); + } - if (!skippedOperations.contains(Operation.CREATE)) { - eventHandlers.put( - EventType.WRITE_ROWS, - (event) -> handleInsert(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.EXT_WRITE_ROWS, - (event) -> handleInsert(partition, effectiveOffsetContext, event)); - } + /** + * @return the connector class type + */ + protected abstract Class getConnectorClass(); - if (!skippedOperations.contains(Operation.UPDATE)) { - eventHandlers.put( - EventType.UPDATE_ROWS, - (event) -> handleUpdate(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.EXT_UPDATE_ROWS, - (event) -> handleUpdate(partition, effectiveOffsetContext, event)); - } + /** + * Handle the GTID event. + * + * @param partition the partition; never null + * @param offsetContext the offset context; never null + * @param event the event + * @param gtidSourceFilter the GTID source filter + * @throws InterruptedException if the thread is interrupted + */ + protected abstract void handleGtidEvent( + P partition, O offsetContext, Event event, Predicate gtidSourceFilter) + throws InterruptedException; - if (!skippedOperations.contains(Operation.DELETE)) { - eventHandlers.put( - EventType.DELETE_ROWS, - (event) -> handleDelete(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.EXT_DELETE_ROWS, - (event) -> handleDelete(partition, effectiveOffsetContext, event)); - } + /** + * Handle the supplied event with a {@link RowsQueryEventData} or {@link AnnotateRowsEventData} + * by recording the original SQL query that generated the event. + * + * @param offsetContext the offset context; never null + * @param event the database change data event to be processed; may not be null + */ + protected abstract void handleRecordingQuery(O offsetContext, Event event); - eventHandlers.put( - EventType.VIEW_CHANGE, (event) -> viewChange(effectiveOffsetContext, event)); - eventHandlers.put( - EventType.XA_PREPARE, (event) -> prepareTransaction(effectiveOffsetContext, event)); - eventHandlers.put( - EventType.XID, - (event) -> handleTransactionCompletion(partition, effectiveOffsetContext, event)); + /** + * @return the event type for reading query events from + */ + protected abstract EventType getIncludeQueryEventType(); - // Conditionally register ROWS_QUERY handler to parse SQL statements. - if (connectorConfig.includeSqlQuery()) { - eventHandlers.put( - EventType.ROWS_QUERY, - (event) -> handleRowsQuery(effectiveOffsetContext, event)); - } + /** + * @return the event type for the global transaction identifier events + */ + protected abstract EventType getGtidEventType(); - BinaryLogClient.EventListener listener; - if (connectorConfig.bufferSizeForStreamingChangeEventSource() == 0) { - listener = (event) -> handleEvent(partition, effectiveOffsetContext, event); - } else { - EventBuffer buffer = - new EventBuffer( - connectorConfig.bufferSizeForStreamingChangeEventSource(), - this, - context); - listener = (event) -> buffer.add(partition, effectiveOffsetContext, event); - } - client.registerEventListener(listener); + /** + * Initialize the connector's global transaction identifier set. + * + * @param value the value to initialize the set based upon + */ + protected abstract void initializeGtidSet(String value); - client.registerLifecycleListener(new ReaderThreadLifecycleListener(effectiveOffsetContext)); - client.registerEventListener((event) -> onEvent(effectiveOffsetContext, event)); - if (LOGGER.isDebugEnabled()) { - client.registerEventListener((event) -> logEvent(effectiveOffsetContext, event)); + private Predicate getGtidDmlSourceFilter() { + if (connectorConfig + .getConfig() + .getBoolean(BinlogConnectorConfig.GTID_SOURCE_FILTER_DML_EVENTS)) { + return connectorConfig.getGtidSourceFilter(); } + return null; + } - final boolean isGtidModeEnabled = connection.isGtidModeEnabled(); - metrics.setIsGtidModeEnabled(isGtidModeEnabled); - - // Get the current GtidSet from MySQL so we can get a filtered/merged GtidSet based off of - // the last Debezium checkpoint. - String availableServerGtidStr = connection.knownGtidSet(); - if (isGtidModeEnabled) { - // The server is using GTIDs, so enable the handler ... - eventHandlers.put( - EventType.GTID, (event) -> handleGtidEvent(effectiveOffsetContext, event)); - - // Now look at the GTID set from the server and what we've previously seen ... - GtidSet availableServerGtidSet = new GtidSet(availableServerGtidStr); + private void handleTransactionCompletion(P partition, O offsetContext, Event event) + throws InterruptedException { + // We are completing the transaction ... + eventDispatcher.dispatchTransactionCommittedEvent( + partition, + offsetContext, + Conversions.toInstantFromMillis(eventTimestamp.toEpochMilli())); + offsetContext.commitTransaction(); + offsetContext.setBinlogThread(-1L); + skipEvent = false; + ignoreDmlEventByGtidSource = false; + } - // also take into account purged GTID logs - GtidSet purgedServerGtidSet = connection.purgedGtidSet(); - LOGGER.info("GTID set purged on server: {}", purgedServerGtidSet); + /** + * If we receive an event for a table that is monitored but whose metadata we don't know, either + * ignore that event or raise a warning or error per the {@link + * BinlogConnectorConfig#INCONSISTENT_SCHEMA_HANDLING_MODE} configuration. + * + * @param partition the partition; never null + * @param offsetContext the offset context; never null + * @param event the binlog event + * @param tableId the table identifier + * @param operation the operation being performed + */ + private void informAboutUnknownTableIfRequired( + P partition, + O offsetContext, + Event event, + TableId tableId, + Envelope.Operation operation) + throws InterruptedException { + if (tableId != null + && connectorConfig.getTableFilters().dataCollectionFilter().isIncluded(tableId)) { + metrics.onErroneousEvent( + partition, "source = " + tableId + ", event " + event, operation); + EventHeaderV4 eventHeader = event.getHeader(); - GtidSet filteredGtidSet = - filterGtidSet( - effectiveOffsetContext, availableServerGtidSet, purgedServerGtidSet); - if (filteredGtidSet != null) { - // We've seen at least some GTIDs, so start reading from the filtered GTID set ... - LOGGER.info("Registering binlog reader with GTID set: {}", filteredGtidSet); - String filteredGtidSetStr = filteredGtidSet.toString(); - client.setGtidSet(filteredGtidSetStr); - effectiveOffsetContext.setCompletedGtidSet(filteredGtidSetStr); - gtidSet = new com.github.shyiko.mysql.binlog.GtidSet(filteredGtidSetStr); + if (inconsistentSchemaHandlingMode == EventProcessingFailureHandlingMode.FAIL) { + LOGGER.error( + "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database schema history topic. Take a new snapshot in this case.{}" + + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", + event, + offsetContext.getOffset(), + tableId, + System.lineSeparator(), + eventHeader.getPosition(), + eventHeader.getNextPosition(), + offsetContext.getSource().binlogFilename()); + throw new DebeziumException( + "Encountered change event for table " + + tableId + + " whose schema isn't known to this connector"); + } else if (inconsistentSchemaHandlingMode == EventProcessingFailureHandlingMode.WARN) { + LOGGER.warn( + "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database schema history topic. Take a new snapshot in this case.{}" + + "The event will be ignored.{}" + + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", + event, + offsetContext.getOffset(), + tableId, + System.lineSeparator(), + System.lineSeparator(), + eventHeader.getPosition(), + eventHeader.getNextPosition(), + offsetContext.getSource().binlogFilename()); } else { - // We've not yet seen any GTIDs, so that means we have to start reading the binlog - // from the beginning ... - client.setBinlogFilename(effectiveOffsetContext.getSource().binlogFilename()); - client.setBinlogPosition(effectiveOffsetContext.getSource().binlogPosition()); - gtidSet = new com.github.shyiko.mysql.binlog.GtidSet(""); + LOGGER.debug( + "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database schema history topic. Take a new snapshot in this case.{}" + + "The event will be ignored.{}" + + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", + event, + offsetContext.getOffset(), + tableId, + System.lineSeparator(), + System.lineSeparator(), + eventHeader.getPosition(), + eventHeader.getNextPosition(), + offsetContext.getSource().binlogFilename()); } } else { - // The server is not using GTIDs, so start reading the binlog based upon where we last - // left off ... - client.setBinlogFilename(effectiveOffsetContext.getSource().binlogFilename()); - client.setBinlogPosition(effectiveOffsetContext.getSource().binlogPosition()); - } - - // We may be restarting in the middle of a transaction, so see how far into the transaction - // we have already processed... - initialEventsToSkip = effectiveOffsetContext.eventsToSkipUponRestart(); - LOGGER.info("Skip {} events on streaming start", initialEventsToSkip); - - // Set the starting row number, which is the next row number to be read ... - startingRowNumber = effectiveOffsetContext.rowsToSkipUponRestart(); - LOGGER.info("Skip {} rows on streaming start", startingRowNumber); - - // Only when we reach the first BEGIN event will we start to skip events ... - skipEvent = false; - - try { - // Start the log reader, which starts background threads ... - if (context.isRunning()) { - long timeout = connectorConfig.getConnectionTimeout().toMillis(); - long started = clock.currentTimeInMillis(); - try { - LOGGER.debug( - "Attempting to establish binlog reader connection with timeout of {} ms", - timeout); - client.connect(timeout); - // Need to wait for keepalive thread to be running, otherwise it can be left - // orphaned - // The problem is with timing. When the close is called too early after connect - // then - // the keepalive thread is not terminated - if (client.isKeepAlive()) { - LOGGER.info("Waiting for keepalive thread to start"); - final Metronome metronome = Metronome.parker(Duration.ofMillis(100), clock); - int waitAttempts = 50; - boolean keepAliveThreadRunning = false; - while (!keepAliveThreadRunning && waitAttempts-- > 0) { - for (Thread t : binaryLogClientThreads.values()) { - if (t.getName().startsWith(KEEPALIVE_THREAD_NAME) && t.isAlive()) { - LOGGER.info("Keepalive thread is running"); - keepAliveThreadRunning = true; - } - } - metronome.pause(); - } - } - } catch (TimeoutException e) { - // If the client thread is interrupted *before* the client could connect, the - // client throws a timeout exception - // The only way we can distinguish this is if we get the timeout exception - // before the specified timeout has - // elapsed, so we simply check this (within 10%) ... - long duration = clock.currentTimeInMillis() - started; - if (duration > (0.9 * timeout)) { - double actualSeconds = TimeUnit.MILLISECONDS.toSeconds(duration); - throw new DebeziumException( - "Timed out after " - + actualSeconds - + " seconds while waiting to connect to MySQL at " - + connectorConfig.hostname() - + ":" - + connectorConfig.port() - + " with user '" - + connectorConfig.username() - + "'", - e); - } - // Otherwise, we were told to shutdown, so we don't care about the timeout - // exception - } catch (AuthenticationException e) { - throw new DebeziumException( - "Failed to authenticate to the MySQL database at " - + connectorConfig.hostname() - + ":" - + connectorConfig.port() - + " with user '" - + connectorConfig.username() - + "'", - e); - } catch (Throwable e) { - throw new DebeziumException( - "Unable to connect to the MySQL database at " - + connectorConfig.hostname() - + ":" - + connectorConfig.port() - + " with user '" - + connectorConfig.username() - + "': " - + e.getMessage(), - e); + if (tableId == null) { + EventData eventData = unwrapData(event); + if (eventData instanceof WriteRowsEventData) { + tableId = + schema.getExcludeTableId(((WriteRowsEventData) eventData).getTableId()); + } else if (eventData instanceof UpdateRowsEventData) { + tableId = + schema.getExcludeTableId( + ((UpdateRowsEventData) eventData).getTableId()); + } else if (eventData instanceof DeleteRowsEventData) { + tableId = + schema.getExcludeTableId( + ((DeleteRowsEventData) eventData).getTableId()); } } - while (context.isRunning()) { - Thread.sleep(100); - } - } finally { - try { - client.disconnect(); - } catch (Exception e) { - LOGGER.info("Exception while stopping binary log client", e); - } + LOGGER.trace("Filtered {} event for {}", event.getHeader().getEventType(), tableId); + metrics.onFilteredEvent(partition, "source = " + tableId, operation); + eventDispatcher.dispatchFilteredEvent(partition, offsetContext); } } - private SSLSocketFactory getBinlogSslSocketFactory( - MySqlConnectorConfig connectorConfig, MySqlConnection connection) { - String acceptedTlsVersion = connection.getSessionVariableForSslVersion(); - if (!isNullOrEmpty(acceptedTlsVersion)) { - SSLMode sslMode = sslModeFor(connectorConfig.sslMode()); - LOGGER.info( - "Enable ssl " - + sslMode - + " mode for connector " - + connectorConfig.getLogicalName()); - - final char[] keyPasswordArray = connection.connectionConfig().sslKeyStorePassword(); - final String keyFilename = connection.connectionConfig().sslKeyStore(); - final char[] trustPasswordArray = connection.connectionConfig().sslTrustStorePassword(); - final String trustFilename = connection.connectionConfig().sslTrustStore(); - KeyManager[] keyManagers = null; - if (keyFilename != null) { - try { - KeyStore ks = connection.loadKeyStore(keyFilename, keyPasswordArray); - - KeyManagerFactory kmf = KeyManagerFactory.getInstance("NewSunX509"); - kmf.init(ks, keyPasswordArray); - - keyManagers = kmf.getKeyManagers(); - } catch (KeyStoreException - | NoSuchAlgorithmException - | UnrecoverableKeyException e) { - throw new DebeziumException("Could not load keystore", e); - } - } - TrustManager[] trustManagers; - try { - KeyStore ks = null; - if (trustFilename != null) { - ks = connection.loadKeyStore(trustFilename, trustPasswordArray); - } - - if (ks == null && (sslMode == SSLMode.PREFERRED || sslMode == SSLMode.REQUIRED)) { - trustManagers = - new TrustManager[] { - new X509TrustManager() { - - @Override - public void checkClientTrusted( - X509Certificate[] x509Certificates, String s) - throws CertificateException {} - - @Override - public void checkServerTrusted( - X509Certificate[] x509Certificates, String s) - throws CertificateException {} - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - } - }; - } else { - TrustManagerFactory tmf = - TrustManagerFactory.getInstance( - TrustManagerFactory.getDefaultAlgorithm()); - tmf.init(ks); - trustManagers = tmf.getTrustManagers(); - } - } catch (KeyStoreException | NoSuchAlgorithmException e) { - throw new DebeziumException("Could not load truststore", e); + /** + * If we receive an event for a table that is monitored but whose metadata we don't know, either + * ignore that event or raise a warning or error per the {@link + * BinlogConnectorConfig#INCONSISTENT_SCHEMA_HANDLING_MODE} configuration. + * + * @param partition the partition; never null + * @param offsetContext the offset context; never null + * @param event the binlog event + * @param tableId the table identifier + * @throws InterruptedException if the thread is interrupted + */ + private void informAboutUnknownTableIfRequired( + P partition, O offsetContext, Event event, TableId tableId) + throws InterruptedException { + informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId, null); + } + + private void validateChangeEventWithTable(Table table, Object[] before, Object[] after) { + if (table != null) { + int columnSize = table.columns().size(); + String message = + "Error processing {} of row in {} because it's different column size with internal schema size {}, but {} size {}, " + + "restart connector with schema recovery mode."; + if (before != null && columnSize != before.length) { + LOGGER.error( + message, "before", table.id().table(), columnSize, "before", before.length); + throw new DebeziumException( + "Error processing row in " + + table.id().table() + + ", internal schema size " + + columnSize + + ", but row size " + + before.length + + " , " + + "restart connector with schema recovery mode."); } - // DBZ-1208 Resembles the logic from the upstream BinaryLogClient, only that - // the accepted TLS version is passed to the constructed factory - final KeyManager[] finalKMS = keyManagers; - return new DefaultSSLSocketFactory(acceptedTlsVersion) { + if (after != null && columnSize != after.length) { + LOGGER.error( + message, "after", table.id().table(), columnSize, "after", after.length); + throw new DebeziumException( + "Error processing row in " + + table.id().table() + + ", internal schema size " + + columnSize + + ", but row size " + + after.length + + " , " + + "restart connector with schema recovery mode."); + } + } + } - @Override - protected void initSSLContext(SSLContext sc) throws GeneralSecurityException { - sc.init(finalKMS, trustManagers, null); + private void handleChange( + P partition, + O offsetContext, + Event event, + Envelope.Operation operation, + Class eventDataClass, + TableIdProvider tableIdProvider, + RowsProvider rowsProvider, + BinlogChangeEmitter changeEmitter, + ChangeEventValidator changeEventValidator) + throws InterruptedException { + if (skipEvent) { + // We can skip this because we should already be at least this far ... + LOGGER.info("Skipping previously processed row event: {}", event); + return; + } + if (ignoreDmlEventByGtidSource) { + LOGGER.debug("Skipping DML event because this GTID source is filtered: {}", event); + return; + } + final T data = unwrapData(event); + final TableId tableId = tableIdProvider.getTableId(data); + final List rows = rowsProvider.getRows(data); + String changeType = operation.name(); + + if (tableId != null && taskContext.getSchema().schemaFor(tableId) != null) { + int count = 0; + int numRows = rows.size(); + if (startingRowNumber < numRows) { + // Iterate rather than index: the row list may be a linked list, in which case + // rows.get(i) turns this loop into O(n^2). + int rowIndex = 0; + for (U row : rows) { + if (rowIndex >= startingRowNumber) { + changeEventValidator.validate(tableId, row); + offsetContext.setRowNumber(rowIndex, numRows); + offsetContext.event(tableId, eventTimestamp); + changeEmitter.emit(tableId, row); + count++; + } + rowIndex++; } - }; + if (LOGGER.isDebugEnabled()) { + if (startingRowNumber != 0) { + LOGGER.debug( + "Emitted {} {} record(s) for last {} row(s) in event: {}", + count, + changeType, + numRows - startingRowNumber, + event); + } else { + LOGGER.debug( + "Emitted {} {} record(s) for event: {}", count, changeType, event); + } + } + offsetContext.changeEventCompleted(); + } else { + // All rows were previously processed ... + LOGGER.debug("Skipping previously processed {} event: {}", changeType, event); + } + } else { + informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId, operation); } + startingRowNumber = 0; + } - return null; + private void waitWhenStreamingPaused(ChangeEventSourceContext context) + throws InterruptedException { + if (context.isPaused()) { + LOGGER.info("Streaming will now pause"); + context.streamingPaused(); + context.waitSnapshotCompletion(); + LOGGER.info("Streaming resumed"); + } } private void logStreamingSourceState() { logStreamingSourceState(Level.ERROR); } - protected void logEvent(MySqlOffsetContext offsetContext, Event event) { + protected void logEvent(O offsetContext, Event event) { LOGGER.trace("Received event: {}", event); } @@ -1395,86 +1457,7 @@ private void logStreamingSourceState(Level severity) { } } - /** - * Apply the include/exclude GTID source filters to the current {@link #source() GTID set} and - * merge them onto the currently available GTID set from a MySQL server. - * - *

    The merging behavior of this method might seem a bit strange at first. It's required in - * order for Debezium to consume a MySQL binlog that has multi-source replication enabled, if a - * failover has to occur. In such a case, the server that Debezium is failed over to might have - * a different set of sources, but still include the sources required for Debezium to continue - * to function. MySQL does not allow downstream replicas to connect if the GTID set does not - * contain GTIDs for all channels that the server is replicating from, even if the server does - * have the data needed by the client. To get around this, we can have Debezium merge its GTID - * set with whatever is on the server, so that MySQL will allow it to connect. See DBZ-143 for details. - * - *

    This method does not mutate any state in the context. - * - * @param availableServerGtidSet the GTID set currently available in the MySQL server - * @param purgedServerGtid the GTID set already purged by the MySQL server - * @return A GTID set meant for consuming from a MySQL binlog; may return null if the SourceInfo - * has no GTIDs and therefore none were filtered - */ - public GtidSet filterGtidSet( - MySqlOffsetContext offsetContext, - GtidSet availableServerGtidSet, - GtidSet purgedServerGtid) { - String gtidStr = offsetContext.gtidSet(); - if (gtidStr == null) { - return null; - } - LOGGER.info("Attempting to generate a filtered GTID set"); - LOGGER.info("GTID set from previous recorded offset: {}", gtidStr); - GtidSet filteredGtidSet = new GtidSet(gtidStr); - Predicate gtidSourceFilter = connectorConfig.gtidSourceFilter(); - if (gtidSourceFilter != null) { - filteredGtidSet = filteredGtidSet.retainAll(gtidSourceFilter); - LOGGER.info( - "GTID set after applying GTID source includes/excludes to previous recorded offset: {}", - filteredGtidSet); - } - LOGGER.info("GTID set available on server: {}", availableServerGtidSet); - - GtidSet mergedGtidSet; - - if (connectorConfig.gtidNewChannelPosition() == GtidNewChannelPosition.EARLIEST) { - LOGGER.info("Using first available positions for new GTID channels"); - final GtidSet relevantAvailableServerGtidSet = - (gtidSourceFilter != null) - ? availableServerGtidSet.retainAll(gtidSourceFilter) - : availableServerGtidSet; - LOGGER.info( - "Relevant GTID set available on server: {}", relevantAvailableServerGtidSet); - - // Since the GTID recorded in the checkpoint represents the CDC-executed records, in - // certain scenarios - // (such as when the startup mode is earliest/timestamp/binlogfile), the recorded GTID - // may not start from - // the beginning. For example, A:300-500. However, during job recovery, we usually only - // need to focus on - // the last consumed point instead of consuming A:1-299. Therefore, some adjustments - // need to be made to the - // recorded offset in the checkpoint, and the available GTID for other MySQL instances - // should be completed. - mergedGtidSet = - GtidUtils.fixOldChannelsGtidSet( - relevantAvailableServerGtidSet, purgedServerGtid, filteredGtidSet); - } else { - LOGGER.info("Using latest positions for new GTID channels"); - mergedGtidSet = - GtidUtils.computeLatestModeGtidSet( - availableServerGtidSet, - purgedServerGtid, - filteredGtidSet, - gtidSourceFilter); - } - - LOGGER.info("Final merged GTID set to use when connecting to MySQL: {}", mergedGtidSet); - return mergedGtidSet; - } - - MySqlStreamingChangeEventSourceMetrics getMetrics() { + BinlogStreamingChangeEventSourceMetrics getMetrics() { return metrics; } @@ -1523,11 +1506,11 @@ protected DebeziumException wrap(Throwable error) { return new DebeziumException(msg, error); } - /** LifecycleListener for Reader Thread. */ - protected final class ReaderThreadLifecycleListener implements LifecycleListener { - private final MySqlOffsetContext offsetContext; + protected final class ReaderThreadLifecycleListener + implements BinaryLogClient.LifecycleListener { + private final O offsetContext; - ReaderThreadLifecycleListener(MySqlOffsetContext offsetContext) { + ReaderThreadLifecycleListener(O offsetContext) { this.offsetContext = offsetContext; } @@ -1557,13 +1540,13 @@ public void onDisconnect(BinaryLogClient client) { public void onConnect(BinaryLogClient client) { // Set up the MDC logging context for this thread ... taskContext.configureLoggingContext("binlog"); - // The event row number will be used when processing the first event ... LOGGER.info( - "Connected to MySQL binlog at {}:{}, starting at {}", - connectorConfig.hostname(), - connectorConfig.port(), + "Connected to binlog at {}:{}, starting at {}", + connectorConfig.getHostName(), + connectorConfig.getPort(), offsetContext); + totalRecordCounter.set(0); } @Override @@ -1597,6 +1580,108 @@ public void onEventDeserializationFailure(BinaryLogClient client, Exception ex) } } + private SSLMode sslModeFor(SecureConnectionMode mode) { + switch (mode) { + case DISABLED: + return SSLMode.DISABLED; + case PREFERRED: + return SSLMode.PREFERRED; + case REQUIRED: + return SSLMode.REQUIRED; + case VERIFY_CA: + return SSLMode.VERIFY_CA; + case VERIFY_IDENTITY: + return SSLMode.VERIFY_IDENTITY; + } + return null; + } + + private SSLSocketFactory getBinlogSslSocketFactory( + BinlogConnectorConfig connectorConfig, BinlogConnectorConnection connection) { + String acceptedTlsVersion = connection.getSessionVariableForSslVersion(); + if (!isNullOrEmpty(acceptedTlsVersion)) { + SSLMode sslMode = sslModeFor(connectorConfig.getSslMode()); + LOGGER.info( + "Enable ssl " + + sslMode + + " mode for connector " + + connectorConfig.getLogicalName()); + + final char[] keyPasswordArray = connection.connectionConfig().sslKeyStorePassword(); + final String keyFilename = connection.connectionConfig().sslKeyStore(); + final char[] trustPasswordArray = connection.connectionConfig().sslTrustStorePassword(); + final String trustFilename = connection.connectionConfig().sslTrustStore(); + KeyManager[] keyManagers = null; + if (keyFilename != null) { + try { + KeyStore ks = connection.loadKeyStore(keyFilename, keyPasswordArray); + + KeyManagerFactory kmf = KeyManagerFactory.getInstance("NewSunX509"); + kmf.init(ks, keyPasswordArray); + + keyManagers = kmf.getKeyManagers(); + } catch (KeyStoreException + | NoSuchAlgorithmException + | UnrecoverableKeyException e) { + throw new DebeziumException("Could not load keystore", e); + } + } + TrustManager[] trustManagers; + try { + KeyStore ks = null; + if (trustFilename != null) { + ks = connection.loadKeyStore(trustFilename, trustPasswordArray); + } + + if (ks == null && (sslMode == SSLMode.PREFERRED || sslMode == SSLMode.REQUIRED)) { + trustManagers = + new TrustManager[] { + new X509TrustManager() { + @Override + public void checkClientTrusted( + X509Certificate[] x509Certificates, String s) + throws CertificateException {} + + @Override + public void checkServerTrusted( + X509Certificate[] x509Certificates, String s) + throws CertificateException {} + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } + }; + } else { + TrustManagerFactory tmf = + TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(ks); + trustManagers = tmf.getTrustManagers(); + } + } catch (KeyStoreException | NoSuchAlgorithmException e) { + throw new DebeziumException("Could not load truststore", e); + } + + // DBZ-1208 Resembles the logic from the upstream BinaryLogClient, only that + // the accepted TLS version is passed to the constructed factory + final KeyManager[] finalKMS = keyManagers; + return new DefaultSSLSocketFactory(acceptedTlsVersion) { + @Override + protected void initSSLContext(SSLContext sc) throws GeneralSecurityException { + sc.init(finalKMS, trustManagers, null); + } + }; + } + return null; + } + + @FunctionalInterface + private interface BinlogChangeEmitter { + void emit(TableId tableId, T data) throws InterruptedException; + } + @FunctionalInterface private interface TableIdProvider { TableId getTableId(E data); @@ -1606,4 +1691,65 @@ private interface TableIdProvider { private interface RowsProvider { List getRows(E data); } + + @FunctionalInterface + private interface ChangeEventValidator { + void validate(TableId tableId, U row); + } + + /** Class that represents a binlog file and position */ + public static class BinlogPosition { + final String filename; + final long position; + + public BinlogPosition(String filename, long position) { + assert filename != null; + + this.filename = filename; + this.position = position; + } + + public String getFilename() { + return filename; + } + + public long getPosition() { + return position; + } + + @Override + public String toString() { + return filename + "/" + position; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + filename.hashCode(); + result = prime * result + (int) (position ^ (position >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + BinlogPosition other = (BinlogPosition) obj; + if (!filename.equals(other.filename)) { + return false; + } + if (position != other.position) { + return false; + } + return true; + } + } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlConnection.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlConnection.java deleted file mode 100644 index aa9699e4eaa..00000000000 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlConnection.java +++ /dev/null @@ -1,810 +0,0 @@ -/* - * Copyright Debezium Authors. - * - * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 - */ - -package io.debezium.connector.mysql; - -import com.mysql.cj.CharsetMapping; -import io.debezium.DebeziumException; -import io.debezium.config.CommonConnectorConfig; -import io.debezium.config.CommonConnectorConfig.EventProcessingFailureHandlingMode; -import io.debezium.config.Configuration; -import io.debezium.config.Configuration.Builder; -import io.debezium.config.Field; -import io.debezium.connector.mysql.MySqlConnectorConfig.SecureConnectionMode; -import io.debezium.connector.mysql.legacy.MySqlJdbcContext.DatabaseLocales; -import io.debezium.jdbc.JdbcConfiguration; -import io.debezium.jdbc.JdbcConnection; -import io.debezium.relational.Column; -import io.debezium.relational.Table; -import io.debezium.relational.TableId; -import io.debezium.relational.history.DatabaseHistory; -import io.debezium.schema.DatabaseSchema; -import io.debezium.util.Strings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.OptionalLong; -import java.util.Properties; - -/** - * Copied from Debezium project(1.9.8.final) to add custom jdbc properties in the jdbc url. The new - * parameter {@link MySqlConnectionConfiguration#MySqlConnectionConfiguration(Configuration config, - * Properties jdbcProperties)} in the constructor of {@link MySqlConnectionConfiguration} will be - * used to generate the jdbc url pattern, and may overwrite the default value. - * - *

    Add field {@link MySqlConnection#urlPattern} in {@link MySqlConnection} and remove old - * pattern. - * - *

    Added MySQL 8.4+ compatible probing fields {@link - * MySqlConnection#MYSQL_CLASSIC_SHOW_BINARY_LOG_STATEMENT}, {@link - * MySqlConnection#MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT}, and {@link - * MySqlConnection#showBinaryLogStatement}. - * - *

    Init {@link MySqlConnection#urlPattern} using the url pattern from {@link - * MySqlConnectionConfiguration}. - * - *

    Generate the connection string by the new field {@link MySqlConnection#urlPattern}. - * - *

    Add new constant and field {@link MySqlConnection#urlPattern} to {@link - * MySqlConnectionConfiguration}. - * - *

    Init new field {@link MySqlConnection#urlPattern} in {@link MySqlConnectionConfiguration}. - * - *

    Add utility methods helping to generate the url pattern and add default values. - * - *

    Added utility method {@link MySqlConnection#getShowBinaryLogStatement} and {@link - * MySqlConnection#probeShowBinaryLogStatement} for MySQL 8.4 compatibility. - */ -public class MySqlConnection extends JdbcConnection { - - private static final Logger LOGGER = LoggerFactory.getLogger(MySqlConnection.class); - - private static final String SQL_SHOW_SYSTEM_VARIABLES = "SHOW VARIABLES"; - private static final String SQL_SHOW_SYSTEM_VARIABLES_CHARACTER_SET = - "SHOW VARIABLES WHERE Variable_name IN ('character_set_server','collation_server')"; - private static final String SQL_SHOW_SESSION_VARIABLE_SSL_VERSION = - "SHOW SESSION STATUS LIKE 'Ssl_version'"; - private static final String QUOTED_CHARACTER = "`"; - - private final Map originalSystemProperties = new HashMap<>(); - private final MySqlConnectionConfiguration connectionConfig; - private final MySqlFieldReader mysqlFieldReader; - - private final String urlPattern; - - private static final String MYSQL_CLASSIC_SHOW_BINARY_LOG_STATEMENT = "SHOW MASTER STATUS"; - private static final String MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT = "SHOW BINARY LOG STATUS"; - private final String showBinaryLogStatement; - - /** - * Creates a new connection using the supplied configuration. - * - * @param connectionConfig {@link MySqlConnectionConfiguration} instance, may not be null. - * @param fieldReader binary or text protocol based readers - */ - public MySqlConnection( - MySqlConnectionConfiguration connectionConfig, MySqlFieldReader fieldReader) { - super( - connectionConfig.jdbcConfig, - connectionConfig.factory(), - QUOTED_CHARACTER, - QUOTED_CHARACTER); - this.connectionConfig = connectionConfig; - this.mysqlFieldReader = fieldReader; - this.urlPattern = connectionConfig.getUrlPattern(); - this.showBinaryLogStatement = probeShowBinaryLogStatement(); - } - - /** - * Creates a new connection using the supplied configuration. - * - * @param connectionConfig {@link MySqlConnectionConfiguration} instance, may not be null. - */ - public MySqlConnection(MySqlConnectionConfiguration connectionConfig) { - this(connectionConfig, new MySqlTextProtocolFieldReader(null)); - } - - @Override - public void close() throws SQLException { - try { - super.close(); - } finally { - // Reset the system properties to their original value ... - originalSystemProperties.forEach( - (name, value) -> { - if (value != null) { - System.setProperty(name, value); - } else { - System.clearProperty(name); - } - }); - } - } - - /** - * Read the MySQL charset-related system variables. - * - * @return the system variables that are related to server character sets; never null - */ - protected Map readMySqlCharsetSystemVariables() { - // Read the system variables from the MySQL instance and get the current database name ... - LOGGER.debug("Reading MySQL charset-related system variables before parsing DDL history."); - return querySystemVariables(SQL_SHOW_SYSTEM_VARIABLES_CHARACTER_SET); - } - - /** - * Read the MySQL system variables. - * - * @return the system variables that are related to server character sets; never null - */ - protected Map readMySqlSystemVariables() { - // Read the system variables from the MySQL instance and get the current database name ... - LOGGER.debug("Reading MySQL system variables"); - return querySystemVariables(SQL_SHOW_SYSTEM_VARIABLES); - } - - private Map querySystemVariables(String statement) { - final Map variables = new HashMap<>(); - try { - query( - statement, - rs -> { - while (rs.next()) { - String varName = rs.getString(1); - String value = rs.getString(2); - if (varName != null && value != null) { - variables.put(varName, value); - LOGGER.debug( - "\t{} = {}", - Strings.pad(varName, 45, ' '), - Strings.pad(value, 45, ' ')); - } - } - }); - } catch (SQLException e) { - throw new DebeziumException("Error reading MySQL variables: " + e.getMessage(), e); - } - - return variables; - } - - protected String setStatementFor(Map variables) { - StringBuilder sb = new StringBuilder("SET "); - boolean first = true; - List varNames = new ArrayList<>(variables.keySet()); - Collections.sort(varNames); - for (String varName : varNames) { - if (first) { - first = false; - } else { - sb.append(", "); - } - sb.append(varName).append("="); - String value = variables.get(varName); - if (value == null) { - value = ""; - } - if (value.contains(",") || value.contains(";")) { - value = "'" + value + "'"; - } - sb.append(value); - } - return sb.append(";").toString(); - } - - protected void setSystemProperty(String property, Field field, boolean showValueInError) { - String value = connectionConfig.originalConfig().getString(field); - if (value != null) { - value = value.trim(); - String existingValue = System.getProperty(property); - if (existingValue == null) { - // There was no existing property ... - String existing = System.setProperty(property, value); - originalSystemProperties.put(property, existing); // the existing value may be null - } else { - existingValue = existingValue.trim(); - if (!existingValue.equalsIgnoreCase(value)) { - // There was an existing property, and the value is different ... - String msg = - "System or JVM property '" - + property - + "' is already defined, but the configuration property '" - + field.name() - + "' defines a different value"; - if (showValueInError) { - msg = - "System or JVM property '" - + property - + "' is already defined as " - + existingValue - + ", but the configuration property '" - + field.name() - + "' defines a different value '" - + value - + "'"; - } - throw new DebeziumException(msg); - } - // Otherwise, there was an existing property, and the value is exactly the same (so - // do nothing!) - } - } - } - - /** - * Read the Ssl Version session variable. - * - * @return the session variables that are related to sessions ssl version - */ - protected String getSessionVariableForSslVersion() { - final String sslVersion = "Ssl_version"; - LOGGER.debug("Reading MySQL Session variable for Ssl Version"); - Map sessionVariables = - querySystemVariables(SQL_SHOW_SESSION_VARIABLE_SSL_VERSION); - if (!sessionVariables.isEmpty() && sessionVariables.containsKey(sslVersion)) { - return sessionVariables.get(sslVersion); - } - return null; - } - - /** - * Determine whether the MySQL server has GTIDs enabled. - * - * @return {@code false} if the server's {@code gtid_mode} is set and is {@code OFF}, or {@code - * true} otherwise - */ - public boolean isGtidModeEnabled() { - try { - return queryAndMap( - "SHOW GLOBAL VARIABLES LIKE 'GTID_MODE'", - rs -> { - if (rs.next()) { - return !"OFF".equalsIgnoreCase(rs.getString(2)); - } - return false; - }); - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking at GTID mode: ", e); - } - } - - /** - * Determine the executed GTID set for MySQL. - * - * @return the string representation of MySQL's GTID sets; never null but an empty string if the - * server does not use GTIDs - */ - public String knownGtidSet() { - try { - return queryAndMap( - showBinaryLogStatement, - rs -> { - if (rs.next() && rs.getMetaData().getColumnCount() > 4) { - return rs.getString( - 5); // GTID set, may be null, blank, or contain a GTID set - } - return ""; - }); - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking at GTID mode: ", e); - } - } - - /** - * Determine the difference between two sets. - * - * @return a subtraction of two GTID sets; never null - */ - public GtidSet subtractGtidSet(GtidSet set1, GtidSet set2) { - try { - return prepareQueryAndMap( - "SELECT GTID_SUBTRACT(?, ?)", - ps -> { - ps.setString(1, set1.toString()); - ps.setString(2, set2.toString()); - }, - rs -> { - if (rs.next()) { - return new GtidSet(rs.getString(1)); - } - return new GtidSet(""); - }); - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking at GTID mode: ", e); - } - } - - /** - * Get the purged GTID values from MySQL (gtid_purged value). - * - * @return A GTID set; may be empty if not using GTIDs or none have been purged yet - */ - public GtidSet purgedGtidSet() { - try { - return queryAndMap( - "SELECT @@global.gtid_purged", - rs -> { - if (rs.next() && rs.getMetaData().getColumnCount() > 0) { - return new GtidSet( - rs.getString( - 1)); // GTID set, may be null, blank, or contain a GTID - // set - } - return new GtidSet(""); - }); - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking at gtid_purged variable: ", - e); - } - } - - /** - * Determine if the current user has the named privilege. Note that if the user has the "ALL" - * privilege this method returns {@code true}. - * - * @param grantName the name of the MySQL privilege; may not be null - * @return {@code true} if the user has the named privilege, or {@code false} otherwise - */ - public boolean userHasPrivileges(String grantName) { - try { - return queryAndMap( - "SHOW GRANTS FOR CURRENT_USER", - rs -> { - while (rs.next()) { - String grants = rs.getString(1); - LOGGER.debug(grants); - if (grants == null) { - return false; - } - grants = grants.toUpperCase(); - if (grants.contains("ALL") - || grants.contains(grantName.toUpperCase())) { - return true; - } - } - return false; - }); - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking at privileges for current user: ", - e); - } - } - - /** - * Determine the earliest binlog filename that is still available in the server. - * - * @return the name of the earliest binlog filename, or null if there are none. - */ - public String earliestBinlogFilename() { - // Accumulate the available binlog filenames ... - List logNames = new ArrayList<>(); - try { - LOGGER.info("Checking all known binlogs from MySQL"); - query( - "SHOW BINARY LOGS", - rs -> { - while (rs.next()) { - logNames.add(rs.getString(1)); - } - }); - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking for binary logs: ", e); - } - - if (logNames.isEmpty()) { - return null; - } - return logNames.get(0); - } - - /** - * Determine whether the MySQL server has the binlog_row_image set to 'FULL'. - * - * @return {@code true} if the server's {@code binlog_row_image} is set to {@code FULL}, or - * {@code false} otherwise - */ - protected boolean isBinlogRowImageFull() { - try { - final String rowImage = - queryAndMap( - "SHOW GLOBAL VARIABLES LIKE 'binlog_row_image'", - rs -> { - if (rs.next()) { - return rs.getString(2); - } - // This setting was introduced in MySQL 5.6+ with default of 'FULL'. - // For older versions, assume 'FULL'. - return "FULL"; - }); - LOGGER.debug("binlog_row_image={}", rowImage); - return "FULL".equalsIgnoreCase(rowImage); - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking at BINLOG_ROW_IMAGE mode: ", - e); - } - } - - /** - * Determine whether the MySQL server has the row-level binlog enabled. - * - * @return {@code true} if the server's {@code binlog_format} is set to {@code ROW}, or {@code - * false} otherwise - */ - protected boolean isBinlogFormatRow() { - try { - final String mode = - queryAndMap( - "SHOW GLOBAL VARIABLES LIKE 'binlog_format'", - rs -> rs.next() ? rs.getString(2) : ""); - LOGGER.debug("binlog_format={}", mode); - return "ROW".equalsIgnoreCase(mode); - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking at BINLOG_FORMAT mode: ", - e); - } - } - - /** - * Query the database server to get the list of the binlog files availble. - * - * @return list of the binlog files - */ - public List availableBinlogFiles() { - List logNames = new ArrayList<>(); - try { - LOGGER.info("Get all known binlogs from MySQL"); - query( - "SHOW BINARY LOGS", - rs -> { - while (rs.next()) { - logNames.add(rs.getString(1)); - } - }); - return logNames; - } catch (SQLException e) { - throw new DebeziumException( - "Unexpected error while connecting to MySQL and looking for binary logs: ", e); - } - } - - public OptionalLong getEstimatedTableSize(TableId tableId) { - try { - // Choose how we create statements based on the # of rows. - // This is approximate and less accurate then COUNT(*), - // but far more efficient for large InnoDB tables. - execute("USE `" + tableId.catalog() + "`;"); - return queryAndMap( - "SHOW TABLE STATUS LIKE '" + tableId.table() + "';", - rs -> { - if (rs.next()) { - return OptionalLong.of((rs.getLong(5))); - } - return OptionalLong.empty(); - }); - } catch (SQLException e) { - LOGGER.debug( - "Error while getting number of rows in table {}: {}", - tableId, - e.getMessage(), - e); - } - return OptionalLong.empty(); - } - - public boolean isTableIdCaseSensitive() { - return !"0" - .equals( - readMySqlSystemVariables() - .get(MySqlSystemVariables.LOWER_CASE_TABLE_NAMES)); - } - - /** - * Read the MySQL default character sets for exisiting databases. - * - * @return the map of database names with their default character sets; never null - */ - protected Map readDatabaseCollations() { - LOGGER.debug("Reading default database charsets"); - try { - return queryAndMap( - "SELECT schema_name, default_character_set_name, default_collation_name FROM information_schema.schemata", - rs -> { - final Map charsets = new HashMap<>(); - while (rs.next()) { - String dbName = rs.getString(1); - String charset = rs.getString(2); - String collation = rs.getString(3); - if (dbName != null && (charset != null || collation != null)) { - charsets.put(dbName, new DatabaseLocales(charset, collation)); - LOGGER.debug( - "\t{} = {}, {}", - Strings.pad(dbName, 45, ' '), - Strings.pad(charset, 45, ' '), - Strings.pad(collation, 45, ' ')); - } - } - return charsets; - }); - } catch (SQLException e) { - throw new DebeziumException( - "Error reading default database charsets: " + e.getMessage(), e); - } - } - - public MySqlConnectionConfiguration connectionConfig() { - return connectionConfig; - } - - public String connectionString() { - return connectionString(urlPattern); - } - - public static String getJavaEncodingForMysqlCharSet(String mysqlCharsetName) { - return CharsetMappingWrapper.getJavaEncodingForMysqlCharSet(mysqlCharsetName); - } - - /** Helper to gain access to protected method. */ - private static final class CharsetMappingWrapper extends CharsetMapping { - static String getJavaEncodingForMysqlCharSet(String mySqlCharsetName) { - return CharsetMapping.getStaticJavaEncodingForMysqlCharset(mySqlCharsetName); - } - } - - /** Connection configuration to create a {@link MySqlConnection}. */ - public static class MySqlConnectionConfiguration { - - protected static final String JDBC_PROPERTY_LEGACY_DATETIME = "useLegacyDatetimeCode"; - protected static final String JDBC_PROPERTY_CONNECTION_TIME_ZONE = "connectionTimeZone"; - protected static final String JDBC_PROPERTY_LEGACY_SERVER_TIME_ZONE = "serverTimezone"; - - private final JdbcConfiguration jdbcConfig; - private final ConnectionFactory factory; - private final Configuration config; - - private static final Properties DEFAULT_JDBC_PROPERTIES = initializeDefaultJdbcProperties(); - private static final String JDBC_URL_PATTERN = - "jdbc:mysql://${hostname}:${port}/?useSSL=${useSSL}&connectTimeout=${connectTimeout}"; - private static final String JDBC_URL_PATTERN_WITH_CUSTOM_USE_SSL = - "jdbc:mysql://${hostname}:${port}/?connectTimeout=${connectTimeout}"; - private final String urlPattern; - - public MySqlConnectionConfiguration(Configuration config) { - this(config, new Properties()); - } - - public MySqlConnectionConfiguration(Configuration config, Properties jdbcProperties) { - // Set up the JDBC connection without actually connecting, with extra MySQL-specific - // properties - // to give us better JDBC database metadata behavior, including using UTF-8 for the - // client-side character encoding - // per https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-reference-charsets.html - this.config = config; - final boolean useSSL = sslModeEnabled(); - final Configuration dbConfig = - config.filter( - x -> - !(x.startsWith( - DatabaseHistory - .CONFIGURATION_FIELD_PREFIX_STRING) - || x.equals( - MySqlConnectorConfig.DATABASE_HISTORY - .name()))) - .edit() - .withDefault( - MySqlConnectorConfig.PORT, - MySqlConnectorConfig.PORT.defaultValue()) - .build() - .subset("database.", true); - - final Builder jdbcConfigBuilder = - dbConfig.edit() - .with( - "connectTimeout", - Long.toString(getConnectionTimeout().toMillis())) - .with("useSSL", Boolean.toString(useSSL)); - - final String legacyDateTime = dbConfig.getString(JDBC_PROPERTY_LEGACY_DATETIME); - if (legacyDateTime == null) { - jdbcConfigBuilder.with(JDBC_PROPERTY_LEGACY_DATETIME, "false"); - } else if ("true".equals(legacyDateTime)) { - LOGGER.warn( - "'{}' is set to 'true'. This setting is not recommended and can result in timezone issues.", - JDBC_PROPERTY_LEGACY_DATETIME); - } - - jdbcConfigBuilder.with( - JDBC_PROPERTY_CONNECTION_TIME_ZONE, determineConnectionTimeZone(dbConfig)); - - this.jdbcConfig = JdbcConfiguration.adapt(jdbcConfigBuilder.build()); - String driverClassName = this.jdbcConfig.getString(MySqlConnectorConfig.JDBC_DRIVER); - this.urlPattern = formatJdbcUrl(jdbcProperties); - DriverManager.getDrivers(); - factory = - JdbcConnection.patternBasedFactory( - urlPattern, driverClassName, getClass().getClassLoader()); - } - - private static String determineConnectionTimeZone(final Configuration dbConfig) { - // Debezium by default expects timezoned data delivered in server timezone - String connectionTimeZone = dbConfig.getString(JDBC_PROPERTY_CONNECTION_TIME_ZONE); - - if (connectionTimeZone != null) { - return connectionTimeZone; - } - - // fall back to legacy property - final String serverTimeZone = dbConfig.getString(JDBC_PROPERTY_LEGACY_SERVER_TIME_ZONE); - if (serverTimeZone != null) { - LOGGER.warn( - "Database configuration option '{}' is set but is obsolete, please use '{}' instead", - JDBC_PROPERTY_LEGACY_SERVER_TIME_ZONE, - JDBC_PROPERTY_CONNECTION_TIME_ZONE); - connectionTimeZone = serverTimeZone; - } - - return connectionTimeZone != null ? connectionTimeZone : "SERVER"; - } - - public JdbcConfiguration config() { - return jdbcConfig; - } - - public Configuration originalConfig() { - return config; - } - - public ConnectionFactory factory() { - return factory; - } - - public String username() { - return config.getString(MySqlConnectorConfig.USER); - } - - public String password() { - return config.getString(MySqlConnectorConfig.PASSWORD); - } - - public String hostname() { - return config.getString(MySqlConnectorConfig.HOSTNAME); - } - - public int port() { - return config.getInteger(MySqlConnectorConfig.PORT); - } - - public SecureConnectionMode sslMode() { - String mode = config.getString(MySqlConnectorConfig.SSL_MODE); - return SecureConnectionMode.parse(mode); - } - - public boolean sslModeEnabled() { - return sslMode() != SecureConnectionMode.DISABLED; - } - - public String sslKeyStore() { - return config.getString(MySqlConnectorConfig.SSL_KEYSTORE); - } - - public char[] sslKeyStorePassword() { - String password = config.getString(MySqlConnectorConfig.SSL_KEYSTORE_PASSWORD); - return Strings.isNullOrBlank(password) ? null : password.toCharArray(); - } - - public String sslTrustStore() { - return config.getString(MySqlConnectorConfig.SSL_TRUSTSTORE); - } - - public char[] sslTrustStorePassword() { - String password = config.getString(MySqlConnectorConfig.SSL_TRUSTSTORE_PASSWORD); - return Strings.isNullOrBlank(password) ? null : password.toCharArray(); - } - - public Duration getConnectionTimeout() { - return Duration.ofMillis(config.getLong(MySqlConnectorConfig.CONNECTION_TIMEOUT_MS)); - } - - public String getUrlPattern() { - return urlPattern; - } - - private String formatJdbcUrl(Properties jdbcProperties) { - Properties combinedProperties = new Properties(); - combinedProperties.putAll(DEFAULT_JDBC_PROPERTIES); - combinedProperties.putAll(jdbcProperties); - - StringBuilder jdbcUrlStringBuilder = - jdbcProperties.getProperty("useSSL") == null - ? new StringBuilder(JDBC_URL_PATTERN) - : new StringBuilder(JDBC_URL_PATTERN_WITH_CUSTOM_USE_SSL); - combinedProperties.forEach( - (key, value) -> { - jdbcUrlStringBuilder.append("&").append(key).append("=").append(value); - }); - - return jdbcUrlStringBuilder.toString(); - } - - private static Properties initializeDefaultJdbcProperties() { - Properties defaultJdbcProperties = new Properties(); - defaultJdbcProperties.setProperty("useInformationSchema", "true"); - defaultJdbcProperties.setProperty("nullCatalogMeansCurrent", "false"); - defaultJdbcProperties.setProperty("useUnicode", "true"); - defaultJdbcProperties.setProperty("zeroDateTimeBehavior", "CONVERT_TO_NULL"); - defaultJdbcProperties.setProperty("characterEncoding", "UTF-8"); - defaultJdbcProperties.setProperty("characterSetResults", "UTF-8"); - return defaultJdbcProperties; - } - - public EventProcessingFailureHandlingMode eventProcessingFailureHandlingMode() { - String mode = - config.getString(CommonConnectorConfig.EVENT_PROCESSING_FAILURE_HANDLING_MODE); - if (mode == null) { - mode = - config.getString( - MySqlConnectorConfig.EVENT_DESERIALIZATION_FAILURE_HANDLING_MODE); - } - return EventProcessingFailureHandlingMode.parse(mode); - } - - public EventProcessingFailureHandlingMode inconsistentSchemaHandlingMode() { - String mode = config.getString(MySqlConnectorConfig.INCONSISTENT_SCHEMA_HANDLING_MODE); - return EventProcessingFailureHandlingMode.parse(mode); - } - } - - @Override - public > Object getColumnValue( - ResultSet rs, int columnIndex, Column column, Table table, T schema) - throws SQLException { - return mysqlFieldReader.readField(rs, columnIndex, column, table); - } - - @Override - public String quotedTableIdString(TableId tableId) { - return tableId.toQuotedString('`'); - } - - public String getShowBinaryLogStatement() { - return showBinaryLogStatement; - } - - private String probeShowBinaryLogStatement() { - LOGGER.info("Probing binary log statement."); - try { - // Attempt to query - query(MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT, rs -> {}); - LOGGER.info( - "Successfully found show binary log statement with `{}`.", - MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT); - return MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT; - } catch (SQLException e) { - LOGGER.info( - "Probing with {} failed, fallback to classic {}. Caused by: {}", - MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT, - MYSQL_CLASSIC_SHOW_BINARY_LOG_STATEMENT, - e.getMessage()); - return MYSQL_CLASSIC_SHOW_BINARY_LOG_STATEMENT; - } - } -} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlDefaultValueConverter.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlDefaultValueConverter.java deleted file mode 100644 index 4f6869633d3..00000000000 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlDefaultValueConverter.java +++ /dev/null @@ -1,527 +0,0 @@ -/* - * Copyright Debezium Authors. - * - * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 - */ - -package io.debezium.connector.mysql; - -import io.debezium.annotation.Immutable; -import io.debezium.relational.Column; -import io.debezium.relational.DefaultValueConverter; -import io.debezium.relational.ValueConverter; -import io.debezium.util.Collect; -import org.apache.kafka.connect.data.Field; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.sql.Timestamp; -import java.sql.Types; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Copied from Debezium project(1.9.8.final) to add BIGINT and SMALLINT to TRIM_DATA_TYPES. Remove - * this until https://issues.redhat.com/browse/DBZ-6824 is fixed in 2.3.3.Final. - * - *

    Line 81 & 82: add BIGINT and SMALLINT to TRIM_DATA_TYPES. - */ -@Immutable -public class MySqlDefaultValueConverter implements DefaultValueConverter { - - private static final Logger LOGGER = LoggerFactory.getLogger(MySqlDefaultValueConverter.class); - - private static final Pattern EPOCH_EQUIVALENT_TIMESTAMP = - Pattern.compile( - "(\\d{4}-\\d{2}-00|\\d{4}-00-\\d{2}|0000-\\d{2}-\\d{2}) (00:00:00(\\.\\d{1,6})?)"); - - private static final Pattern EPOCH_EQUIVALENT_DATE = - Pattern.compile("\\d{4}-\\d{2}-00|\\d{4}-00-\\d{2}|0000-\\d{2}-\\d{2}"); - - private static final String EPOCH_TIMESTAMP = "1970-01-01 00:00:00"; - - private static final String EPOCH_DATE = "1970-01-01"; - - private static final Pattern TIMESTAMP_PATTERN = - Pattern.compile("([0-9]*-[0-9]*-[0-9]*) ([0-9]*:[0-9]*:[0-9]*(\\.([0-9]*))?)"); - - private static final Pattern CHARSET_INTRODUCER_PATTERN = - Pattern.compile("^_[A-Za-z0-9]+'(.*)'$"); - - @Immutable - private static final Set TRIM_DATA_TYPES = - Collect.unmodifiableSet( - Types.TINYINT, - Types.INTEGER, - Types.DATE, - Types.TIMESTAMP, - Types.TIMESTAMP_WITH_TIMEZONE, - Types.TIME, - Types.BOOLEAN, - Types.BIT, - Types.NUMERIC, - Types.DECIMAL, - Types.FLOAT, - Types.DOUBLE, - Types.REAL, - Types.BIGINT, - Types.SMALLINT); - - @Immutable - private static final Set NUMBER_DATA_TYPES = - Collect.unmodifiableSet( - Types.BIT, - Types.TINYINT, - Types.SMALLINT, - Types.INTEGER, - Types.BIGINT, - Types.FLOAT, - Types.REAL, - Types.DOUBLE, - Types.NUMERIC, - Types.DECIMAL); - - private static final DateTimeFormatter ISO_LOCAL_DATE_WITH_OPTIONAL_TIME = - new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ISO_LOCAL_DATE) - .optionalStart() - .appendLiteral(" ") - .append(DateTimeFormatter.ISO_LOCAL_TIME) - .optionalEnd() - .toFormatter(); - - private final MySqlValueConverters converters; - - public MySqlDefaultValueConverter(MySqlValueConverters converters) { - this.converters = converters; - } - - /** - * This interface is used by a DDL parser to convert the string default value to a Java type - * recognized by value converters for a subset of types. - * - * @param column the column definition describing the {@code data} value; never null - * @param defaultValueExpression the default value literal; may be null - * @return value converted to a Java type; optional - */ - @Override - public Optional parseDefaultValue(Column column, String defaultValueExpression) { - Object logicalDefaultValue = convert(column, defaultValueExpression); - if (logicalDefaultValue == null) { - return Optional.empty(); - } - - final SchemaBuilder schemaBuilder = converters.schemaBuilder(column); - if (schemaBuilder == null) { - return Optional.of(logicalDefaultValue); - } - final Schema schema = schemaBuilder.build(); - - // In order to get the valueConverter for this column, we have to create a field; - // The index value -1 in the field will never used when converting default value; - // So we can set any number here; - final Field field = new Field(column.name(), -1, schema); - final ValueConverter valueConverter = converters.converter(column, field); - - return Optional.ofNullable(valueConverter.convert(logicalDefaultValue)); - } - - /** - * Converts a default value from the expected format to a logical object acceptable by the main - * JDBC converter. - * - * @param column column definition - * @param value string formatted default value - * @return value converted to a Java type - */ - public Object convert(Column column, String value) { - if (value == null) { - return value; - } - - // trim non varchar data types before converting - if (TRIM_DATA_TYPES.contains(column.jdbcType())) { - value = value.trim(); - } - - // strip character set introducer on default value expressions - value = stripCharacterSetIntroducer(value); - - // boolean is also INT(1) or TINYINT(1) - if (NUMBER_DATA_TYPES.contains(column.jdbcType()) - && ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value))) { - if (Types.DECIMAL == column.jdbcType() || Types.NUMERIC == column.jdbcType()) { - return convertToDecimal(column, value.equalsIgnoreCase("true") ? "1" : "0"); - } - return value.equalsIgnoreCase("true") ? 1 : 0; - } - switch (column.jdbcType()) { - case Types.DATE: - return convertToLocalDate(column, value); - case Types.TIMESTAMP: - return convertToLocalDateTime(column, value); - case Types.TIMESTAMP_WITH_TIMEZONE: - return convertToTimestamp(column, value); - case Types.TIME: - return convertToDuration(column, value); - case Types.BOOLEAN: - return convertToBoolean(value); - case Types.BIT: - return convertToBits(column, value); - - case Types.NUMERIC: - case Types.DECIMAL: - return convertToDecimal(column, value); - - case Types.FLOAT: - case Types.DOUBLE: - case Types.REAL: - return convertToDouble(value); - } - return value; - } - - /** - * Converts a string object for an object type of {@link LocalDate} or {@link LocalDateTime} in - * case of MySql Date type. If the column definition allows null and default value is - * 0000-00-00, we need return null; else 0000-00-00 will be replaced with 1970-01-01; - * - * @param column the column definition describing the {@code data} value; never null - * @param value the string object to be converted into a {@link LocalDate} type or {@link - * LocalDateTime} in case of MySql Date type; - * @return the converted value; - */ - private Object convertToLocalDate(Column column, String value) { - final boolean zero = - EPOCH_EQUIVALENT_DATE.matcher(value).matches() - || EPOCH_EQUIVALENT_TIMESTAMP.matcher(value).matches() - || "0".equals(value); - - if (zero && column.isOptional()) { - return null; - } - if (zero) { - value = EPOCH_DATE; - } - - try { - return LocalDate.from(ISO_LOCAL_DATE_WITH_OPTIONAL_TIME.parse(value)); - } catch (Exception e) { - LOGGER.warn( - "Invalid default value '{}' for date column '{}'; {}", - value, - column.name(), - e.getMessage()); - if (column.isOptional()) { - return null; - } else { - return LocalDate.from(ISO_LOCAL_DATE_WITH_OPTIONAL_TIME.parse(EPOCH_DATE)); - } - } - } - - /** - * Converts a string object for an object type of {@link LocalDateTime}. If the column - * definition allows null and default value is 0000-00-00 00:00:00, we need return null, else - * 0000-00-00 00:00:00 will be replaced with 1970-01-01 00:00:00; - * - * @param column the column definition describing the {@code data} value; never null - * @param value the string object to be converted into a {@link LocalDateTime} type; - * @return the converted value; - */ - private Object convertToLocalDateTime(Column column, String value) { - final boolean matches = - EPOCH_EQUIVALENT_TIMESTAMP.matcher(value).matches() || "0".equals(value); - if (matches) { - if (column.isOptional()) { - return null; - } - - value = EPOCH_TIMESTAMP; - } - - try { - return LocalDateTime.from(timestampFormat(column.length()).parse(value)); - } catch (Exception e) { - LOGGER.warn( - "Invalid default value '{}' for datetime column '{}'; {}", - value, - column.name(), - e.getMessage()); - if (column.isOptional()) { - return null; - } else { - return LocalDateTime.from(timestampFormat(column.length()).parse(EPOCH_TIMESTAMP)); - } - } - } - - /** - * Converts a string object for an object type of {@link Timestamp}. If the column definition - * allows null and default value is 0000-00-00 00:00:00, we need return null, else 0000-00-00 - * 00:00:00 will be replaced with 1970-01-01 00:00:00; - * - * @param column the column definition describing the {@code data} value; never null - * @param value the string object to be converted into a {@link Timestamp} type; - * @return the converted value; - */ - private Object convertToTimestamp(Column column, String value) { - final boolean matches = - EPOCH_EQUIVALENT_TIMESTAMP.matcher(value).matches() - || "0".equals(value) - || EPOCH_TIMESTAMP.equals(value); - if (matches) { - if (column.isOptional()) { - return null; - } - - return Timestamp.from(Instant.EPOCH); - } - value = cleanTimestamp(value); - return Timestamp.valueOf(value).toInstant().atZone(ZoneId.systemDefault()); - } - - /** - * Converts a string object for an object type of {@link Duration}. - * - * @param column the column definition describing the {@code data} value; never null - * @param value the string object to be converted into a {@link Duration} type; - * @return the converted value; - */ - private Object convertToDuration(Column column, String value) { - Matcher matcher = TIMESTAMP_PATTERN.matcher(value); - if (matcher.matches()) { - value = matcher.group(2); - } - return MySqlValueConverters.stringToDuration(value); - } - - /** - * Converts a string object for an expected JDBC type of {@link Types#DOUBLE}. - * - * @param value the string object to be converted into a {@link Types#DOUBLE} type; - * @return the converted value; - */ - private Object convertToDouble(String value) { - return Double.parseDouble(value); - } - - /** - * Converts a string object for an expected JDBC type of {@link Types#DECIMAL}. - * - * @param column the column definition describing the {@code data} value; never null - * @param value the string object to be converted into a {@link Types#DECIMAL} type; - * @return the converted value; - */ - private Object convertToDecimal(Column column, String value) { - return column.scale().isPresent() - ? new BigDecimal(value).setScale(column.scale().get(), RoundingMode.HALF_UP) - : new BigDecimal(value); - } - - /** - * Converts a string object for an expected JDBC type of {@link Types#BIT}. - * - * @param column the column definition describing the {@code data} value; never null - * @param value the string object to be converted into a {@link Types#BIT} type; - * @return the converted value; - */ - private Object convertToBits(Column column, String value) { - if (column.length() > 1) { - return convertToBits(value); - } - return convertToBit(value); - } - - private Object convertToBit(String value) { - try { - return Short.parseShort(value) != 0; - } catch (NumberFormatException ignore) { - return Boolean.parseBoolean(value); - } - } - - private Object convertToBits(String value) { - int nums = value.length() / Byte.SIZE + (value.length() % Byte.SIZE == 0 ? 0 : 1); - byte[] bytes = new byte[nums]; - for (int i = 0; i < nums; i++) { - int s = value.length() - Byte.SIZE < 0 ? 0 : value.length() - Byte.SIZE; - int e = value.length(); - bytes[nums - i - 1] = (byte) Integer.parseInt(value.substring(s, e), 2); - value = value.substring(0, s); - } - return bytes; - } - - /** - * Converts a string object for an expected JDBC type of {@link Types#BOOLEAN}. - * - * @param value the string object to be converted into a {@link Types#BOOLEAN} type; - * @return the converted value; - */ - private Object convertToBoolean(String value) { - try { - return Integer.parseInt(value) != 0; - } catch (NumberFormatException ignore) { - return Boolean.parseBoolean(value); - } - } - - private DateTimeFormatter timestampFormat(int length) { - final DateTimeFormatterBuilder dtf = - new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd") - .optionalStart() - .appendLiteral(" ") - .append(DateTimeFormatter.ISO_LOCAL_TIME) - .optionalEnd() - .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) - .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) - .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0); - if (length > 0) { - dtf.appendFraction(ChronoField.MICRO_OF_SECOND, 0, length, true); - } - return dtf.toFormatter(); - } - - /** - * Clean input timestamp to yyyy-mm-dd hh:mm:ss[.fffffffff] format. - * - * @param s input timestamp - * @return cleaned timestamp - */ - private String cleanTimestamp(String s) { - if (s == null) { - throw new java.lang.IllegalArgumentException("null string"); - } - - s = s.trim(); - - // clean first dash - s = replaceFirstNonNumericSubstring(s, 0, '-'); - // clean second dash - s = replaceFirstNonNumericSubstring(s, s.indexOf('-') + 1, '-'); - // clean dividing space - s = replaceFirstNonNumericSubstring(s, s.indexOf('-', s.indexOf('-') + 1) + 1, ' '); - if (s.indexOf(' ') != -1) { - // clean first colon - s = replaceFirstNonNumericSubstring(s, s.indexOf(' ') + 1, ':'); - if (s.indexOf(':') != -1) { - // clean second colon - s = replaceFirstNonNumericSubstring(s, s.indexOf(':') + 1, ':'); - } - } - - final int maxMonth = 12; - final int maxDay = 31; - - // Parse the date - int firstDash = s.indexOf('-'); - int secondDash = s.indexOf('-', firstDash + 1); - int dividingSpace = s.indexOf(' '); - - // Parse the time - int firstColon = s.indexOf(':', dividingSpace + 1); - int secondColon = s.indexOf(':', firstColon + 1); - int period = s.indexOf('.', secondColon + 1); - - int year = 0; - int month = 0; - int day = 0; - int hour = 0; - int minute = 0; - int second = 0; - - // Get the date - int len = s.length(); - boolean parsedDate = false; - if (firstDash > 0 && secondDash > firstDash) { - year = Integer.parseInt(s.substring(0, firstDash)); - month = Integer.parseInt(s.substring(firstDash + 1, secondDash)); - if (dividingSpace != -1) { - day = Integer.parseInt(s.substring(secondDash + 1, dividingSpace)); - } else { - day = Integer.parseInt(s.substring(secondDash + 1, len)); - } - - if ((month >= 1 && month <= maxMonth) && (day >= 1 && day <= maxDay)) { - parsedDate = true; - } - } - if (!parsedDate) { - throw new java.lang.IllegalArgumentException("Cannot parse the date from " + s); - } - - // Get the time. Hour, minute, second and colons are all optional - if (dividingSpace != -1 && dividingSpace < len - 1) { - if (firstColon == -1) { - hour = Integer.parseInt(s.substring(dividingSpace + 1, len)); - } else { - hour = Integer.parseInt(s.substring(dividingSpace + 1, firstColon)); - if (firstColon < len - 1) { - if (secondColon == -1) { - minute = Integer.parseInt(s.substring(firstColon + 1, len)); - } else { - minute = Integer.parseInt(s.substring(firstColon + 1, secondColon)); - if (secondColon < len - 1) { - if (period == -1) { - second = Integer.parseInt(s.substring(secondColon + 1, len)); - } else { - second = Integer.parseInt(s.substring(secondColon + 1, period)); - } - } - } - } - } - } - - StringBuilder cleanedTimestamp = new StringBuilder(); - cleanedTimestamp = - cleanedTimestamp.append( - String.format( - "%04d-%02d-%02d %02d:%02d:%02d", - year, month, day, hour, minute, second)); - - if (period != -1 && period < len - 1) { - cleanedTimestamp = cleanedTimestamp.append(".").append(s.substring(period + 1)); - } - - return cleanedTimestamp.toString(); - } - - /** - * Replace the first non-numeric substring. - * - * @param s the original string - * @param startIndex the beginning index, inclusive - * @param c the new character - * @return - */ - private String replaceFirstNonNumericSubstring(String s, int startIndex, char c) { - StringBuilder sb = new StringBuilder(); - sb.append(s.substring(0, startIndex)); - - String rest = s.substring(startIndex); - sb.append(rest.replaceFirst("[^\\d]+", Character.toString(c))); - return sb.toString(); - } - - private String stripCharacterSetIntroducer(String value) { - final Matcher matcher = CHARSET_INTRODUCER_PATTERN.matcher(value); - return !matcher.matches() ? value : matcher.group(1); - } -} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlSnapshotChangeEventSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlSnapshotChangeEventSource.java index 5a4fc26b15e..f449b0ed76e 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlSnapshotChangeEventSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlSnapshotChangeEventSource.java @@ -3,339 +3,83 @@ * * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 */ - package io.debezium.connector.mysql; import io.debezium.DebeziumException; -import io.debezium.connector.SnapshotRecord; -import io.debezium.connector.mysql.legacy.MySqlJdbcContext.DatabaseLocales; -import io.debezium.data.Envelope; +import io.debezium.connector.binlog.BinlogConnectorConfig; +import io.debezium.connector.binlog.BinlogSnapshotChangeEventSource; +import io.debezium.connector.binlog.jdbc.BinlogConnectorConnection; +import io.debezium.connector.mysql.MySqlOffsetContext.Loader; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.function.BlockingConsumer; +import io.debezium.jdbc.MainConnectionProvidingConnectionFactory; import io.debezium.pipeline.EventDispatcher; -import io.debezium.relational.RelationalSnapshotChangeEventSource; -import io.debezium.relational.RelationalTableFilters; -import io.debezium.relational.Table; +import io.debezium.pipeline.notification.NotificationService; import io.debezium.relational.TableId; -import io.debezium.schema.SchemaChangeEvent; +import io.debezium.snapshot.SnapshotterService; import io.debezium.util.Clock; -import io.debezium.util.Collect; -import io.debezium.util.Strings; -import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; -/** - * Copied from Debezium project(1.9.8.Final) to fix MySQL 8.x compatibility. +/* + * Copied from Debezium project(2.7.4.Final). * - *

    Line 338: Use probing methods to determine the statement. + *

    Change 1: the binlog position is read with the statement probed by {@code MySqlConnection} + * (MySQL 8.4 replaced {@code SHOW MASTER STATUS} with {@code SHOW BINARY LOG STATUS}). + * + *

    The snapshot lifecycle patches live in the forked + * {@code io.debezium.connector.binlog.BinlogSnapshotChangeEventSource}, which Debezium 2.7 + * extracted from this class. */ public class MySqlSnapshotChangeEventSource - extends RelationalSnapshotChangeEventSource { + extends BinlogSnapshotChangeEventSource { private static final Logger LOGGER = LoggerFactory.getLogger(MySqlSnapshotChangeEventSource.class); private final MySqlConnectorConfig connectorConfig; - private final MySqlConnection connection; - private long globalLockAcquiredAt = -1; - private long tableLockAcquiredAt = -1; - private final RelationalTableFilters filters; - private final MySqlSnapshotChangeEventSourceMetrics metrics; - private final MySqlDatabaseSchema databaseSchema; - private final List schemaEvents = new ArrayList<>(); - private Set delayedSchemaSnapshotTables = Collections.emptySet(); - private final BlockingConsumer> lastEventProcessor; public MySqlSnapshotChangeEventSource( MySqlConnectorConfig connectorConfig, - MySqlConnection connection, + MainConnectionProvidingConnectionFactory connectionFactory, MySqlDatabaseSchema schema, EventDispatcher dispatcher, Clock clock, MySqlSnapshotChangeEventSourceMetrics metrics, - BlockingConsumer> lastEventProcessor) { - super(connectorConfig, connection, schema, dispatcher, clock, metrics); + BlockingConsumer> lastEventProcessor, + Runnable preSnapshotAction, + NotificationService notificationService, + SnapshotterService snapshotterService) { + super( + connectorConfig, + connectionFactory, + schema, + dispatcher, + clock, + metrics, + lastEventProcessor, + preSnapshotAction, + notificationService, + snapshotterService); this.connectorConfig = connectorConfig; - this.connection = connection; - this.filters = connectorConfig.getTableFilters(); - this.metrics = metrics; - this.databaseSchema = schema; - this.lastEventProcessor = lastEventProcessor; - } - - @Override - protected SnapshottingTask getSnapshottingTask( - MySqlPartition partition, MySqlOffsetContext previousOffset) { - boolean snapshotSchema = true; - boolean snapshotData = true; - - // found a previous offset and the earlier snapshot has completed - if (previousOffset != null && !previousOffset.isSnapshotRunning()) { - LOGGER.info( - "A previous offset indicating a completed snapshot has been found. Neither schema nor data will be snapshotted."); - snapshotSchema = databaseSchema.isStorageInitializationExecuted(); - snapshotData = false; - } else { - LOGGER.info("No previous offset has been found"); - if (connectorConfig.getSnapshotMode().includeData()) { - LOGGER.info( - "According to the connector configuration both schema and data will be snapshotted"); - } else { - LOGGER.info( - "According to the connector configuration only schema will be snapshotted"); - } - snapshotData = connectorConfig.getSnapshotMode().includeData(); - snapshotSchema = connectorConfig.getSnapshotMode().includeSchema(); - } - - return new SnapshottingTask(snapshotSchema, snapshotData); - } - - @Override - protected SnapshotContext prepare(MySqlPartition partition) - throws Exception { - return new MySqlSnapshotContext(partition); - } - - @Override - protected void connectionCreated( - RelationalSnapshotContext snapshotContext) - throws Exception {} - - @Override - protected Set getAllTableIds( - RelationalSnapshotContext ctx) throws Exception { - // ------------------- - // READ DATABASE NAMES - // ------------------- - // Get the list of databases ... - LOGGER.info("Read list of available databases"); - final List databaseNames = new ArrayList<>(); - connection.query( - "SHOW DATABASES", - rs -> { - while (rs.next()) { - databaseNames.add(rs.getString(1)); - } - }); - LOGGER.info("\t list of available databases is: {}", databaseNames); - - // ---------------- - // READ TABLE NAMES - // ---------------- - // Get the list of table IDs for each database. We can't use a prepared statement with - // MySQL, so we have to - // build the SQL statement each time. Although in other cases this might lead to SQL - // injection, in our case - // we are reading the database names from the database and not taking them from the user ... - LOGGER.info("Read list of available tables in each database"); - final Set tableIds = new HashSet<>(); - final Set readableDatabaseNames = new HashSet<>(); - for (String dbName : databaseNames) { - try { - // MySQL sometimes considers some local files as databases (see DBZ-164), - // so we will simply try each one and ignore the problematic ones ... - connection.query( - "SHOW FULL TABLES IN " + quote(dbName) + " where Table_Type = 'BASE TABLE'", - rs -> { - while (rs.next()) { - TableId id = new TableId(dbName, null, rs.getString(1)); - tableIds.add(id); - } - }); - readableDatabaseNames.add(dbName); - } catch (SQLException e) { - // We were unable to execute the query or process the results, so skip this ... - LOGGER.warn( - "\t skipping database '{}' due to error reading tables: {}", - dbName, - e.getMessage()); - } - } - final Set includedDatabaseNames = - readableDatabaseNames.stream() - .filter(filters.databaseFilter()) - .collect(Collectors.toSet()); - LOGGER.info("\tsnapshot continuing with database(s): {}", includedDatabaseNames); - return tableIds; - } - - @Override - protected void lockTablesForSchemaSnapshot( - ChangeEventSourceContext sourceContext, - RelationalSnapshotContext snapshotContext) - throws SQLException, InterruptedException { - // Set the transaction isolation level to REPEATABLE READ. This is the default, but the - // default can be changed - // which is why we explicitly set it here. - // - // With REPEATABLE READ, all SELECT queries within the scope of a transaction (which we - // don't yet have) will read - // from the same MVCC snapshot. Thus each plain (non-locking) SELECT statements within the - // same transaction are - // consistent also with respect to each other. - // - // See: https://dev.mysql.com/doc/refman/5.7/en/set-transaction.html - // See: https://dev.mysql.com/doc/refman/5.7/en/innodb-transaction-isolation-levels.html - // See: https://dev.mysql.com/doc/refman/5.7/en/innodb-consistent-read.html - connection.connection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ); - connection.executeWithoutCommitting( - "SET SESSION lock_wait_timeout=" - + connectorConfig.snapshotLockTimeout().getSeconds()); - try { - connection.executeWithoutCommitting( - "SET SESSION innodb_lock_wait_timeout=" - + connectorConfig.snapshotLockTimeout().getSeconds()); - } catch (SQLException e) { - LOGGER.warn("Unable to set innodb_lock_wait_timeout", e); - } - - // ------------------------------------ - // LOCK TABLES - // ------------------------------------ - // Obtain read lock on all tables. This statement closes all open tables and locks all - // tables - // for all databases with a global read lock, and it prevents ALL updates while we have this - // lock. - // It also ensures that everything we do while we have this lock will be consistent. - if (connectorConfig.getSnapshotLockingMode().usesLocking() - && connectorConfig.useGlobalLock()) { - try { - globalLock(); - metrics.globalLockAcquired(); - } catch (SQLException e) { - LOGGER.info( - "Unable to flush and acquire global read lock, will use table read locks after reading table names"); - // Continue anyway, since RDS (among others) don't allow setting a global lock - assert !isGloballyLocked(); - } - if (connectorConfig.getSnapshotLockingMode().flushResetsIsolationLevel()) { - // FLUSH TABLES resets TX and isolation level - connection.executeWithoutCommitting( - "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); - } - } - } - - @Override - protected void releaseSchemaSnapshotLocks( - RelationalSnapshotContext snapshotContext) - throws SQLException { - if (connectorConfig.getSnapshotLockingMode().usesMinimalLocking()) { - if (isGloballyLocked()) { - globalUnlock(); - } - if (isTablesLocked()) { - // We could not acquire a global read lock and instead had to obtain individual - // table-level read locks - // using 'FLUSH TABLE WITH READ LOCK'. However, if we were to do this, - // the 'UNLOCK TABLES' - // would implicitly commit our active transaction, and this would break our - // consistent snapshot logic. - // Therefore, we cannot unlock the tables here! - // https://dev.mysql.com/doc/refman/5.7/en/flush.html - LOGGER.warn( - "Tables were locked explicitly, but to get a consistent snapshot we cannot release the locks until we've read all tables."); - } - } } @Override - protected void releaseDataSnapshotLocks( - RelationalSnapshotContext snapshotContext) - throws Exception { - if (isGloballyLocked()) { - globalUnlock(); - } - if (isTablesLocked()) { - tableUnlock(); - if (!delayedSchemaSnapshotTables.isEmpty()) { - schemaEvents.clear(); - createSchemaEventsForTables(snapshotContext, delayedSchemaSnapshotTables, false); - - for (Iterator i = schemaEvents.iterator(); i.hasNext(); ) { - final SchemaChangeEvent event = i.next(); - - if (databaseSchema.storeOnlyCapturedTables() - && event.getDatabase() != null - && event.getDatabase().length() != 0 - && !connectorConfig - .getTableFilters() - .databaseFilter() - .test(event.getDatabase())) { - LOGGER.debug( - "Skipping schema event as it belongs to a non-captured database: '{}'", - event); - continue; - } - - LOGGER.debug("Processing schema event {}", event); - - final TableId tableId = - event.getTables().isEmpty() - ? null - : event.getTables().iterator().next().id(); - snapshotContext.offset.event(tableId, getClock().currentTime()); - - if (!i.hasNext()) { - super.lastSnapshotRecord(snapshotContext); - } - - dispatcher.dispatchSchemaChangeEvent( - snapshotContext.partition, - tableId, - (receiver) -> receiver.schemaChangeEvent(event)); - } - - // Make schema available for snapshot source - databaseSchema - .tableIds() - .forEach( - x -> - snapshotContext.tables.overwriteTable( - databaseSchema.tableFor(x))); - } - } + protected MySqlOffsetContext getInitialOffsetContext(BinlogConnectorConfig connectorConfig) { + return MySqlOffsetContext.initial((MySqlConnectorConfig) connectorConfig); } @Override - protected void determineSnapshotOffset( - RelationalSnapshotContext ctx, - MySqlOffsetContext previousOffset) + protected void setOffsetContextBinlogPositionAndGtidDetailsForSnapshot( + MySqlOffsetContext offsetContext, + BinlogConnectorConnection connection, + SnapshotterService snapshotterService) throws Exception { - if (!isGloballyLocked() - && !isTablesLocked() - && connectorConfig.getSnapshotLockingMode().usesLocking()) { - return; - } - if (previousOffset != null) { - ctx.offset = previousOffset; - tryStartingSnapshot(ctx); - return; - } - final MySqlOffsetContext offsetContext = MySqlOffsetContext.initial(connectorConfig); - ctx.offset = offsetContext; LOGGER.info("Read binlog position of MySQL primary server"); - final String showMasterStmt = connection.getShowBinaryLogStatement(); + final String showMasterStmt = ((MySqlConnection) connection).getShowBinaryLogStatement(); connection.query( showMasterStmt, rs -> { @@ -355,12 +99,10 @@ protected void determineSnapshotOffset( binlogFilename, binlogPosition, gtidSet); - } else { - LOGGER.info( - "\t using binlog '{}' at position '{}'", - binlogFilename, - binlogPosition); } + } else if (!snapshotterService.getSnapshotter().shouldStream()) { + LOGGER.warn( + "Failed retrieving binlog position, continuing as streaming CDC wasn't requested"); } else { throw new DebeziumException( "Cannot read the binlog filename and position via '" @@ -368,350 +110,11 @@ protected void determineSnapshotOffset( + "'. Make sure your server is correctly configured"); } }); - tryStartingSnapshot(ctx); - } - - private void addSchemaEvent( - RelationalSnapshotContext snapshotContext, - String database, - String ddl) { - schemaEvents.addAll( - databaseSchema.parseSnapshotDdl( - snapshotContext.partition, - ddl, - database, - snapshotContext.offset, - clock.currentTimeAsInstant())); } @Override - protected void readTableStructure( - ChangeEventSourceContext sourceContext, - RelationalSnapshotContext snapshotContext, - MySqlOffsetContext offsetContext) - throws Exception { - Set capturedSchemaTables; - if (twoPhaseSchemaSnapshot()) { - // Capture schema of captured tables after they are locked - tableLock(snapshotContext); - determineSnapshotOffset(snapshotContext, offsetContext); - capturedSchemaTables = snapshotContext.capturedTables; - LOGGER.info( - "Table level locking is in place, the schema will be capture in two phases, now capturing: {}", - capturedSchemaTables); - delayedSchemaSnapshotTables = - Collect.minus( - snapshotContext.capturedSchemaTables, snapshotContext.capturedTables); - LOGGER.info("Tables for delayed schema capture: {}", delayedSchemaSnapshotTables); - } - if (databaseSchema.storeOnlyCapturedTables()) { - capturedSchemaTables = snapshotContext.capturedTables; - LOGGER.info( - "Only captured tables schema should be captured, capturing: {}", - capturedSchemaTables); - } else { - capturedSchemaTables = snapshotContext.capturedSchemaTables; - LOGGER.info( - "All eligible tables schema should be captured, capturing: {}", - capturedSchemaTables); - } - final Map> tablesToRead = - capturedSchemaTables.stream() - .collect( - Collectors.groupingBy( - TableId::catalog, LinkedHashMap::new, Collectors.toList())); - final Set databases = tablesToRead.keySet(); - - // Record default charset - addSchemaEvent( - snapshotContext, - "", - connection.setStatementFor(connection.readMySqlCharsetSystemVariables())); - - for (TableId tableId : capturedSchemaTables) { - if (!sourceContext.isRunning()) { - throw new InterruptedException( - "Interrupted while emitting initial DROP TABLE events"); - } - addSchemaEvent( - snapshotContext, tableId.catalog(), "DROP TABLE IF EXISTS " + quote(tableId)); - } - - final Map databaseCharsets = connection.readDatabaseCollations(); - for (String database : databases) { - if (!sourceContext.isRunning()) { - throw new InterruptedException( - "Interrupted while reading structure of schema " + databases); - } - - LOGGER.info("Reading structure of database '{}'", database); - addSchemaEvent(snapshotContext, database, "DROP DATABASE IF EXISTS " + quote(database)); - final StringBuilder createDatabaseDddl = - new StringBuilder("CREATE DATABASE " + quote(database)); - final DatabaseLocales defaultDatabaseLocales = databaseCharsets.get(database); - if (defaultDatabaseLocales != null) { - defaultDatabaseLocales.appendToDdlStatement(database, createDatabaseDddl); - } - addSchemaEvent(snapshotContext, database, createDatabaseDddl.toString()); - addSchemaEvent(snapshotContext, database, "USE " + quote(database)); - - createSchemaEventsForTables(snapshotContext, tablesToRead.get(database), true); - } - } - - void createSchemaEventsForTables( - RelationalSnapshotContext snapshotContext, - final Collection tablesToRead, - final boolean firstPhase) - throws SQLException { - for (TableId tableId : tablesToRead) { - if (firstPhase && delayedSchemaSnapshotTables.contains(tableId)) { - continue; - } - connection.query( - "SHOW CREATE TABLE " + quote(tableId), - rs -> { - if (rs.next()) { - addSchemaEvent(snapshotContext, tableId.catalog(), rs.getString(2)); - } - }); - } - } - - private boolean twoPhaseSchemaSnapshot() { - return connectorConfig.getSnapshotLockingMode().usesLocking() && !isGloballyLocked(); - } - - @Override - protected SchemaChangeEvent getCreateTableEvent( - RelationalSnapshotContext snapshotContext, - Table table) - throws SQLException { - return SchemaChangeEvent.ofSnapshotCreate( - snapshotContext.partition, - snapshotContext.offset, - snapshotContext.catalogName, - table); - } - - @Override - protected void complete(SnapshotContext snapshotContext) {} - - /** - * Generate a valid MySQL query string for the specified table and columns - * - * @param tableId the table to generate a query for - * @return a valid query string - */ - @Override - protected Optional getSnapshotSelect( - RelationalSnapshotContext snapshotContext, - TableId tableId, - List columns) { - String snapshotSelectColumns = columns.stream().collect(Collectors.joining(", ")); - - return Optional.of( - String.format( - "SELECT %s FROM `%s`.`%s`", - snapshotSelectColumns, tableId.catalog(), tableId.table())); - } - - private boolean isGloballyLocked() { - return globalLockAcquiredAt != -1; - } - - private boolean isTablesLocked() { - return tableLockAcquiredAt != -1; - } - - private void globalLock() throws SQLException { - LOGGER.info("Flush and obtain global read lock to prevent writes to database"); - connection.executeWithoutCommitting( - connectorConfig.getSnapshotLockingMode().getLockStatement()); - globalLockAcquiredAt = clock.currentTimeInMillis(); - } - - private void globalUnlock() throws SQLException { - LOGGER.info("Releasing global read lock to enable MySQL writes"); - connection.executeWithoutCommitting("UNLOCK TABLES"); - long lockReleased = clock.currentTimeInMillis(); - metrics.globalLockReleased(); - LOGGER.info( - "Writes to MySQL tables prevented for a total of {}", - Strings.duration(lockReleased - globalLockAcquiredAt)); - globalLockAcquiredAt = -1; - } - - private void tableLock( - RelationalSnapshotContext snapshotContext) - throws SQLException { - // ------------------------------------ - // LOCK TABLES and READ BINLOG POSITION - // ------------------------------------ - // We were not able to acquire the global read lock, so instead we have to obtain a read - // lock on each table. - // This requires different privileges than normal, and also means we can't unlock the tables - // without - // implicitly committing our transaction ... - if (!connection.userHasPrivileges("LOCK TABLES")) { - // We don't have the right privileges - throw new DebeziumException( - "User does not have the 'LOCK TABLES' privilege required to obtain a " - + "consistent snapshot by preventing concurrent writes to tables."); - } - // We have the required privileges, so try to lock all of the tables we're interested in ... - LOGGER.info( - "Flush and obtain read lock for {} tables (preventing writes)", - snapshotContext.capturedTables); - if (!snapshotContext.capturedTables.isEmpty()) { - final String tableList = - snapshotContext.capturedTables.stream() - .map(tid -> quote(tid)) - .collect(Collectors.joining(",")); - connection.executeWithoutCommitting("FLUSH TABLES " + tableList + " WITH READ LOCK"); - } - tableLockAcquiredAt = clock.currentTimeInMillis(); - metrics.globalLockAcquired(); - } - - private void tableUnlock() throws SQLException { - LOGGER.info("Releasing table read lock to enable MySQL writes"); - connection.executeWithoutCommitting("UNLOCK TABLES"); - long lockReleased = clock.currentTimeInMillis(); - metrics.globalLockReleased(); - LOGGER.info( - "Writes to MySQL tables prevented for a total of {}", - Strings.duration(lockReleased - tableLockAcquiredAt)); - tableLockAcquiredAt = -1; - } - - private String quote(String dbOrTableName) { - return "`" + dbOrTableName + "`"; - } - - private String quote(TableId id) { - return quote(id.catalog()) + "." + quote(id.table()); - } - - @Override - protected OptionalLong rowCountForTable(TableId tableId) { - return connection.getEstimatedTableSize(tableId); - } - - @Override - protected Statement readTableStatement(OptionalLong rowCount) throws SQLException { - final long largeTableRowCount = connectorConfig.rowCountForLargeTable(); - if (!rowCount.isPresent() - || largeTableRowCount == 0 - || rowCount.getAsLong() <= largeTableRowCount) { - return super.readTableStatement(rowCount); - } - return createStatementWithLargeResultSet(); - } - - /** - * Create a JDBC statement that can be used for large result sets. - * - *

    By default, the MySQL Connector/J driver retrieves all rows for ResultSets and stores them - * in memory. In most cases this is the most efficient way to operate and, due to the design of - * the MySQL network protocol, is easier to implement. However, when ResultSets that have a - * large number of rows or large values, the driver may not be able to allocate heap space in - * the JVM and may result in an {@link OutOfMemoryError}. See DBZ-94 for details. - * - *

    This method handles such cases using the recommended - * technique for MySQL by creating the JDBC {@link Statement} with {@link - * ResultSet#TYPE_FORWARD_ONLY forward-only} cursor and {@link ResultSet#CONCUR_READ_ONLY - * read-only concurrency} flags, and with a {@link Integer#MIN_VALUE minimum value} {@link - * Statement#setFetchSize(int) fetch size hint}. - * - * @return the statement; never null - * @throws SQLException if there is a problem creating the statement - */ - private Statement createStatementWithLargeResultSet() throws SQLException { - int fetchSize = connectorConfig.getSnapshotFetchSize(); - Statement stmt = - connection - .connection() - .createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - stmt.setFetchSize(fetchSize); - return stmt; - } - - /** Mutable context which is populated in the course of snapshotting. */ - private static class MySqlSnapshotContext - extends RelationalSnapshotContext { - - public MySqlSnapshotContext(MySqlPartition partition) throws SQLException { - super(partition, ""); - } - } - - @Override - protected void createSchemaChangeEventsForTables( - ChangeEventSourceContext sourceContext, - RelationalSnapshotContext snapshotContext, - SnapshottingTask snapshottingTask) - throws Exception { - tryStartingSnapshot(snapshotContext); - - for (Iterator i = schemaEvents.iterator(); i.hasNext(); ) { - final SchemaChangeEvent event = i.next(); - if (!sourceContext.isRunning()) { - throw new InterruptedException("Interrupted while processing event " + event); - } - - if (databaseSchema.skipSchemaChangeEvent(event)) { - continue; - } - - LOGGER.debug("Processing schema event {}", event); - - final TableId tableId = - event.getTables().isEmpty() ? null : event.getTables().iterator().next().id(); - snapshotContext.offset.event(tableId, getClock().currentTime()); - - // If data are not snapshotted then the last schema change must set last snapshot flag - if (!snapshottingTask.snapshotData() && !i.hasNext()) { - lastSnapshotRecord(snapshotContext); - } - dispatcher.dispatchSchemaChangeEvent( - snapshotContext.partition, - tableId, - (receiver) -> receiver.schemaChangeEvent(event)); - } - - // Make schema available for snapshot source - databaseSchema - .tableIds() - .forEach(x -> snapshotContext.tables.overwriteTable(databaseSchema.tableFor(x))); - } - - @Override - protected void lastSnapshotRecord( + protected MySqlOffsetContext copyOffset( RelationalSnapshotContext snapshotContext) { - if (delayedSchemaSnapshotTables.isEmpty()) { - super.lastSnapshotRecord(snapshotContext); - } - } - - @Override - protected void postSnapshot() throws InterruptedException { - // We cannot be sure that the last event as the last one - // - last table could be empty - // - data snapshot was not executed - // - the last table schema snaphsotted is not monitored and storing of monitored is disabled - lastEventProcessor.accept( - record -> { - record.sourceOffset().remove(SourceInfo.SNAPSHOT_KEY); - ((Struct) record.value()) - .getStruct(Envelope.FieldName.SOURCE) - .put( - SourceInfo.SNAPSHOT_KEY, - SnapshotRecord.LAST.toString().toLowerCase()); - return record; - }); - super.postSnapshot(); + return new Loader(connectorConfig).load(snapshotContext.offset.getOffset()); } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlValueConverters.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlValueConverters.java deleted file mode 100644 index 421dae75fa6..00000000000 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/MySqlValueConverters.java +++ /dev/null @@ -1,1120 +0,0 @@ -/* - * Copyright Debezium Authors. - * - * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 - */ -package io.debezium.connector.mysql; - -import com.github.shyiko.mysql.binlog.event.deserialization.json.JsonBinary; -import io.debezium.DebeziumException; -import io.debezium.annotation.Immutable; -import io.debezium.config.CommonConnectorConfig.BinaryHandlingMode; -import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser; -import io.debezium.data.Json; -import io.debezium.jdbc.JdbcValueConverters; -import io.debezium.jdbc.TemporalPrecisionMode; -import io.debezium.relational.Column; -import io.debezium.relational.Table; -import io.debezium.relational.ValueConverter; -import io.debezium.time.Year; -import io.debezium.util.Strings; -import org.apache.kafka.connect.data.Decimal; -import org.apache.kafka.connect.data.Field; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.source.SourceRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.math.BigDecimal; -import java.nio.ByteOrder; -import java.nio.charset.Charset; -import java.nio.charset.IllegalCharsetNameException; -import java.nio.charset.StandardCharsets; -import java.sql.Timestamp; -import java.sql.Types; -import java.time.Duration; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.temporal.ChronoField; -import java.time.temporal.ChronoUnit; -import java.time.temporal.Temporal; -import java.time.temporal.TemporalAdjuster; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Copied from Debezium project(1.9.8.Final) to fix FLOAT converted to FLOAT64 type issue. See - * DBZ-3865, DBZ-5843. Remove it when debezium version is upgraded above 2.0.0.Final. - * - *

    Line 240 & 246: add FLOAT type adjustment. - * - *

    Line 485 & 499: add method to convert FLOAT value. - */ -@Immutable -public class MySqlValueConverters extends JdbcValueConverters { - - @FunctionalInterface - public static interface ParsingErrorHandler { - void error(String message, Exception exception); - } - - private static final Logger LOGGER = LoggerFactory.getLogger(MySqlValueConverters.class); - - /** Used to parse values of TIME columns. Format: 000:00:00.000000. */ - private static final Pattern TIME_FIELD_PATTERN = - Pattern.compile("(\\-?[0-9]*):([0-9]*):([0-9]*)(\\.([0-9]*))?"); - - /** Used to parse values of DATE columns. Format: 000-00-00. */ - private static final Pattern DATE_FIELD_PATTERN = Pattern.compile("([0-9]*)-([0-9]*)-([0-9]*)"); - - /** Used to parse values of TIMESTAMP columns. Format: 000-00-00 00:00:00.000. */ - private static final Pattern TIMESTAMP_FIELD_PATTERN = - Pattern.compile("([0-9]*)-([0-9]*)-([0-9]*) .*"); - - /** - * A utility method that adjusts ambiguous 2-digit - * year values of DATETIME, DATE, and TIMESTAMP types using these MySQL-specific rules: - * - *

      - *
    • Year values in the range 00-69 are converted to 2000-2069. - *
    • Year values in the range 70-99 are converted to 1970-1999. - *
    - * - * @param temporal the temporal instance to adjust; may not be null - * @return the possibly adjusted temporal instance; never null - */ - public static Temporal adjustTemporal(Temporal temporal) { - if (temporal.isSupported(ChronoField.YEAR)) { - int year = temporal.get(ChronoField.YEAR); - if (0 <= year && year <= 69) { - temporal = temporal.plus(2000, ChronoUnit.YEARS); - } else if (70 <= year && year <= 99) { - temporal = temporal.plus(1900, ChronoUnit.YEARS); - } - } - return temporal; - } - - private final ParsingErrorHandler parsingErrorHandler; - - /** - * Create a new instance that always uses UTC for the default time zone when_needed converting - * values without timezone information to values that require timezones. - * - *

    - * - * @param decimalMode how {@code DECIMAL} and {@code NUMERIC} values should be treated; may be - * null if {@link io.debezium.jdbc.JdbcValueConverters.DecimalMode#PRECISE} is to be used - * @param temporalPrecisionMode temporal precision mode based on {@link - * io.debezium.jdbc.TemporalPrecisionMode} - * @param bigIntUnsignedMode how {@code BIGINT UNSIGNED} values should be treated; may be null - * if {@link io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode#PRECISE} is to be used - * @param binaryMode how binary columns should be represented - */ - public MySqlValueConverters( - DecimalMode decimalMode, - TemporalPrecisionMode temporalPrecisionMode, - BigIntUnsignedMode bigIntUnsignedMode, - BinaryHandlingMode binaryMode) { - this( - decimalMode, - temporalPrecisionMode, - bigIntUnsignedMode, - binaryMode, - x -> x, - MySqlValueConverters::defaultParsingErrorHandler); - } - - /** - * Create a new instance that always uses UTC for the default time zone when converting values - * without timezone information to values that require timezones. - * - *

    - * - * @param decimalMode how {@code DECIMAL} and {@code NUMERIC} values should be treated; may be - * null if {@link io.debezium.jdbc.JdbcValueConverters.DecimalMode#PRECISE} is to be used - * @param temporalPrecisionMode temporal precision mode based on {@link - * io.debezium.jdbc.TemporalPrecisionMode} - * @param bigIntUnsignedMode how {@code BIGINT UNSIGNED} values should be treated; may be null - * if {@link io.debezium.jdbc.JdbcValueConverters.BigIntUnsignedMode#PRECISE} is to be used - * @param binaryMode how binary columns should be represented - * @param adjuster a temporal adjuster to make a database specific time modification before - * conversion - * @param handler for errors during postponed binlog parsing - */ - public MySqlValueConverters( - DecimalMode decimalMode, - TemporalPrecisionMode temporalPrecisionMode, - BigIntUnsignedMode bigIntUnsignedMode, - BinaryHandlingMode binaryMode, - TemporalAdjuster adjuster, - ParsingErrorHandler parsingErrorHandler) { - super( - decimalMode, - temporalPrecisionMode, - ZoneOffset.UTC, - adjuster, - bigIntUnsignedMode, - binaryMode); - this.parsingErrorHandler = parsingErrorHandler; - } - - @Override - protected ByteOrder byteOrderOfBitType() { - return ByteOrder.BIG_ENDIAN; - } - - @Override - public SchemaBuilder schemaBuilder(Column column) { - // Handle a few MySQL-specific types based upon how they are handled by the MySQL binlog - // client ... - String typeName = column.typeName().toUpperCase(); - if (matches(typeName, "JSON")) { - return Json.builder(); - } - if (matches(typeName, "POINT")) { - return io.debezium.data.geometry.Point.builder(); - } - if (matches(typeName, "GEOMETRY") - || matches(typeName, "LINESTRING") - || matches(typeName, "POLYGON") - || matches(typeName, "MULTIPOINT") - || matches(typeName, "MULTILINESTRING") - || matches(typeName, "MULTIPOLYGON") - || isGeometryCollection(typeName)) { - return io.debezium.data.geometry.Geometry.builder(); - } - if (matches(typeName, "YEAR")) { - return Year.builder(); - } - if (matches(typeName, "ENUM")) { - String commaSeparatedOptions = extractEnumAndSetOptionsAsString(column); - return io.debezium.data.Enum.builder(commaSeparatedOptions); - } - if (matches(typeName, "SET")) { - String commaSeparatedOptions = extractEnumAndSetOptionsAsString(column); - return io.debezium.data.EnumSet.builder(commaSeparatedOptions); - } - if (matches(typeName, "SMALLINT UNSIGNED") - || matches(typeName, "SMALLINT UNSIGNED ZEROFILL") - || matches(typeName, "INT2 UNSIGNED") - || matches(typeName, "INT2 UNSIGNED ZEROFILL")) { - // In order to capture unsigned SMALLINT 16-bit data source, INT32 will be required to - // safely capture all valid values - // Source: - // https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.Type.html - return SchemaBuilder.int32(); - } - if (matches(typeName, "INT UNSIGNED") - || matches(typeName, "INT UNSIGNED ZEROFILL") - || matches(typeName, "INT4 UNSIGNED") - || matches(typeName, "INT4 UNSIGNED ZEROFILL")) { - // In order to capture unsigned INT 32-bit data source, INT64 will be required to safely - // capture all valid values - // Source: - // https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.Type.html - return SchemaBuilder.int64(); - } - if (matches(typeName, "BIGINT UNSIGNED") - || matches(typeName, "BIGINT UNSIGNED ZEROFILL") - || matches(typeName, "INT8 UNSIGNED") - || matches(typeName, "INT8 UNSIGNED ZEROFILL")) { - switch (super.bigIntUnsignedMode) { - case LONG: - return SchemaBuilder.int64(); - case PRECISE: - // In order to capture unsigned INT 64-bit data source, - // org.apache.kafka.connect.data.Decimal:Byte will be required to safely capture - // all valid values with scale of 0 - // Source: - // https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.Type.html - return Decimal.builder(0); - } - } - if ((matches(typeName, "FLOAT") - || matches(typeName, "FLOAT UNSIGNED") - || matches(typeName, "FLOAT UNSIGNED ZEROFILL")) - && !column.scale().isPresent() - && column.length() <= 24) { - return SchemaBuilder.float32(); - } - // Otherwise, let the base class handle it ... - return super.schemaBuilder(column); - } - - @Override - public ValueConverter converter(Column column, Field fieldDefn) { - // Handle a few MySQL-specific types based upon how they are handled by the MySQL binlog - // client ... - String typeName = column.typeName().toUpperCase(); - if (matches(typeName, "JSON")) { - return (data) -> convertJson(column, fieldDefn, data); - } - if (matches(typeName, "GEOMETRY") - || matches(typeName, "LINESTRING") - || matches(typeName, "POLYGON") - || matches(typeName, "MULTIPOINT") - || matches(typeName, "MULTILINESTRING") - || matches(typeName, "MULTIPOLYGON") - || isGeometryCollection(typeName)) { - return (data -> convertGeometry(column, fieldDefn, data)); - } - if (matches(typeName, "POINT")) { - // backwards compatibility - return (data -> convertPoint(column, fieldDefn, data)); - } - if (matches(typeName, "YEAR")) { - return (data) -> convertYearToInt(column, fieldDefn, data); - } - if (matches(typeName, "ENUM")) { - // Build up the character array based upon the column's type ... - List options = extractEnumAndSetOptions(column); - return (data) -> convertEnumToString(options, column, fieldDefn, data); - } - if (matches(typeName, "SET")) { - // Build up the character array based upon the column's type ... - List options = extractEnumAndSetOptions(column); - return (data) -> convertSetToString(options, column, fieldDefn, data); - } - if (matches(typeName, "TINYINT UNSIGNED") - || matches(typeName, "TINYINT UNSIGNED ZEROFILL") - || matches(typeName, "INT1 UNSIGNED") - || matches(typeName, "INT1 UNSIGNED ZEROFILL")) { - // Convert TINYINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary - // settings - return (data) -> convertUnsignedTinyint(column, fieldDefn, data); - } - if (matches(typeName, "SMALLINT UNSIGNED") - || matches(typeName, "SMALLINT UNSIGNED ZEROFILL") - || matches(typeName, "INT2 UNSIGNED") - || matches(typeName, "INT2 UNSIGNED ZEROFILL")) { - // Convert SMALLINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary - // settings - return (data) -> convertUnsignedSmallint(column, fieldDefn, data); - } - if (matches(typeName, "MEDIUMINT UNSIGNED") - || matches(typeName, "MEDIUMINT UNSIGNED ZEROFILL") - || matches(typeName, "INT3 UNSIGNED") - || matches(typeName, "INT3 UNSIGNED ZEROFILL") - || matches(typeName, "MIDDLEINT UNSIGNED") - || matches(typeName, "MIDDLEINT UNSIGNED ZEROFILL")) { - // Convert MEDIUMINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary - // settings - return (data) -> convertUnsignedMediumint(column, fieldDefn, data); - } - if (matches(typeName, "INT UNSIGNED") - || matches(typeName, "INT UNSIGNED ZEROFILL") - || matches(typeName, "INT4 UNSIGNED") - || matches(typeName, "INT4 UNSIGNED ZEROFILL")) { - // Convert INT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary - // settings - return (data) -> convertUnsignedInt(column, fieldDefn, data); - } - if (matches(typeName, "BIGINT UNSIGNED") - || matches(typeName, "BIGINT UNSIGNED ZEROFILL") - || matches(typeName, "INT8 UNSIGNED") - || matches(typeName, "INT8 UNSIGNED ZEROFILL")) { - switch (super.bigIntUnsignedMode) { - case LONG: - return (data) -> convertBigInt(column, fieldDefn, data); - case PRECISE: - // Convert BIGINT UNSIGNED internally from SIGNED to UNSIGNED based on the - // boundary settings - return (data) -> convertUnsignedBigint(column, fieldDefn, data); - } - } - - // We have to convert bytes encoded in the column's character set ... - switch (column.jdbcType()) { - case Types.CHAR: // variable-length - case Types.VARCHAR: // variable-length - case Types.LONGVARCHAR: // variable-length - case Types.CLOB: // variable-length - case Types.NCHAR: // fixed-length - case Types.NVARCHAR: // fixed-length - case Types.LONGNVARCHAR: // fixed-length - case Types.NCLOB: // fixed-length - case Types.DATALINK: - case Types.SQLXML: - Charset charset = charsetFor(column); - if (charset != null) { - logger.debug("Using {} charset by default for column: {}", charset, column); - return (data) -> convertString(column, fieldDefn, charset, data); - } - logger.warn( - "Using UTF-8 charset by default for column without charset: {}", column); - return (data) -> convertString(column, fieldDefn, StandardCharsets.UTF_8, data); - case Types.TIME: - if (adaptiveTimeMicrosecondsPrecisionMode) { - return data -> convertDurationToMicroseconds(column, fieldDefn, data); - } - case Types.TIMESTAMP: - return ((ValueConverter) - (data -> convertTimestampToLocalDateTime(column, fieldDefn, data))) - .and(super.converter(column, fieldDefn)); - default: - break; - } - - // Otherwise, let the base class handle it ... - return super.converter(column, fieldDefn); - } - - /** - * Return the {@link Charset} instance with the MySQL-specific character set name used by the - * given column. - * - * @param column the column in which the character set is used; never null - * @return the Java {@link Charset}, or null if there is no mapping - */ - protected Charset charsetFor(Column column) { - String mySqlCharsetName = column.charsetName(); - if (mySqlCharsetName == null) { - logger.warn("Column is missing a character set: {}", column); - return null; - } - String encoding = MySqlConnection.getJavaEncodingForMysqlCharSet(mySqlCharsetName); - if (encoding == null) { - logger.debug( - "Column uses MySQL character set '{}', which has no mapping to a Java character set, will try it in lowercase", - mySqlCharsetName); - encoding = - MySqlConnection.getJavaEncodingForMysqlCharSet(mySqlCharsetName.toLowerCase()); - } - if (encoding == null) { - logger.warn( - "Column uses MySQL character set '{}', which has no mapping to a Java character set", - mySqlCharsetName); - } else { - try { - return Charset.forName(encoding); - } catch (IllegalCharsetNameException e) { - logger.error( - "Unable to load Java charset '{}' for column with MySQL character set '{}'", - encoding, - mySqlCharsetName); - } - } - return null; - } - - /** - * Convert the {@link String} {@code byte[]} value to a string value used in a {@link - * SourceRecord}. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertJson(Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - "{}", - (r) -> { - if (data instanceof byte[]) { - // The BinlogReader sees these JSON values as binary encoded, so we use the - // binlog client library's utility - // to parse MySQL's internal binary representation into a JSON string, using - // the standard formatter. - - if (((byte[]) data).length == 0) { - r.deliver(column.isOptional() ? null : "{}"); - } else { - try { - r.deliver(JsonBinary.parseAsString((byte[]) data)); - } catch (IOException e) { - parsingErrorHandler.error( - "Failed to parse and read a JSON value on '" - + column - + "' value " - + Arrays.toString((byte[]) data), - e); - r.deliver(column.isOptional() ? null : "{}"); - } - } - } else if (data instanceof String) { - // The SnapshotReader sees JSON values as UTF-8 encoded strings. - r.deliver(data); - } - }); - } - - @Override - protected Object convertSmallInt(Column column, Field fieldDefn, Object data) { - // MySQL allows decimal default values for smallint columns - if (data instanceof String) { - data = Math.round(Double.parseDouble((String) data)); - } - - return super.convertSmallInt(column, fieldDefn, data); - } - - @Override - protected Object convertInteger(Column column, Field fieldDefn, Object data) { - // MySQL allows decimal default values for integer columns - if (data instanceof String) { - data = Math.round(Double.parseDouble((String) data)); - } - - return super.convertInteger(column, fieldDefn, data); - } - - @Override - protected Object convertBigInt(Column column, Field fieldDefn, Object data) { - // MySQL allows decimal default values for bigint columns - if (data instanceof String) { - data = Math.round(Double.parseDouble((String) data)); - } - - return super.convertBigInt(column, fieldDefn, data); - } - - /** - * MySql reports FLOAT(p) values as FLOAT and DOUBLE. A precision from 0 to 23 results in a - * 4-byte single-precision FLOAT column. A precision from 24 to 53 results in an 8-byte - * double-precision DOUBLE column. As of MySQL 8.0.17, the nonstandard FLOAT(M,D) and - * DOUBLE(M,D) syntax is deprecated, and should expect support for it be removed in a future - * version of MySQL. Based on this future, we didn't handle the case. - */ - @Override - protected Object convertFloat(Column column, Field fieldDefn, Object data) { - if (!column.scale().isPresent() && column.length() <= 24) { - return super.convertReal(column, fieldDefn, data); - } else { - return super.convertFloat(column, fieldDefn, data); - } - } - - /** - * Convert the {@link String} or {@code byte[]} value to a string value used in a {@link - * SourceRecord}. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param columnCharset the Java character set in which column byte[] values are encoded; may - * not be null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertString( - Column column, Field fieldDefn, Charset columnCharset, Object data) { - return convertValue( - column, - fieldDefn, - data, - "", - (r) -> { - if (data instanceof byte[]) { - // Decode the binary representation using the given character encoding ... - r.deliver(new String((byte[]) data, columnCharset)); - } else if (data instanceof String) { - r.deliver(data); - } - }); - } - - /** - * Converts a value object for a MySQL {@code YEAR}, which appear in the binlog as an integer - * though returns from the MySQL JDBC driver as either a short or a {@link java.sql.Date}. - * - * @param column the column definition describing the {@code data} value; never null - * @param fieldDefn the field definition; never null - * @param data the data object to be converted into a year literal integer value; never null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - @SuppressWarnings("deprecation") - protected Object convertYearToInt(Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - 0, - (r) -> { - Object mutData = data; - if (data instanceof java.time.Year) { - // The MySQL binlog always returns a Year object ... - r.deliver( - adjustTemporal( - java.time.Year.of( - ((java.time.Year) data).getValue())) - .get(ChronoField.YEAR)); - } else if (data instanceof java.sql.Date) { - // MySQL JDBC driver sometimes returns a Java SQL Date object ... - // year from java.sql.Date is defined as number of years since 1900 - r.deliver(((java.sql.Date) data).getYear() + 1900); - } else if (data instanceof String) { - mutData = Integer.valueOf((String) data); - } - if (mutData instanceof Number) { - // MySQL JDBC driver sometimes returns a short ... - r.deliver( - adjustTemporal(java.time.Year.of(((Number) mutData).intValue())) - .get(ChronoField.YEAR)); - } - }); - } - - /** - * Converts a value object for a MySQL {@code ENUM}, which is represented in the binlog events - * as an integer value containing the index of the enum option. The MySQL JDBC driver returns a - * string containing the option, so this method calculates the same. - * - * @param options the characters that appear in the same order as defined in the column; may not - * be null - * @param column the column definition describing the {@code data} value; never null - * @param fieldDefn the field definition; never null - * @param data the data object to be converted into an {@code ENUM} literal String value - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertEnumToString( - List options, Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - "", - (r) -> { - if (data instanceof String) { - // JDBC should return strings ... - r.deliver(data); - } else if (data instanceof Integer) { - if (options != null) { - // The binlog will contain an int with the 1-based index of the option - // in the enum value ... - int value = ((Integer) data).intValue(); - if (value == 0) { - // an invalid value was specified, which corresponds to the empty - // string '' and an index of 0 - r.deliver(""); - } - int index = value - 1; // 'options' is 0-based - if (index < options.size() && index >= 0) { - r.deliver(options.get(index)); - } - } else { - r.deliver(null); - } - } - }); - } - - /** - * Converts a value object for a MySQL {@code SET}, which is represented in the binlog events - * contain a long number in which every bit corresponds to a different option. The MySQL JDBC - * driver returns a string containing the comma-separated options, so this method calculates the - * same. - * - * @param options the characters that appear in the same order as defined in the column; may not - * be null - * @param column the column definition describing the {@code data} value; never null - * @param fieldDefn the field definition; never null - * @param data the data object to be converted into an {@code SET} literal String value; never - * null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertSetToString( - List options, Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - "", - (r) -> { - if (data instanceof String) { - // JDBC should return strings ... - r.deliver(data); - } else if (data instanceof Long) { - // The binlog will contain a long with the indexes of the options in the set - // value ... - long indexes = ((Long) data).longValue(); - r.deliver(convertSetValue(column, indexes, options)); - } - }); - } - - /** - * Determine if the uppercase form of a column's type exactly matches or begins with the - * specified prefix. Note that this logic works when the column's {@link Column#typeName() type} - * contains the type name followed by parentheses. - * - * @param upperCaseTypeName the upper case form of the column's {@link Column#typeName() type - * name} - * @param upperCaseMatch the upper case form of the expected type or prefix of the type; may not - * be null - * @return {@code true} if the type matches the specified type, or {@code false} otherwise - */ - protected boolean matches(String upperCaseTypeName, String upperCaseMatch) { - if (upperCaseTypeName == null) { - return false; - } - return upperCaseMatch.equals(upperCaseTypeName) - || upperCaseTypeName.startsWith(upperCaseMatch + "("); - } - - /** - * Determine if the uppercase form of a column's type is geometry collection independent of JDBC - * driver or server version. - * - * @param upperCaseTypeName the upper case form of the column's {@link Column#typeName() type - * name} - * @return {@code true} if the type is geometry collection - */ - protected boolean isGeometryCollection(String upperCaseTypeName) { - if (upperCaseTypeName == null) { - return false; - } - - return upperCaseTypeName.equals("GEOMETRYCOLLECTION") - || upperCaseTypeName.equals("GEOMCOLLECTION") - || upperCaseTypeName.endsWith(".GEOMCOLLECTION"); - } - - protected List extractEnumAndSetOptions(Column column) { - return MySqlAntlrDdlParser.extractEnumAndSetOptions(column.enumValues()); - } - - protected String extractEnumAndSetOptionsAsString(Column column) { - return Strings.join(",", extractEnumAndSetOptions(column)); - } - - protected String convertSetValue(Column column, long indexes, List options) { - StringBuilder sb = new StringBuilder(); - int index = 0; - boolean first = true; - int optionLen = options.size(); - while (indexes != 0L) { - if (indexes % 2L != 0) { - if (first) { - first = false; - } else { - sb.append(','); - } - if (index < optionLen) { - sb.append(options.get(index)); - } else { - logger.warn("Found unexpected index '{}' on column {}", index, column); - } - } - ++index; - indexes = indexes >>> 1; - } - return sb.toString(); - } - - /** - * Convert the a value representing a POINT {@code byte[]} value to a Point value used in a - * {@link SourceRecord}. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertPoint(Column column, Field fieldDefn, Object data) { - final MySqlGeometry empty = MySqlGeometry.createEmpty(); - return convertValue( - column, - fieldDefn, - data, - io.debezium.data.geometry.Geometry.createValue( - fieldDefn.schema(), empty.getWkb(), empty.getSrid()), - (r) -> { - if (data instanceof byte[]) { - // The binlog utility sends a byte array for any Geometry type, we will use - // our own binaryParse to parse the byte to WKB, hence - // to the suitable class - MySqlGeometry mySqlGeometry = MySqlGeometry.fromBytes((byte[]) data); - if (mySqlGeometry.isPoint()) { - r.deliver( - io.debezium.data.geometry.Point.createValue( - fieldDefn.schema(), - mySqlGeometry.getWkb(), - mySqlGeometry.getSrid())); - } else { - throw new ConnectException( - "Failed to parse and read a value of type POINT on " + column); - } - } - }); - } - - /** - * Convert the a value representing a GEOMETRY {@code byte[]} value to a Geometry value used in - * a {@link SourceRecord}. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertGeometry(Column column, Field fieldDefn, Object data) { - final MySqlGeometry empty = MySqlGeometry.createEmpty(); - return convertValue( - column, - fieldDefn, - data, - io.debezium.data.geometry.Geometry.createValue( - fieldDefn.schema(), empty.getWkb(), empty.getSrid()), - (r) -> { - if (data instanceof byte[]) { - // The binlog utility sends a byte array for any Geometry type, we will use - // our own binaryParse to parse the byte to WKB, hence - // to the suitable class - if (data instanceof byte[]) { - // The binlog utility sends a byte array for any Geometry type, we will - // use our own binaryParse to parse the byte to WKB, hence - // to the suitable class - MySqlGeometry mySqlGeometry = MySqlGeometry.fromBytes((byte[]) data); - r.deliver( - io.debezium.data.geometry.Geometry.createValue( - fieldDefn.schema(), - mySqlGeometry.getWkb(), - mySqlGeometry.getSrid())); - } - } - }); - } - - @Override - protected byte[] normalizeBinaryData(Column column, byte[] data) { - // DBZ-254 right-pad fixed-length binary column values with 0x00 (zero byte) - if (column.jdbcType() == Types.BINARY && data.length < column.length()) { - data = Arrays.copyOf(data, column.length()); - } - - return super.normalizeBinaryData(column, data); - } - - /** - * Convert the a value representing a Unsigned TINYINT value to the correct Unsigned TINYINT - * representation. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertUnsignedTinyint(Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - (short) 0, - (r) -> { - if (data instanceof Short) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedTinyint((short) data)); - } else if (data instanceof Number) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedTinyint( - ((Number) data).shortValue())); - } else { - // We continue with the original converting method (smallint) since we have - // an unsigned Tinyint - r.deliver(convertSmallInt(column, fieldDefn, data)); - } - }); - } - - /** - * Convert the a value representing a Unsigned SMALLINT value to the correct Unsigned SMALLINT - * representation. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertUnsignedSmallint(Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - 0, - (r) -> { - if (data instanceof Integer) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedSmallint((int) data)); - } else if (data instanceof Number) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedSmallint( - ((Number) data).intValue())); - } else { - // We continue with the original converting method (integer) since we have - // an unsigned Smallint - r.deliver(convertInteger(column, fieldDefn, data)); - } - }); - } - - /** - * Convert the a value representing a Unsigned MEDIUMINT value to the correct Unsigned SMALLINT - * representation. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertUnsignedMediumint(Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - 0, - (r) -> { - if (data instanceof Integer) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedMediumint((int) data)); - } else if (data instanceof Number) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedMediumint( - ((Number) data).intValue())); - } else { - // We continue with the original converting method (integer) since we have - // an unsigned Medium - r.deliver(convertInteger(column, fieldDefn, data)); - } - }); - } - - /** - * Convert the a value representing a Unsigned INT value to the correct Unsigned INT - * representation. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertUnsignedInt(Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - 0L, - (r) -> { - if (data instanceof Long) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedInteger((long) data)); - } else if (data instanceof Number) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedInteger( - ((Number) data).longValue())); - } else { - // We continue with the original converting method (bigint) since we have an - // unsigned Integer - r.deliver(convertBigInt(column, fieldDefn, data)); - } - }); - } - - /** - * Convert the a value representing a Unsigned BIGINT value to the correct Unsigned INT - * representation. - * - * @param column the column in which the value appears - * @param fieldDefn the field definition for the {@link SourceRecord}'s {@link Schema}; never - * null - * @param data the data; may be null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertUnsignedBigint(Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - 0L, - (r) -> { - if (data instanceof BigDecimal) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedBigint( - (BigDecimal) data)); - } else if (data instanceof Number) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedBigint( - new BigDecimal(((Number) data).toString()))); - } else if (data instanceof String) { - r.deliver( - MySqlUnsignedIntegerConverter.convertUnsignedBigint( - new BigDecimal((String) data))); - } else { - r.deliver(convertNumeric(column, fieldDefn, data)); - } - }); - } - - /** - * Converts a value object for an expected type of {@link java.time.Duration} to {@link Long} - * values that represents the time in microseconds. - * - *

    Per the JDBC specification, databases should return {@link java.sql.Time} instances, but - * that's not working because it can only handle Daytime 00:00:00-23:59:59. We use {@link - * java.time.Duration} instead that can handle the range of -838:59:59.000000 to - * 838:59:59.000000 of a MySQL TIME type and transfer data as signed INT64 which reflects the DB - * value converted to microseconds. - * - * @param column the column definition describing the {@code data} value; never null - * @param fieldDefn the field definition; never null - * @param data the data object to be converted into a {@link java.time.Duration} type; never - * null - * @return the converted value, or null if the conversion could not be made and the column - * allows nulls - * @throws IllegalArgumentException if the value could not be converted but the column does not - * allow nulls - */ - protected Object convertDurationToMicroseconds(Column column, Field fieldDefn, Object data) { - return convertValue( - column, - fieldDefn, - data, - 0L, - (r) -> { - try { - if (data instanceof Duration) { - r.deliver(((Duration) data).toNanos() / 1_000); - } - } catch (IllegalArgumentException e) { - } - }); - } - - protected Object convertTimestampToLocalDateTime(Column column, Field fieldDefn, Object data) { - if (data == null && !fieldDefn.schema().isOptional()) { - return null; - } - if (!(data instanceof Timestamp)) { - return data; - } - - return ((Timestamp) data).toLocalDateTime(); - } - - public static Duration stringToDuration(String timeString) { - Matcher matcher = TIME_FIELD_PATTERN.matcher(timeString); - if (!matcher.matches()) { - throw new RuntimeException("Unexpected format for TIME column: " + timeString); - } - - long hours = Long.parseLong(matcher.group(1)); - long minutes = Long.parseLong(matcher.group(2)); - long seconds = Long.parseLong(matcher.group(3)); - long nanoSeconds = 0; - String microSecondsString = matcher.group(5); - if (microSecondsString != null) { - nanoSeconds = Long.parseLong(Strings.justifyLeft(microSecondsString, 9, '0')); - } - - if (hours >= 0) { - return Duration.ofHours(hours) - .plusMinutes(minutes) - .plusSeconds(seconds) - .plusNanos(nanoSeconds); - } else { - return Duration.ofHours(hours) - .minusMinutes(minutes) - .minusSeconds(seconds) - .minusNanos(nanoSeconds); - } - } - - public static LocalDate stringToLocalDate(String dateString, Column column, Table table) { - final Matcher matcher = DATE_FIELD_PATTERN.matcher(dateString); - if (!matcher.matches()) { - throw new RuntimeException("Unexpected format for DATE column: " + dateString); - } - - final int year = Integer.parseInt(matcher.group(1)); - final int month = Integer.parseInt(matcher.group(2)); - final int day = Integer.parseInt(matcher.group(3)); - - if (year == 0 || month == 0 || day == 0) { - LOGGER.warn( - "Invalid value '{}' stored in column '{}' of table '{}' converted to empty value", - dateString, - column.name(), - table.id()); - return null; - } - return LocalDate.of(year, month, day); - } - - public static boolean containsZeroValuesInDatePart( - String timestampString, Column column, Table table) { - final Matcher matcher = TIMESTAMP_FIELD_PATTERN.matcher(timestampString); - if (!matcher.matches()) { - throw new RuntimeException("Unexpected format for DATE column: " + timestampString); - } - - final int year = Integer.parseInt(matcher.group(1)); - final int month = Integer.parseInt(matcher.group(2)); - final int day = Integer.parseInt(matcher.group(3)); - - if (year == 0 || month == 0 || day == 0) { - LOGGER.warn( - "Invalid value '{}' stored in column '{}' of table '{}' converted to empty value", - timestampString, - column.name(), - table.id()); - return true; - } - return false; - } - - public static void defaultParsingErrorHandler(String message, Exception exception) { - throw new DebeziumException(message, exception); - } -} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/antlr/listener/DefaultValueParserListener.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/antlr/listener/DefaultValueParserListener.java deleted file mode 100644 index 563d7b10ad0..00000000000 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/antlr/listener/DefaultValueParserListener.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Debezium Authors. - * - * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 - */ - -package io.debezium.connector.mysql.antlr.listener; - -import io.debezium.ddl.parser.mysql.generated.MySqlParser; -import io.debezium.ddl.parser.mysql.generated.MySqlParserBaseListener; -import io.debezium.relational.ColumnEditor; - -import java.util.concurrent.atomic.AtomicReference; - -/** - * Copied from Debezium project(v1.9.8.Final) to fix - * https://github.com/ververica/flink-cdc-connectors/issues/1506. - * - *

    Line 48~59: use the actual default string value when the sql contains COLLATE. We should - * remove this class after we bumped debezium 2.0 where the - * https://issues.redhat.com/browse/DBZ-5587 has been fixed. - */ -public class DefaultValueParserListener extends MySqlParserBaseListener { - - private final ColumnEditor columnEditor; - private final AtomicReference optionalColumn; - - private boolean converted; - - public DefaultValueParserListener( - ColumnEditor columnEditor, AtomicReference optionalColumn) { - this.columnEditor = columnEditor; - this.optionalColumn = optionalColumn; - this.converted = false; - } - - @Override - public void enterDefaultValue(MySqlParser.DefaultValueContext ctx) { - String sign = ""; - if (ctx.NULL_LITERAL() != null) { - return; - } - if (ctx.unaryOperator() != null) { - sign = ctx.unaryOperator().getText(); - } - if (ctx.constant() != null) { - if (ctx.constant().stringLiteral() != null) { - if (ctx.constant().stringLiteral().COLLATE() == null) { - columnEditor.defaultValueExpression( - sign + unquote(ctx.constant().stringLiteral().getText())); - } else { - columnEditor.defaultValueExpression( - sign - + unquote( - ctx.constant() - .stringLiteral() - .STRING_LITERAL(0) - .getText())); - } - } else if (ctx.constant().decimalLiteral() != null) { - columnEditor.defaultValueExpression( - sign + ctx.constant().decimalLiteral().getText()); - } else if (ctx.constant().BIT_STRING() != null) { - columnEditor.defaultValueExpression( - unquoteBinary(ctx.constant().BIT_STRING().getText())); - } else if (ctx.constant().booleanLiteral() != null) { - columnEditor.defaultValueExpression(ctx.constant().booleanLiteral().getText()); - } else if (ctx.constant().REAL_LITERAL() != null) { - columnEditor.defaultValueExpression(ctx.constant().REAL_LITERAL().getText()); - } - } else if (ctx.currentTimestamp() != null && !ctx.currentTimestamp().isEmpty()) { - if (ctx.currentTimestamp().size() > 1 || (ctx.ON() == null && ctx.UPDATE() == null)) { - final MySqlParser.CurrentTimestampContext currentTimestamp = - ctx.currentTimestamp(0); - if (currentTimestamp.CURRENT_TIMESTAMP() != null - || currentTimestamp.NOW() != null) { - columnEditor.defaultValueExpression("1970-01-01 00:00:00"); - } else { - columnEditor.defaultValueExpression(currentTimestamp.getText()); - } - } - } - exitDefaultValue(true); - super.enterDefaultValue(ctx); - } - - public void exitDefaultValue(boolean skipIfUnknownOptional) { - boolean isOptionalColumn = optionalColumn.get() != null; - if (!converted && (isOptionalColumn || !skipIfUnknownOptional)) { - if (isOptionalColumn) { - columnEditor.optional(optionalColumn.get().booleanValue()); - } - converted = true; - } - } - - private String unquote(String stringLiteral) { - if (stringLiteral != null - && ((stringLiteral.startsWith("'") && stringLiteral.endsWith("'")) - || (stringLiteral.startsWith("\"") && stringLiteral.endsWith("\"")))) { - return stringLiteral.substring(1, stringLiteral.length() - 1); - } - return stringLiteral; - } - - private String unquoteBinary(String stringLiteral) { - return stringLiteral.substring(2, stringLiteral.length() - 1); - } -} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/converters/MysqlDebeziumTimeConverter.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/converters/MysqlDebeziumTimeConverter.java index a05db4ed0c3..10ea88709df 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/converters/MysqlDebeziumTimeConverter.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/converters/MysqlDebeziumTimeConverter.java @@ -50,6 +50,8 @@ * (for example, yyyy-MM-dd) * * @see io.debezium.connector.mysql.converters.TinyIntOneToBooleanConverter + *

    This is a Flink CDC owned class (not a Debezium fork), placed in the {@code io.debezium} + * package for access to package-private Debezium members. */ public class MysqlDebeziumTimeConverter implements CustomConverter { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/GtidUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/gtid/GtidUtils.java similarity index 77% rename from flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/GtidUtils.java rename to flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/gtid/GtidUtils.java index 5956de9b312..513dc4fd4e7 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/GtidUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/gtid/GtidUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package io.debezium.connector.mysql; +package io.debezium.connector.mysql.gtid; import java.util.ArrayList; import java.util.Collections; @@ -25,7 +25,12 @@ import java.util.Map; import java.util.function.Predicate; -/** Utils for handling GTIDs. */ +/** + * Utils for handling GTIDs. * + * + *

    This is a Flink CDC owned class (not a Debezium fork), placed in the {@code io.debezium} + * package for access to package-private Debezium members. + */ public class GtidUtils { /** @@ -40,8 +45,10 @@ public class GtidUtils { * @param checkpointGtidSet the GTID set restored from checkpoint * @return the fixed GTID set for old channels */ - public static GtidSet fixOldChannelsGtidSet( - GtidSet availableServerGtidSet, GtidSet purgedServerGtid, GtidSet checkpointGtidSet) { + public static MySqlGtidSet fixOldChannelsGtidSet( + MySqlGtidSet availableServerGtidSet, + MySqlGtidSet purgedServerGtid, + MySqlGtidSet checkpointGtidSet) { return fixRestoredGtidSet( mergeGtidSetInto( availableServerGtidSet.retainAll( @@ -63,23 +70,23 @@ public static GtidSet fixOldChannelsGtidSet( * @param gtidSourceFilter optional predicate to filter GTID source UUIDs; may be null * @return the merged GTID set suitable for binlog subscription */ - public static GtidSet computeLatestModeGtidSet( - GtidSet availableServerGtidSet, - GtidSet purgedServerGtid, - GtidSet checkpointGtidSet, + public static MySqlGtidSet computeLatestModeGtidSet( + MySqlGtidSet availableServerGtidSet, + MySqlGtidSet purgedServerGtid, + MySqlGtidSet checkpointGtidSet, Predicate gtidSourceFilter) { - final GtidSet relevantAvailableServerGtidSet = + final MySqlGtidSet relevantAvailableServerGtidSet = (gtidSourceFilter != null) ? availableServerGtidSet.retainAll(gtidSourceFilter) : availableServerGtidSet; // Step 1: Fix old channels' GTID ranges - GtidSet fixedOldChannelsGtid = + MySqlGtidSet fixedOldChannelsGtid = fixOldChannelsGtidSet( relevantAvailableServerGtidSet, purgedServerGtid, checkpointGtidSet); // Step 2: For new channels, use server's full GTID to skip all history - GtidSet newChannelsGtid = + MySqlGtidSet newChannelsGtid = relevantAvailableServerGtidSet.retainAll( uuid -> checkpointGtidSet.forServerWithId(uuid) == null); @@ -95,21 +102,22 @@ public static GtidSet computeLatestModeGtidSet( * then it will be adjusted according to the server's GTID set; if it does not exist in the * server's GTID set, it will be directly added to the new GTID set. */ - public static GtidSet fixRestoredGtidSet(GtidSet serverGtidSet, GtidSet restoredGtidSet) { - Map newSet = new HashMap<>(); + public static MySqlGtidSet fixRestoredGtidSet( + MySqlGtidSet serverGtidSet, MySqlGtidSet restoredGtidSet) { + Map newSet = new HashMap<>(); serverGtidSet.getUUIDSets().forEach(uuidSet -> newSet.put(uuidSet.getUUID(), uuidSet)); - for (GtidSet.UUIDSet restoredUuidSet : restoredGtidSet.getUUIDSets()) { - GtidSet.UUIDSet serverUuidSet = newSet.get(restoredUuidSet.getUUID()); + for (MySqlGtidSet.UUIDSet restoredUuidSet : restoredGtidSet.getUUIDSets()) { + MySqlGtidSet.UUIDSet serverUuidSet = newSet.get(restoredUuidSet.getUUID()); if (serverUuidSet != null) { - List serverIntervals = serverUuidSet.getIntervals(); - List restoredIntervals = restoredUuidSet.getIntervals(); + List serverIntervals = serverUuidSet.getIntervals(); + List restoredIntervals = restoredUuidSet.getIntervals(); long earliestRestoredTx = getMinIntervalStart(restoredIntervals); List merged = new ArrayList<>(); // Process each server interval - for (GtidSet.Interval serverInterval : serverIntervals) { + for (MySqlGtidSet.Interval serverInterval : serverIntervals) { // First, check if any part comes before earliest restored if (serverInterval.getStart() < earliestRestoredTx) { long end = Math.min(serverInterval.getEnd(), earliestRestoredTx - 1); @@ -119,7 +127,7 @@ public static GtidSet fixRestoredGtidSet(GtidSet serverGtidSet, GtidSet restored } // Then check for overlaps with restored intervals - for (GtidSet.Interval restoredInterval : restoredIntervals) { + for (MySqlGtidSet.Interval restoredInterval : restoredIntervals) { if (serverInterval.getStart() <= restoredInterval.getEnd() && serverInterval.getEnd() >= restoredInterval.getStart()) { // There's an overlap - add the intersection @@ -138,8 +146,8 @@ public static GtidSet fixRestoredGtidSet(GtidSet serverGtidSet, GtidSet restored } } - GtidSet.UUIDSet mergedUuidSet = - new GtidSet.UUIDSet( + MySqlGtidSet.UUIDSet mergedUuidSet = + new MySqlGtidSet.UUIDSet( new com.github.shyiko.mysql.binlog.GtidSet.UUIDSet( restoredUuidSet.getUUID(), merged)); @@ -148,11 +156,11 @@ public static GtidSet fixRestoredGtidSet(GtidSet serverGtidSet, GtidSet restored newSet.put(restoredUuidSet.getUUID(), restoredUuidSet); } } - return new GtidSet(newSet); + return new MySqlGtidSet(newSet); } - private static long getMinIntervalStart(List intervals) { - return Collections.min(intervals, Comparator.comparingLong(GtidSet.Interval::getStart)) + private static long getMinIntervalStart(List intervals) { + return Collections.min(intervals, Comparator.comparingLong(MySqlGtidSet.Interval::getStart)) .getStart(); } @@ -160,14 +168,14 @@ private static long getMinIntervalStart(List intervals) { * This method merges one GTID set (toMerge) into another (base), without overwriting the * existing elements in the base GTID set. */ - public static GtidSet mergeGtidSetInto(GtidSet base, GtidSet toMerge) { - Map newSet = new HashMap<>(); + public static MySqlGtidSet mergeGtidSetInto(MySqlGtidSet base, MySqlGtidSet toMerge) { + Map newSet = new HashMap<>(); base.getUUIDSets().forEach(uuidSet -> newSet.put(uuidSet.getUUID(), uuidSet)); - for (GtidSet.UUIDSet uuidSet : toMerge.getUUIDSets()) { + for (MySqlGtidSet.UUIDSet uuidSet : toMerge.getUUIDSets()) { if (!newSet.containsKey(uuidSet.getUUID())) { newSet.put(uuidSet.getUUID(), uuidSet); } } - return new GtidSet(newSet); + return new MySqlGtidSet(newSet); } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/jdbc/MySqlConnection.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/jdbc/MySqlConnection.java new file mode 100644 index 00000000000..a9f54910b00 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/jdbc/MySqlConnection.java @@ -0,0 +1,211 @@ +/* + * Copyright Debezium Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ +package io.debezium.connector.mysql.jdbc; + +import io.debezium.DebeziumException; +import io.debezium.connector.binlog.gtid.GtidSet; +import io.debezium.connector.binlog.jdbc.BinlogConnectorConnection; +import io.debezium.connector.binlog.jdbc.BinlogFieldReader; +import io.debezium.connector.mysql.gtid.GtidUtils; +import io.debezium.connector.mysql.gtid.MySqlGtidSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.function.Predicate; + +/** + * Copied from Debezium project(2.7.4.Final). + * + *

    Change 1: MySQL 8.4 removed {@code SHOW MASTER STATUS} in favour of {@code SHOW BINARY LOG + * STATUS}. The statement is probed once, on construction, and exposed through {@link + * #getShowBinaryLogStatement()} so the snapshot source can use the same one; {@link + * #knownGtidSet()} uses it instead of a hard-coded statement. + * + *

    Change 2: {@link #filterGtidSet} honours the {@code gtid.new.channel.position} pass-through + * property (which Debezium 2.0 removed as a config enum), delegating to {@code GtidUtils} for the + * earliest/latest new-channel reconciliation. + * + *

    Change 3: add the single-argument constructor used by Flink CDC's own connection factory. + */ +public class MySqlConnection extends BinlogConnectorConnection { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlConnection.class); + + private static final String MYSQL_CLASSIC_SHOW_BINARY_LOG_STATEMENT = "SHOW MASTER STATUS"; + private static final String MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT = "SHOW BINARY LOG STATUS"; + + private final String showBinaryLogStatement; + + public MySqlConnection(MySqlConnectionConfiguration connectionConfig) { + this(connectionConfig, new MySqlTextProtocolFieldReader(null)); + } + + public MySqlConnection( + MySqlConnectionConfiguration connectionConfig, BinlogFieldReader fieldReader) { + super(connectionConfig, fieldReader); + this.showBinaryLogStatement = probeShowBinaryLogStatement(); + } + + /** + * Returns the statement this server understands for reading the current binary log coordinates. + */ + public String getShowBinaryLogStatement() { + return showBinaryLogStatement; + } + + private String probeShowBinaryLogStatement() { + LOGGER.info("Probing binary log statement."); + try { + query(MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT, rs -> {}); + LOGGER.info( + "Successfully found show binary log statement with `{}`.", + MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT); + return MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT; + } catch (SQLException e) { + LOGGER.info( + "Probing with {} failed, fallback to classic {}. Caused by: {}", + MYSQL_NEW_SHOW_BINARY_LOG_STATEMENT, + MYSQL_CLASSIC_SHOW_BINARY_LOG_STATEMENT, + e.getMessage()); + return MYSQL_CLASSIC_SHOW_BINARY_LOG_STATEMENT; + } + } + + @Override + public boolean isGtidModeEnabled() { + try { + return queryAndMap( + "SHOW GLOBAL VARIABLES LIKE 'GTID_MODE'", + rs -> { + if (rs.next()) { + return "ON".equalsIgnoreCase(rs.getString(2)); + } + return false; + }); + } catch (SQLException e) { + throw new DebeziumException("Unexpected error while looking at GTID mode: ", e); + } + } + + @Override + public GtidSet knownGtidSet() { + try { + return queryAndMap( + showBinaryLogStatement, + rs -> { + if (rs.next() && rs.getMetaData().getColumnCount() > 4) { + return new MySqlGtidSet( + rs.getString( + 5)); // GTID set, may be null, blank, or contain a GTID + // set + } + return new MySqlGtidSet(""); + }); + } catch (SQLException e) { + throw new DebeziumException("Unexpected error while looking at GTID mode: ", e); + } + } + + @Override + public GtidSet subtractGtidSet(GtidSet set1, GtidSet set2) { + try { + return prepareQueryAndMap( + "SELECT GTID_SUBTRACT(?, ?)", + ps -> { + ps.setString(1, set1.toString()); + ps.setString(2, set2.toString()); + }, + rs -> { + if (rs.next()) { + return new MySqlGtidSet(rs.getString(1)); + } + return new MySqlGtidSet(""); + }); + } catch (SQLException e) { + throw new DebeziumException("Unexpected error while executing GTID_SUBTRACT: ", e); + } + } + + @Override + public GtidSet purgedGtidSet() { + try { + return queryAndMap( + "SELECT @@global.gtid_purged", + rs -> { + if (rs.next() && rs.getMetaData().getColumnCount() > 0) { + return new MySqlGtidSet( + rs.getString( + 1)); // GTID set, may be null, blank, or contain a GTID + // set + } + return new MySqlGtidSet(""); + }); + } catch (SQLException e) { + throw new DebeziumException( + "Unexpected error while looking at gtid_purged variable: ", e); + } + } + + @Override + public GtidSet filterGtidSet( + Predicate gtidSourceFilter, + String offsetGtids, + GtidSet availableServerGtidSet, + GtidSet purgedServerGtidSet) { + String gtidStr = offsetGtids; + if (gtidStr == null) { + return null; + } + LOGGER.info("Attempting to generate a filtered GTID set"); + LOGGER.info("GTID set from previous recorded offset: {}", gtidStr); + GtidSet filteredGtidSet = new MySqlGtidSet(gtidStr); + if (gtidSourceFilter != null) { + filteredGtidSet = filteredGtidSet.retainAll(gtidSourceFilter); + LOGGER.info( + "GTID set after applying GTID source includes/excludes to previous recorded offset: {}", + filteredGtidSet); + } + LOGGER.info("GTID set available on server: {}", availableServerGtidSet); + + GtidSet mergedGtidSet; + + // Debezium 2.0 removed the gtid.new.channel.position config enum; Flink CDC keeps the + // behaviour behind the pass-through property of the same name. It defaults to "earliest", + // matching upstream Debezium 2.0. + final String newChannelPosition = + connectionConfig().originalConfig().getString("gtid.new.channel.position"); + final boolean useLatest = "latest".equalsIgnoreCase(newChannelPosition); + + if (!useLatest) { + final GtidSet knownGtidSet = filteredGtidSet; + LOGGER.info("Using first available positions for new GTID channels"); + final GtidSet relevantAvailableServerGtidSet = + (gtidSourceFilter != null) + ? availableServerGtidSet.retainAll(gtidSourceFilter) + : availableServerGtidSet; + LOGGER.info( + "Relevant GTID set available on server: {}", relevantAvailableServerGtidSet); + + mergedGtidSet = + GtidUtils.fixOldChannelsGtidSet( + (MySqlGtidSet) relevantAvailableServerGtidSet, + (MySqlGtidSet) purgedServerGtidSet, + (MySqlGtidSet) knownGtidSet); + } else { + LOGGER.info("Using latest positions for new GTID channels"); + mergedGtidSet = + GtidUtils.computeLatestModeGtidSet( + (MySqlGtidSet) availableServerGtidSet, + (MySqlGtidSet) purgedServerGtidSet, + (MySqlGtidSet) filteredGtidSet, + gtidSourceFilter); + } + + LOGGER.info("Final merged GTID set to use when connecting to MySQL: {}", mergedGtidSet); + return mergedGtidSet; + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/jdbc/MySqlConnectionConfiguration.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/jdbc/MySqlConnectionConfiguration.java new file mode 100644 index 00000000000..cdda7017a34 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/jdbc/MySqlConnectionConfiguration.java @@ -0,0 +1,146 @@ +/* + * Copyright Debezium Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ +package io.debezium.connector.mysql.jdbc; + +import io.debezium.config.Configuration; +import io.debezium.connector.binlog.jdbc.BinlogConnectionConfiguration; +import io.debezium.connector.mysql.MySqlConnectorConfig; +import io.debezium.jdbc.JdbcConfiguration; +import io.debezium.jdbc.JdbcConnection; +import io.debezium.util.Strings; + +import java.util.Properties; + +/** + * Copied from Debezium project(2.7.4.Final). + * + *

    Change 1: add the {@link #MySqlConnectionConfiguration(Configuration, Properties)} + * constructor, which merges user-supplied jdbc properties over {@link #DEFAULT_JDBC_PROPERTIES} and + * builds the url pattern from the result. + * + *

    Change 2: override {@link #factory()} and {@link #getUrlPattern()} to use that pattern rather + * than the fixed {@code URL_PATTERN} of the base class — the base builds its factory inside its own + * constructor, before this class's fields are assigned, so {@code createFactory} is left as-is. + * + *

    Change 3: override {@link #getJdbcConfiguration(Configuration)} to re-add the {@code useSSL} + * and {@code connectTimeout} jdbc properties. The base class resolves {@code connectTimeout} from + * the {@code database.}-prefixed subset only, while Flink CDC passes {@code connect.timeout.ms} at + * the top level, and it dropped {@code useSSL} in favour of {@code sslMode} — Flink CDC keeps + * emitting {@code useSSL} so that user-supplied jdbc properties can still override it. + */ +public class MySqlConnectionConfiguration extends BinlogConnectionConfiguration { + + private static final String JDBC_PROPERTY_CONNECTION_TIME_ZONE = "connectionTimeZone"; + + private static final String JDBC_URL_PATTERN = + "${protocol}://${hostname}:${port}/?useSSL=${useSSL}&connectTimeout=${connectTimeout}"; + + private static final String JDBC_URL_PATTERN_WITH_CUSTOM_USE_SSL = + "${protocol}://${hostname}:${port}/?connectTimeout=${connectTimeout}"; + + private static final Properties DEFAULT_JDBC_PROPERTIES = initializeDefaultJdbcProperties(); + + private final String flinkUrlPattern; + private final JdbcConnection.ConnectionFactory flinkFactory; + public static final String URL_PATTERN = + "${protocol}://${hostname}:${port}/?useInformationSchema=true&nullCatalogMeansCurrent=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=CONVERT_TO_NULL&connectTimeout=${connectTimeout}"; + + public MySqlConnectionConfiguration(Configuration config) { + this(config, new Properties()); + } + + public MySqlConnectionConfiguration(Configuration config, Properties jdbcProperties) { + super(config); + this.flinkUrlPattern = formatJdbcUrl(jdbcProperties); + this.flinkFactory = + JdbcConnection.patternBasedFactory( + flinkUrlPattern, + config.getString(MySqlConnectorConfig.JDBC_DRIVER), + getClass().getClassLoader(), + MySqlConnectorConfig.JDBC_PROTOCOL); + } + + @Override + public JdbcConnection.ConnectionFactory factory() { + return flinkFactory; + } + + private String formatJdbcUrl(Properties jdbcProperties) { + Properties combinedProperties = new Properties(); + combinedProperties.putAll(DEFAULT_JDBC_PROPERTIES); + combinedProperties.putAll(jdbcProperties); + + // when the user supplies their own useSSL, drop the one baked into the pattern so that it + // is not emitted twice + StringBuilder jdbcUrlStringBuilder = + jdbcProperties.getProperty("useSSL") == null + ? new StringBuilder(JDBC_URL_PATTERN) + : new StringBuilder(JDBC_URL_PATTERN_WITH_CUSTOM_USE_SSL); + combinedProperties.forEach( + (key, value) -> + jdbcUrlStringBuilder.append("&").append(key).append("=").append(value)); + return jdbcUrlStringBuilder.toString(); + } + + private static Properties initializeDefaultJdbcProperties() { + Properties defaultJdbcProperties = new Properties(); + defaultJdbcProperties.setProperty("useInformationSchema", "true"); + defaultJdbcProperties.setProperty("nullCatalogMeansCurrent", "false"); + defaultJdbcProperties.setProperty("useUnicode", "true"); + defaultJdbcProperties.setProperty("zeroDateTimeBehavior", "CONVERT_TO_NULL"); + defaultJdbcProperties.setProperty("characterEncoding", "UTF-8"); + defaultJdbcProperties.setProperty("characterSetResults", "UTF-8"); + return defaultJdbcProperties; + } + + @Override + protected String getConnectionTimeZonePropertyName() { + return JDBC_PROPERTY_CONNECTION_TIME_ZONE; + } + + @Override + protected String resolveConnectionTimeZone(Configuration dbConfig) { + // Debezium by default expects time zoned data delivered in server timezone + return Strings.defaultIfBlank( + dbConfig.getString(JDBC_PROPERTY_CONNECTION_TIME_ZONE), "SERVER"); + } + + @Override + protected JdbcConfiguration getJdbcConfiguration(Configuration configuration) { + return JdbcConfiguration.adapt( + super.getJdbcConfiguration(configuration) + .edit() + .with( + "connectTimeout", + Long.toString(getConnectionTimeout(originalConfig()).toMillis())) + .with("useSSL", Boolean.toString(sslModeEnabled())) + .build()); + } + + @Override + protected Configuration.Builder getDatabaseConfiguration(Configuration configuration) { + Configuration.Builder builder = super.getDatabaseConfiguration(configuration); + builder.withDefault( + MySqlConnectorConfig.JDBC_PROTOCOL, + MySqlConnectorConfig.JDBC_PROTOCOL.defaultValue()); + return builder; + } + + @Override + public String getUrlPattern() { + return flinkUrlPattern; + } + + @Override + protected JdbcConnection.ConnectionFactory createFactory(Configuration configuration) { + final String driverClassName = configuration.getString(MySqlConnectorConfig.JDBC_DRIVER); + return JdbcConnection.patternBasedFactory( + URL_PATTERN, + driverClassName, + getClass().getClassLoader(), + MySqlConnectorConfig.JDBC_PROTOCOL); + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/util/ErrorMessageUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/util/ErrorMessageUtils.java index 47efa694f20..5adaac0ac5f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/util/ErrorMessageUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/io/debezium/connector/mysql/util/ErrorMessageUtils.java @@ -19,7 +19,12 @@ import java.util.regex.Pattern; -/** This util tries to optimize error message for some exceptions. */ +/** + * This util tries to optimize error message for some exceptions. * + * + *

    This is a Flink CDC owned class (not a Debezium fork), placed in the {@code io.debezium} + * package for access to package-private Debezium members. + */ public class ErrorMessageUtils { private static final Pattern SERVER_ID_CONFLICT = Pattern.compile( diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/MySqlSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/MySqlSource.java index fb512a1eb9b..5b4fb4de5e0 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/MySqlSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/MySqlSource.java @@ -173,12 +173,17 @@ public DebeziumSourceFunction build() { // connectors, // since it is used as a prefix for all Kafka topic names emanating from this connector. // Only alphanumeric characters and underscores should be used. - props.setProperty("database.server.name", DATABASE_SERVER_NAME); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", DATABASE_SERVER_NAME); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); props.setProperty("database.port", String.valueOf(port)); - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); + // Debezium 2.3 changed the default of "database.ssl.mode" from "disabled" to + // "preferred". Keep the historical Flink CDC behavior (plain connections unless the + // user asks for TLS); user-supplied debezium properties below still override it. + props.setProperty("database.ssl.mode", "disabled"); // debezium use "long" mode to handle unsigned bigint by default, // but it'll cause lose of precise when the value is larger than 2^63, // so use "precise" mode to avoid it. @@ -186,6 +191,12 @@ public DebeziumSourceFunction build() { if (serverId != null) { props.setProperty("database.server.id", String.valueOf(serverId)); + } else { + // Debezium 2.0 made "database.server.id" required (1.9 assigned a random default in + // the 5400-6400 range). Preserve that behavior when no server id is configured. + props.setProperty( + "database.server.id", + String.valueOf(5400 + new java.util.Random().nextInt(1000))); } if (databaseList != null) { props.setProperty("database.include.list", String.join(",", databaseList)); @@ -212,10 +223,13 @@ public DebeziumSourceFunction build() { break; case SPECIFIC_OFFSETS: - // if binlog offset is specified, 'snapshot.mode=schema_only_recovery' must - // be configured. It only snapshots the schemas, not the data, - // and continue binlog reading from the specified offset - props.setProperty("snapshot.mode", "schema_only_recovery"); + // if binlog offset is specified, 'snapshot.mode=recovery' must be + // configured. It only snapshots the schemas, not the data, and continues + // binlog reading from the specified offset. + // Debezium 2.6 kept "schema_only_recovery" as a deprecated alias, but it + // now behaves like "no_data" and no longer recovers the schema history, + // so "recovery" is the mode that preserves the pre-2.6 semantics. + props.setProperty("snapshot.mode", "recovery"); specificOffset = new DebeziumOffset(); Map sourcePartition = new HashMap<>(); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/DebeziumUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/DebeziumUtils.java index f567dde7347..44d8c7d21ca 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/DebeziumUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/DebeziumUtils.java @@ -22,18 +22,21 @@ import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffset; import org.apache.flink.cdc.connectors.mysql.source.utils.TableDiscoveryUtils; import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.TemporaryClassLoaderContext; import com.github.shyiko.mysql.binlog.BinaryLogClient; import com.github.shyiko.mysql.binlog.event.EventData; import com.github.shyiko.mysql.binlog.event.EventHeaderV4; import com.github.shyiko.mysql.binlog.event.RotateEventData; +import io.debezium.config.CommonConnectorConfig; import io.debezium.config.Configuration; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.binlog.jdbc.BinlogSystemVariables; +import io.debezium.connector.binlog.jdbc.ConnectionConfiguration; import io.debezium.connector.mysql.MySqlConnectorConfig; import io.debezium.connector.mysql.MySqlDatabaseSchema; -import io.debezium.connector.mysql.MySqlSystemVariables; -import io.debezium.connector.mysql.MySqlTopicSelector; -import io.debezium.connector.mysql.MySqlValueConverters; +import io.debezium.connector.mysql.jdbc.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnectionConfiguration; +import io.debezium.connector.mysql.jdbc.MySqlValueConverters; import io.debezium.jdbc.JdbcConfiguration; import io.debezium.jdbc.JdbcConnection; import io.debezium.jdbc.JdbcValueConverters; @@ -41,8 +44,8 @@ import io.debezium.relational.Selectors; import io.debezium.relational.TableId; import io.debezium.relational.Tables; -import io.debezium.schema.TopicSelector; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.topic.TopicNamingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,29 +92,40 @@ public static MySqlConnection createMySqlConnection(MySqlSourceConfig sourceConf public static MySqlConnection createMySqlConnection( Configuration dbzConfiguration, Properties jdbcProperties) { return new MySqlConnection( - new MySqlConnection.MySqlConnectionConfiguration(dbzConfiguration, jdbcProperties)); + new MySqlConnectionConfiguration(dbzConfiguration, jdbcProperties)); } /** Creates a new {@link BinaryLogClient} for consuming mysql binlog. */ public static BinaryLogClient createBinaryClient(Configuration dbzConfiguration) { - final MySqlConnectorConfig connectorConfig = new MySqlConnectorConfig(dbzConfiguration); + // Debezium 2.7 moved hostname/port/user/password off MySqlConnectorConfig onto the + // connection configuration. + final MySqlConnectionConfiguration connectionConfig = + new MySqlConnectionConfiguration(dbzConfiguration); return new BinaryLogClient( - connectorConfig.hostname(), - connectorConfig.port(), - connectorConfig.username(), - connectorConfig.password()); + connectionConfig.hostname(), + connectionConfig.port(), + connectionConfig.username(), + connectionConfig.password()); } /** Creates a new {@link MySqlDatabaseSchema} to monitor the latest MySql database schemas. */ public static MySqlDatabaseSchema createMySqlDatabaseSchema( MySqlConnectorConfig dbzMySqlConfig, boolean isTableIdCaseSensitive) { - TopicSelector topicSelector = MySqlTopicSelector.defaultSelector(dbzMySqlConfig); + TopicNamingStrategy topicNamingStrategy; + // Debezium 2.0 resolves classes through the thread context class loader; pin it to the + // loader that loaded Debezium so a stale/closed Flink user class loader cannot break it. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(CommonConnectorConfig.class.getClassLoader())) { + topicNamingStrategy = + dbzMySqlConfig.getTopicNamingStrategy( + MySqlConnectorConfig.TOPIC_NAMING_STRATEGY); + } SchemaNameAdjuster schemaNameAdjuster = SchemaNameAdjuster.create(); MySqlValueConverters valueConverters = getValueConverters(dbzMySqlConfig); return new MySqlDatabaseSchema( dbzMySqlConfig, valueConverters, - topicSelector, + topicNamingStrategy, schemaNameAdjuster, isTableIdCaseSensitive); } @@ -185,7 +199,10 @@ private static MySqlValueConverters getValueConverters(MySqlConnectorConfig dbzM bigIntUnsignedMode, dbzMySqlConfig.binaryHandlingMode(), timeAdjusterEnabled ? MySqlValueConverters::adjustTemporal : x -> x, - MySqlValueConverters::defaultParsingErrorHandler); + // Debezium 2.6 replaced the ParsingErrorHandler argument with + // EventConvertingFailureHandlingMode; 2.7 additionally passes the service registry. + dbzMySqlConfig.getEventConvertingFailureHandlingMode(), + dbzMySqlConfig.getServiceRegistry()); } public static List discoverCapturedTables( @@ -212,7 +229,7 @@ public static boolean isTableIdCaseSensitive(JdbcConnection connection) { return !"0" .equals( readMySqlSystemVariables(connection) - .get(MySqlSystemVariables.LOWER_CASE_TABLE_NAMES)); + .get(BinlogSystemVariables.LOWER_CASE_TABLE_NAMES)); } public static Map readMySqlSystemVariables(JdbcConnection connection) { @@ -244,7 +261,7 @@ private static Map querySystemVariables( public static BinlogOffset findBinlogOffset( long targetMs, MySqlConnection connection, MySqlSourceConfig mySqlSourceConfig) { - MySqlConnection.MySqlConnectionConfiguration config = connection.connectionConfig(); + ConnectionConfiguration config = connection.connectionConfig(); BinaryLogClient client = new BinaryLogClient( config.hostname(), config.port(), config.username(), config.password()); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/EmbeddedFlinkDatabaseHistory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/EmbeddedFlinkDatabaseHistory.java index 293e0c76f46..734f168ae3b 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/EmbeddedFlinkDatabaseHistory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/EmbeddedFlinkDatabaseHistory.java @@ -24,14 +24,15 @@ import io.debezium.relational.TableId; import io.debezium.relational.Tables; import io.debezium.relational.ddl.DdlParser; -import io.debezium.relational.history.DatabaseHistory; -import io.debezium.relational.history.DatabaseHistoryException; -import io.debezium.relational.history.DatabaseHistoryListener; import io.debezium.relational.history.HistoryRecord; import io.debezium.relational.history.HistoryRecordComparator; +import io.debezium.relational.history.SchemaHistory; +import io.debezium.relational.history.SchemaHistoryException; +import io.debezium.relational.history.SchemaHistoryListener; import io.debezium.relational.history.TableChanges; import io.debezium.relational.history.TableChanges.TableChange; +import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -40,19 +41,20 @@ import java.util.concurrent.ConcurrentMap; /** - * A {@link DatabaseHistory} implementation which store the latest table schema in Flink state. + * A {@link SchemaHistory} implementation which store the latest table schema in Flink state. * *

    It stores/recovers history using data offered by {@link MySqlSplitState}. */ -public class EmbeddedFlinkDatabaseHistory implements DatabaseHistory { +public class EmbeddedFlinkDatabaseHistory implements SchemaHistory { - public static final String DATABASE_HISTORY_INSTANCE_NAME = "database.history.instance.name"; + public static final String DATABASE_HISTORY_INSTANCE_NAME = + "schema.history.internal.instance.name"; public static final ConcurrentMap> TABLE_SCHEMAS = new ConcurrentHashMap<>(); private Map tableSchemas; - private DatabaseHistoryListener listener; + private SchemaHistoryListener listener; private boolean storeOnlyMonitoredTablesDdl; private boolean skipUnparseableDDL; @@ -60,10 +62,10 @@ public class EmbeddedFlinkDatabaseHistory implements DatabaseHistory { public void configure( Configuration config, HistoryRecordComparator comparator, - DatabaseHistoryListener listener, + SchemaHistoryListener listener, boolean useCatalogBeforeSchema) { this.listener = listener; - this.storeOnlyMonitoredTablesDdl = config.getBoolean(STORE_ONLY_MONITORED_TABLES_DDL); + this.storeOnlyMonitoredTablesDdl = config.getBoolean(STORE_ONLY_CAPTURED_TABLES_DDL); this.skipUnparseableDDL = config.getBoolean(SKIP_UNPARSEABLE_DDL_STATEMENTS); // recover @@ -74,15 +76,20 @@ public void configure( } } + // Debezium 2.0 wires SchemaHistoryMetrics in as the schema history listener. Its + // started() callback registers a JMX MBean whose name is built only from the connector + // context and the topic prefix, so every parallel subtask sharing a TaskManager JVM asks + // for the very same name. Debezium answers a name clash by sleeping 5 seconds and + // retrying, twelve times, so every reader but the first stalls for up to a minute each + // time it opens a split. Flink CDC publishes its own metrics and never reads these + // MBeans, and Debezium 1.9 did not register them either, so skip the registration. @Override - public void start() { - listener.started(); - } + public void start() {} @Override public void record( Map source, Map position, String databaseName, String ddl) - throws DatabaseHistoryException { + throws SchemaHistoryException { throw new UnsupportedOperationException("should not call here, error"); } @@ -93,10 +100,12 @@ public void record( String databaseName, String schemaName, String ddl, - TableChanges changes) - throws DatabaseHistoryException { + TableChanges changes, + Instant timestamp) + throws SchemaHistoryException { final HistoryRecord record = - new HistoryRecord(source, position, databaseName, schemaName, ddl, changes); + new HistoryRecord( + source, position, databaseName, schemaName, ddl, changes, timestamp); listener.onChangeApplied(record); } @@ -139,12 +148,12 @@ public void initializeStorage() { // do nothing } - @Override + // Debezium 2.2 moved storeOnlyCapturedTables()/skipUnparseableDdlStatements() from the + // SchemaHistory interface to HistorizedDatabaseSchema; kept here as plain helpers. public boolean storeOnlyCapturedTables() { return storeOnlyMonitoredTablesDdl; } - @Override public boolean skipUnparseableDdlStatements() { return skipUnparseableDDL; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/dispatcher/EventDispatcherImpl.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/dispatcher/EventDispatcherImpl.java index 6205c9c7ab6..51bfb89fb64 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/dispatcher/EventDispatcherImpl.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/dispatcher/EventDispatcherImpl.java @@ -28,17 +28,18 @@ import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource; import io.debezium.pipeline.source.spi.EventMetadataProvider; import io.debezium.pipeline.spi.ChangeEventCreator; +import io.debezium.pipeline.spi.OffsetContext; import io.debezium.pipeline.spi.SchemaChangeEventEmitter; import io.debezium.relational.TableId; import io.debezium.relational.history.HistoryRecord; import io.debezium.relational.history.TableChanges; import io.debezium.schema.DataCollectionFilters; -import io.debezium.schema.DataCollectionId; import io.debezium.schema.DatabaseSchema; import io.debezium.schema.HistorizedDatabaseSchema; import io.debezium.schema.SchemaChangeEvent; -import io.debezium.schema.TopicSelector; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.schema.DataCollectionId; +import io.debezium.spi.topic.TopicNamingStrategy; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; @@ -79,13 +80,13 @@ public class EventDispatcherImpl private final HistorizedDatabaseSchema historizedSchema; private final DataCollectionFilters.DataCollectionFilter filter; private final CommonConnectorConfig connectorConfig; - private final TopicSelector topicSelector; + private final TopicNamingStrategy topicNamingStrategy; private final Schema schemaChangeKeySchema; private final Schema schemaChangeValueSchema; public EventDispatcherImpl( CommonConnectorConfig connectorConfig, - TopicSelector topicSelector, + TopicNamingStrategy topicNamingStrategy, DatabaseSchema schema, ChangeEventQueue queue, DataCollectionFilters.DataCollectionFilter filter, @@ -94,7 +95,7 @@ public EventDispatcherImpl( SchemaNameAdjuster schemaNameAdjuster) { super( connectorConfig, - topicSelector, + topicNamingStrategy, schema, queue, filter, @@ -108,7 +109,7 @@ public EventDispatcherImpl( this.filter = filter; this.queue = queue; this.connectorConfig = connectorConfig; - this.topicSelector = topicSelector; + this.topicNamingStrategy = topicNamingStrategy; this.schemaChangeKeySchema = SchemaBuilder.struct() .name( @@ -139,6 +140,7 @@ public ChangeEventQueue getQueue() { @Override public void dispatchSchemaChangeEvent( MySqlPartition partition, + OffsetContext offsetContext, T dataCollectionId, SchemaChangeEventEmitter schemaChangeEventEmitter) throws InterruptedException { @@ -152,7 +154,7 @@ public void dispatchSchemaChangeEvent( IncrementalSnapshotChangeEventSource incrementalEventSource = getIncrementalSnapshotChangeEventSource(); if (incrementalEventSource != null) { - incrementalEventSource.processSchemaChange(partition, dataCollectionId); + incrementalEventSource.processSchemaChange(partition, offsetContext, dataCollectionId); } } @@ -208,7 +210,8 @@ private Struct schemaChangeRecordValue(SchemaChangeEvent event) throws IOExcepti event.getDatabase(), null, event.getDdl(), - event.getTableChanges()); + event.getTableChanges(), + event.getTimestamp()); String historyStr = DOCUMENT_WRITER.write(historyRecord.document()); Struct value = new Struct(schemaChangeValueSchema); @@ -314,7 +317,7 @@ public void schemaChangeEvent(SchemaChangeEvent event) throws InterruptedExcepti historizedSchema.applySchemaChange(event); if (connectorConfig.isSchemaChangesHistoryEnabled()) { try { - final String topicName = topicSelector.getPrimaryTopic(); + final String topicName = connectorConfig.getLogicalName(); final Integer partition = 0; final Struct key = schemaChangeRecordKey(event); final Struct value = schemaChangeRecordValue(event); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/dispatcher/SignalEventDispatcher.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/dispatcher/SignalEventDispatcher.java index bbd4a5d2a6c..4e2a3ad16c1 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/dispatcher/SignalEventDispatcher.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/dispatcher/SignalEventDispatcher.java @@ -22,7 +22,7 @@ import io.debezium.connector.base.ChangeEventQueue; import io.debezium.pipeline.DataChangeEvent; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/SnapshotSplitReader.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/SnapshotSplitReader.java index e3549237065..5634869d63d 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/SnapshotSplitReader.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/SnapshotSplitReader.java @@ -44,7 +44,7 @@ import io.debezium.pipeline.DataChangeEvent; import io.debezium.pipeline.source.spi.ChangeEventSource; import io.debezium.pipeline.spi.SnapshotResult; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; @@ -144,7 +144,7 @@ public void submitSplit(MySqlSplit mySqlSplit) { statefulTaskContext.getDatabaseSchema(), statefulTaskContext.getConnection(), statefulTaskContext.getDispatcher(), - statefulTaskContext.getTopicSelector(), + statefulTaskContext.getTopicNamingStrategy(), statefulTaskContext.getSnapshotReceiver(), StatefulTaskContext.getClock(), currentSnapshotSplit, @@ -176,7 +176,10 @@ private SnapshotResult snapshot( return splitSnapshotReadTask.execute( sourceContext, statefulTaskContext.getMySqlPartition(), - statefulTaskContext.getOffsetContext()); + statefulTaskContext.getOffsetContext(), + splitSnapshotReadTask.getSnapshottingTask( + statefulTaskContext.getMySqlPartition(), + statefulTaskContext.getOffsetContext())); } private void backfill( @@ -262,7 +265,7 @@ private void dispatchBinlogEndEvent(MySqlBinlogSplit backFillBinlogSplit) final SignalEventDispatcher signalEventDispatcher = new SignalEventDispatcher( statefulTaskContext.getOffsetContext().getOffset(), - statefulTaskContext.getTopicSelector().getPrimaryTopic(), + statefulTaskContext.getConnectorConfig().getLogicalName(), statefulTaskContext.getDispatcher().getQueue()); signalEventDispatcher.dispatchWatermarkEvent( backFillBinlogSplit, @@ -477,5 +480,25 @@ public void setHighWatermark(BinlogOffset highWatermark) { public boolean isRunning() { return lowWatermark != null && highWatermark != null; } + + // The following methods are only used by Debezium's signal-based blocking snapshot, + // which Flink CDC does not use, so they are no-ops here. + + @Override + public boolean isPaused() { + return false; + } + + @Override + public void resumeStreaming() throws InterruptedException {} + + @Override + public void waitSnapshotCompletion() throws InterruptedException {} + + @Override + public void streamingPaused() {} + + @Override + public void waitStreamingPaused() throws InterruptedException {} } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/StoppableChangeEventSourceContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/StoppableChangeEventSourceContext.java index 96c5eb6b38e..113c9a3e659 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/StoppableChangeEventSourceContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/StoppableChangeEventSourceContext.java @@ -40,4 +40,24 @@ public void stopChangeEventSource() { public boolean isRunning() { return isRunning; } + + // The following methods are only used by Debezium's signal-based blocking snapshot, + // which Flink CDC does not use, so they are no-ops here. + + @Override + public boolean isPaused() { + return false; + } + + @Override + public void resumeStreaming() throws InterruptedException {} + + @Override + public void waitSnapshotCompletion() throws InterruptedException {} + + @Override + public void streamingPaused() {} + + @Override + public void waitStreamingPaused() throws InterruptedException {} } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlBinlogSplitReadTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlBinlogSplitReadTask.java index 7a6e7757f7d..098036adc38 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlBinlogSplitReadTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlBinlogSplitReadTask.java @@ -24,16 +24,18 @@ import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffset; import org.apache.flink.cdc.connectors.mysql.source.split.MySqlBinlogSplit; import org.apache.flink.cdc.connectors.mysql.source.utils.RecordUtils; +import org.apache.flink.cdc.debezium.internal.SnapshotterServiceFactory; import com.github.shyiko.mysql.binlog.event.Event; import io.debezium.DebeziumException; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.MySqlConnector; import io.debezium.connector.mysql.MySqlConnectorConfig; import io.debezium.connector.mysql.MySqlOffsetContext; import io.debezium.connector.mysql.MySqlPartition; import io.debezium.connector.mysql.MySqlStreamingChangeEventSource; import io.debezium.connector.mysql.MySqlStreamingChangeEventSourceMetrics; import io.debezium.connector.mysql.MySqlTaskContext; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.pipeline.ErrorHandler; import io.debezium.relational.TableId; import io.debezium.util.Clock; @@ -69,7 +71,15 @@ public MySqlBinlogSplitReadTask( MySqlStreamingChangeEventSourceMetrics metrics, MySqlBinlogSplit binlogSplit, Predicate eventFilter) { - super(connectorConfig, connection, dispatcher, errorHandler, clock, taskContext, metrics); + super( + connectorConfig, + connection, + dispatcher, + errorHandler, + clock, + taskContext, + metrics, + SnapshotterServiceFactory.create(connectorConfig, MySqlConnector.class)); this.binlogSplit = binlogSplit; this.eventDispatcher = dispatcher; this.errorHandler = errorHandler; @@ -89,11 +99,14 @@ public void execute( @Override protected void handleEvent( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) { + MySqlPartition partition, + MySqlOffsetContext offsetContext, + ChangeEventSourceContext context, + Event event) { if (!eventFilter.test(event)) { return; } - super.handleEvent(partition, offsetContext, event); + super.handleEvent(partition, offsetContext, context, event); // check do we need to stop for read binlog for snapshot split. if (isBoundedRead()) { final BinlogOffset currentBinlogOffset = diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlSnapshotSplitReadTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlSnapshotSplitReadTask.java index f729f59d05f..d61a9a974c0 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlSnapshotSplitReadTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlSnapshotSplitReadTask.java @@ -28,15 +28,18 @@ import org.apache.flink.cdc.connectors.mysql.source.utils.hooks.SnapshotPhaseHooks; import io.debezium.DebeziumException; -import io.debezium.connector.mysql.MySqlConnection; import io.debezium.connector.mysql.MySqlConnectorConfig; import io.debezium.connector.mysql.MySqlDatabaseSchema; import io.debezium.connector.mysql.MySqlOffsetContext; import io.debezium.connector.mysql.MySqlPartition; -import io.debezium.connector.mysql.MySqlValueConverters; +import io.debezium.connector.mysql.jdbc.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlValueConverters; import io.debezium.pipeline.EventDispatcher; import io.debezium.pipeline.metrics.SnapshotChangeEventSourceMetrics; +import io.debezium.pipeline.notification.NotificationService; +import io.debezium.pipeline.signal.actions.snapshotting.SnapshotConfiguration; import io.debezium.pipeline.source.AbstractSnapshotChangeEventSource; +import io.debezium.pipeline.source.SnapshottingTask; import io.debezium.pipeline.spi.ChangeRecordEmitter; import io.debezium.pipeline.spi.SnapshotResult; import io.debezium.relational.Column; @@ -44,7 +47,8 @@ import io.debezium.relational.SnapshotChangeRecordEmitter; import io.debezium.relational.Table; import io.debezium.relational.TableId; -import io.debezium.schema.TopicSelector; +import io.debezium.schema.SchemaFactory; +import io.debezium.spi.topic.TopicNamingStrategy; import io.debezium.util.Clock; import io.debezium.util.ColumnUtils; import io.debezium.util.Strings; @@ -61,6 +65,7 @@ import java.sql.Types; import java.time.Duration; import java.util.Calendar; +import java.util.Collections; /** Task to read snapshot split of table. */ public class MySqlSnapshotSplitReadTask @@ -72,12 +77,13 @@ public class MySqlSnapshotSplitReadTask private static final Duration LOG_INTERVAL = Duration.ofMillis(10_000); private final MySqlSourceConfig sourceConfig; + private final MySqlConnectorConfig connectorConfig; private final MySqlDatabaseSchema databaseSchema; private final MySqlConnection jdbcConnection; private final EventDispatcherImpl dispatcher; private final Clock clock; private final MySqlSnapshotSplit snapshotSplit; - private final TopicSelector topicSelector; + private final TopicNamingStrategy topicNamingStrategy; private final EventDispatcher.SnapshotReceiver snapshotReceiver; private final SnapshotChangeEventSourceMetrics snapshotChangeEventSourceMetrics; @@ -91,20 +97,28 @@ public MySqlSnapshotSplitReadTask( MySqlDatabaseSchema databaseSchema, MySqlConnection jdbcConnection, EventDispatcherImpl dispatcher, - TopicSelector topicSelector, + TopicNamingStrategy topicNamingStrategy, EventDispatcher.SnapshotReceiver snapshotReceiver, Clock clock, MySqlSnapshotSplit snapshotSplit, SnapshotPhaseHooks hooks, boolean isBackfillSkipped) { - super(connectorConfig, snapshotChangeEventSourceMetrics); + super( + connectorConfig, + snapshotChangeEventSourceMetrics, + new NotificationService<>( + Collections.emptyList(), + connectorConfig, + SchemaFactory.get(), + notification -> {})); this.sourceConfig = sourceConfig; + this.connectorConfig = connectorConfig; this.databaseSchema = databaseSchema; this.jdbcConnection = jdbcConnection; this.dispatcher = dispatcher; this.clock = clock; this.snapshotSplit = snapshotSplit; - this.topicSelector = topicSelector; + this.topicNamingStrategy = topicNamingStrategy; this.snapshotReceiver = snapshotReceiver; this.snapshotChangeEventSourceMetrics = snapshotChangeEventSourceMetrics; this.hooks = hooks; @@ -115,12 +129,12 @@ public MySqlSnapshotSplitReadTask( public SnapshotResult execute( ChangeEventSourceContext context, MySqlPartition partition, - MySqlOffsetContext previousOffset) + MySqlOffsetContext previousOffset, + SnapshottingTask snapshottingTask) throws InterruptedException { - SnapshottingTask snapshottingTask = getSnapshottingTask(partition, previousOffset); final SnapshotContext ctx; try { - ctx = prepare(partition); + ctx = prepare(partition, false); } catch (Exception e) { LOG.error("Failed to initialize snapshot context.", e); throw new RuntimeException(e); @@ -147,7 +161,7 @@ protected SnapshotResult doExecute( final SignalEventDispatcher signalEventDispatcher = new SignalEventDispatcher( previousOffset.getOffset(), - topicSelector.topicNameFor(snapshotSplit.getTableId()), + topicNamingStrategy.dataChangeTopic(snapshotSplit.getTableId()), dispatcher.getQueue()); if (hooks.getPreLowWatermarkAction() != null) { @@ -205,22 +219,35 @@ protected SnapshotResult doExecute( } @Override - protected SnapshottingTask getSnapshottingTask( + public SnapshottingTask getSnapshottingTask( MySqlPartition partition, MySqlOffsetContext previousOffset) { - return new SnapshottingTask(false, true); + return new SnapshottingTask( + false, true, Collections.emptyList(), Collections.emptyMap(), false); + } + + @Override + public SnapshottingTask getBlockingSnapshottingTask( + MySqlPartition partition, + MySqlOffsetContext previousOffset, + SnapshotConfiguration snapshotConfiguration) { + // Debezium 2.6 made this abstract. Flink CDC drives its own split snapshot and never + // runs Debezium's signal-based blocking snapshot, so it behaves like the regular one. + return getSnapshottingTask(partition, previousOffset); } @Override - protected MySqlSnapshotContext prepare(MySqlPartition partition) throws Exception { - return new MySqlSnapshotContext(partition); + protected MySqlSnapshotContext prepare(MySqlPartition partition, boolean onDemand) + throws Exception { + return new MySqlSnapshotContext(partition, onDemand); } private static class MySqlSnapshotContext extends RelationalSnapshotChangeEventSource.RelationalSnapshotContext< MySqlPartition, MySqlOffsetContext> { - public MySqlSnapshotContext(MySqlPartition partition) throws SQLException { - super(partition, ""); + public MySqlSnapshotContext(MySqlPartition partition, boolean onDemand) + throws SQLException { + super(partition, "", onDemand); } } @@ -309,7 +336,7 @@ protected ChangeRecordEmitter getChangeRecordEmitter( MySqlSnapshotContext snapshotContext, TableId tableId, Object[] row) { snapshotContext.offset.event(tableId, clock.currentTime()); return new SnapshotChangeRecordEmitter<>( - snapshotContext.partition, snapshotContext.offset, row, clock); + snapshotContext.partition, snapshotContext.offset, row, clock, connectorConfig); } private Threads.Timer getTableScanLogTimer() { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/context/MySqlErrorHandler.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/context/MySqlErrorHandler.java index 47e3e603326..d77b08e2b11 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/context/MySqlErrorHandler.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/context/MySqlErrorHandler.java @@ -45,6 +45,14 @@ public class MySqlErrorHandler extends ErrorHandler { private static final Pattern NOT_FOUND_TABLE_MSG_PATTERN = Pattern.compile( "Encountered change event for table (.+)\\.(.+) whose schema isn't known to this connector"); + // Debezium <= 2.4 only detected a stale schema deep inside TableSchemaBuilder, which raised a + // ConnectException ending with this text. Debezium 2.5 additionally validates the row size + // against the known schema up-front and raises a DebeziumException matching the pattern below. + private static final String SCHEMA_OUT_OF_SYNC_MSG_SUFFIX = + "internal schema representation is probably out of sync with real database schema"; + private static final Pattern COLUMN_SIZE_MISMATCH_MSG_PATTERN = + Pattern.compile( + "Error processing row in .+, internal schema size \\d+, but row size \\d+"); private final MySqlTaskContext context; private final MySqlSourceConfig sourceConfig; @@ -54,7 +62,7 @@ public MySqlErrorHandler( ChangeEventQueue queue, MySqlTaskContext context, MySqlSourceConfig sourceConfig) { - super(MySqlConnector.class, mySqlConnectorConfig, queue); + super(MySqlConnector.class, mySqlConnectorConfig, queue, null); this.context = context; this.sourceConfig = sourceConfig; } @@ -101,12 +109,19 @@ private Optional extractNotFoundTableId(Throwable t) { } private boolean isSchemaOutOfSyncException(Throwable t) { + if (!sourceConfig.getStartupOptions().isStreamOnly()) { + return false; + } Throwable rootCause = ExceptionUtils.getRootCause(t); - return rootCause instanceof ConnectException - && rootCause - .getMessage() - .endsWith( - "internal schema representation is probably out of sync with real database schema") - && sourceConfig.getStartupOptions().isStreamOnly(); + if (rootCause == null || rootCause.getMessage() == null) { + return false; + } + String message = rootCause.getMessage(); + if (rootCause instanceof ConnectException + && message.endsWith(SCHEMA_OUT_OF_SYNC_MSG_SUFFIX)) { + return true; + } + return rootCause instanceof DebeziumException + && COLUMN_SIZE_MISMATCH_MSG_PATTERN.matcher(message).find(); } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/context/StatefulTaskContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/context/StatefulTaskContext.java index 09a5fe8e85c..c66b36b6182 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/context/StatefulTaskContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/task/context/StatefulTaskContext.java @@ -24,20 +24,22 @@ import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig; import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffset; import org.apache.flink.cdc.connectors.mysql.source.split.MySqlSplit; +import org.apache.flink.util.TemporaryClassLoaderContext; import com.github.shyiko.mysql.binlog.BinaryLogClient; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.AbstractSourceInfo; import io.debezium.connector.base.ChangeEventQueue; -import io.debezium.connector.mysql.GtidSet; -import io.debezium.connector.mysql.GtidUtils; +import io.debezium.connector.binlog.gtid.GtidSet; import io.debezium.connector.mysql.MySqlChangeEventSourceMetricsFactory; -import io.debezium.connector.mysql.MySqlConnection; import io.debezium.connector.mysql.MySqlConnectorConfig; import io.debezium.connector.mysql.MySqlDatabaseSchema; import io.debezium.connector.mysql.MySqlOffsetContext; import io.debezium.connector.mysql.MySqlPartition; import io.debezium.connector.mysql.MySqlStreamingChangeEventSourceMetrics; -import io.debezium.connector.mysql.MySqlTopicSelector; +import io.debezium.connector.mysql.gtid.GtidUtils; +import io.debezium.connector.mysql.gtid.MySqlGtidSet; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.data.Envelope; import io.debezium.pipeline.DataChangeEvent; import io.debezium.pipeline.ErrorHandler; @@ -48,11 +50,11 @@ import io.debezium.pipeline.spi.OffsetContext; import io.debezium.pipeline.spi.Offsets; import io.debezium.relational.TableId; -import io.debezium.schema.DataCollectionId; -import io.debezium.schema.TopicSelector; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.schema.DataCollectionId; +import io.debezium.spi.topic.TopicNamingStrategy; import io.debezium.util.Clock; import io.debezium.util.Collect; -import io.debezium.util.SchemaNameAdjuster; import org.apache.kafka.connect.data.Struct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,7 +90,7 @@ public class StatefulTaskContext implements AutoCloseable { private MySqlTaskContextImpl taskContext; private MySqlOffsetContext offsetContext; private MySqlPartition mySqlPartition; - private TopicSelector topicSelector; + private TopicNamingStrategy topicNamingStrategy; private SnapshotChangeEventSourceMetrics snapshotChangeEventSourceMetrics; private StreamingChangeEventSourceMetrics streamingChangeEventSourceMetrics; private EventDispatcherImpl dispatcher; @@ -110,74 +112,94 @@ public StatefulTaskContext( } public void configure(MySqlSplit mySqlSplit) { - // initial stateful objects - final boolean tableIdCaseInsensitive = connection.isTableIdCaseSensitive(); - this.topicSelector = MySqlTopicSelector.defaultSelector(connectorConfig); - EmbeddedFlinkDatabaseHistory.registerHistory( - sourceConfig - .getDbzConfiguration() - .getString(EmbeddedFlinkDatabaseHistory.DATABASE_HISTORY_INSTANCE_NAME), - mySqlSplit.getTableSchemas().values()); - - Optional.ofNullable(databaseSchema).ifPresent(MySqlDatabaseSchema::close); - this.databaseSchema = - DebeziumUtils.createMySqlDatabaseSchema(connectorConfig, tableIdCaseInsensitive); - - this.mySqlPartition = new MySqlPartition(connectorConfig.getLogicalName()); - - this.offsetContext = - loadStartingOffsetState(new MySqlOffsetContext.Loader(connectorConfig), mySqlSplit); - validateAndLoadDatabaseHistory(offsetContext, databaseSchema); - - this.taskContext = - new MySqlTaskContextImpl(connectorConfig, databaseSchema, binaryLogClient); - - final int queueSize = - mySqlSplit.isSnapshotSplit() - ? sourceConfig.getSplitSize() + DEFAULT_BINLOG_QUEUE_SIZE_IN_SNAPSHOT_SCAN - : connectorConfig.getMaxQueueSize(); - this.queue = - new ChangeEventQueue.Builder() - .pollInterval(connectorConfig.getPollInterval()) - .maxBatchSize(connectorConfig.getMaxBatchSize()) - .maxQueueSize(queueSize) - .maxQueueSizeInBytes(connectorConfig.getMaxQueueSizeInBytes()) - .loggingContextSupplier( - () -> - taskContext.configureLoggingContext( - "mysql-cdc-connector-task")) - // do not buffer any element, we use signal event - // .buffering() - .build(); - this.dispatcher = - new EventDispatcherImpl<>( - connectorConfig, - topicSelector, - databaseSchema, - queue, - connectorConfig.getTableFilters().dataCollectionFilter(), - DataChangeEvent::new, - metadataProvider, - schemaNameAdjuster); - - this.snapshotReceiver = dispatcher.getSnapshotChangeEventReceiver(); - - this.signalEventDispatcher = - new SignalEventDispatcher( - offsetContext.getOffset(), topicSelector.getPrimaryTopic(), queue); - - final MySqlChangeEventSourceMetricsFactory changeEventSourceMetricsFactory = - new MySqlChangeEventSourceMetricsFactory( - new MySqlStreamingChangeEventSourceMetrics( - taskContext, queue, metadataProvider)); - this.snapshotChangeEventSourceMetrics = - changeEventSourceMetricsFactory.getSnapshotMetrics( - taskContext, queue, metadataProvider); - this.streamingChangeEventSourceMetrics = - changeEventSourceMetricsFactory.getStreamingMetrics( - taskContext, queue, metadataProvider); - this.errorHandler = - new MySqlErrorHandler(connectorConfig, queue, taskContext, sourceConfig); + // Debezium 2.0 changed io.debezium.config.Instantiator to resolve classes through the + // thread context class loader. On a Flink task thread that loader can be a user code + // class loader left over from a previous job attempt, which is already closed, so + // Debezium fails with "Trying to access closed classloader" while building its own + // objects. Pin the context class loader to the one that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(CommonConnectorConfig.class.getClassLoader())) { + // initial stateful objects + final boolean tableIdCaseInsensitive = connection.isTableIdCaseSensitive(); + this.topicNamingStrategy = + connectorConfig.getTopicNamingStrategy( + MySqlConnectorConfig.TOPIC_NAMING_STRATEGY); + EmbeddedFlinkDatabaseHistory.registerHistory( + sourceConfig + .getDbzConfiguration() + .getString(EmbeddedFlinkDatabaseHistory.DATABASE_HISTORY_INSTANCE_NAME), + mySqlSplit.getTableSchemas().values()); + + Optional.ofNullable(databaseSchema).ifPresent(MySqlDatabaseSchema::close); + this.databaseSchema = + DebeziumUtils.createMySqlDatabaseSchema( + connectorConfig, tableIdCaseInsensitive); + + this.mySqlPartition = + new MySqlPartition( + connectorConfig.getLogicalName(), + connectorConfig + .getConfig() + .getString( + io.debezium.relational.RelationalDatabaseConnectorConfig + .DATABASE_NAME)); + + this.offsetContext = + loadStartingOffsetState( + new MySqlOffsetContext.Loader(connectorConfig), mySqlSplit); + validateAndLoadDatabaseHistory(offsetContext, databaseSchema); + + this.taskContext = + new MySqlTaskContextImpl(connectorConfig, databaseSchema, binaryLogClient); + + final int queueSize = + mySqlSplit.isSnapshotSplit() + ? sourceConfig.getSplitSize() + + DEFAULT_BINLOG_QUEUE_SIZE_IN_SNAPSHOT_SCAN + : connectorConfig.getMaxQueueSize(); + this.queue = + new ChangeEventQueue.Builder() + .pollInterval(connectorConfig.getPollInterval()) + .maxBatchSize(connectorConfig.getMaxBatchSize()) + .maxQueueSize(queueSize) + .maxQueueSizeInBytes(connectorConfig.getMaxQueueSizeInBytes()) + .loggingContextSupplier( + () -> + taskContext.configureLoggingContext( + "mysql-cdc-connector-task")) + // do not buffer any element, we use signal event + // .buffering() + .build(); + this.dispatcher = + new EventDispatcherImpl<>( + connectorConfig, + topicNamingStrategy, + databaseSchema, + queue, + connectorConfig.getTableFilters().dataCollectionFilter(), + DataChangeEvent::new, + metadataProvider, + schemaNameAdjuster); + + this.snapshotReceiver = dispatcher.getSnapshotChangeEventReceiver(); + + this.signalEventDispatcher = + new SignalEventDispatcher( + offsetContext.getOffset(), connectorConfig.getLogicalName(), queue); + + final MySqlChangeEventSourceMetricsFactory changeEventSourceMetricsFactory = + new MySqlChangeEventSourceMetricsFactory( + new MySqlStreamingChangeEventSourceMetrics( + taskContext, queue, metadataProvider)); + this.snapshotChangeEventSourceMetrics = + changeEventSourceMetricsFactory.getSnapshotMetrics( + taskContext, queue, metadataProvider); + this.streamingChangeEventSourceMetrics = + changeEventSourceMetricsFactory.getStreamingMetrics( + taskContext, queue, metadataProvider); + this.errorHandler = + new MySqlErrorHandler(connectorConfig, queue, taskContext, sourceConfig); + } } private void validateAndLoadDatabaseHistory( @@ -227,7 +249,10 @@ private boolean checkGtidSet(MySqlOffsetContext offset) { return true; // start at beginning ... } - String availableGtidStr = connection.knownGtidSet(); + // Debezium 2.5 changed knownGtidSet() to return a GtidSet rather than its string form. + GtidSet availableGtidSetFromServer = connection.knownGtidSet(); + String availableGtidStr = + availableGtidSetFromServer == null ? null : availableGtidSetFromServer.toString(); if (availableGtidStr == null || availableGtidStr.trim().isEmpty()) { // Last offsets had GTIDs but the server does not use them ... LOG.warn( @@ -236,7 +261,7 @@ private boolean checkGtidSet(MySqlOffsetContext offset) { } // Get the GTID set that is available in the server ... - GtidSet availableGtidSet = new GtidSet(availableGtidStr); + MySqlGtidSet availableGtidSet = new MySqlGtidSet(availableGtidStr); // GTIDs are enabled LOG.info("Merging server GTID set {} with restored GTID set {}", availableGtidSet, gtidStr); @@ -246,7 +271,7 @@ private boolean checkGtidSet(MySqlOffsetContext offset) { // the GTID. This is done to address the issue of being unable to recover from a checkpoint // in certain startup // modes. - GtidSet gtidSet = GtidUtils.fixRestoredGtidSet(availableGtidSet, new GtidSet(gtidStr)); + GtidSet gtidSet = GtidUtils.fixRestoredGtidSet(availableGtidSet, new MySqlGtidSet(gtidStr)); LOG.info("Merged GTID set is {}", gtidSet); if (gtidSet.isContainedWithin(availableGtidSet)) { @@ -428,8 +453,8 @@ public MySqlPartition getMySqlPartition() { return mySqlPartition; } - public TopicSelector getTopicSelector() { - return topicSelector; + public TopicNamingStrategy getTopicNamingStrategy() { + return topicNamingStrategy; } public SnapshotChangeEventSourceMetrics getSnapshotChangeEventSourceMetrics() { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/assigners/MySqlSnapshotSplitAssigner.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/assigners/MySqlSnapshotSplitAssigner.java index f51df018d4d..36a1296886c 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/assigners/MySqlSnapshotSplitAssigner.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/assigners/MySqlSnapshotSplitAssigner.java @@ -179,7 +179,14 @@ private MySqlSnapshotSplitAssigner( this.chunkSplitter = createChunkSplitter(sourceConfig, isTableIdCaseSensitive, chunkSplitterState); this.partition = - new MySqlPartition(sourceConfig.getMySqlConnectorConfig().getLogicalName()); + new MySqlPartition( + sourceConfig.getMySqlConnectorConfig().getLogicalName(), + sourceConfig + .getMySqlConnectorConfig() + .getConfig() + .getString( + io.debezium.relational.RelationalDatabaseConnectorConfig + .DATABASE_NAME)); this.enumeratorContext = enumeratorContext; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfig.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfig.java index 6b87dfef9a7..5499bae66fd 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfig.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfig.java @@ -21,6 +21,7 @@ import org.apache.flink.cdc.connectors.mysql.source.MySqlSource; import org.apache.flink.cdc.connectors.mysql.table.StartupOptions; import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.util.TemporaryClassLoaderContext; import io.debezium.config.Configuration; import io.debezium.connector.mysql.MySqlConnectorConfig; @@ -139,7 +140,13 @@ public class MySqlSourceConfig implements Serializable { this.closeIdleReaders = closeIdleReaders; this.dbzProperties = checkNotNull(dbzProperties); this.dbzConfiguration = Configuration.from(dbzProperties); - this.dbzMySqlConfig = new MySqlConnectorConfig(dbzConfiguration); + // Debezium 2.0 changed io.debezium.config.Instantiator to resolve classes through the + // thread context class loader instead of its own. Unlike the other connectors, this + // config is built eagerly here, which happens during job planning (e.g. in the SQL + // client) where the context class loader does not contain the shaded connector jar, so + // Debezium fails to find its own MySqlSourceInfoStructMaker. Pin the context class + // loader to the one that loaded Debezium for the duration of the call. + this.dbzMySqlConfig = createConnectorConfig(dbzConfiguration); Selectors excludeTableFilter = (excludeTableList == null ? null @@ -268,6 +275,17 @@ public Configuration getDbzConfiguration() { return dbzConfiguration; } + private static MySqlConnectorConfig createConnectorConfig(Configuration dbzConfiguration) { + // Debezium 2.0 resolves classes through the thread context class loader + // (io.debezium.config.Instantiator, and the ServiceLoader based SPIs). On a Flink + // thread that loader may be a user code class loader from a previous job attempt, + // which is already closed. Pin it to the loader that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(MySqlConnectorConfig.class.getClassLoader())) { + return new MySqlConnectorConfig(dbzConfiguration); + } + } + public MySqlConnectorConfig getMySqlConnectorConfig() { return dbzMySqlConfig; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfigFactory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfigFactory.java index 569b62232db..977ed18959f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfigFactory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfigFactory.java @@ -356,7 +356,9 @@ public MySqlSourceConfig createConfig(int subtaskId) { public MySqlSourceConfig createConfig(int subtaskId, String serverName) { checkSupportCheckpointsAfterTasksFinished(closeIdleReaders); Properties props = new Properties(); - props.setProperty("database.server.name", serverName); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix"; getLogicalName() returns + // this value, keeping the {server=} offset partition key unchanged. + props.setProperty("topic.prefix", serverName); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); @@ -366,12 +368,17 @@ public MySqlSourceConfig createConfig(int subtaskId, String serverName) { props.setProperty("database.serverTimezone", serverTimeZone); // database history props.setProperty( - "database.history", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); + "schema.history.internal", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); props.setProperty( - "database.history.instance.name", UUID.randomUUID().toString() + "_" + subtaskId); - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); - props.setProperty("database.history.refer.ddl", String.valueOf(true)); + "schema.history.internal.instance.name", + UUID.randomUUID().toString() + "_" + subtaskId); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.prefer.ddl", String.valueOf(true)); props.setProperty("connect.timeout.ms", String.valueOf(connectTimeout.toMillis())); + // Debezium 2.3 changed the default of "database.ssl.mode" from "disabled" to "preferred". + // Keep the historical Flink CDC behavior (plain connections unless the user asks for TLS); + // user-supplied debezium properties below still override it. + props.setProperty("database.ssl.mode", "disabled"); // the underlying debezium reader should always capture the schema changes and forward them. // Note: the includeSchemaChanges parameter is used to control emitting the schema record, // only DataStream API program need to emit the schema record, the Table API need not @@ -393,6 +400,14 @@ public MySqlSourceConfig createConfig(int subtaskId, String serverName) { if (serverIdRange != null) { int serverId = serverIdRange.getServerId(subtaskId); props.setProperty("database.server.id", String.valueOf(serverId)); + } else { + // Debezium 2.0 made "database.server.id" a required option (1.9 assigned a random + // default in the 5400-6400 range). Preserve that behavior when the user does not + // configure a server id, offsetting by the subtask id to avoid collisions between + // parallel binlog clients. + props.setProperty( + "database.server.id", + String.valueOf(5400 + subtaskId + new java.util.Random().nextInt(1000))); } if (databaseList != null) { props.setProperty("database.include.list", String.join(",", databaseList)); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffset.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffset.java index a7f86b62ea3..66152d9fa57 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffset.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffset.java @@ -21,7 +21,8 @@ import org.apache.flink.cdc.common.annotation.PublicEvolving; import org.apache.flink.cdc.common.annotation.VisibleForTesting; -import io.debezium.connector.mysql.GtidSet; +import io.debezium.connector.binlog.gtid.GtidSet; +import io.debezium.connector.mysql.gtid.MySqlGtidSet; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.connect.errors.ConnectException; @@ -188,8 +189,8 @@ public int compareTo(BinlogOffset that) { // The target offset uses GTIDs, so we ideally compare using GTIDs ... if (StringUtils.isNotEmpty(gtidSetStr)) { // Both have GTIDs, so base the comparison entirely on the GTID sets. - GtidSet gtidSet = new GtidSet(gtidSetStr); - GtidSet targetGtidSet = new GtidSet(targetGtidSetStr); + GtidSet gtidSet = new MySqlGtidSet(gtidSetStr); + GtidSet targetGtidSet = new MySqlGtidSet(targetGtidSetStr); if (gtidSet.equals(targetGtidSet)) { long restartSkipEvents = this.getRestartSkipEvents(); long targetRestartSkipEvents = that.getRestartSkipEvents(); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffsetUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffsetUtils.java index 3c192eaa779..6a718c77ecc 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffsetUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffsetUtils.java @@ -21,7 +21,7 @@ import org.apache.flink.cdc.connectors.mysql.debezium.task.context.StatefulTaskContext; import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; /** Utils for handling {@link BinlogOffset}. */ public class BinlogOffsetUtils { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlSourceReader.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlSourceReader.java index b3f421ceecc..ed2bdfd4e44 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlSourceReader.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlSourceReader.java @@ -48,8 +48,8 @@ import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.Preconditions; -import io.debezium.connector.mysql.MySqlConnection; import io.debezium.connector.mysql.MySqlPartition; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.relational.TableId; import io.debezium.relational.history.TableChanges; import org.slf4j.Logger; @@ -100,7 +100,14 @@ public MySqlSourceReader( this.mySqlSourceReaderContext = context; this.suspendedBinlogSplit = null; this.partition = - new MySqlPartition(sourceConfig.getMySqlConnectorConfig().getLogicalName()); + new MySqlPartition( + sourceConfig.getMySqlConnectorConfig().getLogicalName(), + sourceConfig + .getMySqlConnectorConfig() + .getConfig() + .getString( + io.debezium.relational.RelationalDatabaseConnectorConfig + .DATABASE_NAME)); } @Override diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/OnlineSchemaChangeUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/OnlineSchemaChangeUtils.java index 8bd0f1a5e82..660171ead5d 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/OnlineSchemaChangeUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/OnlineSchemaChangeUtils.java @@ -28,10 +28,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -55,10 +51,17 @@ private OnlineSchemaChangeUtils() {} private static final Pattern OSC_TABLE_ID_PATTERN = Pattern.compile("^_(.*)_(gho|new)$"); /** - * Pattern matching gh-ost delete table ({@code __del}) and pt-osc old table ({@code - * __old}), which are the temporary backup tables created during an online schema change. + * Pattern matching the {@code RENAME TABLE} statement both tools use to swap the shadow table + * in, tolerating the {@code /* gh-ost */} marker comment gh-ost inserts. */ - private static final Pattern OSC_TEMP_TABLE_ID_PATTERN = Pattern.compile("^_(.*)_(del|old)$"); + private static final Pattern RENAME_STATEMENT_PATTERN = + Pattern.compile( + "^rename\\s+(?:/\\*.*?\\*/\\s*)?table\\s+(.+)$", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + /** Pattern matching a single {@code TO } pair inside a rename statement. */ + private static final Pattern RENAME_PAIR_PATTERN = + Pattern.compile("^(\\S+)\\s+to\\s+(\\S+)$", Pattern.CASE_INSENSITIVE); /** * Checks whether the given source record is a gh-ost/pt-osc initiated schema change event by @@ -144,46 +147,35 @@ public static Optional parseOnLineSchemaRenameEvent(SourceRecord record) OBJECT_MAPPER .readTree(value.getString(HISTORY_RECORD_FIELD)) .get(HistoryRecord.Fields.DDL_STATEMENTS) - .asText() - .toLowerCase(); - if (ddl.startsWith("rename table") || ddl.startsWith("rename /* gh-ost */ table")) { - LOG.info("Checking if DDL might be an OSC renaming event... {}", ddl); - List tableNames = - Arrays.asList( - value.getStruct(Envelope.FieldName.SOURCE) - .getString(TABLE_NAME_KEY) - .split(",")); - if (tableNames.size() != 2) { - LOG.info( - "Table name {} is malformed, skip it.", - value.getStruct(Envelope.FieldName.SOURCE).getString(TABLE_NAME_KEY)); - return Optional.empty(); + .asText(); + Matcher statement = RENAME_STATEMENT_PATTERN.matcher(ddl.trim()); + if (!statement.matches()) { + return Optional.empty(); + } + LOG.info("Checking if DDL might be an OSC renaming event... {}", ddl); + // Both tools finish with an atomic + // "RENAME TABLE tbl TO _tbl_del, _tbl_gho TO tbl". Debezium reports that as one + // schema change event per renamed table, so "source.table" carries a single name + // and the pair has to be read back out of the DDL text instead. Keying off the + // shadow table being promoted (_tbl_gho -> tbl) also means this fires exactly + // once, at the point the altered schema becomes the live one. + for (String renaming : statement.group(1).split(",")) { + Matcher pair = RENAME_PAIR_PATTERN.matcher(renaming.trim()); + if (!pair.matches()) { + continue; } - - String renamedFromTableName = - Collections.min(tableNames, Comparator.comparingInt(String::length)); - String renamedToTableName = - Collections.max(tableNames, Comparator.comparingInt(String::length)); - - LOG.info( - "Determined the shorter TableId {} is the renaming source.", - renamedFromTableName); - LOG.info( - "Determined the longer TableId {} is the renaming target.", - renamedToTableName); - - if (OSC_TEMP_TABLE_ID_PATTERN.matcher(renamedToTableName).matches()) { + String renamedFrom = tableName(pair.group(1)); + String renamedTo = tableName(pair.group(2)); + Matcher shadowTable = OSC_TABLE_ID_PATTERN.matcher(renamedFrom); + if (shadowTable.matches() && shadowTable.group(1).equals(renamedTo)) { LOG.info( - "Renamed to TableId name {} matches OSC temporary TableId pattern, yield {}.", - renamedToTableName, - renamedFromTableName); - return Optional.of(renamedFromTableName); + "Shadow table {} was promoted to {}, online schema change is complete.", + renamedFrom, + renamedTo); + return Optional.of(renamedTo); } - - LOG.info( - "Renamed to TableId {} does not match any RegEx pattern, skip it.", - renamedToTableName); } + LOG.info("DDL {} is not an OSC completion renaming, skip it.", ddl); return Optional.empty(); } catch (JsonProcessingException e) { LOG.warn("Failed to parse schema change event {}", value, e); @@ -191,6 +183,13 @@ public static Optional parseOnLineSchemaRenameEvent(SourceRecord record) } } + /** Strips the optional database qualifier and back quotes from a table reference. */ + private static String tableName(String tableReference) { + String unquoted = tableReference.replace("`", "").trim(); + int separator = unquoted.lastIndexOf('.'); + return separator < 0 ? unquoted : unquoted.substring(separator + 1); + } + /** * Peels out a gh-ost/pt-osc mangled {@link TableId} back to the original user-facing one. * diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/RecordUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/RecordUtils.java index 1f9a041d78f..c4f463f88c2 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/RecordUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/RecordUtils.java @@ -30,7 +30,7 @@ import io.debezium.document.DocumentReader; import io.debezium.relational.TableId; import io.debezium.relational.history.HistoryRecord; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/SplitKeyUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/SplitKeyUtils.java index d19de5df573..65efe44c057 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/SplitKeyUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/SplitKeyUtils.java @@ -20,7 +20,7 @@ import org.apache.flink.cdc.connectors.mysql.source.split.FinishedSnapshotSplitInfo; import org.apache.flink.table.types.logical.RowType; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; import org.apache.kafka.connect.data.Struct; import java.math.BigDecimal; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/TableDiscoveryUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/TableDiscoveryUtils.java index 725200531fa..7c586b87f9e 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/TableDiscoveryUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/TableDiscoveryUtils.java @@ -21,8 +21,8 @@ import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig; import org.apache.flink.util.FlinkRuntimeException; -import io.debezium.connector.mysql.MySqlConnection; import io.debezium.connector.mysql.MySqlPartition; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.TableId; import io.debezium.relational.history.TableChanges.TableChange; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/hooks/SnapshotPhaseHook.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/hooks/SnapshotPhaseHook.java index 44eab2e8efe..78e4e621046 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/hooks/SnapshotPhaseHook.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/utils/hooks/SnapshotPhaseHook.java @@ -21,7 +21,7 @@ import org.apache.flink.api.connector.source.SourceSplit; import org.apache.flink.util.function.BiConsumerWithException; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import java.io.Serializable; import java.sql.SQLException; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/FilterGtidSetTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/FilterGtidSetTest.java deleted file mode 100644 index d50f2eea05b..00000000000 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/FilterGtidSetTest.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.debezium.connector.mysql; - -import io.debezium.connector.mysql.MySqlConnectorConfig.GtidNewChannelPosition; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.lang.reflect.Field; -import java.util.function.Predicate; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -/** - * Integration test for {@link MySqlStreamingChangeEventSource#filterGtidSet} to ensure the LATEST - * mode fix (FLINK-39149) cannot regress. - */ -class FilterGtidSetTest { - - @Test - void testFilterGtidSetLatestModeFixesNonContiguousGtid() throws Exception { - MySqlStreamingChangeEventSource source = - createSourceWithConfig(GtidNewChannelPosition.LATEST, null); - - MySqlOffsetContext offsetContext = createOffsetContext("aaa-111:5000-8000"); - GtidSet availableServerGtidSet = new GtidSet("aaa-111:1-10000,bbb-222:1-3000"); - GtidSet purgedServerGtid = new GtidSet(""); - - GtidSet result = - source.filterGtidSet(offsetContext, availableServerGtidSet, purgedServerGtid); - - assertThat(result.toString()).contains("aaa-111:1-8000"); - assertThat(result.toString()).contains("bbb-222:1-3000"); - } - - @Test - void testFilterGtidSetLatestModeWithSourceFilter() throws Exception { - Predicate excludeCcc = uuid -> !uuid.equals("ccc-333"); - MySqlStreamingChangeEventSource source = - createSourceWithConfig(GtidNewChannelPosition.LATEST, excludeCcc); - - MySqlOffsetContext offsetContext = createOffsetContext("aaa-111:5000-8000,bbb-222:1-2000"); - GtidSet availableServerGtidSet = - new GtidSet("aaa-111:1-10000,bbb-222:1-3000,ccc-333:1-5000"); - GtidSet purgedServerGtid = new GtidSet(""); - - GtidSet result = - source.filterGtidSet(offsetContext, availableServerGtidSet, purgedServerGtid); - - assertThat(result.toString()).contains("aaa-111:1-8000"); - assertThat(result.toString()).contains("bbb-222:1-2000"); - assertThat(result.toString()).doesNotContain("ccc-333"); - } - - @Test - void testFilterGtidSetEarliestModeNotAffected() throws Exception { - MySqlStreamingChangeEventSource source = - createSourceWithConfig(GtidNewChannelPosition.EARLIEST, null); - - MySqlOffsetContext offsetContext = createOffsetContext("aaa-111:5000-8000"); - GtidSet availableServerGtidSet = new GtidSet("aaa-111:1-10000,bbb-222:1-3000"); - GtidSet purgedServerGtid = new GtidSet(""); - - GtidSet result = - source.filterGtidSet(offsetContext, availableServerGtidSet, purgedServerGtid); - - assertThat(result.toString()).contains("aaa-111:1-8000"); - assertThat(result.forServerWithId("bbb-222")).isNull(); - } - - @Test - void testFilterGtidSetReturnsNullWhenNoGtid() throws Exception { - MySqlStreamingChangeEventSource source = - createSourceWithConfig(GtidNewChannelPosition.LATEST, null); - - MySqlOffsetContext offsetContext = createOffsetContext(null); - GtidSet availableServerGtidSet = new GtidSet("aaa-111:1-10000"); - GtidSet purgedServerGtid = new GtidSet(""); - - GtidSet result = - source.filterGtidSet(offsetContext, availableServerGtidSet, purgedServerGtid); - - assertThat(result).isNull(); - } - - private static MySqlStreamingChangeEventSource createSourceWithConfig( - GtidNewChannelPosition channelPosition, Predicate gtidSourceFilter) - throws Exception { - MySqlConnectorConfig mockConfig = Mockito.mock(MySqlConnectorConfig.class); - when(mockConfig.gtidNewChannelPosition()).thenReturn(channelPosition); - when(mockConfig.gtidSourceFilter()).thenReturn(gtidSourceFilter); - - MySqlStreamingChangeEventSource source = - Mockito.mock(MySqlStreamingChangeEventSource.class, Mockito.CALLS_REAL_METHODS); - - Field configField = - MySqlStreamingChangeEventSource.class.getDeclaredField("connectorConfig"); - configField.setAccessible(true); - configField.set(source, mockConfig); - - return source; - } - - private static MySqlOffsetContext createOffsetContext(String gtidSetStr) { - MySqlOffsetContext offsetContext = Mockito.mock(MySqlOffsetContext.class); - when(offsetContext.gtidSet()).thenReturn(gtidSetStr); - return offsetContext; - } -} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/GtidUtilsTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/GtidUtilsTest.java deleted file mode 100644 index f0dbe6cd46f..00000000000 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/GtidUtilsTest.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.debezium.connector.mysql; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import java.util.function.Predicate; -import java.util.stream.Stream; - -import static io.debezium.connector.mysql.GtidUtils.computeLatestModeGtidSet; -import static io.debezium.connector.mysql.GtidUtils.fixRestoredGtidSet; -import static io.debezium.connector.mysql.GtidUtils.mergeGtidSetInto; -import static org.assertj.core.api.Assertions.assertThat; - -/** Unit test for {@link GtidUtils}. */ -class GtidUtilsTest { - - @ParameterizedTest(name = "{0}") - @MethodSource("gtidSetsProvider") - void testFixingRestoredGtidSet( - String description, String serverStr, String restoredStr, String expectedStr) { - GtidSet serverGtidSet = new GtidSet(serverStr); - GtidSet restoredGtidSet = new GtidSet(restoredStr); - - GtidSet result = fixRestoredGtidSet(serverGtidSet, restoredGtidSet); - - assertThat(result).hasToString(expectedStr); - } - - private static Stream gtidSetsProvider() { - return Stream.of( - Arguments.of( - "Basic example with a straightforward subset", - "A:1-100", - "A:1-50:63-100", - "A:1-50:63-100"), - Arguments.of( - "Multiple intervals with gaps in restored", - "A:1-100", - "A:45-80:83-90:92-98", - "A:1-80:83-90:92-98"), - Arguments.of( - "Server has disjoint intervals, restored partially overlaps", - "A:1-50:60-90:95-200", - "A:45-50:65-70:96-100", - "A:1-50:65-70:96-100"), - Arguments.of( - "Restored partially covers server range", - "A:1-100:102-200", - "A:106-150:152-200", - "A:1-100:102-150:152-200"), - Arguments.of( - "Restored end exceeds server range", - "A:1-100,B:1-200:205-300", - "A:1-110,B:1-201:210-230:245-305", - "A:1-100,B:1-200:210-230:245-300"), - Arguments.of( - "Multiple UUIDs with different overlaps", - "A:1-100,B:1-50", - "A:45-80,B:30-60,C:1-20", - "A:1-80,B:1-50,C:1-20"), - Arguments.of("Restored starts after server ends", "A:1-100", "A:80-150", "A:1-100"), - Arguments.of( - "Complex overlapping intervals", - "A:1-20:30-50:60-80", - "A:15-35:45-65:75-85", - "A:1-20:30-35:45-50:60-65:75-80")); - } - - /** Tests {@link GtidUtils#computeLatestModeGtidSet} for FLINK-39149. */ - @ParameterizedTest(name = "{0}") - @MethodSource("latestModeGtidSetsProvider") - void testLatestModeGtidMerge( - String description, - String serverGtidStr, - String checkpointGtidStr, - String expectedMergedStr) { - GtidSet serverGtidSet = new GtidSet(serverGtidStr); - GtidSet checkpointGtidSet = new GtidSet(checkpointGtidStr); - - GtidSet mergedGtidSet = - computeLatestModeGtidSet(serverGtidSet, new GtidSet(""), checkpointGtidSet, null); - - assertThat(mergedGtidSet).hasToString(expectedMergedStr); - - // Verify MySQL would not replay pre-checkpoint transactions - GtidSet transactionsToSend = serverGtidSet.subtract(mergedGtidSet); - for (GtidSet.UUIDSet uuidSet : checkpointGtidSet.getUUIDSets()) { - String uuid = uuidSet.getUUID(); - long earliestCheckpointTx = - uuidSet.getIntervals().stream() - .mapToLong(GtidSet.Interval::getStart) - .min() - .orElse(1); - if (earliestCheckpointTx > 1) { - GtidSet.UUIDSet toSendUuidSet = transactionsToSend.forServerWithId(uuid); - if (toSendUuidSet != null) { - for (GtidSet.Interval interval : toSendUuidSet.getIntervals()) { - assertThat(interval.getStart()) - .as( - "Should not replay pre-checkpoint transactions for UUID %s", - uuid) - .isGreaterThan(earliestCheckpointTx); - } - } - } - } - } - - private static Stream latestModeGtidSetsProvider() { - return Stream.of( - Arguments.of( - "Old channel with non-contiguous GTID, new channel present", - "aaa-111:1-10000,bbb-222:1-3000", - "aaa-111:5000-8000", - "aaa-111:1-8000,bbb-222:1-3000"), - Arguments.of( - "Mixed old channels (contiguous and non-contiguous) with new channel", - "aaa-111:1-10000,bbb-222:1-3000,ccc-333:1-5000", - "aaa-111:5000-8000,bbb-222:1-2000", - "aaa-111:1-8000,bbb-222:1-2000,ccc-333:1-5000"), - Arguments.of( - "All old channels, no new channels", - "aaa-111:1-10000,bbb-222:1-3000", - "aaa-111:1-8000,bbb-222:1-2000", - "aaa-111:1-8000,bbb-222:1-2000"), - Arguments.of( - "Contiguous checkpoint GTID, no regression", - "aaa-111:1-10000,bbb-222:1-3000", - "aaa-111:1-8000", - "aaa-111:1-8000,bbb-222:1-3000"), - Arguments.of( - "Only new channels, checkpoint has unknown UUID", - "aaa-111:1-10000,bbb-222:1-3000", - "xxx-999:1-500", - "aaa-111:1-10000,bbb-222:1-3000,xxx-999:1-500")); - } - - @Test - void testMergingGtidSets() { - GtidSet base = new GtidSet("A:1-100"); - GtidSet toMerge = new GtidSet("A:1-10"); - assertThat(mergeGtidSetInto(base, toMerge)).hasToString("A:1-100"); - - base = new GtidSet("A:1-100"); - toMerge = new GtidSet("C:1-10"); - assertThat(mergeGtidSetInto(base, toMerge)).hasToString("A:1-100,C:1-10"); - base = new GtidSet("A:1-100,B:1-100"); - toMerge = new GtidSet("A:1-10,C:1-10"); - assertThat(mergeGtidSetInto(base, toMerge)).hasToString("A:1-100,B:1-100,C:1-10"); - } - - /** Tests {@link GtidUtils#computeLatestModeGtidSet} with {@code gtidSourceFilter}. */ - @Test - void testLatestModeGtidMergeWithSourceFilter() { - GtidSet availableServerGtidSet = - new GtidSet("aaa-111:1-10000,bbb-222:1-3000,ccc-333:1-5000"); - GtidSet checkpointGtidSet = new GtidSet("aaa-111:5000-8000,bbb-222:1-2000"); - Predicate gtidSourceFilter = uuid -> !uuid.equals("ccc-333"); - - GtidSet mergedGtidSet = - computeLatestModeGtidSet( - availableServerGtidSet, - new GtidSet(""), - checkpointGtidSet, - gtidSourceFilter); - - assertThat(mergedGtidSet.toString()).contains("aaa-111:1-8000"); - assertThat(mergedGtidSet.toString()).contains("bbb-222:1-2000"); - assertThat(mergedGtidSet.toString()).doesNotContain("ccc-333"); - } - - /** Tests {@link GtidUtils#computeLatestModeGtidSet} with purged GTID. */ - @Test - void testLatestModeGtidMergeWithPurgedGtid() { - GtidSet availableServerGtidSet = new GtidSet("aaa-111:50-10000,bbb-222:1-3000"); - GtidSet purgedServerGtid = new GtidSet("aaa-111:1-49"); - GtidSet checkpointGtidSet = new GtidSet("aaa-111:5000-8000"); - - GtidSet mergedGtidSet = - computeLatestModeGtidSet( - availableServerGtidSet, purgedServerGtid, checkpointGtidSet, null); - - assertThat(mergedGtidSet.toString()).contains("aaa-111:50-8000"); - assertThat(mergedGtidSet.toString()).contains("bbb-222:1-3000"); - - // Verify no pre-checkpoint replay - GtidSet transactionsToSend = availableServerGtidSet.subtract(mergedGtidSet); - GtidSet.UUIDSet aaaToSend = transactionsToSend.forServerWithId("aaa-111"); - if (aaaToSend != null) { - for (GtidSet.Interval interval : aaaToSend.getIntervals()) { - assertThat(interval.getStart()) - .as("Should not request pre-checkpoint transactions") - .isGreaterThanOrEqualTo(8001); - } - } - } - - /** Tests {@link GtidUtils#computeLatestModeGtidSet} with a completely purged UUID. */ - @Test - void testLatestModeGtidMergeWithFullyPurgedChannel() { - GtidSet availableServerGtidSet = new GtidSet("bbb-222:1-3000"); - GtidSet purgedServerGtid = new GtidSet("aaa-111:1-500"); - GtidSet checkpointGtidSet = new GtidSet("aaa-111:200-400"); - - GtidSet mergedGtidSet = - computeLatestModeGtidSet( - availableServerGtidSet, purgedServerGtid, checkpointGtidSet, null); - - assertThat(mergedGtidSet.toString()).contains("aaa-111:1-400"); - assertThat(mergedGtidSet.toString()).contains("bbb-222:1-3000"); - } -} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/gtid/GtidUtilsTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/gtid/GtidUtilsTest.java new file mode 100644 index 00000000000..e848fbeadc7 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/gtid/GtidUtilsTest.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.debezium.connector.mysql.gtid; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.function.Predicate; +import java.util.stream.Stream; + +import static io.debezium.connector.mysql.gtid.GtidUtils.computeLatestModeGtidSet; +import static io.debezium.connector.mysql.gtid.GtidUtils.fixRestoredGtidSet; +import static io.debezium.connector.mysql.gtid.GtidUtils.mergeGtidSetInto; +import static org.assertj.core.api.Assertions.assertThat; + +/** Unit test for {@link GtidUtils}. */ +class GtidUtilsTest { + + @ParameterizedTest(name = "{0}") + @MethodSource("gtidSetsProvider") + void testFixingRestoredGtidSet( + String description, String serverStr, String restoredStr, String expectedStr) { + MySqlGtidSet serverGtidSet = new MySqlGtidSet(serverStr); + MySqlGtidSet restoredGtidSet = new MySqlGtidSet(restoredStr); + + MySqlGtidSet result = fixRestoredGtidSet(serverGtidSet, restoredGtidSet); + + assertThat(result).hasToString(expectedStr); + } + + private static Stream gtidSetsProvider() { + return Stream.of( + Arguments.of( + "Basic example with a straightforward subset", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-50:63-100", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-50:63-100"), + Arguments.of( + "Multiple intervals with gaps in restored", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100", + "24bc7850-2c16-11e6-a073-0242ac11000a:45-80:83-90:92-98", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-80:83-90:92-98"), + Arguments.of( + "Server has disjoint intervals, restored partially overlaps", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-50:60-90:95-200", + "24bc7850-2c16-11e6-a073-0242ac11000a:45-50:65-70:96-100", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-50:65-70:96-100"), + Arguments.of( + "Restored partially covers server range", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100:102-200", + "24bc7850-2c16-11e6-a073-0242ac11000a:106-150:152-200", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100:102-150:152-200"), + Arguments.of( + "Restored end exceeds server range", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100,24bc7850-2c16-11e6-a073-0242ac11000b:1-200:205-300", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-110,24bc7850-2c16-11e6-a073-0242ac11000b:1-201:210-230:245-305", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100,24bc7850-2c16-11e6-a073-0242ac11000b:1-200:210-230:245-300"), + Arguments.of( + "Multiple UUIDs with different overlaps", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100,24bc7850-2c16-11e6-a073-0242ac11000b:1-50", + "24bc7850-2c16-11e6-a073-0242ac11000a:45-80,24bc7850-2c16-11e6-a073-0242ac11000b:30-60,24bc7850-2c16-11e6-a073-0242ac11000c:1-20", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-80,24bc7850-2c16-11e6-a073-0242ac11000b:1-50,24bc7850-2c16-11e6-a073-0242ac11000c:1-20"), + Arguments.of( + "Restored starts after server ends", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100", + "24bc7850-2c16-11e6-a073-0242ac11000a:80-150", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100"), + Arguments.of( + "Complex overlapping intervals", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-20:30-50:60-80", + "24bc7850-2c16-11e6-a073-0242ac11000a:15-35:45-65:75-85", + "24bc7850-2c16-11e6-a073-0242ac11000a:1-20:30-35:45-50:60-65:75-80")); + } + + /** Tests {@link GtidUtils#computeLatestModeGtidSet} for FLINK-39149. */ + @ParameterizedTest(name = "{0}") + @MethodSource("latestModeGtidSetsProvider") + void testLatestModeGtidMerge( + String description, + String serverGtidStr, + String checkpointGtidStr, + String expectedMergedStr) { + MySqlGtidSet serverGtidSet = new MySqlGtidSet(serverGtidStr); + MySqlGtidSet checkpointGtidSet = new MySqlGtidSet(checkpointGtidStr); + + MySqlGtidSet mergedGtidSet = + computeLatestModeGtidSet( + serverGtidSet, new MySqlGtidSet(""), checkpointGtidSet, null); + + assertThat(mergedGtidSet).hasToString(expectedMergedStr); + + // Verify MySQL would not replay pre-checkpoint transactions + MySqlGtidSet transactionsToSend = serverGtidSet.subtract(mergedGtidSet); + for (MySqlGtidSet.UUIDSet uuidSet : checkpointGtidSet.getUUIDSets()) { + String uuid = uuidSet.getUUID(); + long earliestCheckpointTx = + uuidSet.getIntervals().stream() + .mapToLong(MySqlGtidSet.Interval::getStart) + .min() + .orElse(1); + if (earliestCheckpointTx > 1) { + MySqlGtidSet.UUIDSet toSendUuidSet = transactionsToSend.forServerWithId(uuid); + if (toSendUuidSet != null) { + for (MySqlGtidSet.Interval interval : toSendUuidSet.getIntervals()) { + assertThat(interval.getStart()) + .as( + "Should not replay pre-checkpoint transactions for UUID %s", + uuid) + .isGreaterThan(earliestCheckpointTx); + } + } + } + } + } + + private static Stream latestModeGtidSetsProvider() { + return Stream.of( + Arguments.of( + "Old channel with non-contiguous GTID, new channel present", + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000", + "24bc7850-2c16-11e6-a073-0242ac110001:5000-8000", + "24bc7850-2c16-11e6-a073-0242ac110001:1-8000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000"), + Arguments.of( + "Mixed old channels (contiguous and non-contiguous) with new channel", + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000,24bc7850-2c16-11e6-a073-0242ac110003:1-5000", + "24bc7850-2c16-11e6-a073-0242ac110001:5000-8000,24bc7850-2c16-11e6-a073-0242ac110002:1-2000", + "24bc7850-2c16-11e6-a073-0242ac110001:1-8000,24bc7850-2c16-11e6-a073-0242ac110002:1-2000,24bc7850-2c16-11e6-a073-0242ac110003:1-5000"), + Arguments.of( + "All old channels, no new channels", + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000", + "24bc7850-2c16-11e6-a073-0242ac110001:1-8000,24bc7850-2c16-11e6-a073-0242ac110002:1-2000", + "24bc7850-2c16-11e6-a073-0242ac110001:1-8000,24bc7850-2c16-11e6-a073-0242ac110002:1-2000"), + Arguments.of( + "Contiguous checkpoint GTID, no regression", + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000", + "24bc7850-2c16-11e6-a073-0242ac110001:1-8000", + "24bc7850-2c16-11e6-a073-0242ac110001:1-8000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000"), + Arguments.of( + "Only new channels, checkpoint has unknown UUID", + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000", + "24bc7850-2c16-11e6-a073-0242ac110009:1-500", + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000,24bc7850-2c16-11e6-a073-0242ac110009:1-500")); + } + + @Test + void testMergingGtidSets() { + MySqlGtidSet base = new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac11000a:1-100"); + MySqlGtidSet toMerge = new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac11000a:1-10"); + assertThat(mergeGtidSetInto(base, toMerge)) + .hasToString("24bc7850-2c16-11e6-a073-0242ac11000a:1-100"); + + base = new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac11000a:1-100"); + toMerge = new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac11000c:1-10"); + assertThat(mergeGtidSetInto(base, toMerge)) + .hasToString( + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100,24bc7850-2c16-11e6-a073-0242ac11000c:1-10"); + base = + new MySqlGtidSet( + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100,24bc7850-2c16-11e6-a073-0242ac11000b:1-100"); + toMerge = + new MySqlGtidSet( + "24bc7850-2c16-11e6-a073-0242ac11000a:1-10,24bc7850-2c16-11e6-a073-0242ac11000c:1-10"); + assertThat(mergeGtidSetInto(base, toMerge)) + .hasToString( + "24bc7850-2c16-11e6-a073-0242ac11000a:1-100,24bc7850-2c16-11e6-a073-0242ac11000b:1-100,24bc7850-2c16-11e6-a073-0242ac11000c:1-10"); + } + + /** Tests {@link GtidUtils#computeLatestModeGtidSet} with {@code gtidSourceFilter}. */ + @Test + void testLatestModeGtidMergeWithSourceFilter() { + MySqlGtidSet availableServerGtidSet = + new MySqlGtidSet( + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000,24bc7850-2c16-11e6-a073-0242ac110003:1-5000"); + MySqlGtidSet checkpointGtidSet = + new MySqlGtidSet( + "24bc7850-2c16-11e6-a073-0242ac110001:5000-8000,24bc7850-2c16-11e6-a073-0242ac110002:1-2000"); + Predicate gtidSourceFilter = + uuid -> !uuid.equals("24bc7850-2c16-11e6-a073-0242ac110003"); + + MySqlGtidSet mergedGtidSet = + computeLatestModeGtidSet( + availableServerGtidSet, + new MySqlGtidSet(""), + checkpointGtidSet, + gtidSourceFilter); + + assertThat(mergedGtidSet.toString()) + .contains("24bc7850-2c16-11e6-a073-0242ac110001:1-8000"); + assertThat(mergedGtidSet.toString()) + .contains("24bc7850-2c16-11e6-a073-0242ac110002:1-2000"); + assertThat(mergedGtidSet.toString()).doesNotContain("24bc7850-2c16-11e6-a073-0242ac110003"); + } + + /** Tests {@link GtidUtils#computeLatestModeGtidSet} with purged GTID. */ + @Test + void testLatestModeGtidMergeWithPurgedGtid() { + MySqlGtidSet availableServerGtidSet = + new MySqlGtidSet( + "24bc7850-2c16-11e6-a073-0242ac110001:50-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000"); + MySqlGtidSet purgedServerGtid = + new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac110001:1-49"); + MySqlGtidSet checkpointGtidSet = + new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac110001:5000-8000"); + + MySqlGtidSet mergedGtidSet = + computeLatestModeGtidSet( + availableServerGtidSet, purgedServerGtid, checkpointGtidSet, null); + + assertThat(mergedGtidSet.toString()) + .contains("24bc7850-2c16-11e6-a073-0242ac110001:50-8000"); + assertThat(mergedGtidSet.toString()) + .contains("24bc7850-2c16-11e6-a073-0242ac110002:1-3000"); + + // Verify no pre-checkpoint replay + MySqlGtidSet transactionsToSend = availableServerGtidSet.subtract(mergedGtidSet); + MySqlGtidSet.UUIDSet aaaToSend = + transactionsToSend.forServerWithId("24bc7850-2c16-11e6-a073-0242ac110001"); + if (aaaToSend != null) { + for (MySqlGtidSet.Interval interval : aaaToSend.getIntervals()) { + assertThat(interval.getStart()) + .as("Should not request pre-checkpoint transactions") + .isGreaterThanOrEqualTo(8001); + } + } + } + + /** Tests {@link GtidUtils#computeLatestModeGtidSet} with a completely purged UUID. */ + @Test + void testLatestModeGtidMergeWithFullyPurgedChannel() { + MySqlGtidSet availableServerGtidSet = + new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac110002:1-3000"); + MySqlGtidSet purgedServerGtid = + new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac110001:1-500"); + MySqlGtidSet checkpointGtidSet = + new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac110001:200-400"); + + MySqlGtidSet mergedGtidSet = + computeLatestModeGtidSet( + availableServerGtidSet, purgedServerGtid, checkpointGtidSet, null); + + assertThat(mergedGtidSet.toString()).contains("24bc7850-2c16-11e6-a073-0242ac110001:1-400"); + assertThat(mergedGtidSet.toString()) + .contains("24bc7850-2c16-11e6-a073-0242ac110002:1-3000"); + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/jdbc/FilterGtidSetTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/jdbc/FilterGtidSetTest.java new file mode 100644 index 00000000000..83d9abc4317 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/io/debezium/connector/mysql/jdbc/FilterGtidSetTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.debezium.connector.mysql.jdbc; + +import io.debezium.config.Configuration; +import io.debezium.connector.binlog.gtid.GtidSet; +import io.debezium.connector.binlog.jdbc.BinlogConnectionConfiguration; +import io.debezium.connector.mysql.gtid.MySqlGtidSet; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.util.function.Predicate; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +/** + * Integration test for {@link MySqlConnection#filterGtidSet} to ensure the LATEST mode fix + * (FLINK-39149) cannot regress. + * + *

    Debezium 2.5 moved the GTID merge from the streaming source onto the connection, so the test + * drives {@link MySqlConnection} rather than {@code MySqlStreamingChangeEventSource}. + */ +class FilterGtidSetTest { + + @Test + void testFilterGtidSetLatestModeFixesNonContiguousGtid() throws Exception { + MySqlConnection connection = createConnectionWithConfig("latest"); + GtidSet availableServerGtidSet = + new MySqlGtidSet( + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000"); + GtidSet purgedServerGtid = new MySqlGtidSet(""); + + GtidSet result = + connection.filterGtidSet( + null, + "24bc7850-2c16-11e6-a073-0242ac110001:5000-8000", + availableServerGtidSet, + purgedServerGtid); + + assertThat(result.toString()).contains("24bc7850-2c16-11e6-a073-0242ac110001:1-8000"); + assertThat(result.toString()).contains("24bc7850-2c16-11e6-a073-0242ac110002:1-3000"); + } + + @Test + void testFilterGtidSetLatestModeWithSourceFilter() throws Exception { + Predicate excludeCcc = uuid -> !uuid.equals("24bc7850-2c16-11e6-a073-0242ac110003"); + MySqlConnection connection = createConnectionWithConfig("latest"); + GtidSet availableServerGtidSet = + new MySqlGtidSet( + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000,24bc7850-2c16-11e6-a073-0242ac110003:1-5000"); + GtidSet purgedServerGtid = new MySqlGtidSet(""); + + GtidSet result = + connection.filterGtidSet( + excludeCcc, + "24bc7850-2c16-11e6-a073-0242ac110001:5000-8000,24bc7850-2c16-11e6-a073-0242ac110002:1-2000", + availableServerGtidSet, + purgedServerGtid); + + assertThat(result.toString()).contains("24bc7850-2c16-11e6-a073-0242ac110001:1-8000"); + assertThat(result.toString()).contains("24bc7850-2c16-11e6-a073-0242ac110002:1-2000"); + assertThat(result.toString()).doesNotContain("24bc7850-2c16-11e6-a073-0242ac110003"); + } + + @Test + void testFilterGtidSetEarliestModeNotAffected() throws Exception { + MySqlConnection connection = createConnectionWithConfig("earliest"); + GtidSet availableServerGtidSet = + new MySqlGtidSet( + "24bc7850-2c16-11e6-a073-0242ac110001:1-10000,24bc7850-2c16-11e6-a073-0242ac110002:1-3000"); + GtidSet purgedServerGtid = new MySqlGtidSet(""); + + GtidSet result = + connection.filterGtidSet( + null, + "24bc7850-2c16-11e6-a073-0242ac110001:5000-8000", + availableServerGtidSet, + purgedServerGtid); + + assertThat(result.toString()).contains("24bc7850-2c16-11e6-a073-0242ac110001:1-8000"); + assertThat(((MySqlGtidSet) result).forServerWithId("24bc7850-2c16-11e6-a073-0242ac110002")) + .isNull(); + } + + @Test + void testFilterGtidSetReturnsNullWhenNoGtid() throws Exception { + MySqlConnection connection = createConnectionWithConfig("latest"); + GtidSet availableServerGtidSet = + new MySqlGtidSet("24bc7850-2c16-11e6-a073-0242ac110001:1-10000"); + GtidSet purgedServerGtid = new MySqlGtidSet(""); + + GtidSet result = + connection.filterGtidSet(null, null, availableServerGtidSet, purgedServerGtid); + + assertThat(result).isNull(); + } + + private static MySqlConnection createConnectionWithConfig(String channelPosition) + throws Exception { + Configuration dbzConfig = + Configuration.create().with("gtid.new.channel.position", channelPosition).build(); + + BinlogConnectionConfiguration mockConnectionConfig = + Mockito.mock(BinlogConnectionConfiguration.class); + when(mockConnectionConfig.originalConfig()).thenReturn(dbzConfig); + + MySqlConnection connection = + Mockito.mock(MySqlConnection.class, Mockito.CALLS_REAL_METHODS); + + Field configField = + Class.forName("io.debezium.connector.binlog.jdbc.BinlogConnectorConnection") + .getDeclaredField("connectionConfig"); + configField.setAccessible(true); + configField.set(connection, mockConnectionConfig); + + return connection; + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/LegacyMySqlSourceTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/LegacyMySqlSourceTest.java index e02c1cc2dff..20a9a5f8743 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/LegacyMySqlSourceTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/LegacyMySqlSourceTest.java @@ -839,7 +839,8 @@ void testChooseDatabase(boolean useLegacyImplementation) throws Exception { "test", "test", "CREATE TABLE test(a int)", - null) + null, + java.time.Instant.now()) .document(); historyState.add(writer.write(document)); } @@ -1328,6 +1329,16 @@ public void collect(T t) { private static class MockedTable implements Table { + @Override + public io.debezium.relational.Attribute attributeWithName(String name) { + return null; + } + + @Override + public java.util.List attributes() { + return java.util.Collections.emptyList(); + } + private static final Table INSTANCE = new MockedTable(); private MockedTable() {} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/DebeziumUtilsTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/DebeziumUtilsTest.java index eeae681ea7b..fb0dd4b1fbe 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/DebeziumUtilsTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/DebeziumUtilsTest.java @@ -21,7 +21,7 @@ import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfigFactory; import org.apache.flink.cdc.connectors.mysql.table.StartupOptions; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/BinlogSplitReaderTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/BinlogSplitReaderTest.java index e7fd2c4cbb9..1304d3d905f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/BinlogSplitReaderTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/BinlogSplitReaderTest.java @@ -49,10 +49,10 @@ import com.github.shyiko.mysql.binlog.event.EventHeaderV4; import com.github.shyiko.mysql.binlog.event.EventType; import com.github.shyiko.mysql.binlog.event.WriteRowsEventData; -import io.debezium.connector.mysql.MySqlConnection; import io.debezium.connector.mysql.MySqlConnectorConfig; import io.debezium.connector.mysql.MySqlOffsetContext; import io.debezium.connector.mysql.MySqlPartition; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.jdbc.JdbcConnection; import io.debezium.pipeline.spi.OffsetContext; import io.debezium.relational.TableId; @@ -1255,7 +1255,7 @@ private MySqlBinlogSplit createBinlogSplit(MySqlSourceConfig sourceConfig) throw Map tableSchemas = TableDiscoveryUtils.discoverSchemaForCapturedTables( new MySqlPartition( - sourceConfig.getMySqlConnectorConfig().getLogicalName()), + sourceConfig.getMySqlConnectorConfig().getLogicalName(), null), sourceConfig, jdbc); return MySqlBinlogSplit.fillTableSchemas( diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/SnapshotSplitReaderTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/SnapshotSplitReaderTest.java index 6598ed12674..1b123549f26 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/SnapshotSplitReaderTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/SnapshotSplitReaderTest.java @@ -33,7 +33,7 @@ import org.apache.flink.util.FlinkRuntimeException; import com.github.shyiko.mysql.binlog.BinaryLogClient; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.TableId; import org.apache.kafka.connect.source.SourceRecord; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlMultipleTablesRenamingITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlMultipleTablesRenamingITCase.java index 9297b1ff6d2..4cd45a01077 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlMultipleTablesRenamingITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlMultipleTablesRenamingITCase.java @@ -39,7 +39,7 @@ import org.apache.flink.streaming.api.operators.collect.CollectSinkOperatorFactory; import org.apache.flink.streaming.api.operators.collect.CollectStreamSink; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlSourceITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlSourceITCase.java index 62f51e49512..7198021bcf1 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlSourceITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlSourceITCase.java @@ -69,7 +69,7 @@ import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.Collector; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.jdbc.JdbcConnection; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.connect.source.SourceRecord; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/NewlyAddedTableITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/NewlyAddedTableITCase.java index 01bf8c1a550..861ac7b9f28 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/NewlyAddedTableITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/NewlyAddedTableITCase.java @@ -50,7 +50,7 @@ import org.apache.flink.types.RowUtils; import org.apache.flink.util.ExceptionUtils; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.jdbc.JdbcConnection; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.AfterEach; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/SpecificStartingOffsetITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/SpecificStartingOffsetITCase.java index 1a97a86db1e..e06cad07d65 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/SpecificStartingOffsetITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/SpecificStartingOffsetITCase.java @@ -44,7 +44,7 @@ import org.apache.flink.test.junit5.MiniClusterExtension; import org.apache.flink.util.FlinkRuntimeException; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/TableExclusionDuringSnapshotIT.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/TableExclusionDuringSnapshotIT.java index 2a305871408..4ccc9cfd887 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/TableExclusionDuringSnapshotIT.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/TableExclusionDuringSnapshotIT.java @@ -42,7 +42,7 @@ import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.ExceptionUtils; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/assigners/state/PendingSplitsStateSerializerTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/assigners/state/PendingSplitsStateSerializerTest.java index 60abb56ff16..2589504847a 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/assigners/state/PendingSplitsStateSerializerTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/assigners/state/PendingSplitsStateSerializerTest.java @@ -244,6 +244,16 @@ private static Map getTestTableSchema(TableId /** An implementation for {@link Table} which is used for tests. */ private static class TestTableImpl implements Table { + @Override + public io.debezium.relational.Attribute attributeWithName(String name) { + return null; + } + + @Override + public java.util.List attributes() { + return java.util.Collections.emptyList(); + } + private final TableId tableId; public TestTableImpl(TableId tableId) { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffsetTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffsetTest.java index 5767a6562a2..3b4c55afe03 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffsetTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/offset/BinlogOffsetTest.java @@ -22,8 +22,12 @@ /** Unit test for {@link BinlogOffset}. */ public class BinlogOffsetTest { - public static final String PART_OF_GTID_SET_1 = "abcd:1-4"; - public static final String PART_OF_GTID_SET_2 = "efgh:1-10"; + // Real server UUIDs: since mysql-binlog-connector-java 0.28 (Debezium 2.4) GtidSet parses the + // source id with UUID.fromString, so placeholder ids are rejected. + public static final String SERVER_UUID_1 = "24bc7850-2c16-11e6-a073-0242ac110002"; + public static final String SERVER_UUID_2 = "24bc7850-2c16-11e6-a073-0242ac110003"; + public static final String PART_OF_GTID_SET_1 = SERVER_UUID_1 + ":1-4"; + public static final String PART_OF_GTID_SET_2 = SERVER_UUID_2 + ":1-10"; public static final String FULL_GTID_SET = String.join(",", PART_OF_GTID_SET_1, PART_OF_GTID_SET_2); @@ -54,7 +58,7 @@ public void testCompareToWithGtidSet() { BinlogOffset offset3 = BinlogOffset.builder().setGtidSet(PART_OF_GTID_SET_1).build(); BinlogOffset offset4 = BinlogOffset.builder() - .setGtidSet("abcd:1-5") // Contains offset3's GTID set + .setGtidSet(SERVER_UUID_1 + ":1-5") // Contains offset3's GTID set .build(); // offset3 should be before offset4 @@ -69,7 +73,7 @@ public void testCompareToWithGtidSet() { .build(); offset4 = BinlogOffset.builder() - .setGtidSet("abcd:1-5") // Contains offset3's GTID set + .setGtidSet(SERVER_UUID_1 + ":1-5") // Contains offset3's GTID set .setBinlogFilePosition("binlog.001", 23) .build(); assertCompareTo(offset3, offset4, -1); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlRecordEmitterTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlRecordEmitterTest.java index 22167e0e5cc..21d7439e1b8 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlRecordEmitterTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlRecordEmitterTest.java @@ -32,11 +32,9 @@ import io.debezium.config.Configuration; import io.debezium.connector.mysql.MySqlConnectorConfig; import io.debezium.heartbeat.Heartbeat; -import io.debezium.heartbeat.HeartbeatFactory; +import io.debezium.heartbeat.HeartbeatImpl; import io.debezium.jdbc.JdbcConfiguration; -import io.debezium.relational.TableId; -import io.debezium.schema.TopicSelector; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; @@ -49,8 +47,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import static io.debezium.config.CommonConnectorConfig.TRANSACTION_TOPIC; -import static io.debezium.connector.mysql.MySqlConnectorConfig.SERVER_NAME; +import static io.debezium.config.CommonConnectorConfig.TOPIC_PREFIX; /** Unit test for {@link org.apache.flink.cdc.connectors.mysql.source.reader.MySqlRecordEmitter}. */ class MySqlRecordEmitterTest { @@ -60,18 +57,21 @@ void testHeartbeatEventHandling() throws Exception { Configuration dezConf = JdbcConfiguration.create() .with(Heartbeat.HEARTBEAT_INTERVAL, 100) - .with(TRANSACTION_TOPIC, "fake-topic") - .with(SERVER_NAME, "mysql_binlog_source") + .with(TOPIC_PREFIX, "mysql_binlog_source") .build(); MySqlConnectorConfig mySqlConfig = new MySqlConnectorConfig(dezConf); - HeartbeatFactory heartbeatFactory = - new HeartbeatFactory<>( - new MySqlConnectorConfig(dezConf), - TopicSelector.defaultSelector( - mySqlConfig, (id, prefix, delimiter) -> "fake-topic"), + // Construct a plain HeartbeatImpl directly. Debezium 2.0's HeartbeatFactory would route to + // a DatabaseHeartbeatImpl (which needs a live JDBC connection) because MySqlConnectorConfig + // reports a non-null heartbeat action query; this unit test only needs a heartbeat record. + Heartbeat heartbeat = + new HeartbeatImpl( + mySqlConfig.getHeartbeatInterval(), + mySqlConfig + .getTopicNamingStrategy(MySqlConnectorConfig.TOPIC_NAMING_STRATEGY) + .heartbeatTopic(), + mySqlConfig.getLogicalName(), SchemaNameAdjuster.create()); - Heartbeat heartbeat = heartbeatFactory.createHeartbeat(); BinlogOffset fakeOffset = BinlogOffset.ofBinlogFilePosition("fake-file", 15213L); MySqlRecordEmitter recordEmitter = createRecordEmitter(); MySqlBinlogSplitState splitState = createBinlogSplitState(); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlSourceReaderTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlSourceReaderTest.java index f2fc6426697..4588be3a8e3 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlSourceReaderTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/reader/MySqlSourceReaderTest.java @@ -55,8 +55,8 @@ import org.apache.flink.util.Collector; import org.apache.flink.util.Preconditions; -import io.debezium.connector.mysql.MySqlConnection; import io.debezium.connector.mysql.MySqlPartition; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.document.Array; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.TableId; @@ -135,7 +135,7 @@ void testRemoveTableUsingStateFromSnapshotPhase() throws Exception { Map tableSchemas = TableDiscoveryUtils.discoverSchemaForCapturedTables( new MySqlPartition( - sourceConfig.getMySqlConnectorConfig().getLogicalName()), + sourceConfig.getMySqlConnectorConfig().getLogicalName(), null), sourceConfig, jdbc); TableId tableId0 = new TableId(customerDatabase.getDatabaseName(), null, "customers"); @@ -230,7 +230,7 @@ void testFinishedUnackedSplitsUsingStateFromSnapshotPhase(boolean skipBackFill) Map tableSchemas = TableDiscoveryUtils.discoverSchemaForCapturedTables( new MySqlPartition( - sourceConfig.getMySqlConnectorConfig().getLogicalName()), + sourceConfig.getMySqlConnectorConfig().getLogicalName(), null), sourceConfig, jdbc); TableId tableId = new TableId(customerDatabase.getDatabaseName(), null, "customers"); @@ -330,7 +330,7 @@ void testBinlogReadFailoverCrossTransaction() throws Exception { Map tableSchemas = TableDiscoveryUtils.discoverSchemaForCapturedTables( new MySqlPartition( - sourceConfig.getMySqlConnectorConfig().getLogicalName()), + sourceConfig.getMySqlConnectorConfig().getLogicalName(), null), sourceConfig, jdbc); binlogSplit = diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/split/MySqlBinlogSplitTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/split/MySqlBinlogSplitTest.java index ff310d367e8..9696ef56d93 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/split/MySqlBinlogSplitTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/split/MySqlBinlogSplitTest.java @@ -176,6 +176,17 @@ public void duplicateSplitInfo() { /** A mock implementation for {@link Table} which is used for unit tests. */ private static class MockTable implements Table { + + @Override + public io.debezium.relational.Attribute attributeWithName(String name) { + return null; + } + + @Override + public java.util.List attributes() { + return java.util.Collections.emptyList(); + } + private final TableId tableId; public MockTable(TableId tableId) { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/io/debezium/connector/binlog/BinlogStreamingChangeEventSource.java similarity index 65% rename from flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java rename to flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/io/debezium/connector/binlog/BinlogStreamingChangeEventSource.java index 14b3d96f147..19e1c6fba64 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/io/debezium/connector/binlog/BinlogStreamingChangeEventSource.java @@ -3,22 +3,21 @@ * * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 */ - -package io.debezium.connector.mysql; +package io.debezium.connector.binlog; import com.github.shyiko.mysql.binlog.BinaryLogClient; -import com.github.shyiko.mysql.binlog.BinaryLogClient.LifecycleListener; +import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData; import com.github.shyiko.mysql.binlog.event.DeleteRowsEventData; import com.github.shyiko.mysql.binlog.event.Event; import com.github.shyiko.mysql.binlog.event.EventData; import com.github.shyiko.mysql.binlog.event.EventHeader; import com.github.shyiko.mysql.binlog.event.EventHeaderV4; import com.github.shyiko.mysql.binlog.event.EventType; -import com.github.shyiko.mysql.binlog.event.GtidEventData; import com.github.shyiko.mysql.binlog.event.QueryEventData; import com.github.shyiko.mysql.binlog.event.RotateEventData; import com.github.shyiko.mysql.binlog.event.RowsQueryEventData; import com.github.shyiko.mysql.binlog.event.TableMapEventData; +import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData; import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData; import com.github.shyiko.mysql.binlog.event.WriteRowsEventData; import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException; @@ -34,20 +33,31 @@ import io.debezium.annotation.SingleThreadAccess; import io.debezium.config.CommonConnectorConfig.EventProcessingFailureHandlingMode; import io.debezium.config.Configuration; -import io.debezium.connector.mysql.MySqlConnectorConfig.GtidNewChannelPosition; -import io.debezium.connector.mysql.MySqlConnectorConfig.SecureConnectionMode; +import io.debezium.connector.binlog.BinlogConnectorConfig.SecureConnectionMode; +import io.debezium.connector.binlog.event.EventDataDeserializationExceptionData; +import io.debezium.connector.binlog.event.RowDeserializers; +import io.debezium.connector.binlog.event.StopEventDataDeserializer; +import io.debezium.connector.binlog.event.TransactionPayloadDeserializer; +import io.debezium.connector.binlog.gtid.GtidSet; +import io.debezium.connector.binlog.jdbc.BinlogConnectorConnection; +import io.debezium.connector.binlog.metrics.BinlogStreamingChangeEventSourceMetrics; import io.debezium.connector.mysql.util.ErrorMessageUtils; -import io.debezium.data.Envelope.Operation; +import io.debezium.data.Envelope; import io.debezium.function.BlockingConsumer; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.EventDispatcher; import io.debezium.pipeline.source.spi.StreamingChangeEventSource; +import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.schema.SchemaChangeEvent; +import io.debezium.snapshot.SnapshotterService; +import io.debezium.snapshot.mode.NeverSnapshotter; +import io.debezium.time.Conversions; import io.debezium.util.Clock; import io.debezium.util.Metronome; import io.debezium.util.Strings; import io.debezium.util.Threads; +import org.apache.kafka.connect.source.SourceConnector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; @@ -84,182 +94,438 @@ import static io.debezium.util.Strings.isNullOrEmpty; /** - * Copied from mysql-cdc to be compatible with OceanBase Binlog Service. + * An abstract common implementation of {@link StreamingChangeEventSource} for binlog-based + * connectors. + */ +/** + * Copied from Debezium project(2.7.4.Final). + * + *

    Change 1: the effective offset context is initialised at the top of {@code execute} — + * Debezium's coordinator calls {@code init(offsetContext)} before {@code execute}, but Flink CDC + * drives this source directly and never calls {@code init}. * - *

    Copied from Debezium project(1.9.8.Final) to fix - * https://github.com/ververica/flink-cdc-connectors/issues/1944. + *

    Change 2: {@code handleChange} iterates the row list instead of indexing it — the list may be + * a linked list, in which case {@code get(i)} turns the loop into O(n^2). * - *

    Line 1438-1449 : Adjust GTID merging logic to support recovering from job which previously - * specifying starting offset on start. Uses {@link GtidUtils#fixOldChannelsGtidSet} for shared - * EARLIEST/LATEST logic. + *

    Change 3: {@code wrap()} runs the message through {@code ErrorMessageUtils} to keep the error + * text bounded. * - *

    Line 1450-1458 : Fix LATEST mode GTID merging to avoid replaying pre-checkpoint transactions - * when checkpoint GTID has non-contiguous ranges. Delegates to {@link - * GtidUtils#computeLatestModeGtidSet}. See FLINK-39149. + *

    Change 4 (OceanBase): skip {@code offsetContext.completeEvent()} for HEARTBEAT events emitted + * by the OceanBase Binlog Service. * - *

    Line 1496 : Add more error details for some exceptions. + *

    Change 5: when the offset carries no GTID set but does pin a binlog file/position, keep the + * binlog client in file/position mode instead of switching it to GTID mode. Debezium always + * switches to GTID mode here because upstream only reaches this branch with an empty binlog + * filename; Flink CDC reaches it whenever a job starts from a specific binlog offset, and there + * {@code BinaryLogClient} would stream from the GTID set (ignoring the requested position) or, with + * an empty GTID set, fail with an NPE in {@code DumpBinaryLogGtidCommand} — it only materialises + * its internal GtidSet when the binlog filename is empty. * - *

    Line 956-968 : Use iterator instead of index-based loop to avoid O(n²) complexity when - * processing LinkedList rows in handleChange method. See FLINK-38846. + *

    Debezium 2.7 split the shared binlog engine out of the MySQL connector, so these patches moved + * here from {@code io.debezium.connector.mysql.MySqlStreamingChangeEventSource}, which is now a + * thin subclass and no longer forked. */ -public class MySqlStreamingChangeEventSource - implements StreamingChangeEventSource { +public abstract class BinlogStreamingChangeEventSource< + P extends BinlogPartition, O extends BinlogOffsetContext> + implements StreamingChangeEventSource { private static final Logger LOGGER = - LoggerFactory.getLogger(MySqlStreamingChangeEventSource.class); + LoggerFactory.getLogger(BinlogStreamingChangeEventSource.class); private static final String KEEPALIVE_THREAD_NAME = "blc-keepalive"; + private static final String SET_STATEMENT_REGEX = "SET STATEMENT .* FOR"; - private final EnumMap> eventHandlers = - new EnumMap<>(EventType.class); private final BinaryLogClient client; - private final MySqlStreamingChangeEventSourceMetrics metrics; + private final BinlogStreamingChangeEventSourceMetrics metrics; + // todo: can we go back to accessing schema via task context? issue is all the generics :/ + private final BinlogTaskContext taskContext; + private final BinlogDatabaseSchema schema; + private final BinlogConnectorConfig connectorConfig; + private final BinlogConnectorConnection connection; + private final EventDispatcher eventDispatcher; private final Clock clock; + private final ErrorHandler errorHandler; private final EventProcessingFailureHandlingMode eventDeserializationFailureHandlingMode; private final EventProcessingFailureHandlingMode inconsistentSchemaHandlingMode; + private final SnapshotterService snapshotterService; + private final Predicate gtidDmlSourceFilter; + private final boolean isGtidModeEnabled; + private final AtomicLong totalRecordCounter = new AtomicLong(); + private final Map binaryLogClientThreads = new ConcurrentHashMap<>(4); + private final EnumMap> eventHandlers = + new EnumMap<>(EventType.class); + private final float heartbeatIntervalFactor = 0.8f; private int startingRowNumber = 0; private long initialEventsToSkip = 0L; private boolean skipEvent = false; private boolean ignoreDmlEventByGtidSource = false; - private final Predicate gtidDmlSourceFilter; - private final AtomicLong totalRecordCounter = new AtomicLong(); private volatile Map lastOffset = null; - private com.github.shyiko.mysql.binlog.GtidSet gtidSet; - private final float heartbeatIntervalFactor = 0.8f; - private final Map binaryLogClientThreads = new ConcurrentHashMap<>(4); - private final MySqlTaskContext taskContext; - private final MySqlConnectorConfig connectorConfig; - private final MySqlConnection connection; - private final EventDispatcher eventDispatcher; - private final ErrorHandler errorHandler; + private O effectiveOffsetContext; @SingleThreadAccess("binlog client thread") - private Instant eventTimestamp; + protected Instant eventTimestamp; - /** Describe binlog position. */ - public static class BinlogPosition { - final String filename; - final long position; + public BinlogStreamingChangeEventSource( + BinlogConnectorConfig connectorConfig, + BinlogConnectorConnection connection, + EventDispatcher dispatcher, + ErrorHandler errorHandler, + Clock clock, + BinlogTaskContext taskContext, + BinlogDatabaseSchema schema, + BinlogStreamingChangeEventSourceMetrics metrics, + SnapshotterService snapshotterService) { + this.taskContext = taskContext; + this.connectorConfig = connectorConfig; + this.schema = schema; + this.connection = connection; + this.clock = clock; + this.eventDispatcher = dispatcher; + this.errorHandler = errorHandler; + this.metrics = metrics; + this.eventDeserializationFailureHandlingMode = + connectorConfig.getEventProcessingFailureHandlingMode(); + this.inconsistentSchemaHandlingMode = + connectorConfig.getInconsistentSchemaFailureHandlingMode(); + this.snapshotterService = snapshotterService; + this.client = + createBinaryLogClient( + taskContext, connectorConfig, binaryLogClientThreads, connection); + this.gtidDmlSourceFilter = getGtidDmlSourceFilter(); + this.isGtidModeEnabled = connection.isGtidModeEnabled(); + } - public BinlogPosition(String filename, long position) { - assert filename != null; + @Override + public void execute(ChangeEventSourceContext context, P partition, O offsetContext) + throws InterruptedException { + // Debezium's ChangeEventSourceCoordinator calls init(offsetContext) before execute(...) to + // set the effective offset context. Flink CDC drives this source directly and never calls + // init, so fall back to the offset context it passes in. + if (getOffsetContext() == null) { + setEffectiveOffsetContext(offsetContext); + } + if (!(snapshotterService.getSnapshotter() instanceof NeverSnapshotter)) { + taskContext.getSchema().assureNonEmptySchema(); + } + final Set skippedOperations = connectorConfig.getSkippedOperations(); - this.filename = filename; - this.position = position; + // Register our event handlers ... + eventHandlers.put( + EventType.STOP, (event) -> handleServerStop(effectiveOffsetContext, event)); + eventHandlers.put( + EventType.HEARTBEAT, + (event) -> handleServerHeartbeat(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.INCIDENT, + (event) -> handleServerIncident(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.ROTATE, (event) -> handleRotateLogsEvent(effectiveOffsetContext, event)); + eventHandlers.put( + EventType.TABLE_MAP, + (event) -> handleUpdateTableMetadata(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.QUERY, + (event) -> handleQueryEvent(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.TRANSACTION_PAYLOAD, + (event) -> handleTransactionPayload(partition, effectiveOffsetContext, event)); + + if (!skippedOperations.contains(Envelope.Operation.CREATE)) { + eventHandlers.put( + EventType.WRITE_ROWS, + (event) -> handleInsert(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.EXT_WRITE_ROWS, + (event) -> handleInsert(partition, effectiveOffsetContext, event)); } - public String getFilename() { - return filename; + if (!skippedOperations.contains(Envelope.Operation.UPDATE)) { + eventHandlers.put( + EventType.UPDATE_ROWS, + (event) -> handleUpdate(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.EXT_UPDATE_ROWS, + (event) -> handleUpdate(partition, effectiveOffsetContext, event)); } - public long getPosition() { - return position; + if (!skippedOperations.contains(Envelope.Operation.DELETE)) { + eventHandlers.put( + EventType.DELETE_ROWS, + (event) -> handleDelete(partition, effectiveOffsetContext, event)); + eventHandlers.put( + EventType.EXT_DELETE_ROWS, + (event) -> handleDelete(partition, effectiveOffsetContext, event)); } - @Override - public String toString() { - return filename + "/" + position; + eventHandlers.put( + EventType.VIEW_CHANGE, (event) -> viewChange(effectiveOffsetContext, event)); + eventHandlers.put( + EventType.XA_PREPARE, (event) -> prepareTransaction(effectiveOffsetContext, event)); + eventHandlers.put( + EventType.XID, + (event) -> handleTransactionCompletion(partition, effectiveOffsetContext, event)); + + // Conditionally register ROWS_QUERY handler to parse SQL statements. + if (connectorConfig.isSqlQueryIncluded()) { + final EventType eventType = getIncludeQueryEventType(); + eventHandlers.put( + eventType, (event) -> handleRecordingQuery(effectiveOffsetContext, event)); } - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + filename.hashCode(); - result = prime * result + (int) (position ^ (position >>> 32)); - return result; + BinaryLogClient.EventListener listener; + if (connectorConfig.getBufferSizeForStreamingChangeEventSource() == 0) { + listener = (event) -> handleEvent(partition, effectiveOffsetContext, context, event); + } else { + EventBuffer buffer = + new EventBuffer<>( + connectorConfig.getBufferSizeForStreamingChangeEventSource(), + this, + context); + listener = (event) -> buffer.add(partition, effectiveOffsetContext, event); } + client.registerEventListener(listener); - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; + client.registerLifecycleListener(new ReaderThreadLifecycleListener(effectiveOffsetContext)); + client.registerEventListener((event) -> onEvent(effectiveOffsetContext, event)); + if (LOGGER.isDebugEnabled()) { + client.registerEventListener((event) -> logEvent(effectiveOffsetContext, event)); + } + + metrics.setIsGtidModeEnabled(isGtidModeEnabled); + + // Get the current GtidSet from MySQL so we can get a filtered/merged GtidSet based off of + // the last Debezium checkpoint. + if (isGtidModeEnabled) { + // The server is using GTIDs, so enable the handler ... + eventHandlers.put( + getGtidEventType(), + (event) -> + handleGtidEvent( + partition, effectiveOffsetContext, event, gtidDmlSourceFilter)); + + // Now look at the GTID set from the server and what we've previously seen ... + GtidSet availableServerGtidSet = connection.knownGtidSet(); + + // also take into account purged GTID logs + GtidSet purgedServerGtidSet = connection.purgedGtidSet(); + LOGGER.info("GTID set purged on server: '{}'", purgedServerGtidSet); + + final GtidSet filteredGtidSet = + connection.filterGtidSet( + connectorConfig.getGtidSourceFilter(), + effectiveOffsetContext.gtidSet(), + availableServerGtidSet, + purgedServerGtidSet); + + if (filteredGtidSet != null) { + // We've seen at least some GTIDs, so start reading from the filtered GTID set ... + LOGGER.info("Registering binlog reader with GTID set: '{}'", filteredGtidSet); + String filteredGtidSetStr = filteredGtidSet.toString(); + client.setGtidSet(filteredGtidSetStr); + effectiveOffsetContext.setCompletedGtidSet(filteredGtidSetStr); + initializeGtidSet(filteredGtidSetStr); + } else { + // We've not yet seen any GTIDs, so that means we have to start reading the binlog + // from the beginning ... + final String startBinlogFilename = + effectiveOffsetContext.getSource().binlogFilename(); + client.setBinlogFilename(startBinlogFilename); + client.setBinlogPosition(effectiveOffsetContext.getSource().binlogPosition()); + if (!Strings.isNullOrEmpty(startBinlogFilename)) { + LOGGER.info( + "No GTID stored in the offset, registering binlog reader with binlog file '{}' and position {}.", + startBinlogFilename, + effectiveOffsetContext.getSource().binlogPosition()); + initializeGtidSet(""); + } else if (purgedServerGtidSet == null || purgedServerGtidSet.isEmpty()) { + LOGGER.info( + "No GTID stored in the offset, registering binlog reader with empty GTID set."); + client.setGtidSet(""); + initializeGtidSet(""); + } else { + LOGGER.info( + "No GTID stored in the offset, but there is non-empty purged GTID set. Registering binlog reader with purged GTID set: '{}'", + purgedServerGtidSet.toString()); + client.setGtidSet(purgedServerGtidSet.toString()); + // We don't have stored any GTID in the offset, so start from empty GTID set. + initializeGtidSet(""); + } } - if (getClass() != obj.getClass()) { - return false; + } else { + // The server is not using GTIDs, so start reading the binlog based upon where we last + // left off ... + client.setBinlogFilename(effectiveOffsetContext.getSource().binlogFilename()); + client.setBinlogPosition(effectiveOffsetContext.getSource().binlogPosition()); + } + + // We may be restarting in the middle of a transaction, so see how far into the transaction + // we have already processed... + initialEventsToSkip = effectiveOffsetContext.eventsToSkipUponRestart(); + LOGGER.info("Skip {} events on streaming start", initialEventsToSkip); + + // Set the starting row number, which is the next row number to be read ... + startingRowNumber = effectiveOffsetContext.rowsToSkipUponRestart(); + LOGGER.info("Skip {} rows on streaming start", startingRowNumber); + + // Only when we reach the first BEGIN event will we start to skip events ... + skipEvent = false; + + try { + // Start the log reader, which starts background threads ... + if (context.isRunning()) { + long timeout = connectorConfig.getConnectionTimeout().toMillis(); + long started = clock.currentTimeInMillis(); + try { + LOGGER.debug( + "Attempting to establish binlog reader connection with timeout of {} ms", + timeout); + client.connect(timeout); + // Need to wait for keepalive thread to be running, otherwise it can be left + // orphaned + // The problem is with timing. When the close is called too early after connect + // then + // the keepalive thread is not terminated + if (client.isKeepAlive()) { + LOGGER.info("Waiting for keepalive thread to start"); + final Metronome metronome = Metronome.parker(Duration.ofMillis(100), clock); + int waitAttempts = 50; + boolean keepAliveThreadRunning = false; + while (!keepAliveThreadRunning && waitAttempts-- > 0) { + for (Thread t : binaryLogClientThreads.values()) { + if (t.getName().startsWith(KEEPALIVE_THREAD_NAME) && t.isAlive()) { + LOGGER.info("Keepalive thread is running"); + keepAliveThreadRunning = true; + } + } + metronome.pause(); + } + } + } catch (TimeoutException e) { + // If the client thread is interrupted *before* the client could connect, the + // client throws a timeout exception + // The only way we can distinguish this is if we get the timeout exception + // before the specified timeout has + // elapsed, so we simply check this (within 10%) ... + long duration = clock.currentTimeInMillis() - started; + if (duration > (0.9 * timeout)) { + double actualSeconds = TimeUnit.MILLISECONDS.toSeconds(duration); + throw new DebeziumException( + String.format( + "Timed out after %f seconds while waiting to connect to database at %s:%d with user '%s'", + actualSeconds, + connectorConfig.getHostName(), + connectorConfig.getPort(), + connectorConfig.getUserName()), + e); + } + // Otherwise, we were told to shutdown, so we don't care about the timeout + // exception + } catch (AuthenticationException e) { + throw new DebeziumException( + String.format( + "Failed to authenticate to the database at %s:%d with user '%s'", + connectorConfig.getHostName(), + connectorConfig.getPort(), + connectorConfig.getUserName()), + e); + } catch (Throwable e) { + throw new DebeziumException( + String.format( + "Unable to connect to the database at %s:%d with user '%s': %s", + connectorConfig.getHostName(), + connectorConfig.getPort(), + connectorConfig.getUserName(), + e.getMessage()), + e); + } } - BinlogPosition other = (BinlogPosition) obj; - if (!filename.equals(other.filename)) { - return false; + while (context.isRunning()) { + Thread.sleep(100); + waitWhenStreamingPaused(context); } - if (position != other.position) { - return false; + } finally { + try { + client.disconnect(); + } catch (Exception e) { + LOGGER.info("Exception while stopping binary log client", e); } - return true; } } - @FunctionalInterface - private interface BinlogChangeEmitter { - void emit(TableId tableId, T data) throws InterruptedException; + @Override + public O getOffsetContext() { + return effectiveOffsetContext; } - public MySqlStreamingChangeEventSource( - MySqlConnectorConfig connectorConfig, - MySqlConnection connection, - EventDispatcher dispatcher, - ErrorHandler errorHandler, - Clock clock, - MySqlTaskContext taskContext, - MySqlStreamingChangeEventSourceMetrics metrics) { + protected void setEffectiveOffsetContext(O offsetContext) { + this.effectiveOffsetContext = offsetContext; + } - this.taskContext = taskContext; - this.connectorConfig = connectorConfig; - this.connection = connection; - this.clock = clock; - this.eventDispatcher = dispatcher; - this.errorHandler = errorHandler; - this.metrics = metrics; + protected void setIgnoreDmlEventByGtidSource(boolean ignoreDmlEventByGtidSource) { + this.ignoreDmlEventByGtidSource = ignoreDmlEventByGtidSource; + } - eventDeserializationFailureHandlingMode = - connectorConfig.getEventProcessingFailureHandlingMode(); - inconsistentSchemaHandlingMode = connectorConfig.inconsistentSchemaFailureHandlingMode(); + protected void setGtidChanged(String gtid) { + metrics.onGtidChange(gtid); + } + + protected boolean isGtidModeEnabled() { + return isGtidModeEnabled; + } - // Set up the log reader ... - client = taskContext.getBinaryLogClient(); - // BinaryLogClient will overwrite thread names later + // todo: perhaps refactor back out to a binary log configurator instance? + protected BinaryLogClient createBinaryLogClient( + BinlogTaskContext taskContext, + BinlogConnectorConfig connectorConfig, + Map clientThreads, + BinlogConnectorConnection connection) { + final BinaryLogClient client = taskContext.getBinaryLogClient(); client.setThreadFactory( Threads.threadFactory( - MySqlConnector.class, + getConnectorClass(), connectorConfig.getLogicalName(), "binlog-client", false, false, - x -> binaryLogClientThreads.put(x.getName(), x))); - client.setServerId(connectorConfig.serverId()); - client.setSSLMode(sslModeFor(connectorConfig.sslMode())); - if (connectorConfig.sslModeEnabled()) { + x -> clientThreads.put(x.getName(), x))); + client.setServerId(connectorConfig.getServerId()); + + client.setSSLMode(sslModeFor(connectorConfig.getSslMode())); + if (connectorConfig.isSslModeEnabled()) { SSLSocketFactory sslSocketFactory = getBinlogSslSocketFactory(connectorConfig, connection); if (sslSocketFactory != null) { client.setSslSocketFactory(sslSocketFactory); } } + client.setUseNonGracefulDisconnect(connectorConfig.usesNonGracefulDisconnect()); + + configureReplicaCompatibility(client); + Configuration configuration = connectorConfig.getConfig(); - client.setKeepAlive(configuration.getBoolean(MySqlConnectorConfig.KEEP_ALIVE)); final long keepAliveInterval = - configuration.getLong(MySqlConnectorConfig.KEEP_ALIVE_INTERVAL_MS); + configuration.getLong(BinlogConnectorConfig.KEEP_ALIVE_INTERVAL_MS); + + client.setKeepAlive(configuration.getBoolean(BinlogConnectorConfig.KEEP_ALIVE)); client.setKeepAliveInterval(keepAliveInterval); - // Considering heartbeatInterval should be less than keepAliveInterval, we use the + + // Considering heartbeatInterval should be less than keepAliveInterval, use the // heartbeatIntervalFactor - // multiply by keepAliveInterval and set the result value to heartbeatInterval.The default - // value of heartbeatIntervalFactor - // is 0.8, and we believe the left time (0.2 * keepAliveInterval) is enough to process the - // packet received from the MySQL server. + // multiply by keepAliveInterval and set the result value to heartbeatInterval. The default + // value of + // heartbeatIntervalFactor is 0.0, and we believe the left time (0.2 * keepAliveInterval) is + // enough + // to process the packet received from the database server. client.setHeartbeatInterval((long) (keepAliveInterval * heartbeatIntervalFactor)); + client.setEventDeserializer(createEventDeserializer()); + return client; + } - boolean filterDmlEventsByGtidSource = - configuration.getBoolean(MySqlConnectorConfig.GTID_SOURCE_FILTER_DML_EVENTS); - gtidDmlSourceFilter = - filterDmlEventsByGtidSource ? connectorConfig.gtidSourceFilter() : null; + protected void configureReplicaCompatibility(BinaryLogClient client) {} - // Set up the event deserializer with additional type(s) ... - final Map tableMapEventByTableId = - new HashMap(); + protected EventDeserializer createEventDeserializer() { + final Map tableMapEventByTableId = new HashMap<>(); EventDeserializer eventDeserializer = new EventDeserializer() { @Override @@ -276,6 +542,28 @@ public Event nextEvent(ByteArrayInputStream inputStream) throws IOException { tableMapEvent.getTableId(), tableMapEvent); } + // DBZ-2663 Handle for transaction payload and capture the table map + // event and add it to the map + if (event.getHeader().getEventType() == EventType.TRANSACTION_PAYLOAD) { + TransactionPayloadEventData transactionPayloadEventData = + event.getData(); + /** + * Loop over the uncompressed events in the transaction payload + * event and add the table map event in the map of table events + */ + for (Event uncompressedEvent : + transactionPayloadEventData.getUncompressedEvents()) { + if (uncompressedEvent.getHeader().getEventType() + == EventType.TABLE_MAP + && uncompressedEvent.getData() != null) { + TableMapEventData tableMapEvent = + uncompressedEvent.getData(); + tableMapEventByTableId.put( + tableMapEvent.getTableId(), tableMapEvent); + } + } + } + // DBZ-5126 Clean cache on rotate event to prevent it from growing // indefinitely. if (event.getHeader().getEventType() == EventType.ROTATE @@ -318,35 +606,46 @@ public Event nextEvent(ByteArrayInputStream inputStream) throws IOException { } }; - // Add our custom deserializers ... eventDeserializer.setEventDataDeserializer(EventType.STOP, new StopEventDataDeserializer()); eventDeserializer.setEventDataDeserializer(EventType.GTID, new GtidEventDataDeserializer()); eventDeserializer.setEventDataDeserializer( EventType.WRITE_ROWS, - new RowDeserializers.WriteRowsDeserializer(tableMapEventByTableId)); + new RowDeserializers.WriteRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode)); eventDeserializer.setEventDataDeserializer( EventType.UPDATE_ROWS, - new RowDeserializers.UpdateRowsDeserializer(tableMapEventByTableId)); + new RowDeserializers.UpdateRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode)); eventDeserializer.setEventDataDeserializer( EventType.DELETE_ROWS, - new RowDeserializers.DeleteRowsDeserializer(tableMapEventByTableId)); + new RowDeserializers.DeleteRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode)); eventDeserializer.setEventDataDeserializer( EventType.EXT_WRITE_ROWS, - new RowDeserializers.WriteRowsDeserializer(tableMapEventByTableId) + new RowDeserializers.WriteRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode) .setMayContainExtraInformation(true)); eventDeserializer.setEventDataDeserializer( EventType.EXT_UPDATE_ROWS, - new RowDeserializers.UpdateRowsDeserializer(tableMapEventByTableId) + new RowDeserializers.UpdateRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode) .setMayContainExtraInformation(true)); eventDeserializer.setEventDataDeserializer( EventType.EXT_DELETE_ROWS, - new RowDeserializers.DeleteRowsDeserializer(tableMapEventByTableId) + new RowDeserializers.DeleteRowsDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode) .setMayContainExtraInformation(true)); - client.setEventDeserializer(eventDeserializer); + eventDeserializer.setEventDataDeserializer( + EventType.TRANSACTION_PAYLOAD, + new TransactionPayloadDeserializer( + tableMapEventByTableId, eventDeserializationFailureHandlingMode)); + + return eventDeserializer; } - protected void onEvent(MySqlOffsetContext offsetContext, Event event) { + protected void onEvent(O offsetContext, Event event) { long ts = 0; + totalRecordCounter.incrementAndGet(); if (event.getHeader().getEventType() == EventType.HEARTBEAT) { // HEARTBEAT events have no timestamp but are fired only when @@ -365,31 +664,26 @@ protected void onEvent(MySqlOffsetContext offsetContext, Event event) { return; } - ts = clock.currentTimeInMillis() - eventTs; + setEventTimestamp(event, eventTs); + + ts = clock.currentTimeInMillis() - eventTimestamp.toEpochMilli(); LOGGER.trace("Current milliseconds behind source: {} ms", ts); metrics.setMilliSecondsBehindSource(ts); } - protected void ignoreEvent(MySqlOffsetContext offsetContext, Event event) { + protected abstract void setEventTimestamp(Event event, long eventTs); + + protected void ignoreEvent(O offsetContext, Event event) { LOGGER.trace("Ignoring event due to missing handler: {}", event); } protected void handleEvent( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) { + P partition, O offsetContext, ChangeEventSourceContext context, Event event) { if (event == null) { return; } final EventHeader eventHeader = event.getHeader(); - // Update the source offset info. Note that the client returns the value in *milliseconds*, - // even though the binlog - // contains only *seconds* precision ... - // HEARTBEAT events have no timestamp; only set the timestamp if the event is not a - // HEARTBEAT - eventTimestamp = - !eventHeader.getEventType().equals(EventType.HEARTBEAT) - ? Instant.ofEpochMilli(eventHeader.getTimestamp()) - : null; offsetContext.setBinlogServerId(eventHeader.getServerId()); final EventType eventType = eventHeader.getEventType(); @@ -413,6 +707,8 @@ protected void handleEvent( // If there is a handler for this event, forward the event to it ... try { + waitWhenStreamingPaused(context); + // Forward the event to the handler ... eventHandlers .getOrDefault(eventType, (e) -> ignoreEvent(offsetContext, e)) @@ -422,7 +718,7 @@ protected void handleEvent( eventDispatcher.dispatchHeartbeatEvent(partition, offsetContext); // Capture that we've completed another event ... - // Skip HeratBeat event for OceanBase Binlog Service + // Skip HeartBeat event for OceanBase Binlog Service if (eventType != EventType.HEARTBEAT) { offsetContext.completeEvent(); } @@ -466,11 +762,11 @@ protected T unwrapData(Event event) { } /** - * Handle the supplied event that signals that mysqld has stopped. + * Handle the supplied event that signals that the database has stopped. * * @param event the server stopped event to be processed; may not be null */ - protected void handleServerStop(MySqlOffsetContext offsetContext, Event event) { + protected void handleServerStop(O offsetContext, Event event) { LOGGER.debug("Server stopped: {}", event); } @@ -480,8 +776,7 @@ protected void handleServerStop(MySqlOffsetContext offsetContext, Event event) { * * @param event the server stopped event to be processed; may not be null */ - protected void handleServerHeartbeat( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleServerHeartbeat(P partition, O offsetContext, Event event) throws InterruptedException { LOGGER.trace("Server heartbeat: {}", event); eventDispatcher.dispatchServerHeartbeatEvent(partition, offsetContext); @@ -494,8 +789,7 @@ protected void handleServerHeartbeat( * * @param event the server stopped event to be processed; may not be null */ - protected void handleServerIncident( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) { + protected void handleServerIncident(P partition, O offsetContext, Event event) { if (event.getData() instanceof EventDataDeserializationExceptionData) { metrics.onErroneousEvent(partition, "source = " + event); EventDataDeserializationExceptionData data = event.getData(); @@ -541,91 +835,34 @@ protected void handleServerIncident( /** * Handle the supplied event with a {@link RotateEventData} that signals the logs are being * rotated. This means that either the server was restarted, or the binlog has transitioned to a - * new file. In either case, subsequent table numbers will be different than those seen to this + * new file. In either case, subsequent table numbers will be different from those seen to this * point. * * @param event the database change data event to be processed; may not be null */ - protected void handleRotateLogsEvent(MySqlOffsetContext offsetContext, Event event) { + protected void handleRotateLogsEvent(O offsetContext, Event event) { LOGGER.debug("Rotating logs: {}", event); RotateEventData command = unwrapData(event); assert command != null; - taskContext.getSchema().clearTableMappings(); - } - - /** - * Handle the supplied event with a {@link GtidEventData} that signals the beginning of a GTID - * transaction. We don't yet know whether this transaction contains any events we're interested - * in, but we have to record it so that we know the position of this event and know we've - * processed the binlog to this point. - * - *

    Note that this captures the current GTID and complete GTID set, regardless of whether the - * connector is {@link MySqlTaskContext#gtidSourceFilter() filtering} the GTID set upon - * connection. We do this because we actually want to capture all GTID set values found in the - * binlog, whether or not we process them. However, only when we connect do we actually want to - * pass to MySQL only those GTID ranges that are applicable per the configuration. - * - * @param event the GTID event to be processed; may not be null - */ - protected void handleGtidEvent(MySqlOffsetContext offsetContext, Event event) { - LOGGER.debug("GTID transaction: {}", event); - GtidEventData gtidEvent = unwrapData(event); - String gtid = gtidEvent.getGtid(); - gtidSet.add(gtid); - offsetContext.startGtid(gtid, gtidSet.toString()); // rather than use the client's GTID set - ignoreDmlEventByGtidSource = false; - if (gtidDmlSourceFilter != null && gtid != null) { - String uuid = gtid.trim().substring(0, gtid.indexOf(":")); - if (!gtidDmlSourceFilter.test(uuid)) { - ignoreDmlEventByGtidSource = true; - } - } - metrics.onGtidChange(gtid); - } - - /** - * Handle the supplied event with an {@link RowsQueryEventData} by recording the original SQL - * query that generated the event. - * - * @param event the database change data event to be processed; may not be null - */ - protected void handleRowsQuery(MySqlOffsetContext offsetContext, Event event) { - // Unwrap the RowsQueryEvent - final RowsQueryEventData lastRowsQueryEventData = unwrapData(event); - - // Set the query on the source - offsetContext.setQuery(lastRowsQueryEventData.getQuery()); + schema.clearTableMappings(); } /** * Handle the supplied event with an {@link QueryEventData} by possibly recording the DDL - * statements as changes in the MySQL schemas. + * statements as changes in the database schemas. * * @param partition the partition in which the even occurred * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while recording the DDL statements */ - protected void handleQueryEvent( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleQueryEvent(P partition, O offsetContext, Event event) throws InterruptedException { + Instant eventTime = Conversions.toInstantFromMillis(eventTimestamp.toEpochMilli()); QueryEventData command = unwrapData(event); LOGGER.debug("Received query command: {}", event); String sql = command.getSql().trim(); if (sql.equalsIgnoreCase("BEGIN")) { - // We are starting a new transaction ... - offsetContext.startNextTransaction(); - eventDispatcher.dispatchTransactionStartedEvent( - partition, offsetContext.getTransactionId(), offsetContext); - offsetContext.setBinlogThread(command.getThreadId()); - if (initialEventsToSkip != 0) { - LOGGER.debug( - "Restarting partially-processed transaction; change events will not be created for the first {} events plus {} more rows in the next event", - initialEventsToSkip, - startingRowNumber); - // We are restarting, so we need to skip the events in this transaction that we - // processed previously... - skipEvent = true; - } + handleTransactionBegin(partition, offsetContext, event, command.getThreadId()); return; } if (sql.equalsIgnoreCase("COMMIT")) { @@ -633,13 +870,14 @@ protected void handleQueryEvent( return; } - String upperCasedStatementBegin = Strings.getBegin(sql, 7).toUpperCase(); + String upperCasedStatementBegin = + Strings.getBegin(removeSetStatement(sql), 7).toUpperCase(); if (upperCasedStatementBegin.startsWith("XA ")) { // This is an XA transaction, and we currently ignore these and do nothing ... return; } - if (connectorConfig.getDdlFilter().test(sql)) { + if (schema.ddlFilter().test(sql)) { LOGGER.debug("DDL '{}' was filtered out of processing", sql); return; } @@ -656,21 +894,15 @@ protected void handleQueryEvent( // We have hit a ROLLBACK which is not supported LOGGER.warn( "Rollback statements cannot be handled without binlog buffering, the connector will fail. Please check '{}' to see how to enable buffering", - MySqlConnectorConfig.BUFFER_SIZE_FOR_BINLOG_READER.name()); + BinlogConnectorConfig.BUFFER_SIZE_FOR_BINLOG_READER.name()); } final List schemaChangeEvents = - taskContext - .getSchema() - .parseStreamingDdl( - partition, - sql, - command.getDatabase(), - offsetContext, - clock.currentTimeAsInstant()); + schema.parseStreamingDdl( + partition, sql, command.getDatabase(), offsetContext, eventTime); try { for (SchemaChangeEvent schemaChangeEvent : schemaChangeEvents) { - if (taskContext.getSchema().skipSchemaChangeEvent(schemaChangeEvent)) { + if (schema.skipSchemaChangeEvent(schemaChangeEvent)) { continue; } @@ -678,8 +910,28 @@ protected void handleQueryEvent( schemaChangeEvent.getTables().isEmpty() ? null : schemaChangeEvent.getTables().iterator().next().id(); + if (tableId != null + && !connectorConfig + .getSkippedOperations() + .contains(Envelope.Operation.TRUNCATE) + && schemaChangeEvent + .getType() + .equals(SchemaChangeEvent.SchemaChangeEventType.TRUNCATE)) { + eventDispatcher.dispatchDataChangeEvent( + partition, + tableId, + new BinlogChangeRecordEmitter<>( + partition, + offsetContext, + clock, + Envelope.Operation.TRUNCATE, + null, + null, + connectorConfig)); + } eventDispatcher.dispatchSchemaChangeEvent( partition, + offsetContext, tableId, (receiver) -> { try { @@ -694,15 +946,8 @@ protected void handleQueryEvent( } } - private void handleTransactionCompletion( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) - throws InterruptedException { - // We are completing the transaction ... - eventDispatcher.dispatchTransactionCommittedEvent(partition, offsetContext); - offsetContext.commitTransaction(); - offsetContext.setBinlogThread(-1L); - skipEvent = false; - ignoreDmlEventByGtidSource = false; + private String removeSetStatement(String sql) { + return sql.replaceAll(SET_STATEMENT_REGEX, "").trim(); } /** @@ -718,17 +963,18 @@ private void handleTransactionCompletion( *

  • MySQL rotates to a new binary log file, even if the table structure does not change. * * + * @param partition the partition; never null + * @param offsetContext the offset context; never null * @param event the update event; never null */ - protected void handleUpdateTableMetadata( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleUpdateTableMetadata(P partition, O offsetContext, Event event) throws InterruptedException { TableMapEventData metadata = unwrapData(event); long tableNumber = metadata.getTableId(); String databaseName = metadata.getDatabase(); String tableName = metadata.getTable(); TableId tableId = new TableId(databaseName, null, tableName); - if (taskContext.getSchema().assignTableNumber(tableNumber, tableId)) { + if (schema.assignTableNumber(tableNumber, tableId)) { LOGGER.debug("Received update table metadata event: {}", event); } else { informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId); @@ -736,103 +982,31 @@ protected void handleUpdateTableMetadata( } /** - * If we receive an event for a table that is monitored but whose metadata we don't know, either - * ignore that event or raise a warning or error as per the {@link - * MySqlConnectorConfig#INCONSISTENT_SCHEMA_HANDLING_MODE} configuration. + * Handle an event of type TRANSACTION_PAYLOAD_EVENT + * + *

    This method should be called whenever a transaction payload event is encountered by the + * mysql binlog connector. A Transaction payload event is propagated from the binlog connector + * when compression is turned on over binlog. This method loops over the individual events in + * the compressed binlog and calls the respective atomic event handlers. + * + * @param partition the partition; never null + * @param offsetContext the offset context; never null + * @param event the transaction payload event; never null */ - private void informAboutUnknownTableIfRequired( - MySqlPartition partition, - MySqlOffsetContext offsetContext, - Event event, - TableId tableId, - Operation operation) + protected void handleTransactionPayload(P partition, O offsetContext, Event event) throws InterruptedException { - if (tableId != null - && connectorConfig.getTableFilters().dataCollectionFilter().isIncluded(tableId)) { - metrics.onErroneousEvent( - partition, "source = " + tableId + ", event " + event, operation); - EventHeaderV4 eventHeader = event.getHeader(); - - if (inconsistentSchemaHandlingMode == EventProcessingFailureHandlingMode.FAIL) { - LOGGER.error( - "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database history topic. Take a new snapshot in this case.{}" - + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", - event, - offsetContext.getOffset(), - tableId, - System.lineSeparator(), - eventHeader.getPosition(), - eventHeader.getNextPosition(), - offsetContext.getSource().binlogFilename()); - throw new DebeziumException( - "Encountered change event for table " - + tableId - + " whose schema isn't known to this connector"); - } else if (inconsistentSchemaHandlingMode == EventProcessingFailureHandlingMode.WARN) { - LOGGER.warn( - "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database history topic. Take a new snapshot in this case.{}" - + "The event will be ignored.{}" - + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", - event, - offsetContext.getOffset(), - tableId, - System.lineSeparator(), - System.lineSeparator(), - eventHeader.getPosition(), - eventHeader.getNextPosition(), - offsetContext.getSource().binlogFilename()); - } else { - LOGGER.debug( - "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database history topic. Take a new snapshot in this case.{}" - + "The event will be ignored.{}" - + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", - event, - offsetContext.getOffset(), - tableId, - System.lineSeparator(), - System.lineSeparator(), - eventHeader.getPosition(), - eventHeader.getNextPosition(), - offsetContext.getSource().binlogFilename()); - } - } else { - if (tableId == null) { - EventData eventData = unwrapData(event); - if (eventData instanceof WriteRowsEventData) { - tableId = - taskContext - .getSchema() - .getExcludeTableId( - ((WriteRowsEventData) eventData).getTableId()); - } else if (eventData instanceof UpdateRowsEventData) { - tableId = - taskContext - .getSchema() - .getExcludeTableId( - ((UpdateRowsEventData) eventData).getTableId()); - } else if (eventData instanceof DeleteRowsEventData) { - tableId = - taskContext - .getSchema() - .getExcludeTableId( - ((DeleteRowsEventData) eventData).getTableId()); - } - } - LOGGER.trace("Filtered {} event for {}", event.getHeader().getEventType(), tableId); - metrics.onFilteredEvent(partition, "source = " + tableId, operation); - eventDispatcher.dispatchFilteredEvent(partition, offsetContext); + TransactionPayloadEventData transactionPayloadEventData = event.getData(); + // Loop over the uncompressed events in the transaction payload event and add the table map + // event in the map of table events + for (Event uncompressedEvent : transactionPayloadEventData.getUncompressedEvents()) { + eventHandlers + .getOrDefault( + uncompressedEvent.getHeader().getEventType(), + (e) -> ignoreEvent(offsetContext, uncompressedEvent)) + .accept(uncompressedEvent); } } - private void informAboutUnknownTableIfRequired( - MySqlPartition partition, - MySqlOffsetContext offsetContext, - Event event, - TableId tableId) - throws InterruptedException { - informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId, null); - } - /** * Generate source records for the supplied event with an {@link WriteRowsEventData}. * @@ -840,28 +1014,31 @@ private void informAboutUnknownTableIfRequired( * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void handleInsert( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleInsert(P partition, O offsetContext, Event event) throws InterruptedException { handleChange( partition, offsetContext, event, - Operation.CREATE, + Envelope.Operation.CREATE, WriteRowsEventData.class, - x -> taskContext.getSchema().getTableId(x.getTableId()), + x -> schema.getTableId(x.getTableId()), WriteRowsEventData::getRows, (tableId, row) -> eventDispatcher.dispatchDataChangeEvent( partition, tableId, - new MySqlChangeRecordEmitter( + new BinlogChangeRecordEmitter<>( partition, offsetContext, clock, - Operation.CREATE, + Envelope.Operation.CREATE, null, - row))); + row, + connectorConfig)), + (tableId, row) -> + validateChangeEventWithTable( + taskContext.getSchema().tableFor(tableId), null, row)); } /** @@ -871,28 +1048,33 @@ protected void handleInsert( * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void handleUpdate( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleUpdate(P partition, O offsetContext, Event event) throws InterruptedException { handleChange( partition, offsetContext, event, - Operation.UPDATE, + Envelope.Operation.UPDATE, UpdateRowsEventData.class, - x -> taskContext.getSchema().getTableId(x.getTableId()), + x -> schema.getTableId(x.getTableId()), UpdateRowsEventData::getRows, (tableId, row) -> eventDispatcher.dispatchDataChangeEvent( partition, tableId, - new MySqlChangeRecordEmitter( + new BinlogChangeRecordEmitter<>( partition, offsetContext, clock, - Operation.UPDATE, + Envelope.Operation.UPDATE, row.getKey(), - row.getValue()))); + row.getValue(), + connectorConfig)), + (tableId, row) -> + validateChangeEventWithTable( + taskContext.getSchema().tableFor(tableId), + row.getKey(), + row.getValue())); } /** @@ -902,93 +1084,31 @@ protected void handleUpdate( * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void handleDelete( - MySqlPartition partition, MySqlOffsetContext offsetContext, Event event) + protected void handleDelete(P partition, O offsetContext, Event event) throws InterruptedException { handleChange( partition, offsetContext, event, - Operation.DELETE, + Envelope.Operation.DELETE, DeleteRowsEventData.class, - x -> taskContext.getSchema().getTableId(x.getTableId()), + x -> schema.getTableId(x.getTableId()), DeleteRowsEventData::getRows, (tableId, row) -> eventDispatcher.dispatchDataChangeEvent( partition, tableId, - new MySqlChangeRecordEmitter( + new BinlogChangeRecordEmitter<>( partition, offsetContext, clock, - Operation.DELETE, + Envelope.Operation.DELETE, row, - null))); - } - - private void handleChange( - MySqlPartition partition, - MySqlOffsetContext offsetContext, - Event event, - Operation operation, - Class eventDataClass, - TableIdProvider tableIdProvider, - RowsProvider rowsProvider, - BinlogChangeEmitter changeEmitter) - throws InterruptedException { - if (skipEvent) { - // We can skip this because we should already be at least this far ... - LOGGER.info("Skipping previously processed row event: {}", event); - return; - } - if (ignoreDmlEventByGtidSource) { - LOGGER.debug("Skipping DML event because this GTID source is filtered: {}", event); - return; - } - final T data = unwrapData(event); - final TableId tableId = tableIdProvider.getTableId(data); - final List rows = rowsProvider.getRows(data); - String changeType = operation.name(); - - if (tableId != null && taskContext.getSchema().schemaFor(tableId) != null) { - int count = 0; - int numRows = rows.size(); - if (startingRowNumber < numRows) { - // Use iterator to avoid O(n²) complexity when rows is a LinkedList - // (mysql-binlog-connector-java uses LinkedList in WriteRowsEventDataDeserializer - // and DeleteRowsEventDataDeserializer) - int rowIndex = 0; - for (U rowData : rows) { - if (rowIndex >= startingRowNumber) { - offsetContext.setRowNumber(rowIndex, numRows); - offsetContext.event(tableId, eventTimestamp); - changeEmitter.emit(tableId, rowData); - count++; - } - rowIndex++; - } - if (LOGGER.isDebugEnabled()) { - if (startingRowNumber != 0) { - LOGGER.debug( - "Emitted {} {} record(s) for last {} row(s) in event: {}", - count, - changeType, - numRows - startingRowNumber, - event); - } else { - LOGGER.debug( - "Emitted {} {} record(s) for event: {}", count, changeType, event); - } - } - offsetContext.changeEventCompleted(); - } else { - // All rows were previously processed ... - LOGGER.debug("Skipping previously processed {} event: {}", changeType, event); - } - } else { - informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId, operation); - } - startingRowNumber = 0; + null, + connectorConfig)), + (tableId, row) -> + validateChangeEventWithTable( + taskContext.getSchema().tableFor(tableId), row, null)); } /** @@ -997,8 +1117,7 @@ private void handleChange( * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void viewChange(MySqlOffsetContext offsetContext, Event event) - throws InterruptedException { + protected void viewChange(O offsetContext, Event event) throws InterruptedException { LOGGER.debug("View Change event: {}", event); // do nothing } @@ -1009,357 +1128,319 @@ protected void viewChange(MySqlOffsetContext offsetContext, Event event) * @param event the database change data event to be processed; may not be null * @throws InterruptedException if this thread is interrupted while blocking */ - protected void prepareTransaction(MySqlOffsetContext offsetContext, Event event) - throws InterruptedException { + protected void prepareTransaction(O offsetContext, Event event) throws InterruptedException { LOGGER.debug("XA Prepare event: {}", event); // do nothing } - private SSLMode sslModeFor(SecureConnectionMode mode) { - switch (mode) { - case DISABLED: - return SSLMode.DISABLED; - case PREFERRED: - return SSLMode.PREFERRED; - case REQUIRED: - return SSLMode.REQUIRED; - case VERIFY_CA: - return SSLMode.VERIFY_CA; - case VERIFY_IDENTITY: - return SSLMode.VERIFY_IDENTITY; - } - return null; - } - - @Override - public void execute( - ChangeEventSourceContext context, - MySqlPartition partition, - MySqlOffsetContext offsetContext) + protected void handleTransactionBegin(P partition, O offsetContext, Event event, Long threadId) throws InterruptedException { - if (!connectorConfig.getSnapshotMode().shouldStream()) { - LOGGER.info( - "Streaming is disabled for snapshot mode {}", - connectorConfig.getSnapshotMode()); - return; + Instant eventTime = Conversions.toInstantFromMillis(eventTimestamp.toEpochMilli()); + // We are starting a new transaction ... + offsetContext.startNextTransaction(); + eventDispatcher.dispatchTransactionStartedEvent( + partition, offsetContext.getTransactionId(), offsetContext, eventTime); + if (threadId != null) { + offsetContext.setBinlogThread(threadId); } - if (connectorConfig.getSnapshotMode() != MySqlConnectorConfig.SnapshotMode.NEVER) { - taskContext.getSchema().assureNonEmptySchema(); + if (initialEventsToSkip != 0) { + LOGGER.debug( + "Restarting partially-processed transaction; change events will not be created for the first {} events plus {} more rows in the next event", + initialEventsToSkip, + startingRowNumber); + // We are restarting, so we need to skip the events in this transaction that we + // processed previously... + skipEvent = true; } - final Set skippedOperations = connectorConfig.getSkippedOperations(); - - final MySqlOffsetContext effectiveOffsetContext = - offsetContext != null ? offsetContext : MySqlOffsetContext.initial(connectorConfig); - - // Register our event handlers ... - eventHandlers.put( - EventType.STOP, (event) -> handleServerStop(effectiveOffsetContext, event)); - eventHandlers.put( - EventType.HEARTBEAT, - (event) -> handleServerHeartbeat(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.INCIDENT, - (event) -> handleServerIncident(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.ROTATE, (event) -> handleRotateLogsEvent(effectiveOffsetContext, event)); - eventHandlers.put( - EventType.TABLE_MAP, - (event) -> handleUpdateTableMetadata(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.QUERY, - (event) -> handleQueryEvent(partition, effectiveOffsetContext, event)); + } - if (!skippedOperations.contains(Operation.CREATE)) { - eventHandlers.put( - EventType.WRITE_ROWS, - (event) -> handleInsert(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.EXT_WRITE_ROWS, - (event) -> handleInsert(partition, effectiveOffsetContext, event)); - } + /** + * @return the connector class type + */ + protected abstract Class getConnectorClass(); - if (!skippedOperations.contains(Operation.UPDATE)) { - eventHandlers.put( - EventType.UPDATE_ROWS, - (event) -> handleUpdate(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.EXT_UPDATE_ROWS, - (event) -> handleUpdate(partition, effectiveOffsetContext, event)); - } + /** + * Handle the GTID event. + * + * @param partition the partition; never null + * @param offsetContext the offset context; never null + * @param event the event + * @param gtidSourceFilter the GTID source filter + * @throws InterruptedException if the thread is interrupted + */ + protected abstract void handleGtidEvent( + P partition, O offsetContext, Event event, Predicate gtidSourceFilter) + throws InterruptedException; - if (!skippedOperations.contains(Operation.DELETE)) { - eventHandlers.put( - EventType.DELETE_ROWS, - (event) -> handleDelete(partition, effectiveOffsetContext, event)); - eventHandlers.put( - EventType.EXT_DELETE_ROWS, - (event) -> handleDelete(partition, effectiveOffsetContext, event)); - } + /** + * Handle the supplied event with a {@link RowsQueryEventData} or {@link AnnotateRowsEventData} + * by recording the original SQL query that generated the event. + * + * @param offsetContext the offset context; never null + * @param event the database change data event to be processed; may not be null + */ + protected abstract void handleRecordingQuery(O offsetContext, Event event); - eventHandlers.put( - EventType.VIEW_CHANGE, (event) -> viewChange(effectiveOffsetContext, event)); - eventHandlers.put( - EventType.XA_PREPARE, (event) -> prepareTransaction(effectiveOffsetContext, event)); - eventHandlers.put( - EventType.XID, - (event) -> handleTransactionCompletion(partition, effectiveOffsetContext, event)); + /** + * @return the event type for reading query events from + */ + protected abstract EventType getIncludeQueryEventType(); - // Conditionally register ROWS_QUERY handler to parse SQL statements. - if (connectorConfig.includeSqlQuery()) { - eventHandlers.put( - EventType.ROWS_QUERY, - (event) -> handleRowsQuery(effectiveOffsetContext, event)); - } + /** + * @return the event type for the global transaction identifier events + */ + protected abstract EventType getGtidEventType(); - BinaryLogClient.EventListener listener; - if (connectorConfig.bufferSizeForStreamingChangeEventSource() == 0) { - listener = (event) -> handleEvent(partition, effectiveOffsetContext, event); - } else { - EventBuffer buffer = - new EventBuffer( - connectorConfig.bufferSizeForStreamingChangeEventSource(), - this, - context); - listener = (event) -> buffer.add(partition, effectiveOffsetContext, event); - } - client.registerEventListener(listener); + /** + * Initialize the connector's global transaction identifier set. + * + * @param value the value to initialize the set based upon + */ + protected abstract void initializeGtidSet(String value); - client.registerLifecycleListener(new ReaderThreadLifecycleListener(effectiveOffsetContext)); - client.registerEventListener((event) -> onEvent(effectiveOffsetContext, event)); - if (LOGGER.isDebugEnabled()) { - client.registerEventListener((event) -> logEvent(effectiveOffsetContext, event)); + private Predicate getGtidDmlSourceFilter() { + if (connectorConfig + .getConfig() + .getBoolean(BinlogConnectorConfig.GTID_SOURCE_FILTER_DML_EVENTS)) { + return connectorConfig.getGtidSourceFilter(); } + return null; + } - final boolean isGtidModeEnabled = connection.isGtidModeEnabled(); - metrics.setIsGtidModeEnabled(isGtidModeEnabled); - - // Get the current GtidSet from MySQL so we can get a filtered/merged GtidSet based off of - // the last Debezium checkpoint. - String availableServerGtidStr = connection.knownGtidSet(); - if (isGtidModeEnabled) { - // The server is using GTIDs, so enable the handler ... - eventHandlers.put( - EventType.GTID, (event) -> handleGtidEvent(effectiveOffsetContext, event)); - - // Now look at the GTID set from the server and what we've previously seen ... - GtidSet availableServerGtidSet = new GtidSet(availableServerGtidStr); + private void handleTransactionCompletion(P partition, O offsetContext, Event event) + throws InterruptedException { + // We are completing the transaction ... + eventDispatcher.dispatchTransactionCommittedEvent( + partition, + offsetContext, + Conversions.toInstantFromMillis(eventTimestamp.toEpochMilli())); + offsetContext.commitTransaction(); + offsetContext.setBinlogThread(-1L); + skipEvent = false; + ignoreDmlEventByGtidSource = false; + } - // also take into account purged GTID logs - GtidSet purgedServerGtidSet = connection.purgedGtidSet(); - LOGGER.info("GTID set purged on server: {}", purgedServerGtidSet); + /** + * If we receive an event for a table that is monitored but whose metadata we don't know, either + * ignore that event or raise a warning or error per the {@link + * BinlogConnectorConfig#INCONSISTENT_SCHEMA_HANDLING_MODE} configuration. + * + * @param partition the partition; never null + * @param offsetContext the offset context; never null + * @param event the binlog event + * @param tableId the table identifier + * @param operation the operation being performed + */ + private void informAboutUnknownTableIfRequired( + P partition, + O offsetContext, + Event event, + TableId tableId, + Envelope.Operation operation) + throws InterruptedException { + if (tableId != null + && connectorConfig.getTableFilters().dataCollectionFilter().isIncluded(tableId)) { + metrics.onErroneousEvent( + partition, "source = " + tableId + ", event " + event, operation); + EventHeaderV4 eventHeader = event.getHeader(); - GtidSet filteredGtidSet = - filterGtidSet( - effectiveOffsetContext, availableServerGtidSet, purgedServerGtidSet); - if (filteredGtidSet != null) { - // We've seen at least some GTIDs, so start reading from the filtered GTID set ... - LOGGER.info("Registering binlog reader with GTID set: {}", filteredGtidSet); - String filteredGtidSetStr = filteredGtidSet.toString(); - client.setGtidSet(filteredGtidSetStr); - effectiveOffsetContext.setCompletedGtidSet(filteredGtidSetStr); - gtidSet = new com.github.shyiko.mysql.binlog.GtidSet(filteredGtidSetStr); + if (inconsistentSchemaHandlingMode == EventProcessingFailureHandlingMode.FAIL) { + LOGGER.error( + "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database schema history topic. Take a new snapshot in this case.{}" + + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", + event, + offsetContext.getOffset(), + tableId, + System.lineSeparator(), + eventHeader.getPosition(), + eventHeader.getNextPosition(), + offsetContext.getSource().binlogFilename()); + throw new DebeziumException( + "Encountered change event for table " + + tableId + + " whose schema isn't known to this connector"); + } else if (inconsistentSchemaHandlingMode == EventProcessingFailureHandlingMode.WARN) { + LOGGER.warn( + "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database schema history topic. Take a new snapshot in this case.{}" + + "The event will be ignored.{}" + + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", + event, + offsetContext.getOffset(), + tableId, + System.lineSeparator(), + System.lineSeparator(), + eventHeader.getPosition(), + eventHeader.getNextPosition(), + offsetContext.getSource().binlogFilename()); } else { - // We've not yet seen any GTIDs, so that means we have to start reading the binlog - // from the beginning ... - client.setBinlogFilename(effectiveOffsetContext.getSource().binlogFilename()); - client.setBinlogPosition(effectiveOffsetContext.getSource().binlogPosition()); - gtidSet = new com.github.shyiko.mysql.binlog.GtidSet(""); + LOGGER.debug( + "Encountered change event '{}' at offset {} for table {} whose schema isn't known to this connector. One possible cause is an incomplete database schema history topic. Take a new snapshot in this case.{}" + + "The event will be ignored.{}" + + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", + event, + offsetContext.getOffset(), + tableId, + System.lineSeparator(), + System.lineSeparator(), + eventHeader.getPosition(), + eventHeader.getNextPosition(), + offsetContext.getSource().binlogFilename()); } } else { - // The server is not using GTIDs, so start reading the binlog based upon where we last - // left off ... - client.setBinlogFilename(effectiveOffsetContext.getSource().binlogFilename()); - client.setBinlogPosition(effectiveOffsetContext.getSource().binlogPosition()); - } - - // We may be restarting in the middle of a transaction, so see how far into the transaction - // we have already processed... - initialEventsToSkip = effectiveOffsetContext.eventsToSkipUponRestart(); - LOGGER.info("Skip {} events on streaming start", initialEventsToSkip); - - // Set the starting row number, which is the next row number to be read ... - startingRowNumber = effectiveOffsetContext.rowsToSkipUponRestart(); - LOGGER.info("Skip {} rows on streaming start", startingRowNumber); - - // Only when we reach the first BEGIN event will we start to skip events ... - skipEvent = false; - - try { - // Start the log reader, which starts background threads ... - if (context.isRunning()) { - long timeout = connectorConfig.getConnectionTimeout().toMillis(); - long started = clock.currentTimeInMillis(); - try { - LOGGER.debug( - "Attempting to establish binlog reader connection with timeout of {} ms", - timeout); - client.connect(timeout); - // Need to wait for keepalive thread to be running, otherwise it can be left - // orphaned - // The problem is with timing. When the close is called too early after connect - // then - // the keepalive thread is not terminated - if (client.isKeepAlive()) { - LOGGER.info("Waiting for keepalive thread to start"); - final Metronome metronome = Metronome.parker(Duration.ofMillis(100), clock); - int waitAttempts = 50; - boolean keepAliveThreadRunning = false; - while (!keepAliveThreadRunning && waitAttempts-- > 0) { - for (Thread t : binaryLogClientThreads.values()) { - if (t.getName().startsWith(KEEPALIVE_THREAD_NAME) && t.isAlive()) { - LOGGER.info("Keepalive thread is running"); - keepAliveThreadRunning = true; - } - } - metronome.pause(); - } - } - } catch (TimeoutException e) { - // If the client thread is interrupted *before* the client could connect, the - // client throws a timeout exception - // The only way we can distinguish this is if we get the timeout exception - // before the specified timeout has - // elapsed, so we simply check this (within 10%) ... - long duration = clock.currentTimeInMillis() - started; - if (duration > (0.9 * timeout)) { - double actualSeconds = TimeUnit.MILLISECONDS.toSeconds(duration); - throw new DebeziumException( - "Timed out after " - + actualSeconds - + " seconds while waiting to connect to MySQL at " - + connectorConfig.hostname() - + ":" - + connectorConfig.port() - + " with user '" - + connectorConfig.username() - + "'", - e); - } - // Otherwise, we were told to shutdown, so we don't care about the timeout - // exception - } catch (AuthenticationException e) { - throw new DebeziumException( - "Failed to authenticate to the MySQL database at " - + connectorConfig.hostname() - + ":" - + connectorConfig.port() - + " with user '" - + connectorConfig.username() - + "'", - e); - } catch (Throwable e) { - throw new DebeziumException( - "Unable to connect to the MySQL database at " - + connectorConfig.hostname() - + ":" - + connectorConfig.port() - + " with user '" - + connectorConfig.username() - + "': " - + e.getMessage(), - e); + if (tableId == null) { + EventData eventData = unwrapData(event); + if (eventData instanceof WriteRowsEventData) { + tableId = + schema.getExcludeTableId(((WriteRowsEventData) eventData).getTableId()); + } else if (eventData instanceof UpdateRowsEventData) { + tableId = + schema.getExcludeTableId( + ((UpdateRowsEventData) eventData).getTableId()); + } else if (eventData instanceof DeleteRowsEventData) { + tableId = + schema.getExcludeTableId( + ((DeleteRowsEventData) eventData).getTableId()); } } - while (context.isRunning()) { - Thread.sleep(100); - } - } finally { - try { - client.disconnect(); - } catch (Exception e) { - LOGGER.info("Exception while stopping binary log client", e); - } + LOGGER.trace("Filtered {} event for {}", event.getHeader().getEventType(), tableId); + metrics.onFilteredEvent(partition, "source = " + tableId, operation); + eventDispatcher.dispatchFilteredEvent(partition, offsetContext); } } - private SSLSocketFactory getBinlogSslSocketFactory( - MySqlConnectorConfig connectorConfig, MySqlConnection connection) { - String acceptedTlsVersion = connection.getSessionVariableForSslVersion(); - if (!isNullOrEmpty(acceptedTlsVersion)) { - SSLMode sslMode = sslModeFor(connectorConfig.sslMode()); - LOGGER.info( - "Enable ssl " - + sslMode - + " mode for connector " - + connectorConfig.getLogicalName()); - - final char[] keyPasswordArray = connection.connectionConfig().sslKeyStorePassword(); - final String keyFilename = connection.connectionConfig().sslKeyStore(); - final char[] trustPasswordArray = connection.connectionConfig().sslTrustStorePassword(); - final String trustFilename = connection.connectionConfig().sslTrustStore(); - KeyManager[] keyManagers = null; - if (keyFilename != null) { - try { - KeyStore ks = connection.loadKeyStore(keyFilename, keyPasswordArray); - - KeyManagerFactory kmf = KeyManagerFactory.getInstance("NewSunX509"); - kmf.init(ks, keyPasswordArray); - - keyManagers = kmf.getKeyManagers(); - } catch (KeyStoreException - | NoSuchAlgorithmException - | UnrecoverableKeyException e) { - throw new DebeziumException("Could not load keystore", e); - } - } - TrustManager[] trustManagers; - try { - KeyStore ks = null; - if (trustFilename != null) { - ks = connection.loadKeyStore(trustFilename, trustPasswordArray); - } - - if (ks == null && (sslMode == SSLMode.PREFERRED || sslMode == SSLMode.REQUIRED)) { - trustManagers = - new TrustManager[] { - new X509TrustManager() { - - @Override - public void checkClientTrusted( - X509Certificate[] x509Certificates, String s) - throws CertificateException {} - - @Override - public void checkServerTrusted( - X509Certificate[] x509Certificates, String s) - throws CertificateException {} - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - } - }; - } else { - TrustManagerFactory tmf = - TrustManagerFactory.getInstance( - TrustManagerFactory.getDefaultAlgorithm()); - tmf.init(ks); - trustManagers = tmf.getTrustManagers(); - } - } catch (KeyStoreException | NoSuchAlgorithmException e) { - throw new DebeziumException("Could not load truststore", e); + /** + * If we receive an event for a table that is monitored but whose metadata we don't know, either + * ignore that event or raise a warning or error per the {@link + * BinlogConnectorConfig#INCONSISTENT_SCHEMA_HANDLING_MODE} configuration. + * + * @param partition the partition; never null + * @param offsetContext the offset context; never null + * @param event the binlog event + * @param tableId the table identifier + * @throws InterruptedException if the thread is interrupted + */ + private void informAboutUnknownTableIfRequired( + P partition, O offsetContext, Event event, TableId tableId) + throws InterruptedException { + informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId, null); + } + + private void validateChangeEventWithTable(Table table, Object[] before, Object[] after) { + if (table != null) { + int columnSize = table.columns().size(); + String message = + "Error processing {} of row in {} because it's different column size with internal schema size {}, but {} size {}, " + + "restart connector with schema recovery mode."; + if (before != null && columnSize != before.length) { + LOGGER.error( + message, "before", table.id().table(), columnSize, "before", before.length); + throw new DebeziumException( + "Error processing row in " + + table.id().table() + + ", internal schema size " + + columnSize + + ", but row size " + + before.length + + " , " + + "restart connector with schema recovery mode."); } - // DBZ-1208 Resembles the logic from the upstream BinaryLogClient, only that - // the accepted TLS version is passed to the constructed factory - final KeyManager[] finalKMS = keyManagers; - return new DefaultSSLSocketFactory(acceptedTlsVersion) { + if (after != null && columnSize != after.length) { + LOGGER.error( + message, "after", table.id().table(), columnSize, "after", after.length); + throw new DebeziumException( + "Error processing row in " + + table.id().table() + + ", internal schema size " + + columnSize + + ", but row size " + + after.length + + " , " + + "restart connector with schema recovery mode."); + } + } + } - @Override - protected void initSSLContext(SSLContext sc) throws GeneralSecurityException { - sc.init(finalKMS, trustManagers, null); + private void handleChange( + P partition, + O offsetContext, + Event event, + Envelope.Operation operation, + Class eventDataClass, + TableIdProvider tableIdProvider, + RowsProvider rowsProvider, + BinlogChangeEmitter changeEmitter, + ChangeEventValidator changeEventValidator) + throws InterruptedException { + if (skipEvent) { + // We can skip this because we should already be at least this far ... + LOGGER.info("Skipping previously processed row event: {}", event); + return; + } + if (ignoreDmlEventByGtidSource) { + LOGGER.debug("Skipping DML event because this GTID source is filtered: {}", event); + return; + } + final T data = unwrapData(event); + final TableId tableId = tableIdProvider.getTableId(data); + final List rows = rowsProvider.getRows(data); + String changeType = operation.name(); + + if (tableId != null && taskContext.getSchema().schemaFor(tableId) != null) { + int count = 0; + int numRows = rows.size(); + if (startingRowNumber < numRows) { + // Iterate rather than index: the row list may be a linked list, in which case + // rows.get(i) turns this loop into O(n^2). + int rowIndex = 0; + for (U row : rows) { + if (rowIndex >= startingRowNumber) { + changeEventValidator.validate(tableId, row); + offsetContext.setRowNumber(rowIndex, numRows); + offsetContext.event(tableId, eventTimestamp); + changeEmitter.emit(tableId, row); + count++; + } + rowIndex++; } - }; + if (LOGGER.isDebugEnabled()) { + if (startingRowNumber != 0) { + LOGGER.debug( + "Emitted {} {} record(s) for last {} row(s) in event: {}", + count, + changeType, + numRows - startingRowNumber, + event); + } else { + LOGGER.debug( + "Emitted {} {} record(s) for event: {}", count, changeType, event); + } + } + offsetContext.changeEventCompleted(); + } else { + // All rows were previously processed ... + LOGGER.debug("Skipping previously processed {} event: {}", changeType, event); + } + } else { + informAboutUnknownTableIfRequired(partition, offsetContext, event, tableId, operation); } + startingRowNumber = 0; + } - return null; + private void waitWhenStreamingPaused(ChangeEventSourceContext context) + throws InterruptedException { + if (context.isPaused()) { + LOGGER.info("Streaming will now pause"); + context.streamingPaused(); + context.waitSnapshotCompletion(); + LOGGER.info("Streaming resumed"); + } } private void logStreamingSourceState() { logStreamingSourceState(Level.ERROR); } - protected void logEvent(MySqlOffsetContext offsetContext, Event event) { + protected void logEvent(O offsetContext, Event event) { LOGGER.trace("Received event: {}", event); } @@ -1382,86 +1463,7 @@ private void logStreamingSourceState(Level severity) { } } - /** - * Apply the include/exclude GTID source filters to the current {@link #source() GTID set} and - * merge them onto the currently available GTID set from a MySQL server. - * - *

    The merging behavior of this method might seem a bit strange at first. It's required in - * order for Debezium to consume a MySQL binlog that has multi-source replication enabled, if a - * failover has to occur. In such a case, the server that Debezium is failed over to might have - * a different set of sources, but still include the sources required for Debezium to continue - * to function. MySQL does not allow downstream replicas to connect if the GTID set does not - * contain GTIDs for all channels that the server is replicating from, even if the server does - * have the data needed by the client. To get around this, we can have Debezium merge its GTID - * set with whatever is on the server, so that MySQL will allow it to connect. See DBZ-143 for details. - * - *

    This method does not mutate any state in the context. - * - * @param availableServerGtidSet the GTID set currently available in the MySQL server - * @param purgedServerGtid the GTID set already purged by the MySQL server - * @return A GTID set meant for consuming from a MySQL binlog; may return null if the SourceInfo - * has no GTIDs and therefore none were filtered - */ - public GtidSet filterGtidSet( - MySqlOffsetContext offsetContext, - GtidSet availableServerGtidSet, - GtidSet purgedServerGtid) { - String gtidStr = offsetContext.gtidSet(); - if (gtidStr == null) { - return null; - } - LOGGER.info("Attempting to generate a filtered GTID set"); - LOGGER.info("GTID set from previous recorded offset: {}", gtidStr); - GtidSet filteredGtidSet = new GtidSet(gtidStr); - Predicate gtidSourceFilter = connectorConfig.gtidSourceFilter(); - if (gtidSourceFilter != null) { - filteredGtidSet = filteredGtidSet.retainAll(gtidSourceFilter); - LOGGER.info( - "GTID set after applying GTID source includes/excludes to previous recorded offset: {}", - filteredGtidSet); - } - LOGGER.info("GTID set available on server: {}", availableServerGtidSet); - - GtidSet mergedGtidSet; - - if (connectorConfig.gtidNewChannelPosition() == GtidNewChannelPosition.EARLIEST) { - LOGGER.info("Using first available positions for new GTID channels"); - final GtidSet relevantAvailableServerGtidSet = - (gtidSourceFilter != null) - ? availableServerGtidSet.retainAll(gtidSourceFilter) - : availableServerGtidSet; - LOGGER.info( - "Relevant GTID set available on server: {}", relevantAvailableServerGtidSet); - - // Since the GTID recorded in the checkpoint represents the CDC-executed records, in - // certain scenarios - // (such as when the startup mode is earliest/timestamp/binlogfile), the recorded GTID - // may not start from - // the beginning. For example, A:300-500. However, during job recovery, we usually only - // need to focus on - // the last consumed point instead of consuming A:1-299. Therefore, some adjustments - // need to be made to the - // recorded offset in the checkpoint, and the available GTID for other MySQL instances - // should be completed. - mergedGtidSet = - GtidUtils.fixOldChannelsGtidSet( - relevantAvailableServerGtidSet, purgedServerGtid, filteredGtidSet); - } else { - LOGGER.info("Using latest positions for new GTID channels"); - mergedGtidSet = - GtidUtils.computeLatestModeGtidSet( - availableServerGtidSet, - purgedServerGtid, - filteredGtidSet, - gtidSourceFilter); - } - - LOGGER.info("Final merged GTID set to use when connecting to MySQL: {}", mergedGtidSet); - return mergedGtidSet; - } - - MySqlStreamingChangeEventSourceMetrics getMetrics() { + BinlogStreamingChangeEventSourceMetrics getMetrics() { return metrics; } @@ -1510,11 +1512,11 @@ protected DebeziumException wrap(Throwable error) { return new DebeziumException(msg, error); } - /** LifecycleListener for Reader Thread. */ - protected final class ReaderThreadLifecycleListener implements LifecycleListener { - private final MySqlOffsetContext offsetContext; + protected final class ReaderThreadLifecycleListener + implements BinaryLogClient.LifecycleListener { + private final O offsetContext; - ReaderThreadLifecycleListener(MySqlOffsetContext offsetContext) { + ReaderThreadLifecycleListener(O offsetContext) { this.offsetContext = offsetContext; } @@ -1544,13 +1546,13 @@ public void onDisconnect(BinaryLogClient client) { public void onConnect(BinaryLogClient client) { // Set up the MDC logging context for this thread ... taskContext.configureLoggingContext("binlog"); - // The event row number will be used when processing the first event ... LOGGER.info( - "Connected to MySQL binlog at {}:{}, starting at {}", - connectorConfig.hostname(), - connectorConfig.port(), + "Connected to binlog at {}:{}, starting at {}", + connectorConfig.getHostName(), + connectorConfig.getPort(), offsetContext); + totalRecordCounter.set(0); } @Override @@ -1584,6 +1586,108 @@ public void onEventDeserializationFailure(BinaryLogClient client, Exception ex) } } + private SSLMode sslModeFor(SecureConnectionMode mode) { + switch (mode) { + case DISABLED: + return SSLMode.DISABLED; + case PREFERRED: + return SSLMode.PREFERRED; + case REQUIRED: + return SSLMode.REQUIRED; + case VERIFY_CA: + return SSLMode.VERIFY_CA; + case VERIFY_IDENTITY: + return SSLMode.VERIFY_IDENTITY; + } + return null; + } + + private SSLSocketFactory getBinlogSslSocketFactory( + BinlogConnectorConfig connectorConfig, BinlogConnectorConnection connection) { + String acceptedTlsVersion = connection.getSessionVariableForSslVersion(); + if (!isNullOrEmpty(acceptedTlsVersion)) { + SSLMode sslMode = sslModeFor(connectorConfig.getSslMode()); + LOGGER.info( + "Enable ssl " + + sslMode + + " mode for connector " + + connectorConfig.getLogicalName()); + + final char[] keyPasswordArray = connection.connectionConfig().sslKeyStorePassword(); + final String keyFilename = connection.connectionConfig().sslKeyStore(); + final char[] trustPasswordArray = connection.connectionConfig().sslTrustStorePassword(); + final String trustFilename = connection.connectionConfig().sslTrustStore(); + KeyManager[] keyManagers = null; + if (keyFilename != null) { + try { + KeyStore ks = connection.loadKeyStore(keyFilename, keyPasswordArray); + + KeyManagerFactory kmf = KeyManagerFactory.getInstance("NewSunX509"); + kmf.init(ks, keyPasswordArray); + + keyManagers = kmf.getKeyManagers(); + } catch (KeyStoreException + | NoSuchAlgorithmException + | UnrecoverableKeyException e) { + throw new DebeziumException("Could not load keystore", e); + } + } + TrustManager[] trustManagers; + try { + KeyStore ks = null; + if (trustFilename != null) { + ks = connection.loadKeyStore(trustFilename, trustPasswordArray); + } + + if (ks == null && (sslMode == SSLMode.PREFERRED || sslMode == SSLMode.REQUIRED)) { + trustManagers = + new TrustManager[] { + new X509TrustManager() { + @Override + public void checkClientTrusted( + X509Certificate[] x509Certificates, String s) + throws CertificateException {} + + @Override + public void checkServerTrusted( + X509Certificate[] x509Certificates, String s) + throws CertificateException {} + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } + }; + } else { + TrustManagerFactory tmf = + TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(ks); + trustManagers = tmf.getTrustManagers(); + } + } catch (KeyStoreException | NoSuchAlgorithmException e) { + throw new DebeziumException("Could not load truststore", e); + } + + // DBZ-1208 Resembles the logic from the upstream BinaryLogClient, only that + // the accepted TLS version is passed to the constructed factory + final KeyManager[] finalKMS = keyManagers; + return new DefaultSSLSocketFactory(acceptedTlsVersion) { + @Override + protected void initSSLContext(SSLContext sc) throws GeneralSecurityException { + sc.init(finalKMS, trustManagers, null); + } + }; + } + return null; + } + + @FunctionalInterface + private interface BinlogChangeEmitter { + void emit(TableId tableId, T data) throws InterruptedException; + } + @FunctionalInterface private interface TableIdProvider { TableId getTableId(E data); @@ -1593,4 +1697,65 @@ private interface TableIdProvider { private interface RowsProvider { List getRows(E data); } + + @FunctionalInterface + private interface ChangeEventValidator { + void validate(TableId tableId, U row); + } + + /** Class that represents a binlog file and position */ + public static class BinlogPosition { + final String filename; + final long position; + + public BinlogPosition(String filename, long position) { + assert filename != null; + + this.filename = filename; + this.position = position; + } + + public String getFilename() { + return filename; + } + + public long getPosition() { + return position; + } + + @Override + public String toString() { + return filename + "/" + position; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + filename.hashCode(); + result = prime * result + (int) (position ^ (position >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + BinlogPosition other = (BinlogPosition) obj; + if (!filename.equals(other.filename)) { + return false; + } + if (position != other.position) { + return false; + } + return true; + } + } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/test/java/org/apache/flink/cdc/connectors/oceanbase/OceanBaseFailoverITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/test/java/org/apache/flink/cdc/connectors/oceanbase/OceanBaseFailoverITCase.java index 55a1b7dd2ce..562a87b877f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/test/java/org/apache/flink/cdc/connectors/oceanbase/OceanBaseFailoverITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/test/java/org/apache/flink/cdc/connectors/oceanbase/OceanBaseFailoverITCase.java @@ -32,7 +32,7 @@ import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.StringUtils; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import io.debezium.jdbc.JdbcConnection; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/test/java/org/apache/flink/cdc/connectors/oceanbase/OceanBaseSourceTestBase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/test/java/org/apache/flink/cdc/connectors/oceanbase/OceanBaseSourceTestBase.java index fed82bb6308..89606efd118 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/test/java/org/apache/flink/cdc/connectors/oceanbase/OceanBaseSourceTestBase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/test/java/org/apache/flink/cdc/connectors/oceanbase/OceanBaseSourceTestBase.java @@ -25,7 +25,7 @@ import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; -import io.debezium.connector.mysql.MySqlConnection; +import io.debezium.connector.mysql.jdbc.MySqlConnection; import org.apache.commons.lang3.StringUtils; import org.assertj.core.api.Assertions; import org.slf4j.Logger; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/antlr/listener/ColumnDefinitionParserListener.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/antlr/listener/ColumnDefinitionParserListener.java new file mode 100644 index 00000000000..0f08a3e9d00 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/antlr/listener/ColumnDefinitionParserListener.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.debezium.connector.oracle.antlr.listener; + +import io.debezium.antlr.DataTypeResolver; +import io.debezium.connector.oracle.antlr.OracleDdlParser; +import io.debezium.ddl.parser.oracle.generated.PlSqlParser; +import io.debezium.relational.Column; +import io.debezium.relational.ColumnEditor; +import io.debezium.relational.TableEditor; +import oracle.jdbc.OracleTypes; +import org.antlr.v4.runtime.tree.ParseTreeListener; + +import java.sql.Types; +import java.util.List; + +/** + * Parser listener that parses column definitions of Oracle DDL statements. + * + *

    Copied from Debezium project (2.7.4.Final). + * + *

    Change: when a column is declared with a schema-qualified object type, Debezium recognises + * only {@code MDSYS.SDO_GEOMETRY} and otherwise leaves the column editor untouched, so the + * resulting column has a {@code null} type name. Since Debezium 2.2 the field-name selector always + * wraps its namer in a {@code FieldNameCache}, whose backing map calls {@link + * io.debezium.relational.Column#equals(Object)}, and that dereferences {@code typeName()} without a + * null check -- so such a column throws a {@link NullPointerException} while the table schema is + * built. Recognise {@code SYS.XMLTYPE} as well, mirroring what the Oracle pipeline connector's own + * listener already does. + */ +public class ColumnDefinitionParserListener extends BaseParserListener { + + private final OracleDdlParser parser; + private final DataTypeResolver dataTypeResolver; + private final TableEditor tableEditor; + private final List listeners; + private ColumnEditor columnEditor; + + ColumnDefinitionParserListener( + final TableEditor tableEditor, + final ColumnEditor columnEditor, + OracleDdlParser parser, + List listeners) { + this.tableEditor = tableEditor; + this.columnEditor = columnEditor; + this.parser = parser; + this.dataTypeResolver = parser.dataTypeResolver(); + this.listeners = listeners; + } + + void setColumnEditor(ColumnEditor columnEditor) { + this.columnEditor = columnEditor; + } + + Column getColumn() { + return columnEditor.create(); + } + + @Override + public void enterColumn_definition(PlSqlParser.Column_definitionContext ctx) { + resolveColumnDataType(ctx); + if (ctx.DEFAULT() != null) { + columnEditor.defaultValueExpression(ctx.column_default_value().getText()); + } + super.enterColumn_definition(ctx); + } + + @Override + public void enterPrimary_key_clause(PlSqlParser.Primary_key_clauseContext ctx) { + // this rule will be parsed only if no primary key is set in a table + // otherwise the statement can't be executed due to multiple primary key error + columnEditor.optional(false); + tableEditor.addColumn(columnEditor.create()); + tableEditor.setPrimaryKeyNames(columnEditor.name()); + super.enterPrimary_key_clause(ctx); + } + + @Override + public void enterModify_col_properties(PlSqlParser.Modify_col_propertiesContext ctx) { + // Scale should always get unset + // It should be parsed by the data type resolver, if its applicable + // This standardizes the handling of scale when data types shift from one to another + columnEditor.unsetScale(); + + resolveColumnDataType(ctx); + if (ctx.DEFAULT() != null) { + columnEditor.defaultValueExpression(ctx.column_default_value().getText()); + } + super.enterModify_col_properties(ctx); + } + + // todo use dataTypeResolver instead + private void resolveColumnDataType(PlSqlParser.Column_definitionContext ctx) { + columnEditor.name(getColumnName(ctx.column_name())); + + boolean hasNotNullConstraint = + ctx.inline_constraint().stream().anyMatch(c -> c.NOT() != null); + columnEditor.optional(!hasNotNullConstraint); + + if (ctx.datatype() == null) { + if (ctx.type_name() != null + && "MDSYS.SDO_GEOMETRY" + .equalsIgnoreCase(ctx.type_name().getText().replace("\"", ""))) { + columnEditor.jdbcType(Types.STRUCT).type("MDSYS.SDO_GEOMETRY"); + } else if (ctx.type_name() != null + && "SYS.XMLTYPE" + .equalsIgnoreCase(ctx.type_name().getText().replace("\"", ""))) { + columnEditor.jdbcType(Types.SQLXML).type("SYS.XMLTYPE"); + } + } else { + resolveColumnDataType(ctx.datatype()); + } + } + + private void resolveColumnDataType(PlSqlParser.Modify_col_propertiesContext ctx) { + columnEditor.name(getColumnName(ctx.column_name())); + + resolveColumnDataType(ctx.datatype()); + + boolean hasNullConstraint = + ctx.inline_constraint().stream().anyMatch(c -> c.NULL_() != null); + boolean hasNotNullConstraint = + ctx.inline_constraint().stream().anyMatch(c -> c.NOT() != null); + if (hasNotNullConstraint && columnEditor.isOptional()) { + columnEditor.optional(false); + } else if (hasNullConstraint && !columnEditor.isOptional()) { + columnEditor.optional(true); + } + } + + private void resolveColumnDataType(PlSqlParser.DatatypeContext ctx) { + // If the context is null, there is nothing this method can resolve and it is safe to return + if (ctx == null) { + return; + } + + if (ctx.native_datatype_element() != null) { + PlSqlParser.Precision_partContext precisionPart = ctx.precision_part(); + if (ctx.native_datatype_element().INT() != null + || ctx.native_datatype_element().INTEGER() != null + || ctx.native_datatype_element().SMALLINT() != null + || ctx.native_datatype_element().NUMERIC() != null + || ctx.native_datatype_element().DECIMAL() != null) { + // NUMERIC and DECIMAL types have by default zero scale + columnEditor.jdbcType(Types.NUMERIC).type("NUMBER"); + + if (precisionPart == null) { + columnEditor.length(38).scale(0); + } else { + setPrecision(precisionPart, columnEditor); + setScale(precisionPart, columnEditor); + } + } else if (ctx.native_datatype_element().DATE() != null) { + // JDBC driver reports type as timestamp but name DATE + columnEditor.jdbcType(Types.TIMESTAMP).type("DATE"); + } else if (ctx.native_datatype_element().TIMESTAMP() != null) { + if (ctx.WITH() != null && ctx.TIME() != null && ctx.ZONE() != null) { + if (ctx.LOCAL() != null) { + columnEditor + .jdbcType(OracleTypes.TIMESTAMPLTZ) + .type("TIMESTAMP WITH LOCAL TIME ZONE"); + } else { + columnEditor + .jdbcType(OracleTypes.TIMESTAMPTZ) + .type("TIMESTAMP WITH TIME ZONE"); + } + } else { + columnEditor.jdbcType(Types.TIMESTAMP).type("TIMESTAMP"); + } + + if (precisionPart == null) { + columnEditor.length(6); + } else { + setPrecision(precisionPart, columnEditor); + } + } + // VARCHAR is the same as VARCHAR2 in Oracle + else if (ctx.native_datatype_element().VARCHAR2() != null + || ctx.native_datatype_element().VARCHAR() != null) { + columnEditor.jdbcType(Types.VARCHAR).type("VARCHAR2"); + + if (precisionPart == null) { + columnEditor.length(getVarCharDefaultLength()); + } else { + setPrecision(precisionPart, columnEditor); + } + } else if (ctx.native_datatype_element().NVARCHAR2() != null) { + columnEditor.jdbcType(Types.NVARCHAR).type("NVARCHAR2"); + + if (precisionPart == null) { + columnEditor.length(getVarCharDefaultLength()); + } else { + setPrecision(precisionPart, columnEditor); + } + } else if (ctx.native_datatype_element().CHAR() != null + || ctx.native_datatype_element().CHARACTER() != null) { + columnEditor.jdbcType(Types.CHAR).type("CHAR").length(1); + + if (precisionPart != null) { + setPrecision(precisionPart, columnEditor); + } + } else if (ctx.native_datatype_element().NCHAR() != null) { + columnEditor.jdbcType(Types.NCHAR).type("NCHAR").length(1); + + if (precisionPart != null) { + setPrecision(precisionPart, columnEditor); + } + } else if (ctx.native_datatype_element().BINARY_FLOAT() != null) { + columnEditor.jdbcType(OracleTypes.BINARY_FLOAT).type("BINARY_FLOAT"); + } else if (ctx.native_datatype_element().BINARY_DOUBLE() != null) { + columnEditor.jdbcType(OracleTypes.BINARY_DOUBLE).type("BINARY_DOUBLE"); + } + // PRECISION keyword is mandatory + else if (ctx.native_datatype_element().FLOAT() != null + || (ctx.native_datatype_element().DOUBLE() != null + && ctx.native_datatype_element().PRECISION() != null)) { + columnEditor.jdbcType(Types.FLOAT).type("FLOAT").length(126); + + // TODO float's precision is about bits not decimal digits; should be ok for now to + // over-size + if (precisionPart != null) { + setPrecision(precisionPart, columnEditor); + } + } else if (ctx.native_datatype_element().REAL() != null) { + columnEditor + .jdbcType(Types.FLOAT) + .type("FLOAT") + // TODO float's precision is about bits not decimal digits; should be ok for + // now to over-size + .length(63); + } else if (ctx.native_datatype_element().NUMBER() != null) { + columnEditor.jdbcType(Types.NUMERIC).type("NUMBER"); + + if (precisionPart == null) { + columnEditor.length(38); + } else { + if (precisionPart.ASTERISK() != null) { + // when asterisk is used, explicitly set precision to 38 + columnEditor.length(38); + } else { + setPrecision(precisionPart, columnEditor); + } + setScale(precisionPart, columnEditor); + } + } else if (ctx.native_datatype_element().BLOB() != null) { + columnEditor.jdbcType(Types.BLOB).type("BLOB"); + } else if (ctx.native_datatype_element().CLOB() != null) { + columnEditor.jdbcType(Types.CLOB).type("CLOB"); + } else if (ctx.native_datatype_element().NCLOB() != null) { + columnEditor.jdbcType(Types.NCLOB).type("NCLOB"); + } else if (ctx.native_datatype_element().RAW() != null) { + columnEditor.jdbcType(OracleTypes.RAW).type("RAW"); + + setPrecision(precisionPart, columnEditor); + } else if (ctx.native_datatype_element().SDO_GEOMETRY() != null) { + // Allows the registration of new SDO_GEOMETRY columns via an CREATE/ALTER TABLE + // This is the same registration of the column that is resolved during JDBC metadata + // inspection. + columnEditor.jdbcType(OracleTypes.OTHER).type("SDO_GEOMETRY").length(1); + } else if (ctx.native_datatype_element().ROWID() != null) { + columnEditor.jdbcType(Types.VARCHAR).type("ROWID"); + } else if (ctx.native_datatype_element().XMLTYPE() != null) { + columnEditor.jdbcType(OracleTypes.SQLXML).type("XMLTYPE"); + } else { + columnEditor + .jdbcType(OracleTypes.OTHER) + .type(ctx.native_datatype_element().getText()); + } + } else if (ctx.INTERVAL() != null + && ctx.YEAR() != null + && ctx.TO() != null + && ctx.MONTH() != null) { + columnEditor.jdbcType(OracleTypes.INTERVALYM).type("INTERVAL YEAR TO MONTH").length(2); + if (!ctx.expression().isEmpty()) { + columnEditor.length(Integer.valueOf((ctx.expression(0).getText()))); + } + } else if (ctx.INTERVAL() != null + && ctx.DAY() != null + && ctx.TO() != null + && ctx.SECOND() != null) { + columnEditor + .jdbcType(OracleTypes.INTERVALDS) + .type("INTERVAL DAY TO SECOND") + .length(2) + .scale(6); + for (final PlSqlParser.ExpressionContext e : ctx.expression()) { + if (e.getSourceInterval().startsAfter(ctx.TO().getSourceInterval())) { + columnEditor.scale(Integer.valueOf(e.getText())); + } else { + columnEditor.length(Integer.valueOf(e.getText())); + } + } + if (!ctx.expression().isEmpty()) { + columnEditor.length(Integer.valueOf((ctx.expression(0).getText()))); + } + } else { + columnEditor.jdbcType(OracleTypes.OTHER).type(ctx.getText()); + } + } + + private int getVarCharDefaultLength() { + // TODO replace with value from select name, value from v$parameter where + // name='max_string_size'; + return 4000; + } + + private void setPrecision( + PlSqlParser.Precision_partContext precisionPart, ColumnEditor columnEditor) { + columnEditor.length(Integer.valueOf(precisionPart.numeric(0).getText())); + } + + private void setScale( + PlSqlParser.Precision_partContext precisionPart, ColumnEditor columnEditor) { + if (precisionPart.numeric().size() > 1) { + columnEditor.scale(Integer.valueOf(precisionPart.numeric(1).getText())); + } else if (precisionPart.numeric_negative() != null) { + columnEditor.scale(Integer.valueOf(precisionPart.numeric_negative().getText())); + } else { + columnEditor.scale(0); + } + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerAdapter.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerAdapter.java index 481051e8a58..6c74af4f76c 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerAdapter.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerAdapter.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package io.debezium.connector.oracle.logminer; import io.debezium.DebeziumException; import io.debezium.config.Configuration; +import io.debezium.connector.base.ChangeEventQueueMetrics; import io.debezium.connector.oracle.AbstractStreamingAdapter; import io.debezium.connector.oracle.OracleConnection; import io.debezium.connector.oracle.OracleConnectorConfig; @@ -26,19 +26,20 @@ import io.debezium.connector.oracle.OracleDatabaseSchema; import io.debezium.connector.oracle.OracleOffsetContext; import io.debezium.connector.oracle.OraclePartition; -import io.debezium.connector.oracle.OracleStreamingChangeEventSourceMetrics; import io.debezium.connector.oracle.OracleTaskContext; import io.debezium.connector.oracle.Scn; import io.debezium.document.Document; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.EventDispatcher; import io.debezium.pipeline.source.snapshot.incremental.SignalBasedIncrementalSnapshotContext; +import io.debezium.pipeline.source.spi.EventMetadataProvider; import io.debezium.pipeline.source.spi.StreamingChangeEventSource; import io.debezium.pipeline.spi.OffsetContext; import io.debezium.pipeline.txmetadata.TransactionContext; import io.debezium.relational.RelationalSnapshotChangeEventSource.RelationalSnapshotContext; import io.debezium.relational.TableId; import io.debezium.relational.history.HistoryRecordComparator; +import io.debezium.snapshot.SnapshotterService; import io.debezium.util.Clock; import io.debezium.util.HexConverter; import io.debezium.util.Strings; @@ -59,12 +60,13 @@ import java.util.stream.Collectors; /** - * Copied from Debezium 1.9.8.Final. + * Copied from Debezium project(2.7.4.Final). * - *

    Line 356: Replace < condition with <= to be able to catch ongoing transactions during snapshot - * if current SCN points to START/INSERT/DELETE/UPDATE event. + *

    Change 1: the pending-transaction probe uses {@code START_SCN <=} rather than {@code <}, so a + * transaction starting exactly at the current SCN is not missed. */ -public class LogMinerAdapter extends AbstractStreamingAdapter { +public class LogMinerAdapter + extends AbstractStreamingAdapter { private static final Duration GET_TRANSACTION_SCN_PAUSE = Duration.ofSeconds(1); @@ -107,7 +109,8 @@ public StreamingChangeEventSource getSourc OracleDatabaseSchema schema, OracleTaskContext taskContext, Configuration jdbcConfig, - OracleStreamingChangeEventSourceMetrics streamingMetrics) { + LogMinerStreamingChangeEventSourceMetrics streamingMetrics, + SnapshotterService snapshotterService) { return new LogMinerStreamingChangeEventSource( connectorConfig, connection, @@ -116,7 +119,18 @@ public StreamingChangeEventSource getSourc clock, schema, jdbcConfig, - streamingMetrics); + streamingMetrics, + snapshotterService); + } + + @Override + public LogMinerStreamingChangeEventSourceMetrics getStreamingMetrics( + OracleTaskContext taskContext, + ChangeEventQueueMetrics changeEventQueueMetrics, + EventMetadataProvider metadataProvider, + OracleConnectorConfig connectorConfig) { + return new LogMinerStreamingChangeEventSourceMetrics( + taskContext, changeEventQueueMetrics, metadataProvider, connectorConfig); } @Override @@ -148,9 +162,7 @@ public OracleOffsetContext determineSnapshotOffset( // that prevents switching from a PDB to the root CDB and if invoking the LogMiner APIs on // such a connection, the use of commit/rollback by LogMiner will drop/invalidate the save // point as well. A separate connection is necessary to preserve the save point. - try (OracleConnection conn = - new OracleConnection( - connection.config(), () -> getClass().getClassLoader(), false)) { + try (OracleConnection conn = new OracleConnection(connection.config(), false)) { conn.setAutoCommit(false); if (!Strings.isNullOrEmpty(connectorConfig.getPdbName())) { // The next stage cannot be run within the PDB, reset the connection to the CDB. @@ -161,6 +173,18 @@ public OracleOffsetContext determineSnapshotOffset( } } + @Override + public Scn getOffsetScn(OracleOffsetContext offsetContext) { + return offsetContext.getScn(); + } + + @Override + public OracleOffsetContext copyOffset( + OracleConnectorConfig connectorConfig, OracleOffsetContext offsetContext) { + return new LogMinerOracleOffsetContextLoader(connectorConfig) + .load(offsetContext.getOffset()); + } + private Optional getCurrentScn(Scn latestTableDdlScn, OracleConnection connection) throws SQLException { final String query = "SELECT CURRENT_SCN FROM V$DATABASE"; @@ -197,21 +221,6 @@ private Optional getPendingTransactions( try (Statement s = connection.connection().createStatement(); ResultSet rs = s.executeQuery(query)) { - List results = new ArrayList<>(); - Statement s2 = connection.connection().createStatement(); - ResultSet rs2 = - s2.executeQuery( - "SELECT t.START_SCN, t.START_SCNB, t.DEPENDENT_SCN FROM V$TRANSACTION t"); - while (rs2.next()) { - results.add( - String.join( - " | ", rs2.getString(1), rs2.getString(2), rs2.getString(3))); - } - if (!results.isEmpty()) { - LOGGER.info("NOT EMPTY TRSNASSS: {}", results); - } - rs2.close(); - while (rs.next()) { if (currentScn == null) { // Only need to set this once per iteration @@ -219,10 +228,22 @@ private Optional getPendingTransactions( } final String pendingTxStartScn = rs.getString(3); if (!Strings.isNullOrEmpty(pendingTxStartScn)) { - // There is a pending transaction, capture state - transactions.put( - HexConverter.convertToHexString(rs.getBytes(2)), - Scn.valueOf(pendingTxStartScn)); + final String transactionId = + HexConverter.convertToHexString(rs.getBytes(2)); + final Scn transactionStartScn = Scn.valueOf(pendingTxStartScn); + // There is a use case where if the archive logs do not contain sufficient + // logs where the + // transaction started, LogMiner will return a value of 0 as the START_SCN + // and this can + // unintentionally cause starting from the beginning of time. + if (transactionStartScn.compareTo(Scn.ONE) > 0) { + // There is a pending transaction, capture state + transactions.put(transactionId, transactionStartScn); + } else { + LOGGER.warn( + "Unable to determine the start SCN, transaction {} will not be included", + transactionId); + } } } } catch (SQLException e) { @@ -321,8 +342,8 @@ private void stopSession(OracleConnection connection) throws SQLException { private Scn getOldestScnAvailableInLogs( OracleConnectorConfig config, OracleConnection connection) throws SQLException { - final Duration archiveLogRetention = config.getLogMiningArchiveLogRetention(); - final String archiveLogDestinationName = config.getLogMiningArchiveDestinationName(); + final Duration archiveLogRetention = config.getArchiveLogRetention(); + final String archiveLogDestinationName = config.getArchiveLogDestinationName(); return connection.queryAndMap( SqlUtils.oldestFirstChangeQuery(archiveLogRetention, archiveLogDestinationName), rs -> { @@ -339,13 +360,8 @@ private Scn getOldestScnAvailableInLogs( private List getOrderedLogsFromScn( OracleConnectorConfig config, Scn sinceScn, OracleConnection connection) throws SQLException { - return LogMinerHelper.getLogFilesForOffsetScn( - connection, - sinceScn, - config.getLogMiningArchiveLogRetention(), - config.isArchiveLogOnlyMode(), - config.getLogMiningArchiveDestinationName()) - .stream() + final LogFileCollector collector = new LogFileCollector(config, connection); + return collector.getLogs(sinceScn).stream() .sorted(Comparator.comparing(LogFile::getSequence)) .collect(Collectors.toList()); } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerChangeRecordEmitter.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerChangeRecordEmitter.java index 4969c1abdca..82e69ffd05d 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerChangeRecordEmitter.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerChangeRecordEmitter.java @@ -1,9 +1,19 @@ /* - * Copyright Debezium Authors. + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.debezium.connector.oracle.logminer; import io.debezium.DebeziumException; @@ -11,25 +21,37 @@ import io.debezium.connector.oracle.OracleConnectorConfig; import io.debezium.connector.oracle.OracleDatabaseSchema; import io.debezium.connector.oracle.logminer.events.EventType; +import io.debezium.connector.oracle.util.TimestampUtils; import io.debezium.data.Envelope.Operation; import io.debezium.pipeline.spi.OffsetContext; import io.debezium.pipeline.spi.Partition; +import io.debezium.relational.Column; import io.debezium.relational.Table; import io.debezium.util.Clock; +import io.debezium.util.Strings; +import oracle.jdbc.OracleTypes; import oracle.sql.ROWID; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.header.ConnectHeaders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.sql.Connection; import java.util.Optional; /** - * Copied from Debezium 1.9.8.Final. Emits change records based on an event read from Oracle - * LogMiner. + * Emits change records based on an event read from Oracle LogMiner. * - *

    This class add RowId and overrides the emit methods to put rowId in the header. + *

    Copied from Debezium project(2.7.4.Final).. + * + *

    Change 1: both constructors gain a trailing {@code String rowId} argument, and {@code + * getEmitConnectHeaders()} attaches it as a {@code ROWID} Connect header on every emitted record + * (hook added by the forked {@code RelationalChangeRecordEmitter}). */ public class LogMinerChangeRecordEmitter extends BaseChangeRecordEmitter { + private static final Logger LOGGER = LoggerFactory.getLogger(LogMinerChangeRecordEmitter.class); + private final Operation operation; private final String rowId; @@ -79,6 +101,7 @@ private static Operation getOperation(EventType eventType) { return Operation.CREATE; case UPDATE: case SELECT_LOB_LOCATOR: + case XML_BEGIN: return Operation.UPDATE; case DELETE: return Operation.DELETE; @@ -92,6 +115,33 @@ public Operation getOperation() { return operation; } + @Override + protected Object convertReselectPrimaryKeyColumn( + Connection connection, Column column, Object value) { + if (value instanceof String) { + // LogMiner raw values are always string; otherwise generally null + switch (column.jdbcType()) { + case OracleTypes.TIMESTAMP: + case OracleTypes.DATE: + final String formattedTimestamp = + TimestampUtils.toSqlCompliantFunctionCall((String) value); + if (!Strings.isNullOrBlank(formattedTimestamp)) { + value = convertValueViaQuery(connection, formattedTimestamp); + } + break; + case OracleTypes.INTERVALYM: + case OracleTypes.INTERVALDS: + // LogMiner provides these values in SQL-compliant query fragments + value = convertValueViaQuery(connection, (String) value); + break; + default: + // no -op + break; + } + } + return value; + } + @Override protected Optional getEmitConnectHeaders() { ConnectHeaders headers = new ConnectHeaders(); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerStreamingChangeEventSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerStreamingChangeEventSource.java index 5993cadee71..13873712581 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerStreamingChangeEventSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/LogMinerStreamingChangeEventSource.java @@ -14,29 +14,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package io.debezium.connector.oracle.logminer; import io.debezium.DebeziumException; import io.debezium.config.Configuration; import io.debezium.connector.oracle.OracleConnection; import io.debezium.connector.oracle.OracleConnectorConfig; +import io.debezium.connector.oracle.OracleConnectorConfig.LogMiningBufferType; import io.debezium.connector.oracle.OracleDatabaseSchema; import io.debezium.connector.oracle.OracleOffsetContext; import io.debezium.connector.oracle.OraclePartition; -import io.debezium.connector.oracle.OracleStreamingChangeEventSourceMetrics; +import io.debezium.connector.oracle.RedoThreadState.RedoThread; import io.debezium.connector.oracle.Scn; import io.debezium.connector.oracle.logminer.logwriter.CommitLogWriterFlushStrategy; import io.debezium.connector.oracle.logminer.logwriter.LogWriterFlushStrategy; import io.debezium.connector.oracle.logminer.logwriter.RacCommitLogWriterFlushStrategy; +import io.debezium.connector.oracle.logminer.logwriter.ReadOnlyLogWriterFlushStrategy; import io.debezium.connector.oracle.logminer.processor.LogMinerEventProcessor; import io.debezium.jdbc.JdbcConfiguration; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.source.snapshot.incremental.SignalBasedIncrementalSnapshotContext; import io.debezium.pipeline.source.spi.StreamingChangeEventSource; +import io.debezium.pipeline.txmetadata.TransactionContext; import io.debezium.relational.Column; import io.debezium.relational.Table; import io.debezium.relational.TableId; +import io.debezium.snapshot.SnapshotterService; import io.debezium.util.Clock; import io.debezium.util.Metronome; import io.debezium.util.Stopwatch; @@ -44,6 +48,7 @@ import org.slf4j.LoggerFactory; import java.math.BigInteger; +import java.sql.CallableStatement; import java.sql.SQLException; import java.text.DecimalFormat; import java.time.Duration; @@ -60,15 +65,14 @@ import java.util.Set; import java.util.stream.Collectors; -import static io.debezium.connector.oracle.logminer.LogMinerHelper.logError; -import static io.debezium.connector.oracle.logminer.LogMinerHelper.setLogFilesForMining; - /** - * Copied from Debezium 1.9.8.Final. A {@link StreamingChangeEventSource} based on Oracle's LogMiner - * utility. The event handler loop is executed in a separate executor. + * A {@link StreamingChangeEventSource} based on Oracle's LogMiner utility. The event handler loop + * is executed in a separate executor. * - *

    Diff: Make createProcessor method as protected to produce a LogMinerEventProcessor with - * enhanced processRow method to distinguish whether is bounded. + *

    Copied from Debezium project(2.7.4.Final). + * + *

    Change 1: {@code createProcessor} is protected (upstream: private) so {@code + * EventProcessorFactory} can supply its own processor. */ public class LogMinerStreamingChangeEventSource implements StreamingChangeEventSource { @@ -78,6 +82,7 @@ public class LogMinerStreamingChangeEventSource private static final int MAXIMUM_NAME_LENGTH = 30; private static final String ALL_COLUMN_LOGGING = "ALL COLUMN LOGGING"; private static final int MINING_START_RETRIES = 5; + private static final Long SMALL_REDO_LOG_WARNING = 524_288_000L; private final OracleConnection jdbcConnection; private final EventDispatcher dispatcher; @@ -86,21 +91,23 @@ public class LogMinerStreamingChangeEventSource private final JdbcConfiguration jdbcConfiguration; private final OracleConnectorConfig.LogMiningStrategy strategy; private final ErrorHandler errorHandler; - private final boolean isContinuousMining; - private final OracleStreamingChangeEventSourceMetrics streamingMetrics; + private final LogMinerStreamingChangeEventSourceMetrics streamingMetrics; private final OracleConnectorConfig connectorConfig; private final Duration archiveLogRetention; private final boolean archiveLogOnlyMode; private final String archiveDestinationName; - private final int logFileQueryMaxRetries; - private final Duration initialDelay; - private final Duration maxDelay; + private final LogFileCollector logCollector; + private final boolean continuousMining; private Scn startScn; // startScn is the **exclusive** lower bound for mining - private Scn endScn; + private Scn endScn = Scn.NULL; private Scn snapshotScn; private List currentLogFiles; private List currentRedoLogSequences; + private OracleOffsetContext effectiveOffset; + private int currentBatchSize; + private long currentSleepTime; + private final SnapshotterService snapshotterService; public LogMinerStreamingChangeEventSource( OracleConnectorConfig connectorConfig, @@ -110,27 +117,48 @@ public LogMinerStreamingChangeEventSource( Clock clock, OracleDatabaseSchema schema, Configuration jdbcConfig, - OracleStreamingChangeEventSourceMetrics streamingMetrics) { + LogMinerStreamingChangeEventSourceMetrics streamingMetrics, + SnapshotterService snapshotterService) { this.jdbcConnection = jdbcConnection; this.dispatcher = dispatcher; this.clock = clock; this.schema = schema; this.connectorConfig = connectorConfig; this.strategy = connectorConfig.getLogMiningStrategy(); - this.isContinuousMining = connectorConfig.isContinuousMining(); this.errorHandler = errorHandler; this.streamingMetrics = streamingMetrics; this.jdbcConfiguration = JdbcConfiguration.adapt(jdbcConfig); - this.archiveLogRetention = connectorConfig.getLogMiningArchiveLogRetention(); + this.archiveLogRetention = connectorConfig.getArchiveLogRetention(); this.archiveLogOnlyMode = connectorConfig.isArchiveLogOnlyMode(); - this.archiveDestinationName = connectorConfig.getLogMiningArchiveDestinationName(); - this.logFileQueryMaxRetries = connectorConfig.getMaximumNumberOfLogQueryRetries(); - this.initialDelay = connectorConfig.getLogMiningInitialDelay(); - this.maxDelay = connectorConfig.getLogMiningMaxDelay(); + this.archiveDestinationName = connectorConfig.getArchiveLogDestinationName(); + this.currentBatchSize = connectorConfig.getLogMiningBatchSizeDefault(); + this.currentSleepTime = connectorConfig.getLogMiningSleepTimeDefault().toMillis(); + this.continuousMining = connectorConfig.isLogMiningContinuousMining(); + + this.snapshotterService = snapshotterService; + + this.streamingMetrics.setBatchSize(this.currentBatchSize); + this.streamingMetrics.setSleepTime(this.currentSleepTime); + + this.logCollector = new LogFileCollector(connectorConfig, jdbcConnection); + } + + @Override + public void init(OracleOffsetContext offsetContext) throws InterruptedException { + this.effectiveOffset = offsetContext == null ? emptyContext() : offsetContext; + } + + private OracleOffsetContext emptyContext() { + return OracleOffsetContext.create() + .logicalName(connectorConfig) + .snapshotPendingTransactions(Collections.emptyMap()) + .transactionContext(new TransactionContext()) + .incrementalSnapshotContext(new SignalBasedIncrementalSnapshotContext<>()) + .build(); } /** - * This is the loop to get changes from LogMiner. + * This is the loop to get changes from LogMiner * * @param context change event source context */ @@ -139,17 +167,22 @@ public void execute( ChangeEventSourceContext context, OraclePartition partition, OracleOffsetContext offsetContext) { - if (!connectorConfig.getSnapshotMode().shouldStream()) { - LOGGER.info("Streaming is not enabled in current configuration"); - return; - } + try { - // We explicitly expect auto-commit to be disabled - jdbcConnection.setAutoCommit(false); - startScn = offsetContext.getScn(); + prepareConnection(false); + + this.effectiveOffset = offsetContext; + startScn = connectorConfig.getAdapter().getOffsetScn(this.effectiveOffset); snapshotScn = offsetContext.getSnapshotScn(); - Scn firstScn = getFirstScnInLogs(jdbcConnection); + Scn firstScn = + jdbcConnection + .getFirstScnInLogs(archiveLogRetention, archiveDestinationName) + .orElseThrow( + () -> + new DebeziumException( + "Failed to calculate oldest SCN available in logs")); + if (startScn.compareTo(snapshotScn) == 0) { // This is the initial run of the streaming change event source. // We need to compute the correct start offset for mining. That is not the snapshot @@ -161,7 +194,7 @@ public void execute( } try (LogWriterFlushStrategy flushStrategy = resolveFlushStrategy()) { - if (!isContinuousMining && startScn.compareTo(firstScn.subtract(Scn.ONE)) < 0) { + if (!continuousMining && startScn.compareTo(firstScn.subtract(Scn.ONE)) < 0) { // startScn is the exclusive lower bound, so must be >= (firstScn - 1) throw new DebeziumException( "Online REDO LOG files or archive log files do not contain the offset scn " @@ -169,8 +202,8 @@ public void execute( + ". Please perform a new snapshot."); } - setNlsSessionParameters(jdbcConnection); checkDatabaseAndTableState(jdbcConnection, connectorConfig.getPdbName(), schema); + logOnlineRedoLogSizes(connectorConfig); try (LogMinerEventProcessor processor = createProcessor(context, partition, offsetContext)) { @@ -186,7 +219,7 @@ public void execute( while (context.isRunning()) { // Calculate time difference before each mining session to detect time zone // offset changes (e.g. DST) on database server - streamingMetrics.calculateTimeDifference( + streamingMetrics.setDatabaseTimeDifference( getDatabaseSystemTime(jdbcConnection)); if (archiveLogOnlyMode @@ -195,7 +228,13 @@ public void execute( } Instant start = Instant.now(); - endScn = calculateEndScn(jdbcConnection, startScn, endScn); + + endScn = calculateUpperBounds(startScn, endScn); + if (endScn.isNull()) { + LOGGER.debug("Requested delay of mining by one iteration"); + pauseBetweenMiningSessions(); + continue; + } // This is a small window where when archive log only mode has completely // caught up to the last @@ -235,6 +274,9 @@ public void execute( // flush PGA. // At this point we use a new mining session endMiningSession(jdbcConnection, offsetContext); + if (connectorConfig.isLogMiningRestartConnection()) { + prepareConnection(true); + } initializeRedoLogsForMining(jdbcConnection, true, startScn); // log switch or restart required, re-create a new stop watch @@ -247,18 +289,26 @@ public void execute( retryAttempts++; } else { retryAttempts = 1; - streamingMetrics.setCurrentBatchProcessingTime( + startScn = processor.process(startScn, endScn); + streamingMetrics.setLastBatchProcessingDuration( Duration.between(start, Instant.now())); captureSessionMemoryStatistics(jdbcConnection); - startScn = processor.process(partition, startScn, endScn); } pauseBetweenMiningSessions(); } + + if (context.isPaused()) { + LOGGER.info("Streaming will now pause"); + context.streamingPaused(); + context.waitSnapshotCompletion(); + LOGGER.info("Streaming resumed"); + } } } } } catch (Throwable t) { - logError(streamingMetrics, "Mining session stopped due to the {}", t); + LOGGER.error("Mining session stopped due to error.", t); + streamingMetrics.incrementErrorCount(); errorHandler.setProducerThrowable(t); } finally { LOGGER.info("startScn={}, endScn={}", startScn, endScn); @@ -267,6 +317,45 @@ public void execute( } } + private void prepareConnection(boolean closeAndReconnect) throws SQLException { + if (closeAndReconnect) { + // Close and reconnect + LOGGER.debug( + "Log switch or maximum session threshold detected, restarting Oracle JDBC connection."); + jdbcConnection.close(); + } + + // We explicitly expect auto-commit to be disabled + jdbcConnection.setAutoCommit(false); + setNlsSessionParameters(jdbcConnection); + } + + private void logOnlineRedoLogSizes(OracleConnectorConfig config) throws SQLException { + jdbcConnection.query( + "SELECT GROUP#, BYTES FROM V$LOG ORDER BY 1", + rs -> { + LOGGER.info("Redo Log Group Sizes:"); + boolean potentiallySmallLogs = false; + while (rs.next()) { + long logSize = rs.getLong(2); + if (logSize < SMALL_REDO_LOG_WARNING) { + potentiallySmallLogs = true; + } + LOGGER.info("\tGroup #{}: {} bytes", rs.getInt(1), logSize); + } + if (config.getAdapter().getType().equals(LogMinerAdapter.TYPE)) { + if (config.getLogMiningStrategy() + == OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO) { + if (potentiallySmallLogs) { + LOGGER.warn( + "Redo logs may be sized too small using the default mining strategy, " + + "consider increasing redo log sizes to a minimum of 500MB."); + } + } + } + }); + } + /** * Computes the start SCN for the first mining session. * @@ -325,7 +414,7 @@ private void computeStartScnForFirstMiningSession( } // set start SCN to minScn - if (minScn.compareTo(startScn) <= 0) { + if (minScn.compareTo(startScn) < 0) { LOGGER.info( "Resetting start SCN from {} (snapshot SCN) to {} (start of oldest complete pending transaction)", startScn, @@ -364,8 +453,7 @@ protected LogMinerEventProcessor createProcessor( ChangeEventSourceContext context, OraclePartition partition, OracleOffsetContext offsetContext) { - final OracleConnectorConfig.LogMiningBufferType bufferType = - connectorConfig.getLogMiningBufferType(); + final LogMiningBufferType bufferType = connectorConfig.getLogMiningBufferType(); return bufferType.createProcessor( context, connectorConfig, @@ -377,65 +465,31 @@ protected LogMinerEventProcessor createProcessor( streamingMetrics); } - /** - * Gets the first system change number in both archive and redo logs. - * - * @param connection database connection, should not be {@code null} - * @return the oldest system change number - * @throws SQLException if a database exception occurred - * @throws DebeziumException if the oldest system change number cannot be found due to no logs - * available - */ - private Scn getFirstScnInLogs(OracleConnection connection) throws SQLException { - String oldestScn = - connection.singleOptionalValue( - SqlUtils.oldestFirstChangeQuery( - archiveLogRetention, archiveDestinationName), - rs -> rs.getString(1)); - if (oldestScn == null) { - throw new DebeziumException("Failed to calculate oldest SCN available in logs"); - } - LOGGER.trace("Oldest SCN in logs is '{}'", oldestScn); - return Scn.valueOf(oldestScn); - } - private void initializeRedoLogsForMining( OracleConnection connection, boolean postEndMiningSession, Scn startScn) throws SQLException { - if (!postEndMiningSession) { - if (OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO.equals(strategy)) { - buildDataDictionary(connection); - } - if (!isContinuousMining) { - currentLogFiles = - setLogFilesForMining( - connection, - startScn, - archiveLogRetention, - archiveLogOnlyMode, - archiveDestinationName, - logFileQueryMaxRetries, - initialDelay, - maxDelay); - currentRedoLogSequences = getCurrentLogFileSequences(currentLogFiles); - } - } else { - if (!isContinuousMining) { - if (OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO.equals(strategy)) { - buildDataDictionary(connection); + if (!continuousMining) { + connection.removeAllLogFilesFromLogMinerSession(); + } + + if ((!postEndMiningSession || !continuousMining) + && OracleConnectorConfig.LogMiningStrategy.CATALOG_IN_REDO.equals(strategy)) { + buildDataDictionary(connection); + } + + if (!continuousMining) { + // Collect logs and add them to the session + currentLogFiles = logCollector.getLogs(startScn); + for (LogFile logFile : currentLogFiles) { + LOGGER.trace("Adding log file {} to the mining session.", logFile.getFileName()); + String addLogFileStatement = + SqlUtils.addLogFileStatement("DBMS_LOGMNR.ADDFILE", logFile.getFileName()); + try (CallableStatement statement = + connection.connection(false).prepareCall(addLogFileStatement)) { + statement.execute(); } - currentLogFiles = - setLogFilesForMining( - connection, - startScn, - archiveLogRetention, - archiveLogOnlyMode, - archiveDestinationName, - logFileQueryMaxRetries, - initialDelay, - maxDelay); - currentRedoLogSequences = getCurrentLogFileSequences(currentLogFiles); } + currentRedoLogSequences = getCurrentLogFileSequences(currentLogFiles); } updateRedoLogMetrics(); @@ -458,7 +512,7 @@ private List getCurrentLogFileSequences(List logFiles) { } /** - * Get the maximum archive log SCN. + * Get the maximum archive log SCN * * @param logFiles the current logs that are part of the mining session * @return the maximum system change number from the archive logs @@ -563,8 +617,8 @@ private void updateRedoLogMetrics() throws SQLException { }); final Set fileNames = getCurrentRedoLogFiles(jdbcConnection); - streamingMetrics.setCurrentLogFileName(fileNames); - streamingMetrics.setRedoLogStatus(logStatuses); + streamingMetrics.setCurrentLogFileNames(fileNames); + streamingMetrics.setRedoLogStatuses(logStatuses); } /** @@ -610,8 +664,7 @@ private List getCurrentRedoLogSequences() throws SQLException { } private void pauseBetweenMiningSessions() throws InterruptedException { - Duration period = - Duration.ofMillis(streamingMetrics.getMillisecondToSleepBetweenMiningQuery()); + Duration period = Duration.ofMillis(streamingMetrics.getSleepTimeInMilliseconds()); Metronome.sleeper(period, clock).pause(); } @@ -622,14 +675,14 @@ private void pauseBetweenMiningSessions() throws InterruptedException { * @throws SQLException if a database exception occurred */ private void setNlsSessionParameters(OracleConnection connection) throws SQLException { - final String nlsSessionParameters = + final String NLS_SESSION_PARAMETERS = "ALTER SESSION SET " + " NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS'" + " NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'" + " NLS_TIMESTAMP_TZ_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM'" + " NLS_NUMERIC_CHARACTERS = '.,'"; - connection.executeWithoutCommitting(nlsSessionParameters); + connection.executeWithoutCommitting(NLS_SESSION_PARAMETERS); // This is necessary so that TIMESTAMP WITH LOCAL TIME ZONE is returned in UTC connection.executeWithoutCommitting("ALTER SESSION SET TIME_ZONE = '00:00'"); } @@ -665,22 +718,26 @@ private OffsetDateTime getDatabaseSystemTime(OracleConnection connection) throws public boolean startMiningSession( OracleConnection connection, Scn startScn, Scn endScn, int attempts) throws SQLException { - LOGGER.trace( - "Starting mining session startScn={}, endScn={}, strategy={}, continuous={}", + LOGGER.debug( + "Starting mining session startScn={}, endScn={}, strategy={}, continuous={}, attempts={}/{}", startScn, endScn, strategy, - isContinuousMining); + continuousMining, + attempts, + MINING_START_RETRIES); try { Instant start = Instant.now(); // NOTE: we treat startSCN as the _exclusive_ lower bound for mining, // whereas START_LOGMNR takes an _inclusive_ lower bound, hence the increment. connection.executeWithoutCommitting( SqlUtils.startLogMinerStatement( - startScn.add(Scn.ONE), endScn, strategy, isContinuousMining)); - streamingMetrics.addCurrentMiningSessionStart(Duration.between(start, Instant.now())); + startScn.add(Scn.ONE), endScn, strategy, continuousMining)); + streamingMetrics.setLastMiningSessionStartDuration( + Duration.between(start, Instant.now())); return true; } catch (SQLException e) { + LogMinerDatabaseStateWriter.writeLogMinerStartParameters(connection); if (e.getErrorCode() == 1291 || e.getMessage().startsWith("ORA-01291")) { if (attempts <= MINING_START_RETRIES) { LOGGER.warn("Failed to start Oracle LogMiner session, retrying..."); @@ -690,6 +747,7 @@ public boolean startMiningSession( "Failed to start Oracle LogMiner after '{}' attempts.", MINING_START_RETRIES, e); + LogMinerDatabaseStateWriter.writeLogMinerLogFailures(connection); } LOGGER.error("Got exception when starting mining session.", e); // Capture the database state before throwing the exception up @@ -715,7 +773,7 @@ public void endMiningSession(OracleConnection connection, OracleOffsetContext of endScn, offsetContext.getScn(), strategy, - isContinuousMining); + continuousMining); connection.executeWithoutCommitting("BEGIN SYS.DBMS_LOGMNR.END_LOGMNR(); END;"); } catch (SQLException e) { if (e.getMessage().toUpperCase().contains("ORA-01307")) { @@ -727,108 +785,265 @@ public void endMiningSession(OracleConnection connection, OracleOffsetContext of } } + private void updateBatchSize(boolean increment) { + int batchSizeMin = connectorConfig.getLogMiningBatchSizeMin(); + int batchSizeMax = connectorConfig.getLogMiningBatchSizeMax(); + if (increment && currentBatchSize < batchSizeMax) { + currentBatchSize = Math.min(currentBatchSize + batchSizeMin, batchSizeMax); + if (currentBatchSize == batchSizeMax) { + LOGGER.info( + "The connector is now using the maximum batch size {} when querying the LogMiner view.{}", + currentBatchSize, + connectorConfig.isLobEnabled() + ? "" + : " This could be indicate of a large SCN gap."); + } + } else if (!increment && currentBatchSize > batchSizeMin) { + currentBatchSize = Math.max(currentBatchSize - batchSizeMin, batchSizeMin); + } + + if (currentBatchSize != batchSizeMax) { + LOGGER.debug("Updated batch size window, using batch size {}", currentBatchSize); + } + + streamingMetrics.setBatchSize(currentBatchSize); + } + + private void updateSleepTime(boolean increment) { + if (increment && currentSleepTime < connectorConfig.getLogMiningSleepTimeMax().toMillis()) { + currentSleepTime += connectorConfig.getLogMiningSleepTimeIncrement().toMillis(); + } else if (currentSleepTime > connectorConfig.getLogMiningSleepTimeMin().toMillis()) { + currentSleepTime -= connectorConfig.getLogMiningSleepTimeIncrement().toMillis(); + } + + LOGGER.debug("Updated sleep time window, using sleep time {}.", currentSleepTime); + streamingMetrics.setSleepTime(currentSleepTime); + } + /** - * Calculates the mining session's end system change number. - * - *

    This calculation is based upon a sliding window algorithm to where if the connector is - * falling behind, the mining session's end point will be calculated based on the batch size and - * either be increased up to the maximum batch size or reduced to as low as the minimum batch - * size. + * Calculates the upper mining range SCN boundary * - *

    Additionally, this method calculates and maintains a sliding algorithm for the sleep time - * between the mining sessions, increasing the pause up to the maximum sleep time if the - * connector is not behind or is mining too quick and reducing the pause down to the mimum sleep - * time if the connector has fallen behind and needs to catch-up faster. - * - * @param connection database connection, should not be {@code null} - * @param startScn upcoming mining session's starting change number, should not be {@code null} - * @param prevEndScn last mining session's ending system change number, can be {@code null} - * @return the ending system change number to be used for the upcoming mining session, never - * {@code null} + * @param startScn the starting position for the SCN mining range + * @param prevEndScn the previous iteration's end position for the SCN mining range + * @return the newly calculated upper boundary, or {@link Scn#NULL} if the loop should pause. * @throws SQLException if the current max system change number cannot be obtained from the * database */ - private Scn calculateEndScn(OracleConnection connection, Scn startScn, Scn prevEndScn) - throws SQLException { - Scn currentScn = + private Scn calculateUpperBounds(Scn startScn, Scn prevEndScn) throws SQLException { + Scn upperBoundsScn = archiveLogOnlyMode ? getMaxArchiveLogScn(currentLogFiles) - : connection.getCurrentScn(); - streamingMetrics.setCurrentScn(currentScn); - - // Add the current batch size to the starting system change number - final Scn currentBatchSizeScn = Scn.valueOf(streamingMetrics.getBatchSize()); - Scn topScnToMine = startScn.add(currentBatchSizeScn); - - // Control adjusting batch size - boolean topMiningScnInFarFuture = false; - if (topScnToMine.subtract(currentScn).compareTo(currentBatchSizeScn) > 0) { - streamingMetrics.changeBatchSize(false, connectorConfig.isLobEnabled()); - topMiningScnInFarFuture = true; + : jdbcConnection.getCurrentScn(); + streamingMetrics.setCurrentScn(jdbcConnection.getCurrentScn()); + + final Scn batchUpperBoundsScn = startScn.add(Scn.valueOf(streamingMetrics.getBatchSize())); + final Scn defaultBatchSize = Scn.valueOf(connectorConfig.getLogMiningBatchSizeDefault()); + + // Initially set the upper bounds based on batch size + // The following logic will alter this value as needed based on specific rules + Scn result = batchUpperBoundsScn; + + // Check if the batch upper bounds is greater than the current upper bounds + // If it isn't, there is no need to update the batch size + boolean batchUpperBoundsScnAfterCurrentScn = false; + if (batchUpperBoundsScn.subtract(upperBoundsScn).compareTo(defaultBatchSize) > 0) { + // Don't update the batch size, batch upper bounds currently large enough + updateBatchSize(false); + batchUpperBoundsScnAfterCurrentScn = true; } - if (currentScn.subtract(topScnToMine).compareTo(currentBatchSizeScn) > 0) { - streamingMetrics.changeBatchSize(true, connectorConfig.isLobEnabled()); + + if (upperBoundsScn.subtract(batchUpperBoundsScn).compareTo(defaultBatchSize) > 0) { + // Update batch size because the database upper position is greater than the batch size + updateBatchSize(true); } - // Control sleep time to reduce database impact - if (currentScn.compareTo(topScnToMine) < 0) { - if (!topMiningScnInFarFuture) { - streamingMetrics.changeSleepingTime(true); + if (upperBoundsScn.compareTo(batchUpperBoundsScn) < 0) { + if (!batchUpperBoundsScnAfterCurrentScn) { + updateSleepTime(true); } - LOGGER.debug("Using current SCN {} as end SCN.", currentScn); - return currentScn; + // Batch upperbounds greater than database max possible read position. + // Cap it at the max possible database read position + LOGGER.debug( + "Batch upper bounds {} exceeds maximum read position, capping to {}.", + batchUpperBoundsScn, + upperBoundsScn); + result = upperBoundsScn; } else { - if (prevEndScn != null && topScnToMine.compareTo(prevEndScn) <= 0) { + if (!prevEndScn.isNull() && batchUpperBoundsScn.compareTo(prevEndScn) <= 0) { + // Batch size is too small, make a large leap and use current SCN LOGGER.debug( - "Max batch size too small, using current SCN {} as end SCN.", currentScn); - return currentScn; - } - streamingMetrics.changeSleepingTime(false); - if (topScnToMine.compareTo(startScn) < 0) { - LOGGER.debug( - "Top SCN calculation resulted in end before start SCN, using current SCN {} as end SCN.", - currentScn); - return currentScn; - } - - if (prevEndScn != null) { - final Scn deltaScn = currentScn.subtract(prevEndScn); - if (deltaScn.compareTo( - Scn.valueOf( - connectorConfig.getLogMiningScnGapDetectionGapSizeMin())) - > 0) { - Optional prevEndScnTimestamp = - connection.getScnToTimestamp(prevEndScn); - if (prevEndScnTimestamp.isPresent()) { - Optional currentScnTimestamp = - connection.getScnToTimestamp(currentScn); - if (currentScnTimestamp.isPresent()) { - long timeDeltaMs = - ChronoUnit.MILLIS.between( - prevEndScnTimestamp.get(), currentScnTimestamp.get()); - if (timeDeltaMs - < connectorConfig - .getLogMiningScnGapDetectionTimeIntervalMaxMs()) { - LOGGER.warn( - "Detected possible SCN gap, using current SCN, startSCN {}, prevEndScn {} timestamp {}, current SCN {} timestamp {}.", - startScn, - prevEndScn, - prevEndScnTimestamp.get(), - currentScn, - currentScnTimestamp.get()); - return currentScn; + "Batch size upper bounds {} too small, using maximum read position {} instead.", + batchUpperBoundsScn, + upperBoundsScn); + result = upperBoundsScn; + } else { + updateSleepTime(false); + if (batchUpperBoundsScn.compareTo(startScn) < 0) { + // Batch SCN calculation resulted in a value before start SCN, fallback to max + // read position + LOGGER.debug( + "Batch upper bounds {} is before start SCN {}, fallback to maximum read position {}.", + batchUpperBoundsScn, + startScn, + upperBoundsScn); + result = upperBoundsScn; + } else if (!prevEndScn.isNull()) { + final Scn deltaScn = upperBoundsScn.subtract(prevEndScn); + if (deltaScn.compareTo( + Scn.valueOf( + connectorConfig + .getLogMiningScnGapDetectionGapSizeMin())) + > 0) { + Optional prevEndScnTimestamp = + jdbcConnection.getScnToTimestamp(prevEndScn); + if (prevEndScnTimestamp.isPresent()) { + Optional upperBoundsScnTimestamp = + jdbcConnection.getScnToTimestamp(upperBoundsScn); + if (upperBoundsScnTimestamp.isPresent()) { + long deltaTime = + ChronoUnit.MILLIS.between( + prevEndScnTimestamp.get(), + upperBoundsScnTimestamp.get()); + if (deltaTime + < connectorConfig + .getLogMiningScnGapDetectionTimeIntervalMaxMs()) { + LOGGER.warn( + "Detected possible SCN gap, using upperBounds SCN, startSCN {}, prevEndSCN {}, timestamp {}, upperBounds SCN {} timestamp {}.", + startScn, + prevEndScn, + prevEndScnTimestamp.get(), + upperBoundsScn, + upperBoundsScnTimestamp.get()); + result = upperBoundsScn; + } } } } } } + } + // If the connector is configured with maximum SCN deviation, apply the deviation time. + // This rolls the current maximum read SCN position back based on the deviation duration. + final Duration deviation = connectorConfig.getLogMiningMaxScnDeviation(); + if (!deviation.isZero()) { + Optional deviatedScn = calculateDeviatedEndScn(startScn, result, deviation); + if (deviatedScn.isEmpty()) { + return Scn.NULL; + } LOGGER.debug( - "Using Top SCN calculation {} as end SCN. currentScn {}, startScn {}", - topScnToMine, - currentScn, - startScn); - return topScnToMine; + "Adjusted upper bounds {} based on deviation to {}.", + result, + deviatedScn.get()); + result = deviatedScn.get(); + } + + // Retrieve the redo thread state and get the minimum flushed SCN across all open redo + // threads + final Scn minOpenRedoThreadLastScn = + jdbcConnection.getRedoThreadState().getThreads().stream() + .filter(RedoThread::isOpen) + .map(RedoThread::getLastRedoScn) + .min(Scn::compareTo) + .orElse(Scn.NULL); + + // If there is a minimum flushed SCN across Open redo threads, and it is before the + // currently + // assigned maximum read position, we should attempt to cap the maximum read position based + // on the redo thread data. + if (!minOpenRedoThreadLastScn.isNull()) { + if (minOpenRedoThreadLastScn.compareTo(result) < 0) { + // There are situations where on first start-up that the startScn may be higher + // than the last flushed redo thread SCN, in which case we should delay by one + // iteration until the startScn is before the minOpenRedoThreadLastScn + if (minOpenRedoThreadLastScn.compareTo(startScn) < 0) { + return Scn.NULL; + } + LOGGER.debug( + "Adjusting upper bounds {} to minimum read thread flush SCN {}.", + result, + minOpenRedoThreadLastScn); + result = minOpenRedoThreadLastScn; + } + } + + if (result.compareTo(startScn) <= 0) { + // Final sanity check to prevent ORA-01281: SCN range specified is invalid + LOGGER.debug( + "Final upper bounds {} matches start read position, delay required.", result); + return Scn.NULL; + } + + LOGGER.debug("Final upper bounds range is {}.", result); + return result; + } + + /** + * Calculates the deviated end scn based on the scn range and deviation. + * + * @param lowerboundsScn the mining range's lower bounds + * @param upperboundsScn the mining range's upper bounds + * @param deviation the time deviation + * @return an optional that contains the deviated scn or empty if the operation should be + * performed again + */ + private Optional calculateDeviatedEndScn( + Scn lowerboundsScn, Scn upperboundsScn, Duration deviation) { + if (archiveLogOnlyMode) { + // When archive-only mode is enabled, deviation should be ignored, even when enabled. + return Optional.of(upperboundsScn); + } + + final Optional calculatedDeviatedEndScn = getDeviatedMaxScn(upperboundsScn, deviation); + if (calculatedDeviatedEndScn.isEmpty() || calculatedDeviatedEndScn.get().isNull()) { + // This happens only if the deviation calculation is outside the flashback/undo area or + // an exception was thrown. + // In this case we have no choice but to use the upper bounds as a fallback. + LOGGER.warn( + "Mining session end SCN deviation calculation is outside undo space, using upperbounds {}. If this continues, " + + "consider lowering the value of the '{}' configuration property.", + upperboundsScn, + OracleConnectorConfig.LOG_MINING_MAX_SCN_DEVIATION_MS.name()); + return Optional.of(upperboundsScn); + } else if (calculatedDeviatedEndScn.get().compareTo(lowerboundsScn) <= 0) { + // This should also force the outer loop to recall this method again. + LOGGER.debug( + "Mining session end SCN deviation as {}, outside of mining range, recalculating.", + calculatedDeviatedEndScn.get()); + return Optional.empty(); + } else { + // Calculated SCN is after lower bounds and within flashback/undo area, safe to return. + return calculatedDeviatedEndScn; + } + } + + /** + * Uses the provided Upperbound SCN and deviation to calculate an SCN that happened in the past + * at a time based on Oracle's {@code TIMESTAMP_TO_SCN} and {@code SCN_TO_TIMESTAMP} functions. + * + * @param upperboundsScn the upper bound system change number, should not be {@code null} + * @param deviation the time deviation to be applied, should not be {@code null} + * @return the newly calculated Scn + */ + private Optional getDeviatedMaxScn(Scn upperboundsScn, Duration deviation) { + try { + final Scn currentScn = jdbcConnection.getCurrentScn(); + final Optional currentInstant = jdbcConnection.getScnToTimestamp(currentScn); + final Optional upperInstant = jdbcConnection.getScnToTimestamp(upperboundsScn); + if (currentInstant.isPresent() && upperInstant.isPresent()) { + // If the upper bounds satisfies the deviation time + if (Duration.between(upperInstant.get(), currentInstant.get()).compareTo(deviation) + >= 0) { + LOGGER.trace( + "Upper bounds {} is within deviation period, using it.", + upperboundsScn); + return Optional.of(upperboundsScn); + } + } + return Optional.of(jdbcConnection.getScnAdjustedByTime(upperboundsScn, deviation)); + } catch (SQLException e) { + LOGGER.warn("Failed to calculate deviated max SCN value from {}.", upperboundsScn); + return Optional.empty(); } } @@ -994,8 +1209,12 @@ private boolean isTableAllColumnsSupplementalLoggingEnabled( // A table can be defined with multiple logging groups, hence why this check needs to // iterate // multiple returned rows to see whether ALL_COLUMN_LOGGING is part of the set. - return connection.queryAndMap( - SqlUtils.tableSupplementalLoggingCheckQuery(tableId), + return connection.prepareQueryAndMap( + SqlUtils.tableSupplementalLoggingCheckQuery(), + ps -> { + ps.setString(1, tableId.schema()); + ps.setString(2, tableId.table()); + }, rs -> { while (rs.next()) { if (ALL_COLUMN_LOGGING.equals(rs.getString(2))) { @@ -1012,11 +1231,14 @@ private boolean isTableAllColumnsSupplementalLoggingEnabled( * @return the strategy to be used to flush Oracle's LGWR process, never {@code null}. */ private LogWriterFlushStrategy resolveFlushStrategy() { + if (connectorConfig.isLogMiningReadOnly()) { + return new ReadOnlyLogWriterFlushStrategy(); + } if (connectorConfig.isRacSystem()) { return new RacCommitLogWriterFlushStrategy( connectorConfig, jdbcConfiguration, streamingMetrics); } - return new CommitLogWriterFlushStrategy(jdbcConnection); + return new CommitLogWriterFlushStrategy(connectorConfig, jdbcConnection); } /** @@ -1061,13 +1283,14 @@ private boolean waitForStartScnInArchiveLogs(ChangeEventSourceContext context, S * @throws SQLException if a database exception occurred */ private boolean isStartScnInArchiveLogs(Scn startScn) throws SQLException { - List logs = - LogMinerHelper.getLogFilesForOffsetScn( - jdbcConnection, - startScn, - archiveLogRetention, - archiveLogOnlyMode, - archiveDestinationName); + final List logs; + try { + logs = logCollector.getLogs(startScn); + } catch (LogFileNotFoundException e) { + // In this specific case, it's safe to ignore the error. + // This identifies that the check should simply be re-executed. + return false; + } return logs.stream() .anyMatch( l -> @@ -1077,7 +1300,12 @@ private boolean isStartScnInArchiveLogs(Scn startScn) throws SQLException { } @Override - public void commitOffset(Map offset) { + public void commitOffset(Map partition, Map offset) { // nothing to do } + + @Override + public OracleOffsetContext getOffsetContext() { + return effectiveOffset; + } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/processor/AbstractLogMinerEventProcessor.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/processor/AbstractLogMinerEventProcessor.java index 8a0b516678d..990905050a3 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/processor/AbstractLogMinerEventProcessor.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/io/debezium/connector/oracle/logminer/processor/AbstractLogMinerEventProcessor.java @@ -3,10 +3,10 @@ * * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 */ - package io.debezium.connector.oracle.logminer.processor; import io.debezium.DebeziumException; +import io.debezium.annotation.VisibleForTesting; import io.debezium.connector.oracle.OracleConnection; import io.debezium.connector.oracle.OracleConnection.NonRelationalTableException; import io.debezium.connector.oracle.OracleConnectorConfig; @@ -14,32 +14,47 @@ import io.debezium.connector.oracle.OracleOffsetContext; import io.debezium.connector.oracle.OraclePartition; import io.debezium.connector.oracle.OracleSchemaChangeEventEmitter; -import io.debezium.connector.oracle.OracleStreamingChangeEventSourceMetrics; import io.debezium.connector.oracle.Scn; import io.debezium.connector.oracle.logminer.LogMinerChangeRecordEmitter; +import io.debezium.connector.oracle.logminer.LogMinerQueryBuilder; +import io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSourceMetrics; +import io.debezium.connector.oracle.logminer.SqlUtils; import io.debezium.connector.oracle.logminer.events.DmlEvent; import io.debezium.connector.oracle.logminer.events.EventType; import io.debezium.connector.oracle.logminer.events.LobEraseEvent; import io.debezium.connector.oracle.logminer.events.LobWriteEvent; import io.debezium.connector.oracle.logminer.events.LogMinerEvent; import io.debezium.connector.oracle.logminer.events.LogMinerEventRow; +import io.debezium.connector.oracle.logminer.events.RedoSqlDmlEvent; import io.debezium.connector.oracle.logminer.events.SelectLobLocatorEvent; import io.debezium.connector.oracle.logminer.events.TruncateEvent; +import io.debezium.connector.oracle.logminer.events.XmlBeginEvent; +import io.debezium.connector.oracle.logminer.events.XmlEndEvent; +import io.debezium.connector.oracle.logminer.events.XmlWriteEvent; +import io.debezium.connector.oracle.logminer.logwriter.LogWriterFlushStrategy; import io.debezium.connector.oracle.logminer.parser.DmlParserException; +import io.debezium.connector.oracle.logminer.parser.LogMinerColumnResolverDmlParser; import io.debezium.connector.oracle.logminer.parser.LogMinerDmlEntry; import io.debezium.connector.oracle.logminer.parser.LogMinerDmlEntryImpl; import io.debezium.connector.oracle.logminer.parser.LogMinerDmlParser; import io.debezium.connector.oracle.logminer.parser.SelectLobParser; +import io.debezium.connector.oracle.logminer.parser.XmlBeginParser; import io.debezium.data.Envelope; import io.debezium.pipeline.EventDispatcher; import io.debezium.pipeline.source.spi.ChangeEventSource.ChangeEventSourceContext; import io.debezium.relational.Table; import io.debezium.relational.TableId; +import io.debezium.relational.Tables; +import io.debezium.text.ParsingException; import io.debezium.util.Clock; import io.debezium.util.Strings; +import oracle.sql.RAW; +import org.infinispan.commons.util.CloseableIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -47,26 +62,37 @@ import java.time.Instant; import java.time.ZoneOffset; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** - * Copied from Debezium 1.9.8.Final. + * An abstract implementation of {@link LogMinerEventProcessor} that all processors should extend. * - *

    An abstract implementation of {@link LogMinerEventProcessor} that all processors should - * extend. + *

    Copied from Debezium project(2.7.4.Final). * - *

    Lines 411, 423: pass ROWID to LogMinerChangeRecordEmitter. + *

    Change 1: pass {@code dmlEvent.getRowId()} to the {@link LogMinerChangeRecordEmitter} so the + * ROWID is emitted as a change-event header. */ public abstract class AbstractLogMinerEventProcessor implements LogMinerEventProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractLogMinerEventProcessor.class); + private static final Logger ABANDONED_DETAILS_LOGGER = + LoggerFactory.getLogger( + AbstractLogMinerEventProcessor.class.getName() + ".AbandonedDetails"); private static final String NO_SEQUENCE_TRX_ID_SUFFIX = "ffffffff"; + private static final String XML_WRITE_PREAMBLE = "XML_REDO := "; + private static final String XML_WRITE_PREAMBLE_NULL = XML_WRITE_PREAMBLE + "NULL"; + protected final OracleConnection jdbcConnection; private final ChangeEventSourceContext context; private final OracleConnectorConfig connectorConfig; @@ -74,17 +100,24 @@ public abstract class AbstractLogMinerEventProcessor dispatcher; - private final OracleStreamingChangeEventSourceMetrics metrics; + private final LogMinerStreamingChangeEventSourceMetrics metrics; private final LogMinerDmlParser dmlParser; + private final LogMinerColumnResolverDmlParser reconstructColumnDmlParser; private final SelectLobParser selectLobParser; + private final XmlBeginParser xmlBeginParser; + private final Tables.TableFilter tableFilter; protected final Counters counters; + protected final String sqlQuery; private Scn currentOffsetScn = Scn.NULL; private Map currentOffsetCommitScns = new HashMap<>(); + private Instant lastProcessedScnChangeTime = null; private Scn lastProcessedScn = Scn.NULL; private boolean sequenceUnavailable = false; + private final Set abandonedTransactionsCache = new HashSet<>(); + public AbstractLogMinerEventProcessor( ChangeEventSourceContext context, OracleConnectorConfig connectorConfig, @@ -92,7 +125,8 @@ public AbstractLogMinerEventProcessor( OraclePartition partition, OracleOffsetContext offsetContext, EventDispatcher dispatcher, - OracleStreamingChangeEventSourceMetrics metrics) { + LogMinerStreamingChangeEventSourceMetrics metrics, + OracleConnection jdbcConnection) { this.context = context; this.connectorConfig = connectorConfig; this.schema = schema; @@ -100,9 +134,18 @@ public AbstractLogMinerEventProcessor( this.offsetContext = offsetContext; this.dispatcher = dispatcher; this.metrics = metrics; + this.tableFilter = connectorConfig.getTableFilters().dataCollectionFilter(); this.counters = new Counters(); this.dmlParser = new LogMinerDmlParser(); + this.reconstructColumnDmlParser = new LogMinerColumnResolverDmlParser(); this.selectLobParser = new SelectLobParser(); + this.xmlBeginParser = new XmlBeginParser(); + this.sqlQuery = LogMinerQueryBuilder.build(connectorConfig); + this.jdbcConnection = jdbcConnection; + } + + protected Set getAbandonedTransactionsCache() { + return abandonedTransactionsCache; } protected OracleConnectorConfig getConfig() { @@ -142,6 +185,15 @@ protected Scn getLastProcessedScn() { return lastProcessedScn; } + /** + * Return the last processed system change number's change time. + * + * @return the last processed system change number change time, may be {@code null} + */ + protected Instant getLastProcessedScnChangeTime() { + return lastProcessedScnChangeTime; + } + /** * Returns the {@code TransactionCache} implementation. * @@ -178,21 +230,19 @@ protected boolean isTrxIdRawValue() { } @Override - public Scn process(OraclePartition partition, Scn startScn, Scn endScn) - throws SQLException, InterruptedException { + public Scn process(Scn startScn, Scn endScn) throws SQLException, InterruptedException { counters.reset(); try (PreparedStatement statement = createQueryStatement()) { LOGGER.debug("Fetching results for SCN [{}, {}]", startScn, endScn); - statement.setFetchSize(getConfig().getLogMiningViewFetchSize()); + statement.setFetchSize(getConfig().getQueryFetchSize()); statement.setFetchDirection(ResultSet.FETCH_FORWARD); statement.setString(1, startScn.toString()); statement.setString(2, endScn.toString()); Instant queryStart = Instant.now(); try (ResultSet resultSet = statement.executeQuery()) { - metrics.setLastDurationOfBatchCapturing( - Duration.between(queryStart, Instant.now())); + metrics.setLastDurationOfFetchQuery(Duration.between(queryStart, Instant.now())); Instant startProcessTime = Instant.now(); processResults(this.partition, resultSet); @@ -220,15 +270,40 @@ public Scn process(OraclePartition partition, Scn startScn, Scn endScn) offsetContext.getScn(), offsetContext.getCommitScn(), metrics.getNumberOfActiveTransactions(), - metrics.getMillisecondToSleepBetweenMiningQuery()); + metrics.getSleepTimeInMilliseconds()); + + if (metrics.getNumberOfActiveTransactions() > 0 && LOGGER.isDebugEnabled()) { + // This is wrapped in try-with-resources specifically for Infinispan performance + try (Stream stream = getTransactionCache().values().stream()) { + LOGGER.debug( + "All active transactions: {}", + stream.map(t -> t.getTransactionId() + " (" + t.getStartScn() + ")") + .collect(Collectors.joining(","))); + } + } - metrics.addProcessedRows(counters.rows); - return calculateNewStartScn( - endScn, offsetContext.getCommitScn().getMaxCommittedScn()); + metrics.setLastProcessedRowsCount(counters.rows); + + if (counters.rows == 0) { + // When no rows are processed, don't advance the SCN + return startScn; + } else { + return calculateNewStartScn( + endScn, offsetContext.getCommitScn().getMaxCommittedScn()); + } } } } + /** + * Get the LogMiner query that will be used to fetch results. + * + * @return the SQL query to use, never {@code null} + */ + protected String getQueryString() { + return sqlQuery; + } + /** * Create the JDBC query that will be used to fetch the mining result set. * @@ -277,6 +352,7 @@ protected void processRow(OraclePartition partition, LogMinerEventRow row) throws SQLException, InterruptedException { if (!row.getEventType().equals(EventType.MISSING_SCN)) { lastProcessedScn = row.getScn(); + lastProcessedScnChangeTime = row.getChangeTime(); } // filter out all events that are captured as part of the initial snapshot if (row.getScn().compareTo(offsetContext.getSnapshotScn()) < 0) { @@ -291,6 +367,50 @@ protected void processRow(OraclePartition partition, LogMinerEventRow row) return; } } + + // Check whether the row has a table reference and if so, is the reference included by the + // filter. + // If the reference isn't included, the row will be skipped entirely. + if (row.getTableId() != null) { + if (LogWriterFlushStrategy.isFlushTable( + row.getTableId(), + connectorConfig.getJdbcConfig().getUser(), + connectorConfig.getLogMiningFlushTableName())) { + LOGGER.trace("Skipped change associated with flush table '{}'", row.getTableId()); + return; + } + + // DDL events get filtered inside the DDL handler + // We do the non-DDL ones here to cover multiple switch handlers in one place. + if (!EventType.DDL.equals(row.getEventType()) + && !tableFilter.isIncluded(row.getTableId())) { + if (isUsingHybridStrategy()) { + if (!isTableLookupByObjectIdRequired(row)) { + LOGGER.trace( + "Skipping change associated with table '{}' which does not match filters.", + row.getTableId()); + return; + } + // Special use case where the table has been dropped and purged, and we are + // processing an + // old event for the table that comes prior to the drop. + LOGGER.debug( + "Found DML for dropped table in history with object-id based table name {}.", + row.getTableId().table()); + final TableId tableId = schema.getTableIdByObjectId(row.getObjectId(), null); + if (tableId != null) { + row.setTableId(tableId); + } + } + if (!tableFilter.isIncluded(row.getTableId())) { + LOGGER.trace( + "Skipping change associated with table '{}' which does not match filters.", + row.getTableId()); + return; + } + } + } + switch (row.getEventType()) { case MISSING_SCN: handleMissingScn(row); @@ -315,6 +435,15 @@ protected void processRow(OraclePartition partition, LogMinerEventRow row) case LOB_ERASE: handleLobErase(row); break; + case XML_BEGIN: + handleXmlBegin(row); + break; + case XML_WRITE: + handleXmlWrite(row); + break; + case XML_END: + handleXmlEnd(row); + break; case INSERT: case UPDATE: case DELETE: @@ -342,17 +471,21 @@ protected void handleMissingScn(LogMinerEventRow row) { */ protected void handleStart(LogMinerEventRow row) { final String transactionId = row.getTransactionId(); - final AbstractTransaction transaction = getTransactionCache().get(transactionId); + final T transaction = getTransactionCache().get(transactionId); if (transaction == null && !isRecentlyProcessed(transactionId)) { getTransactionCache().put(transactionId, createTransaction(row)); - metrics.setActiveTransactions(getTransactionCache().size()); + metrics.setActiveTransactionCount(getTransactionCache().size()); } else if (transaction != null && !isRecentlyProcessed(transactionId)) { LOGGER.trace( "Transaction {} is not yet committed and START event detected.", transactionId); - transaction.start(); + resetTransactionToStart(transaction); } } + protected void resetTransactionToStart(T transaction) { + transaction.start(); + } + /** * Handle processing a LogMinerEventRow for a {@code COMMIT} event. * @@ -369,63 +502,81 @@ protected void handleCommit(OraclePartition partition, LogMinerEventRow row) final T transaction = getAndRemoveTransactionFromCache(transactionId); if (transaction == null) { - LOGGER.trace("Transaction {} not found, commit skipped.", transactionId); - return; + if (!offsetContext.getCommitScn().hasCommitAlreadyBeenHandled(row)) { + LOGGER.debug( + "Transaction {} not found in cache with SCN {}, no events to commit.", + transactionId, + row.getScn()); + } + handleCommitNotFoundInBuffer(row); } - // Calculate the smallest SCN that remains in the transaction cache - final Scn smallestScn = getTransactionCacheMinimumScn(); - metrics.setOldestScn(smallestScn.isNull() ? Scn.valueOf(-1) : smallestScn); - + final Scn smallestScn = calculateSmallestScn(); final Scn commitScn = row.getScn(); if (offsetContext.getCommitScn().hasCommitAlreadyBeenHandled(row)) { - final Scn lastCommittedScn = - offsetContext.getCommitScn().getCommitScnForRedoThread(row.getThread()); - LOGGER.debug( - "Transaction {} has already been processed. " - + "Offset Commit SCN {}, Transaction Commit SCN {}, Last Seen Commit SCN {}.", - transactionId, - offsetContext.getCommitScn(), - commitScn, - lastCommittedScn); - removeTransactionAndEventsFromCache(transaction); - metrics.setActiveTransactions(getTransactionCache().size()); + if (transaction != null) { + if (transaction.getNumberOfEvents() > 0) { + final Scn lastCommittedScn = + offsetContext.getCommitScn().getCommitScnForRedoThread(row.getThread()); + LOGGER.debug( + "Transaction {} has already been processed. " + + "Offset Commit SCN {}, Transaction Commit SCN {}, Last Seen Commit SCN {}.", + transactionId, + offsetContext.getCommitScn(), + commitScn, + lastCommittedScn); + } + cleanupAfterTransactionRemovedFromCache(transaction, false); + metrics.setActiveTransactionCount(getTransactionCache().size()); + } return; } counters.commitCount++; - int numEvents = getTransactionEventCount(transaction); - LOGGER.trace("Commit (smallest SCN {}) {}", smallestScn, row); - LOGGER.trace("Transaction {} has {} events", transactionId, numEvents); - - final ZoneOffset databaseOffset = metrics.getDatabaseOffset(); + int numEvents = (transaction == null) ? 0 : getTransactionEventCount(transaction); + LOGGER.debug( + "Committing transaction {} with {} events (scn: {}, oldest buffer scn: {}): {}", + transactionId, + numEvents, + row.getScn(), + smallestScn, + row); - final boolean skipExcludedUserName = isTransactionUserExcluded(transaction); - TransactionCommitConsumer.Handler delegate = - new TransactionCommitConsumer.Handler() { - private int numEvents = getTransactionEventCount(transaction); + // When a COMMIT is received, regardless of the number of events it has, it still + // must be recorded in the commit scn for the node to guarantee updates to the + // offsets. This must be done prior to dispatching the transaction-commit or the + // heartbeat event that follows commit dispatch. + offsetContext.getCommitScn().recordCommit(row); - @Override - public void accept(LogMinerEvent event, long eventsProcessed) - throws InterruptedException { + Instant start = Instant.now(); + boolean dispatchTransactionCommittedEvent = false; + if (numEvents > 0) { + final boolean skipExcludedUserName = isTransactionUserExcluded(transaction); + dispatchTransactionCommittedEvent = !skipExcludedUserName; + final ZoneOffset databaseOffset = metrics.getDatabaseOffset(); + TransactionCommitConsumer.Handler delegate = + (event, eventsProcessed) -> { // Update SCN in offset context only if processed SCN less than SCN of other // transactions if (smallestScn.isNull() || commitScn.compareTo(smallestScn) < 0) { offsetContext.setScn(event.getScn()); - metrics.setOldestScn(event.getScn()); + metrics.setOldestScnDetails(event.getScn(), event.getChangeTime()); } offsetContext.setEventScn(event.getScn()); offsetContext.setTransactionId(transactionId); + offsetContext.setUserName(transaction.getUserName()); offsetContext.setSourceTime( event.getChangeTime() .minusSeconds(databaseOffset.getTotalSeconds())); offsetContext.setTableId(event.getTableId()); offsetContext.setRedoThread(row.getThread()); - if (eventsProcessed == numEvents) { - // reached the last event update the commit scn in the offsets - offsetContext.getCommitScn().recordCommit(row); + offsetContext.setRsId(event.getRsId()); + offsetContext.setRowId(event.getRowId()); + + if (event instanceof RedoSqlDmlEvent) { + offsetContext.setRedoSql(((RedoSqlDmlEvent) event).getRedoSql()); } final DmlEvent dmlEvent = (DmlEvent) event; @@ -463,14 +614,13 @@ public void accept(LogMinerEvent event, long eventsProcessed) dispatcher.dispatchDataChangeEvent( partition, event.getTableId(), logMinerChangeRecordEmitter); } - } - }; - Instant start = Instant.now(); - int dispatchedEventCount = 0; - if (numEvents > 0) { + // Clear redo SQL + offsetContext.setRedoSql(null); + }; try (TransactionCommitConsumer commitConsumer = new TransactionCommitConsumer(delegate, connectorConfig, schema)) { + int dispatchedEventCount = 0; final Iterator iterator = getTransactionEventIterator(transaction); while (iterator.hasNext()) { if (!context.isRunning()) { @@ -488,25 +638,78 @@ public void accept(LogMinerEvent event, long eventsProcessed) } offsetContext.setEventScn(commitScn); - if (getTransactionEventCount(transaction) > 0 && !skipExcludedUserName) { - dispatcher.dispatchTransactionCommittedEvent(partition, offsetContext); + offsetContext.setRsId(row.getRsId()); + offsetContext.setRowId(""); + + if (dispatchTransactionCommittedEvent) { + dispatcher.dispatchTransactionCommittedEvent( + partition, offsetContext, transaction.getChangeTime()); } else { dispatcher.dispatchHeartbeatEvent(partition, offsetContext); } - metrics.calculateLagMetrics(row.getChangeTime()); + metrics.calculateLagFromSource(row.getChangeTime()); - finalizeTransactionCommit(transactionId, commitScn); - removeTransactionAndEventsFromCache(transaction); + if (transaction != null) { + finalizeTransactionCommit(transactionId, commitScn); + cleanupAfterTransactionRemovedFromCache(transaction, false); + metrics.setActiveTransactionCount(getTransactionCache().size()); + } - metrics.incrementCommittedTransactions(); - metrics.setActiveTransactions(getTransactionCache().size()); - metrics.incrementCommittedDmlCount(dispatchedEventCount); - metrics.setCommittedScn(commitScn); + metrics.incrementCommittedTransactionCount(); + metrics.setCommitScn(commitScn); metrics.setOffsetScn(offsetContext.getScn()); metrics.setLastCommitDuration(Duration.between(start, Instant.now())); } + /** + * Calculate the smallest SCN that remains in the transaction cache. + * + * @return the smallest SCN + */ + private Scn calculateSmallestScn() { + final Optional oldestTransaction = getOldestTransactionInCache(); + final Scn smallestScn; + if (oldestTransaction.isPresent()) { + smallestScn = oldestTransaction.get().getStartScn(); + metrics.setOldestScnDetails(smallestScn, oldestTransaction.get().getChangeTime()); + } else { + smallestScn = Scn.NULL; + metrics.setOldestScnDetails(Scn.valueOf(-1), null); + } + return smallestScn; + } + + /** + * Allow for post-processing of a transaction commit in the stream that was not found in the + * transaction buffer, perhaps because it aged out due to retention policies. + * + * @param row the result set row + */ + protected void handleCommitNotFoundInBuffer(LogMinerEventRow row) { + // In the event the transaction was prematurely removed due to retention policy, when we do + // find + // the transaction's commit in the logs in the future, we should remove the entry if it + // exists + // to avoid any potential memory-leak with the cache. + abandonedTransactionsCache.remove(row.getTransactionId()); + } + + /** + * Allow for post-processing of a transaction rollback in the stream that was not found in the + * transaction buffer, perhaps because it aged out due to retention policies. + * + * @param row the result set row + */ + protected void handleRollbackNotFoundInBuffer(LogMinerEventRow row) { + // In the event the transaction was prematurely removed due to retention policy, when we do + // find + // the transaction's rollback in the logs in the future, we should remove the entry if it + // exists + // to avoid any potential memory-leak with the cache. + abandonedTransactionsCache.remove(row.getTransactionId()); + } + /** * Gets a transaction instance from the transaction cache while also removing its cache entry. * @@ -516,11 +719,19 @@ public void accept(LogMinerEvent event, long eventsProcessed) protected abstract T getAndRemoveTransactionFromCache(String transactionId); /** - * Removes the transaction and all its associated event entries from the connector's caches. + * Removes the items associated with the transaction (e.g. events if they are stored + * independently). * * @param transaction the transaction instance, should never be {@code null} + * @param isAbandoned whether the removal is because transaction is being abandoned */ - protected abstract void removeTransactionAndEventsFromCache(T transaction); + protected void cleanupAfterTransactionRemovedFromCache(T transaction, boolean isAbandoned) { + if (isAbandoned) { + abandonedTransactionsCache.add(transaction.getTransactionId()); + } else { + abandonedTransactionsCache.remove(transaction.getTransactionId()); + } + } /** * Get an iterator over the events that are part of the specified transaction. @@ -538,6 +749,13 @@ public void accept(LogMinerEvent event, long eventsProcessed) */ protected abstract void finalizeTransactionCommit(String transactionId, Scn commitScn); + /** + * Returns only the first transaction id in the transaction buffer. + * + * @return the first active transaction in the buffer, or {@code null} if there is none. + */ + protected abstract String getFirstActiveTransactionKey(); + /** * Check whether the supplied username associated with the specified transaction is excluded. * @@ -552,7 +770,7 @@ protected boolean isTransactionUserExcluded(T transaction) { } else if (connectorConfig .getLogMiningUsernameExcludes() .contains(transaction.getUserName())) { - LOGGER.trace("Skipped transaction with excluded username {}", transaction); + LOGGER.debug("Skipped transaction with excluded username {}", transaction); return true; } } @@ -566,21 +784,22 @@ protected boolean isTransactionUserExcluded(T transaction) { */ protected void handleRollback(LogMinerEventRow row) { if (getTransactionCache().containsKey(row.getTransactionId())) { - LOGGER.trace("Transaction {} was rolled back.", row.getTransactionId()); + LOGGER.debug("Transaction {} was rolled back.", row.getTransactionId()); finalizeTransactionRollback(row.getTransactionId(), row.getScn()); - metrics.setActiveTransactions(getTransactionCache().size()); - metrics.incrementRolledBackTransactions(); - metrics.addRolledBackTransactionId(row.getTransactionId()); - counters.rollbackCount++; + metrics.setActiveTransactionCount(getTransactionCache().size()); } else { - LOGGER.trace( - "Could not rollback transaction {}, was not found in cache.", + LOGGER.debug( + "Transaction {} not found in cache, no events to rollback.", row.getTransactionId()); + handleRollbackNotFoundInBuffer(row); } + metrics.incrementRolledBackTransactionCount(); + metrics.addRolledBackTransactionId(row.getTransactionId()); + counters.rollbackCount++; } /** - * Finalizes the rollback the specified transaction. + * Finalizes the rollback the specified transaction * * @param transactionId the unique transaction identifier, never {@code null} * @param rollbackScn the rollback transaction's system change number, never {@code null} @@ -594,6 +813,30 @@ protected void handleRollback(LogMinerEventRow row) { * @throws InterruptedException if the event dispatcher is interrupted sending the event */ protected void handleSchemaChange(LogMinerEventRow row) throws InterruptedException { + if (row.getTableId() != null) { + boolean isExcluded = + connectorConfig.getLogMiningSchemaChangesUsernameExcludes().stream() + .anyMatch(userName -> userName.equalsIgnoreCase(row.getUserName())); + if (isExcluded) { + LOGGER.trace( + "User '{}' is in schema change exclusions, DDL '{}' skipped.", + row.getUserName(), + row.getRedoSql()); + return; + } else if (row.getInfo() != null && row.getInfo().startsWith("INTERNAL DDL")) { + // Internal DDL operations are skipped. + LOGGER.trace("Internal DDL '{}' skipped", row.getRedoSql()); + return; + } + + if (schema.storeOnlyCapturedTables() && !tableFilter.isIncluded(row.getTableId())) { + LOGGER.trace( + "Skipping DDL associated with table '{}', schema history only stores included tables only.", + row.getTableId()); + return; + } + } + if (hasSchemaChangeBeenSeen(row)) { LOGGER.trace( "DDL: Scn {}, SQL '{}' has already been processed, skipped.", @@ -625,7 +868,7 @@ protected void handleSchemaChange(LogMinerEventRow row) throws InterruptedExcept // The DDL isn't wrapped in a transaction, fast-forward the lower boundary advanceLowerScnBoundary = true; } else if (activeTransactions == 1) { - final String transactionId = getTransactionCache().keySet().iterator().next(); + final String transactionId = getFirstActiveTransactionKey(); if (transactionId.equals(row.getTransactionId())) { // The row's transaction is the current and only active transaction. advanceLowerScnBoundary = true; @@ -646,8 +889,12 @@ protected void handleSchemaChange(LogMinerEventRow row) throws InterruptedExcept offsetContext.setEventScn(row.getScn()); offsetContext.setRedoThread(row.getThread()); + offsetContext.setRsId(row.getRsId()); + offsetContext.setRowId(""); + dispatcher.dispatchSchemaChangeEvent( partition, + offsetContext, tableId, new OracleSchemaChangeEventEmitter( getConfig(), @@ -656,6 +903,10 @@ protected void handleSchemaChange(LogMinerEventRow row) throws InterruptedExcept tableId, tableId.catalog(), tableId.schema(), + row.getObjectId(), + // ALTER TABLE does not populate the data object id, pass object id on + // purpose + row.getObjectId(), row.getRedoSql(), getSchema(), row.getChangeTime(), @@ -666,6 +917,18 @@ protected void handleSchemaChange(LogMinerEventRow row) throws InterruptedExcept private void processTruncateEvent(LogMinerEventRow row) { LOGGER.debug("Handling truncate event"); + + try { + // Truncate event is being treated as a DML. + Table table = getTableForDataEvent(row); + if (table == null) { + return; + } + } catch (SQLException | InterruptedException e) { + LOGGER.warn("Failed to process truncate event.", e); + return; + } + addToTransaction( row.getTransactionId(), row, @@ -689,7 +952,7 @@ protected void handleSelectLobLocator(LogMinerEventRow row) { return; } - LOGGER.trace("SEL_LOB_LOCATOR: {}", row); + LOGGER.debug("SEL_LOB_LOCATOR: {}", row); final TableId tableId = row.getTableId(); final Table table = getSchema().tableFor(tableId); if (table == null) { @@ -713,7 +976,7 @@ protected void handleSelectLobLocator(LogMinerEventRow row) { selectLobParser.isBinary()); }); - metrics.incrementRegisteredDmlCount(); + metrics.incrementTotalChangesCount(); } /** @@ -730,7 +993,7 @@ protected void handleLobWrite(LogMinerEventRow row) { return; } - LOGGER.trace( + LOGGER.debug( "LOB_WRITE: scn={}, tableId={}, changeTime={}, transactionId={}", row.getScn(), row.getTableId(), @@ -766,7 +1029,7 @@ private void handleLobErase(LogMinerEventRow row) { return; } - LOGGER.trace("LOB_ERASE: {}", row); + LOGGER.debug("LOB_ERASE: {}", row); final TableId tableId = row.getTableId(); final Table table = getSchema().tableFor(tableId); if (table == null) { @@ -777,6 +1040,128 @@ private void handleLobErase(LogMinerEventRow row) { addToTransaction(row.getTransactionId(), row, () -> new LobEraseEvent(row)); } + private void handleXmlBegin(LogMinerEventRow row) { + if (!getConfig().isLobEnabled()) { + LOGGER.trace("LOB support is disabled, XML_BEGIN '{}' skipped.", row.getRedoSql()); + return; + } + + LOGGER.trace("XML_BEGIN: {}", row); + final TableId tableId = row.getTableId(); + final Table table = getSchema().tableFor(tableId); + if (table == null) { + LOGGER.warn("XML_BEGIN for table '{}' is not known, skipped.", tableId); + return; + } + + addToTransaction( + row.getTransactionId(), + row, + () -> { + final LogMinerDmlEntry dmlEntry = xmlBeginParser.parse(row.getRedoSql(), table); + dmlEntry.setObjectName(row.getTableName()); + dmlEntry.setObjectOwner(row.getTablespaceName()); + return new XmlBeginEvent(row, dmlEntry, xmlBeginParser.getColumnName()); + }); + } + + private void handleXmlWrite(LogMinerEventRow row) { + if (!getConfig().isLobEnabled()) { + LOGGER.trace("LOB support is disabled, XML_WRITE '{}' skipped.", row.getRedoSql()); + return; + } + + LOGGER.trace("XML_WRITE: {}", row); + final TableId tableId = row.getTableId(); + final Table table = getSchema().tableFor(tableId); + if (table == null) { + LOGGER.warn("XML_WRITE for table '{}' is not known, skipped.", tableId); + return; + } + + addToTransaction(row.getTransactionId(), row, () -> getXmlWriteEventFromRow(row)); + } + + private XmlWriteEvent getXmlWriteEventFromRow(LogMinerEventRow row) { + final String sql = row.getRedoSql(); + if (!sql.startsWith(XML_WRITE_PREAMBLE)) { + throw new ParsingException( + null, "XML write operation does not start with XML_REDO preamble"); + } + + try { + final String xml; + if (XML_WRITE_PREAMBLE_NULL.equals(sql)) { + // The XML field is being explicitly set to NULL + return new XmlWriteEvent(row, null, 0); + } else if (sql.charAt(XML_WRITE_PREAMBLE.length()) == '\'') { + // The XML is not provided as HEXTORAW, which means it was likely stored inline as a + // VARCHAR column data type because the text is relatively short, i.e. short CLOB. + int lastQuoteIndex = sql.lastIndexOf('\''); + if (lastQuoteIndex == -1) { + throw new IllegalStateException("Failed to find end of XML document"); + } + // indices here remove leading and trailing single quotes + xml = sql.substring(XML_WRITE_PREAMBLE.length() + 1, lastQuoteIndex); + } else { + // The XML is provided as HEXTORAW, which means that it was stored out of bands in + // LOB storage and not inline in the data page. The contents of the XML will + // require being decoded. + int lastParenIndex = sql.lastIndexOf(')'); + if (lastParenIndex == -1) { + throw new IllegalStateException("Failed to find end of XML document"); + } + + // indices are meant to preserve the prefix function call and suffix parenthesis + String xmlHex = sql.substring(XML_WRITE_PREAMBLE.length(), lastParenIndex + 1); + + // NOTE: Oracle generates a small bug here where the initial row starts the function + // argument with a single quote but the last entry to fulfill the data does not + // end-quote the argument, but rather simply stops with a parenthesis. + if (!xmlHex.startsWith("HEXTORAW('") || !xmlHex.endsWith(")")) { + throw new IllegalStateException("Invalid HEXTORAW XML decoded data"); + } else { + if (xmlHex.endsWith("')")) { + // Handles situation when Oracle fixes bug + xmlHex = xmlHex.substring(10, xmlHex.length() - 2); + } else { + // Compensates for the bug + xmlHex = xmlHex.substring(10, xmlHex.length() - 1); + } + } + + xml = new String(RAW.hexString2Bytes(xmlHex), StandardCharsets.UTF_8); + } + + int lastColonIndex = sql.lastIndexOf(':'); + if (lastColonIndex == -1) { + throw new IllegalStateException("Failed to find XML document length"); + } + + final Integer length = Integer.parseInt(sql.substring(lastColonIndex + 1).trim()); + return new XmlWriteEvent(row, xml, length); + } catch (Exception e) { + throw new ParsingException(null, "Failed to parse XML write data", e); + } + } + + private void handleXmlEnd(LogMinerEventRow row) { + if (!getConfig().isLobEnabled()) { + LOGGER.trace("LOB support is disabled, XML_END skipped."); + return; + } + + LOGGER.trace("XML_END: {}", row); + final TableId tableId = row.getTableId(); + final Table table = getSchema().tableFor(tableId); + if (table == null) { + LOGGER.warn("XM_END for table ' {}' is not known, skipped.", tableId); + return; + } + + addToTransaction(row.getTransactionId(), row, () -> new XmlEndEvent(row)); + } + /** * Handle processing a LogMinerEventRow for a {@code INSERT}, {@code UPDATE}, or {@code DELETE} * event. @@ -790,7 +1175,7 @@ protected void handleDataEvent(LogMinerEventRow row) throws SQLException, Interr return; } - LOGGER.trace("DML: {}", row); + LOGGER.debug("DML: {}", row); LOGGER.trace("\t{}", row.getRedoSql()); // Oracle LogMiner reports LONG data types as STATUS=2 on UPDATE statements but there is no @@ -798,24 +1183,29 @@ protected void handleDataEvent(LogMinerEventRow row) throws SQLException, Interr // so to be backward compatible, we only explicitly trigger this behavior if there is an // error reason for STATUS=2 in the INFO column as well as STATUS=2. if (row.getStatus() == 2 && !Strings.isNullOrBlank(row.getInfo())) { - // The SQL in the SQL_REDO column is not valid and cannot be parsed. - switch (connectorConfig.getEventProcessingFailureHandlingMode()) { - case FAIL: - LOGGER.error("Oracle LogMiner is unable to re-construct the SQL for '{}'", row); - throw new DebeziumException( - "Oracle failed to re-construct redo SQL '" + row.getRedoSql() + "'"); - case WARN: - LOGGER.warn( - "Oracle LogMiner event '{}' cannot be parsed. This event will be ignored and skipped.", - row); - return; - default: - // In this case, we explicitly log the situation in "debug" only and not as an - // error/warn. - LOGGER.debug( - "Oracle LogMiner event '{}' cannot be parsed. This event will be ignored and skipped.", - row); - return; + if (!isUsingHybridStrategy() || (isUsingHybridStrategy() && !isTableKnown(row))) { + // The SQL in the SQL_REDO column is not valid and cannot be parsed. + switch (connectorConfig.getEventProcessingFailureHandlingMode()) { + case FAIL: + LOGGER.error( + "Oracle LogMiner is unable to re-construct the SQL for '{}'", row); + throw new DebeziumException( + "Oracle failed to re-construct redo SQL '" + + row.getRedoSql() + + "'"); + case WARN: + LOGGER.warn( + "Oracle LogMiner event '{}' cannot be parsed. This event will be ignored and skipped.", + row); + return; + default: + // In this case, we explicitly log the situation in "debug" only and not as + // an error/warn. + LOGGER.debug( + "Oracle LogMiner event '{}' cannot be parsed. This event will be ignored and skipped.", + row); + return; + } } } @@ -851,14 +1241,16 @@ protected void handleDataEvent(LogMinerEventRow row) throws SQLException, Interr row.getTransactionId(), row, () -> { - final LogMinerDmlEntry dmlEntry = - parseDmlStatement(row.getRedoSql(), table, row.getTransactionId()); + final LogMinerDmlEntry dmlEntry = parseDmlStatement(row, table); dmlEntry.setObjectName(row.getTableName()); dmlEntry.setObjectOwner(row.getTablespaceName()); + if (connectorConfig.isLogMiningIncludeRedoSql()) { + return new RedoSqlDmlEvent(row, dmlEntry, row.getRedoSql()); + } return new DmlEvent(row, dmlEntry); }); - metrics.incrementRegisteredDmlCount(); + metrics.incrementTotalChangesCount(); } protected void handleUnsupportedEvent(LogMinerEventRow row) { @@ -869,6 +1261,7 @@ protected void handleUnsupportedEvent(LogMinerEventRow row) { row.getTransactionId(), row.getScn(), row.getThread()); + LOGGER.debug("\t{}", row); } } @@ -894,8 +1287,10 @@ protected void warnPotentiallyStuckScn( previousOffsetScn, previousOffsetCommitScns); metrics.incrementScnFreezeCount(); + counters.stuckCount = 0; } } else { + metrics.setScnFreezeCount(0); counters.stuckCount = 0; } } @@ -903,7 +1298,7 @@ protected void warnPotentiallyStuckScn( private Table getTableForDataEvent(LogMinerEventRow row) throws SQLException, InterruptedException { - final TableId tableId = row.getTableId(); + final TableId tableId = getTableIdForDataEvent(row); Table table = getSchema().tableFor(tableId); if (table == null) { if (!getConfig().getTableFilters().dataCollectionFilter().isIncluded(tableId)) { @@ -916,6 +1311,58 @@ private Table getTableForDataEvent(LogMinerEventRow row) return table; } + private TableId getTableIdForDataEvent(LogMinerEventRow row) throws SQLException { + TableId tableId = row.getTableId(); + if (isUsingHybridStrategy()) { + if (tableId.table().startsWith("BIN$")) { + // Object was dropped but has not been purged. + try (OracleConnection connection = + new OracleConnection(connectorConfig.getJdbcConfig())) { + return connection.prepareQueryAndMap( + "SELECT OWNER, ORIGINAL_NAME FROM DBA_RECYCLEBIN WHERE OBJECT_NAME=?", + ps -> ps.setString(1, row.getTableId().table()), + rs -> { + if (rs.next()) { + return new TableId( + row.getTableId().catalog(), + rs.getString(1), + rs.getString(2)); + } + return row.getTableId(); + }); + } + } else if (tableId.table().equalsIgnoreCase("UNKNOWN")) { + // Object has been dropped and purged. + final TableId resolvedTableId = + schema.getTableIdByObjectId(row.getObjectId(), row.getDataObjectId()); + if (resolvedTableId != null) { + return resolvedTableId; + } + throw new DebeziumException( + "Failed to resolve UNKNOWN table name by object id lookup"); + } + } + return tableId; + } + + private boolean isTableKnown(LogMinerEventRow row) { + return !row.getTableId().table().equalsIgnoreCase("UNKNOWN"); + } + + private boolean isTableLookupByObjectIdRequired(LogMinerEventRow row) { + final String tableName = row.getTableId().table(); + if (tableName.startsWith("OBJ# ")) { + // This is a table that has been dropped and purged + return true; + } else if (tableName.startsWith("BIN$") + && tableName.endsWith("==$0") + && tableName.length() == 30) { + // This is a table that has been dropped, but not yet purged from the RECYCLEBIN + return true; + } + return false; + } + /** * Checks whether the result-set has any more data available. When a new row is available, the * streaming metrics is updated with the fetch timings. @@ -929,7 +1376,7 @@ private boolean hasNextWithMetricsUpdate(ResultSet resultSet) throws SQLExceptio boolean result = false; try { if (resultSet.next()) { - metrics.addCurrentResultSetNext(Duration.between(start, Instant.now())); + metrics.setLastResultSetNextDuration(Duration.between(start, Instant.now())); result = true; } @@ -998,85 +1445,85 @@ protected abstract void addToTransaction( * @throws SQLException if a database exception occurred * @throws InterruptedException if the event dispatch was interrupted */ - private Table dispatchSchemaChangeEventAndGetTableForNewCapturedTable( + @VisibleForTesting + public Table dispatchSchemaChangeEventAndGetTableForNewCapturedTable( TableId tableId, OracleOffsetContext offsetContext, EventDispatcher dispatcher) throws SQLException, InterruptedException { - final String tableDdl; - try { - tableDdl = getTableMetadataDdl(tableId); + LOGGER.warn( + "Obtaining schema for table {}, which should be already loaded, this may signal potential bug in fetching table schemas.", + tableId); + try (OracleConnection connection = + new OracleConnection(connectorConfig.getJdbcConfig(), false)) { + connection.setAutoCommit(false); + if (!Strings.isNullOrBlank(connectorConfig.getPdbName())) { + connection.setSessionToPdb(connectorConfig.getPdbName()); + } + + final String tableDdl = getTableMetadataDdl(connection, tableId); + final Long objectId = connection.getTableObjectId(tableId); + final Long dataObjectId = connection.getTableDataObjectId(tableId); + + dispatcher.dispatchSchemaChangeEvent( + partition, + offsetContext, + tableId, + new OracleSchemaChangeEventEmitter( + connectorConfig, + partition, + offsetContext, + tableId, + tableId.catalog(), + tableId.schema(), + objectId, + dataObjectId, + tableDdl, + getSchema(), + Instant.now(), + metrics, + null)); + + return getSchema().tableFor(tableId); } catch (NonRelationalTableException e) { LOGGER.warn("Table {} is not a relational table and will be skipped.", tableId); metrics.incrementWarningCount(); return null; } - - LOGGER.info("Table '{}' is new and will now be captured.", tableId); - offsetContext.event(tableId, Instant.now()); - dispatcher.dispatchSchemaChangeEvent( - partition, - tableId, - new OracleSchemaChangeEventEmitter( - connectorConfig, - partition, - offsetContext, - tableId, - tableId.catalog(), - tableId.schema(), - tableDdl, - getSchema(), - Instant.now(), - metrics, - null)); - - return getSchema().tableFor(tableId); } /** * Get the specified table's create DDL statement. * + * @param connection the connection to be used, must not be {@code null} * @param tableId the table identifier, must not be {@code null} * @return the table's create DDL statement, never {@code null} * @throws SQLException if an exception occurred obtaining the DDL statement * @throws NonRelationalTableException if the table is not a relational table */ - private String getTableMetadataDdl(TableId tableId) + @VisibleForTesting + public String getTableMetadataDdl(OracleConnection connection, TableId tableId) throws SQLException, NonRelationalTableException { counters.tableMetadataCount++; LOGGER.info("Getting database metadata for table '{}'", tableId); - // A separate connection must be used for this out-of-bands query while processing LogMiner - // results. - // This should have negligible overhead since this use case should happen rarely. - try (OracleConnection connection = - new OracleConnection( - connectorConfig.getJdbcConfig(), - () -> getClass().getClassLoader(), - false)) { - connection.setAutoCommit(false); - final String pdbName = getConfig().getPdbName(); - if (pdbName != null) { - connection.setSessionToPdb(pdbName); - } - return connection.getTableMetadataDdl(tableId); - } + return connection.getTableMetadataDdl(tableId); } /** * Parse a DML redo SQL statement. * - * @param redoSql the redo SQL statement + * @param row the result set row * @param table the table the SQL statement is for - * @param transactionId the associated transaction id for the SQL statement * @return a parse object for the redo SQL statement */ - private LogMinerDmlEntry parseDmlStatement(String redoSql, Table table, String transactionId) { + private LogMinerDmlEntry parseDmlStatement(LogMinerEventRow row, Table table) { LogMinerDmlEntry dmlEntry; + final String redoSql = row.getRedoSql(); try { Instant parseStart = Instant.now(); - dmlEntry = dmlParser.parse(redoSql, table); - metrics.addCurrentParseTime(Duration.between(parseStart, Instant.now())); + dmlEntry = resolveParser(row).parse(redoSql, table); + metrics.setLastParseTimeDuration(Duration.between(parseStart, Instant.now())); } catch (DmlParserException e) { String message = "DML statement couldn't be parsed." @@ -1097,7 +1544,21 @@ private LogMinerDmlEntry parseDmlStatement(String redoSql, Table table, String t return dmlEntry; } - private static final Pattern LOB_WRITE_SQL_PATTERN = + private LogMinerDmlParser resolveParser(LogMinerEventRow row) { + if (row.getStatus() == 2 + && !Strings.isNullOrBlank(row.getInfo()) + && isUsingHybridStrategy()) { + return reconstructColumnDmlParser; + } + return dmlParser; + } + + private boolean isUsingHybridStrategy() { + return OracleConnectorConfig.LogMiningStrategy.HYBRID.equals( + connectorConfig.getLogMiningStrategy()); + } + + private static Pattern LOB_WRITE_SQL_PATTERN = Pattern.compile( "(?s).* := ((?:HEXTORAW\\()?'.*'(?:\\))?);\\s*dbms_lob.write\\([^,]+,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,[^,]+\\);.*"); @@ -1125,6 +1586,14 @@ private ParsedLobWriteSql parseLobWriteSql(String sql) { } int length = Integer.parseInt(m.group(2)); int offset = Integer.parseInt(m.group(3)) - 1; // Oracle uses 1-based offsets + + // Double check whether Oracle may have escaped single-quotes in the SQL data. + // This avoids unintended truncation during the LOB merge phase during the commit + // logic handled by TransactionCommitConsumer. + if (data.contains("''")) { + data = data.replaceAll("''", "'"); + } + return new ParsedLobWriteSql(offset, length, data); } @@ -1133,10 +1602,10 @@ private class ParsedLobWriteSql { final int length; final String data; - ParsedLobWriteSql(int offset, int length, String data) { - this.offset = offset; - this.length = length; - this.data = data; + ParsedLobWriteSql(int _offset, int _length, String _data) { + offset = _offset; + length = _length; + data = _data; } } @@ -1147,6 +1616,13 @@ private class ParsedLobWriteSql { */ protected abstract Scn getTransactionCacheMinimumScn(); + /** + * Get the oldest transaction in the cache. + * + * @return the oldest transaction in the cache or maybe {@code null} if cache is empty + */ + protected abstract Optional getOldestTransactionInCache(); + /** * Returns whether the transaction id has no sequence number component. * @@ -1158,13 +1634,13 @@ private class ParsedLobWriteSql { *

  • Sequence number of the transaction that generated the change * * - *

    When Oracle LogMiner mines records, it is possible that when an undo operation is - * detected, often the product of a constraint violation, the LogMiner row will have the same - * explicit XID (transaction id) as the source operation that we should undo; however, if the - * record to be undone was mined in a prior iteration, Oracle LogMiner won't be able to make a - * link back to the full transaction's sequence number, therefore the XID value for the undo row - * will contain only the undo segment number and slot number, setting the sequence number to - * 4294967295 (aka -1 or 0xFFFFFFFF). + * When Oracle LogMiner mines records, it is possible that when an undo operation is detected, + * often the product of a constraint violation, the LogMiner row will have the same explicit XID + * (transaction id) as the source operation that we should undo; however, if the record to be + * undone was mined in a prior iteration, Oracle LogMiner won't be able to make a link back to + * the full transaction's sequence number, therefore the XID value for the undo row will contain + * only the undo segment number and slot number, setting the sequence number to 4294967295 (aka + * -1 or 0xFFFFFFFF). * *

    This method explicitly checks if the provided transaction id has the no sequence sentinel * value and if so, returns {@code true}; otherwise returns {@code false}. @@ -1180,9 +1656,212 @@ protected String getTransactionIdPrefix(String transactionId) { return transactionId.substring(0, 8); } - /** Wrapper for all counter variables. */ - protected class Counters { - public int stuckCount; + protected boolean isTransactionOverEventThreshold(T transaction) { + if (getConfig().getLogMiningBufferTransactionEventsThreshold() == 0) { + return false; + } + return getTransactionEventCount(transaction) + >= getConfig().getLogMiningBufferTransactionEventsThreshold(); + } + + protected void abandonTransactionOverEventThreshold(T transaction) { + LOGGER.warn( + "Transaction {} exceeds maximum allowed number of events, transaction will be abandoned.", + transaction.getTransactionId()); + metrics.incrementWarningCount(); + getAndRemoveTransactionFromCache(transaction.getTransactionId()); + abandonedTransactionsCache.add(transaction.getTransactionId()); + metrics.incrementOversizedTransactionCount(); + } + + @Override + public void abandonTransactions(Duration retention) throws InterruptedException { + if (!Duration.ZERO.equals(retention)) { + Optional lastScnToAbandonTransactions = + getLastScnToAbandon(jdbcConnection, retention); + if (lastScnToAbandonTransactions.isPresent()) { + Scn thresholdScn = lastScnToAbandonTransactions.get(); + Scn smallestScn = getTransactionCacheMinimumScn(); + if (!smallestScn.isNull() && thresholdScn.compareTo(smallestScn) >= 0) { + boolean first = true; + Iterator> iterator = + getTransactionCache().entrySet().iterator(); + try { + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (entry.getValue().getStartScn().compareTo(thresholdScn) <= 0) { + if (first) { + LOGGER.warn( + "All transactions with SCN <= {} will be abandoned.", + thresholdScn); + if (LOGGER.isDebugEnabled()) { + try (Stream s = + getTransactionCache().keySet().stream()) { + LOGGER.debug( + "List of transactions in the cache before transactions being abandoned: [{}]", + s.collect(Collectors.joining(","))); + } + } + first = false; + } + LOGGER.warn( + "Transaction {} (start SCN {}, change time {}, redo thread {}, {} events{}) is being abandoned.", + entry.getKey(), + entry.getValue().getStartScn(), + entry.getValue().getChangeTime(), + entry.getValue().getRedoThreadId(), + entry.getValue().getNumberOfEvents(), + getLoggedAbandonedTransactionTableNames(entry.getValue())); + + cleanupAfterTransactionRemovedFromCache(entry.getValue(), true); + iterator.remove(); + + metrics.addAbandonedTransactionId(entry.getKey()); + metrics.setActiveTransactionCount(getTransactionCache().size()); + } + } + } finally { + if (iterator instanceof CloseableIterator) { + ((CloseableIterator>) iterator).close(); + } + } + if (LOGGER.isDebugEnabled()) { + try (Stream s = getTransactionCache().keySet().stream()) { + LOGGER.debug( + "List of transactions in the cache after transactions being abandoned: [{}]", + s.collect(Collectors.joining(","))); + } + } + + // Update the oldest scn metric are transaction abandonment + final Optional oldestTransaction = getOldestTransactionInCache(); + if (oldestTransaction.isPresent()) { + final T transaction = oldestTransaction.get(); + metrics.setOldestScnDetails( + transaction.getStartScn(), transaction.getChangeTime()); + } else { + metrics.setOldestScnDetails(Scn.NULL, null); + } + + offsetContext.setScn(thresholdScn); + } + dispatcher.dispatchHeartbeatEvent(partition, offsetContext); + } + } + } + + /** + * Calculates a list of tables that participate in the abandoned transaction, but only if the + * {@link #ABANDONED_DETAILS_LOGGER} logger has {@code DEBUG} logging enabled. + * + * @param transaction the transaction to inspect events for + * @return details about abandoned transactions, or an empty string if logging level is INFO or + * higher. + */ + protected String getLoggedAbandonedTransactionTableNames(T transaction) { + if (ABANDONED_DETAILS_LOGGER.isDebugEnabled()) { + final Set tableNames = new HashSet<>(); + final Iterator eventIterator = getTransactionEventIterator(transaction); + while (eventIterator.hasNext()) { + final LogMinerEvent event = eventIterator.next(); + tableNames.add(event.getTableId().identifier()); + } + return String.format( + ", %d tables [%s]", tableNames.size(), String.join(",", tableNames)); + } + return ""; + } + + /** + * Calculates the SCN as a watermark to abandon for long running transactions. The criteria is + * do not let the offset SCN expire from archives older the specified retention hours. + * + * @param connection database connection, should not be {@code null} + * @param retention duration to tolerate long running transactions before being abandoned, must + * not be {@code null} + * @return an optional system change number as the watermark for transaction buffer abandonment + */ + protected Optional getLastScnToAbandon(OracleConnection connection, Duration retention) { + try { + if (getLastProcessedScn().isNull()) { + return Optional.empty(); + } + BigInteger scnToAbandon = + connection.singleOptionalValue( + SqlUtils.getScnByTimeDeltaQuery(getLastProcessedScn(), retention), + rs -> rs.getBigDecimal(1).toBigInteger()); + return Optional.of(new Scn(scnToAbandon)); + } catch (SQLException e) { + // This can happen when the last processed SCN has aged out of the UNDO_RETENTION. + // In this case, we use a fallback in order to calculate the SCN based on the + // change times in the transaction cache. + if (getLastProcessedScnChangeTime() != null) { + final Scn calculatedLastScn = + getLastScnToAbandonFallbackByTransactionChangeTime(retention); + if (!calculatedLastScn.isNull()) { + return Optional.of(calculatedLastScn); + } + } + + // Both SCN database calculation and fallback failed, log error. + LOGGER.error( + String.format( + "Cannot fetch SCN %s by given duration to calculate SCN to abandon", + getLastProcessedScn()), + e); + metrics.incrementErrorCount(); + return Optional.empty(); + } + } + + /** + * Calculates the last system change number to abandon by directly examining the transaction + * buffer cache and comparing the transaction start time to the most recent last processed + * change time and comparing the difference to the configured transaction retention policy. + * + * @param retention duration to tolerate long-running transactions before being abandoned, must + * not be {@code null} + * @return the system change number to consider for transaction abandonment, never {@code null} + */ + private Scn getLastScnToAbandonFallbackByTransactionChangeTime(Duration retention) { + LOGGER.debug( + "Getting abandon SCN breakpoint based on change time {} (retention {} minutes).", + getLastProcessedScnChangeTime(), + retention.toMinutes()); + + Scn calculatedLastScn = Scn.NULL; + for (Transaction transaction : getTransactionCache().values()) { + final Instant changeTime = transaction.getChangeTime(); + final long diffMinutes = + Duration.between(getLastProcessedScnChangeTime(), changeTime).abs().toMinutes(); + if (diffMinutes > 0 && diffMinutes > retention.toMinutes()) { + // We either now will capture the transaction's SCN because it is the first detected + // transaction + // outside the configured retention period or the transaction has a start SCN that + // is more recent + // than the current calculated SCN but is still outside the configured retention + // period. + LOGGER.debug( + "Transaction {} with SCN {} started at {}, age is {} minutes.", + transaction.getTransactionId(), + transaction.getStartScn(), + changeTime, + diffMinutes); + if (calculatedLastScn.isNull() + || calculatedLastScn.compareTo(transaction.getStartScn()) < 0) { + calculatedLastScn = transaction.getStartScn(); + } + } + } + + return calculatedLastScn; + } + + /** Wrapper for all counter variables */ + protected static class Counters { + public int + stuckCount; // it will be reset after 25 mining session iterations or if offset SCN + // changes public int dmlCount; public int ddlCount; public int insertCount; @@ -1194,7 +1873,6 @@ protected class Counters { public long rows; public void reset() { - stuckCount = 0; dmlCount = 0; ddlCount = 0; insertCount = 0; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/OracleSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/OracleSource.java index cb56a5165f0..6b9bb88563f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/OracleSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/OracleSource.java @@ -146,7 +146,8 @@ public DebeziumSourceFunction build() { // for all Kafka topic names emanating from this connector. Only alphanumeric characters // and // underscores should be used. - props.setProperty("database.server.name", DATABASE_SERVER_NAME); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", DATABASE_SERVER_NAME); props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); @@ -159,7 +160,7 @@ public DebeziumSourceFunction build() { if (port != null) { props.setProperty("database.port", String.valueOf(port)); } - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); props.setProperty("database.dbname", checkNotNull(database)); if (schemaList != null) { props.setProperty("schema.include.list", String.join(",", schemaList)); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/connection/OracleSourceConnection.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/connection/OracleSourceConnection.java index f47f770fd6d..914fc9357e1 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/connection/OracleSourceConnection.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/connection/OracleSourceConnection.java @@ -21,8 +21,6 @@ import io.debezium.jdbc.JdbcConfiguration; import io.debezium.relational.Column; import io.debezium.relational.Table; -import io.debezium.relational.TableId; -import io.debezium.schema.DatabaseSchema; import java.sql.ResultSet; import java.sql.SQLException; @@ -34,12 +32,13 @@ public class OracleSourceConnection extends OracleConnection { public OracleSourceConnection( JdbcConfiguration config, Supplier classLoaderSupplier) { - super(config, classLoaderSupplier); + // Debezium 2.0 removed the classloader-supplier constructor; the classloader is now + // resolved internally. The supplier is kept in this signature for caller compatibility. + super(config); } @Override - public > Object getColumnValue( - ResultSet rs, int columnIndex, Column column, Table table, T schema) + public Object getColumnValue(ResultSet rs, int columnIndex, Column column, Table table) throws SQLException { try { if (rs.getObject(columnIndex) == null) { @@ -54,7 +53,7 @@ public > Object getColumnValue( return rs.getObject(columnIndex); } } catch (SQLException e) { - return super.getColumnValue(rs, columnIndex, column, table, schema); + return super.getColumnValue(rs, columnIndex, column, table); } } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfig.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfig.java index 26152d52c09..3ae1e116777 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfig.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfig.java @@ -19,6 +19,7 @@ import org.apache.flink.cdc.connectors.base.config.JdbcSourceConfig; import org.apache.flink.cdc.connectors.base.options.StartupOptions; +import org.apache.flink.util.TemporaryClassLoaderContext; import io.debezium.config.Configuration; import io.debezium.connector.oracle.OracleConnectorConfig; @@ -98,7 +99,14 @@ public OracleSourceConfig( @Override public OracleConnectorConfig getDbzConnectorConfig() { - return new OracleConnectorConfig(getDbzConfiguration()); + // Debezium 2.0 resolves classes through the thread context class loader + // (io.debezium.config.Instantiator, and the ServiceLoader based SPIs). On a Flink + // thread that loader may be a user code class loader from a previous job attempt, + // which is already closed. Pin it to the loader that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(OracleConnectorConfig.class.getClassLoader())) { + return new OracleConnectorConfig(getDbzConfiguration()); + } } public Configuration getOriginDbzConnectorConfig() { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactory.java index a255b380626..6fd865bee75 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactory.java @@ -71,17 +71,19 @@ public OracleSourceConfig create(int subtaskId) { // for all Kafka topic names emanating from this connector. Only alphanumeric characters // and // underscores should be used. - props.setProperty("database.server.name", DATABASE_SERVER_NAME); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", DATABASE_SERVER_NAME); props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); props.setProperty("database.dbname", checkNotNull(databaseList.get(0))); // database history props.setProperty( - "database.history", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); - props.setProperty("database.history.instance.name", UUID.randomUUID() + "_" + subtaskId); - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); - props.setProperty("database.history.refer.ddl", String.valueOf(true)); + "schema.history.internal", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); + props.setProperty( + "schema.history.internal.instance.name", UUID.randomUUID() + "_" + subtaskId); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.prefer.ddl", String.valueOf(true)); props.setProperty("connect.timeout.ms", String.valueOf(connectTimeout.toMillis())); // disable tombstones props.setProperty("tombstones.on.delete", String.valueOf(false)); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/EventProcessorFactory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/EventProcessorFactory.java index 00e8d91d4c3..e1823277c48 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/EventProcessorFactory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/EventProcessorFactory.java @@ -29,7 +29,7 @@ import io.debezium.connector.oracle.OracleDatabaseSchema; import io.debezium.connector.oracle.OracleOffsetContext; import io.debezium.connector.oracle.OraclePartition; -import io.debezium.connector.oracle.OracleStreamingChangeEventSourceMetrics; +import io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSourceMetrics; import io.debezium.connector.oracle.logminer.events.LogMinerEventRow; import io.debezium.connector.oracle.logminer.processor.LogMinerEventProcessor; import io.debezium.connector.oracle.logminer.processor.infinispan.EmbeddedInfinispanLogMinerEventProcessor; @@ -63,7 +63,7 @@ public static LogMinerEventProcessor createProcessor( OraclePartition partition, OracleOffsetContext offsetContext, OracleDatabaseSchema schema, - OracleStreamingChangeEventSourceMetrics metrics, + LogMinerStreamingChangeEventSourceMetrics metrics, ErrorHandler errorHandler, StreamSplit redoLogSplit) { final OracleConnectorConfig.LogMiningBufferType bufferType = @@ -134,7 +134,7 @@ public CDCMemoryLogMinerEventProcessor( OraclePartition partition, OracleOffsetContext offsetContext, OracleDatabaseSchema schema, - OracleStreamingChangeEventSourceMetrics metrics, + LogMinerStreamingChangeEventSourceMetrics metrics, ErrorHandler errorHandler, StreamSplit redoLogSplit) { super( @@ -184,7 +184,7 @@ public CDCEmbeddedInfinispanLogMinerEventProcessor( OraclePartition partition, OracleOffsetContext offsetContext, OracleDatabaseSchema schema, - OracleStreamingChangeEventSourceMetrics metrics, + LogMinerStreamingChangeEventSourceMetrics metrics, ErrorHandler errorHandler, StreamSplit redoLogSplit) { super( @@ -234,7 +234,7 @@ public CDCRemoteInfinispanLogMinerEventProcessor( OraclePartition partition, OracleOffsetContext offsetContext, OracleDatabaseSchema schema, - OracleStreamingChangeEventSourceMetrics metrics, + LogMinerStreamingChangeEventSourceMetrics metrics, ErrorHandler errorHandler, StreamSplit redoLogSplit) { super( diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleScanFetchTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleScanFetchTask.java index 4ce09a1e1c6..188079e669d 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleScanFetchTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleScanFetchTask.java @@ -31,7 +31,10 @@ import io.debezium.connector.oracle.logminer.LogMinerOracleOffsetContextLoader; import io.debezium.heartbeat.Heartbeat; import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.notification.NotificationService; +import io.debezium.pipeline.signal.actions.snapshotting.SnapshotConfiguration; import io.debezium.pipeline.source.AbstractSnapshotChangeEventSource; +import io.debezium.pipeline.source.SnapshottingTask; import io.debezium.pipeline.source.spi.SnapshotProgressListener; import io.debezium.pipeline.spi.ChangeRecordEmitter; import io.debezium.pipeline.spi.SnapshotResult; @@ -39,6 +42,7 @@ import io.debezium.relational.SnapshotChangeRecordEmitter; import io.debezium.relational.Table; import io.debezium.relational.TableId; +import io.debezium.schema.SchemaFactory; import io.debezium.util.Clock; import io.debezium.util.ColumnUtils; import io.debezium.util.Strings; @@ -51,6 +55,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Duration; +import java.util.Collections; import static org.apache.flink.cdc.connectors.oracle.source.reader.fetch.OracleStreamFetchTask.RedoLogSplitReadTask; import static org.apache.flink.cdc.connectors.oracle.source.utils.OracleUtils.buildSplitScanQuery; @@ -81,7 +86,10 @@ protected void executeDataSnapshot(Context context) throws Exception { snapshotSplitReadTask.execute( changeEventSourceContext, sourceFetchContext.getPartition(), - sourceFetchContext.getOffsetContext()); + sourceFetchContext.getOffsetContext(), + snapshotSplitReadTask.getSnapshottingTask( + sourceFetchContext.getPartition(), + sourceFetchContext.getOffsetContext())); if (!snapshotResult.isCompletedOrSkipped()) { taskRunning = false; throw new IllegalStateException( @@ -171,7 +179,14 @@ public OracleSnapshotSplitReadTask( OracleConnection jdbcConnection, EventDispatcher eventDispatcher, SnapshotSplit snapshotSplit) { - super(connectorConfig, snapshotProgressListener); + super( + connectorConfig, + snapshotProgressListener, + new NotificationService<>( + Collections.emptyList(), + connectorConfig, + SchemaFactory.get(), + notification -> {})); this.offsetContext = previousOffset; this.connectorConfig = connectorConfig; this.databaseSchema = databaseSchema; @@ -186,12 +201,12 @@ public OracleSnapshotSplitReadTask( public SnapshotResult execute( ChangeEventSourceContext context, OraclePartition partition, - OracleOffsetContext previousOffset) + OracleOffsetContext previousOffset, + SnapshottingTask snapshottingTask) throws InterruptedException { - SnapshottingTask snapshottingTask = getSnapshottingTask(partition, previousOffset); final SnapshotContext ctx; try { - ctx = prepare(partition); + ctx = prepare(partition, false); } catch (Exception e) { LOG.error("Failed to initialize snapshot context.", e); throw new RuntimeException(e); @@ -220,23 +235,35 @@ protected SnapshotResult doExecute( } @Override - protected SnapshottingTask getSnapshottingTask( + public SnapshottingTask getSnapshottingTask( OraclePartition partition, OracleOffsetContext previousOffset) { - return new SnapshottingTask(false, true); + return new SnapshottingTask( + false, true, Collections.emptyList(), Collections.emptyMap(), false); + } + + @Override + public SnapshottingTask getBlockingSnapshottingTask( + OraclePartition partition, + OracleOffsetContext previousOffset, + SnapshotConfiguration snapshotConfiguration) { + // Debezium 2.6 made this abstract. Flink CDC drives its own split snapshot and never + // runs Debezium's signal-based blocking snapshot, so it behaves like the regular one. + return getSnapshottingTask(partition, previousOffset); } @Override protected SnapshotContext prepare( - OraclePartition partition) throws Exception { - return new OracleSnapshotContext(partition); + OraclePartition partition, boolean onDemand) throws Exception { + return new OracleSnapshotContext(partition, onDemand); } private static class OracleSnapshotContext extends RelationalSnapshotChangeEventSource.RelationalSnapshotContext< OraclePartition, OracleOffsetContext> { - public OracleSnapshotContext(OraclePartition partition) throws SQLException { - super(partition, ""); + public OracleSnapshotContext(OraclePartition partition, boolean onDemand) + throws SQLException { + super(partition, "", onDemand); } } @@ -293,8 +320,7 @@ private void createDataEventsForTable( while (rs.next()) { rows++; - final Object[] row = - jdbcConnection.rowToArray(table, databaseSchema, rs, columnArray); + final Object[] row = jdbcConnection.rowToArray(table, rs, columnArray); if (logTimer.expired()) { long stop = clock.currentTimeInMillis(); LOG.info( @@ -328,7 +354,7 @@ protected ChangeRecordEmitter getChangeRecordEmitter( Object[] row) { snapshotContext.offset.event(tableId, clock.currentTime()); return new SnapshotChangeRecordEmitter<>( - snapshotContext.partition, snapshotContext.offset, row, clock); + snapshotContext.partition, snapshotContext.offset, row, clock, connectorConfig); } private Threads.Timer getTableScanLogTimer() { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleSourceFetchTaskContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleSourceFetchTaskContext.java index 59bd957932c..ea94b2adc2f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleSourceFetchTaskContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleSourceFetchTaskContext.java @@ -35,6 +35,7 @@ import org.apache.flink.cdc.connectors.oracle.util.ChunkUtils; import org.apache.flink.table.types.logical.RowType; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.base.ChangeEventQueue; import io.debezium.connector.oracle.OracleChangeEventSourceMetricsFactory; import io.debezium.connector.oracle.OracleConnectorConfig; @@ -42,11 +43,10 @@ import io.debezium.connector.oracle.OracleErrorHandler; import io.debezium.connector.oracle.OracleOffsetContext; import io.debezium.connector.oracle.OraclePartition; -import io.debezium.connector.oracle.OracleStreamingChangeEventSourceMetrics; import io.debezium.connector.oracle.OracleTaskContext; -import io.debezium.connector.oracle.OracleTopicSelector; import io.debezium.connector.oracle.SourceInfo; import io.debezium.connector.oracle.logminer.LogMinerOracleOffsetContextLoader; +import io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSourceMetrics; import io.debezium.data.Envelope; import io.debezium.pipeline.DataChangeEvent; import io.debezium.pipeline.ErrorHandler; @@ -57,8 +57,8 @@ import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.relational.Tables; -import io.debezium.schema.DataCollectionId; -import io.debezium.schema.TopicSelector; +import io.debezium.spi.schema.DataCollectionId; +import io.debezium.spi.topic.TopicNamingStrategy; import io.debezium.util.Collect; import oracle.sql.ROWID; import org.apache.kafka.connect.data.Struct; @@ -88,8 +88,8 @@ public class OracleSourceFetchTaskContext extends JdbcSourceFetchTaskContext { private OraclePartition partition; private SnapshotChangeEventSourceMetrics snapshotChangeEventSourceMetrics; - private OracleStreamingChangeEventSourceMetrics streamingChangeEventSourceMetrics; - private TopicSelector topicSelector; + private LogMinerStreamingChangeEventSourceMetrics streamingChangeEventSourceMetrics; + private TopicNamingStrategy topicSelector; private JdbcSourceEventDispatcher dispatcher; private ChangeEventQueue queue; private OracleErrorHandler errorHandler; @@ -105,7 +105,8 @@ public OracleSourceFetchTaskContext( public void configure(SourceSplitBase sourceSplitBase) { // initial stateful objects final OracleConnectorConfig connectorConfig = getDbzConnectorConfig(); - this.topicSelector = OracleTopicSelector.defaultSelector(connectorConfig); + this.topicSelector = + connectorConfig.getTopicNamingStrategy(CommonConnectorConfig.TOPIC_NAMING_STRATEGY); EmbeddedFlinkDatabaseHistory.registerHistory( sourceConfig .getDbzConfiguration() @@ -116,7 +117,14 @@ public void configure(SourceSplitBase sourceSplitBase) { this.offsetContext = loadStartingOffsetState( new LogMinerOracleOffsetContextLoader(connectorConfig), sourceSplitBase); - this.partition = new OraclePartition(connectorConfig.getLogicalName()); + // Debezium 2.0 OraclePartition requires the database name (PDB name when present). + final String partitionDatabaseName = + (connectorConfig.getPdbName() == null + || connectorConfig.getPdbName().trim().isEmpty()) + ? connectorConfig.getDatabaseName() + : connectorConfig.getPdbName(); + this.partition = + new OraclePartition(connectorConfig.getLogicalName(), partitionDatabaseName); validateAndLoadDatabaseHistory(offsetContext, databaseSchema); this.taskContext = new OracleTaskContext(connectorConfig, databaseSchema); @@ -151,16 +159,16 @@ public void configure(SourceSplitBase sourceSplitBase) { final OracleChangeEventSourceMetricsFactory changeEventSourceMetricsFactory = new OracleChangeEventSourceMetricsFactory( - new OracleStreamingChangeEventSourceMetrics( + new LogMinerStreamingChangeEventSourceMetrics( taskContext, queue, metadataProvider, connectorConfig)); this.snapshotChangeEventSourceMetrics = changeEventSourceMetricsFactory.getSnapshotMetrics( taskContext, queue, metadataProvider); this.streamingChangeEventSourceMetrics = - (OracleStreamingChangeEventSourceMetrics) + (LogMinerStreamingChangeEventSourceMetrics) changeEventSourceMetricsFactory.getStreamingMetrics( taskContext, queue, metadataProvider); - this.errorHandler = new OracleErrorHandler(connectorConfig, queue); + this.errorHandler = new OracleErrorHandler(connectorConfig, queue, null); } @Override @@ -186,7 +194,7 @@ public SnapshotChangeEventSourceMetrics getSnapshotChangeEventS return snapshotChangeEventSourceMetrics; } - public OracleStreamingChangeEventSourceMetrics getStreamingChangeEventSourceMetrics() { + public LogMinerStreamingChangeEventSourceMetrics getStreamingChangeEventSourceMetrics() { return streamingChangeEventSourceMetrics; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleStreamFetchTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleStreamFetchTask.java index 95fbefdd818..61b3ac2e7e3 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleStreamFetchTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/OracleStreamFetchTask.java @@ -22,15 +22,17 @@ import org.apache.flink.cdc.connectors.base.source.meta.split.SourceSplitBase; import org.apache.flink.cdc.connectors.base.source.meta.split.StreamSplit; import org.apache.flink.cdc.connectors.base.source.reader.external.FetchTask; +import org.apache.flink.cdc.debezium.internal.SnapshotterServiceFactory; import io.debezium.config.Configuration; import io.debezium.connector.oracle.OracleConnection; +import io.debezium.connector.oracle.OracleConnector; import io.debezium.connector.oracle.OracleConnectorConfig; import io.debezium.connector.oracle.OracleDatabaseSchema; import io.debezium.connector.oracle.OracleOffsetContext; import io.debezium.connector.oracle.OraclePartition; -import io.debezium.connector.oracle.OracleStreamingChangeEventSourceMetrics; import io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSource; +import io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSourceMetrics; import io.debezium.connector.oracle.logminer.processor.LogMinerEventProcessor; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.EventDispatcher; @@ -104,7 +106,7 @@ public static class RedoLogSplitReadTask extends LogMinerStreamingChangeEventSou private final OracleDatabaseSchema schema; - private final OracleStreamingChangeEventSourceMetrics metrics; + private final LogMinerStreamingChangeEventSourceMetrics metrics; public RedoLogSplitReadTask( OracleConnectorConfig connectorConfig, @@ -114,7 +116,7 @@ public RedoLogSplitReadTask( ErrorHandler errorHandler, OracleDatabaseSchema schema, Configuration jdbcConfig, - OracleStreamingChangeEventSourceMetrics metrics, + LogMinerStreamingChangeEventSourceMetrics metrics, StreamSplit redoLogSplit) { super( connectorConfig, @@ -124,7 +126,8 @@ public RedoLogSplitReadTask( Clock.SYSTEM, schema, jdbcConfig, - metrics); + metrics, + SnapshotterServiceFactory.create(connectorConfig, OracleConnector.class)); this.redoLogSplit = redoLogSplit; this.eventDispatcher = eventDispatcher; this.watermarkDispatcher = watermarkDispatcher; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/StoppableChangeEventSourceContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/StoppableChangeEventSourceContext.java index 14bd8324abb..8d2bc3951ca 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/StoppableChangeEventSourceContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/StoppableChangeEventSourceContext.java @@ -36,4 +36,24 @@ public void stopChangeEventSource() { public boolean isRunning() { return isRunning; } + + // The following methods are only used by Debezium's signal-based blocking snapshot, + // which Flink CDC does not use, so they are no-ops here. + + @Override + public boolean isPaused() { + return false; + } + + @Override + public void resumeStreaming() throws InterruptedException {} + + @Override + public void waitSnapshotCompletion() throws InterruptedException {} + + @Override + public void streamingPaused() {} + + @Override + public void waitStreamingPaused() throws InterruptedException {} } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/utils/OracleSchema.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/utils/OracleSchema.java index 64630cd20c9..ec48ada32aa 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/utils/OracleSchema.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/utils/OracleSchema.java @@ -29,9 +29,10 @@ import java.sql.SQLException; import java.util.HashMap; -import java.util.HashSet; +import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Objects; +import java.util.stream.Collectors; /** A component used to get schema by table path. */ public class OracleSchema { @@ -55,29 +56,86 @@ public TableChange getTableSchema(JdbcConnection jdbc, TableId tableId) { private TableChange readTableSchema(JdbcConnection jdbc, TableId tableId) { OracleConnection oracleConnection = (OracleConnection) jdbc; - Set tableIdSet = new HashSet<>(); - tableIdSet.add(tableId); - final Map tableChangeMap = new HashMap<>(); - Tables tables = new Tables(); - tables.overwriteTable(tables.editOrCreateTable(tableId).create()); + // Debezium 2.0 removed OracleConnection#readSchemaForCapturedTables, whose contract was to + // look the table up by the caller's own TableId and store the result under that same id. + // Core's JdbcConnection#readSchema behaves differently in two ways that both matter here: + // it rebuilds the id from the JDBC metadata (catalog, schema, table) and stores the table + // under *that* id, and it applies the table filter to the rebuilt id rather than to the + // requested one. Callers in Flink CDC identify an Oracle table as + // TableId(catalog=owner, schema=null, table=name) - see OracleSchemaUtils#toDbzTableId - + // which never equals the rebuilt TableId(catalog, owner, name), so an exact-equality + // filter matches nothing and a lookup by the requested id finds nothing. + // + // The removed method was also lenient about that "owner" in a way callers rely on: it + // passed the value straight to DatabaseMetaData#getColumns as the catalog, which Oracle + // ignores, together with a null schema pattern - so it effectively searched every schema + // and keyed the result by the requested id. OracleMetadataAccessorITCase depends on that, + // asking for the container's database name (ORCLCDB) rather than the real owner + // (DEBEZIUM). Try the owner as a schema first, which is correct and cheap when callers + // pass a real owner, and only fall back to the schema-wide search when that finds + // nothing. + final String owner = tableId.schema() != null ? tableId.schema() : tableId.catalog(); + Table table = readTable(oracleConnection, tableId, owner); + if (table == null) { + table = readTable(oracleConnection, tableId, null); + } + if (table == null) { + throw new FlinkRuntimeException( + String.format("Can't obtain schema for table %s ", tableId)); + } + + // Restore the pre-Debezium-2.0 behavior of returning a table carrying the requested id. + return new TableChange( + TableChanges.TableChangeType.CREATE, table.edit().tableId(tableId).create()); + } + + /** + * Reads the table from JDBC metadata, restricted to {@code schemaPattern} when it is non-null. + * Returns {@code null} when no unambiguous match exists. + */ + private Table readTable( + OracleConnection oracleConnection, TableId tableId, String schemaPattern) { + final Tables tables = new Tables(); try { - oracleConnection.readSchemaForCapturedTables( - tables, tableId.catalog(), tableId.schema(), null, false, tableIdSet); - Table table = tables.forTable(tableId); - TableChange tableChange = new TableChange(TableChanges.TableChangeType.CREATE, table); - tableChangeMap.put(tableId, tableChange); + // Oracle exposes the owner as the JDBC schema and has no meaningful catalog; passing + // the owner as the catalog (as an unqualified read would) returns no rows at all. + oracleConnection.readSchema( + tables, + null, + schemaPattern, + Tables.TableFilter.fromPredicate( + id -> matches(id, schemaPattern, tableId.table())), + null, + false); } catch (SQLException e) { throw new FlinkRuntimeException( String.format("Failed to read schema for table %s ", tableId), e); } - if (!tableChangeMap.containsKey(tableId)) { + List candidates = + tables.tableIds().stream() + .filter(id -> matches(id, schemaPattern, tableId.table())) + .map(tables::forTable) + .filter(Objects::nonNull) + .filter(t -> !t.columns().isEmpty()) + .collect(Collectors.toList()); + + if (candidates.size() > 1) { throw new FlinkRuntimeException( - String.format("Can't obtain schema for table %s ", tableId)); + String.format( + "Table %s is ambiguous, it matches %s. Qualify it with its owner.", + tableId, + candidates.stream() + .map(t -> t.id().toString()) + .collect(Collectors.toList()))); } + return candidates.isEmpty() ? null : candidates.get(0); + } - return tableChangeMap.get(tableId); + private static boolean matches(TableId candidate, String owner, String tableName) { + return Objects.equals(candidate.table(), tableName) + && (owner == null || owner.equals(candidate.schema())); } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/utils/OracleUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/utils/OracleUtils.java index b87217a8eae..72d3c3c325e 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/utils/OracleUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/utils/OracleUtils.java @@ -21,17 +21,18 @@ import org.apache.flink.cdc.connectors.oracle.source.OracleSourceValueConverters; import org.apache.flink.cdc.connectors.oracle.source.meta.offset.RedoLogOffset; import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.TemporaryClassLoaderContext; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.oracle.OracleConnectorConfig; import io.debezium.connector.oracle.OracleDatabaseSchema; import io.debezium.connector.oracle.OracleDefaultValueConverter; -import io.debezium.connector.oracle.OracleTopicSelector; import io.debezium.connector.oracle.OracleValueConverters; import io.debezium.connector.oracle.StreamingAdapter; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.TableId; -import io.debezium.schema.TopicSelector; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.topic.TopicNamingStrategy; import org.apache.kafka.connect.source.SourceRecord; import java.sql.Connection; @@ -201,7 +202,15 @@ public static PreparedStatement readTableSplitDataStatement( /** Creates a new {@link OracleDatabaseSchema} to monitor the latest oracle database schemas. */ public static OracleDatabaseSchema createOracleDatabaseSchema( OracleConnectorConfig dbzOracleConfig, OracleSourceConnection oracleConnection) { - TopicSelector topicSelector = OracleTopicSelector.defaultSelector(dbzOracleConfig); + TopicNamingStrategy topicSelector; + // Debezium 2.0 resolves classes through the thread context class loader; pin it to the + // loader that loaded Debezium so a stale/closed Flink user class loader cannot break it. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(CommonConnectorConfig.class.getClassLoader())) { + topicSelector = + dbzOracleConfig.getTopicNamingStrategy( + CommonConnectorConfig.TOPIC_NAMING_STRATEGY); + } SchemaNameAdjuster schemaNameAdjuster = SchemaNameAdjuster.create(); OracleValueConverters oracleValueConverters = new OracleSourceValueConverters(dbzOracleConfig, oracleConnection); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/OracleSourceTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/OracleSourceTest.java index ae891ace340..3e233a60758 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/OracleSourceTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/OracleSourceTest.java @@ -571,7 +571,8 @@ private OracleSource.Builder basicSourceBuilder() { Properties debeziumProperties = new Properties(); debeziumProperties.setProperty("debezium.log.mining.strategy", "online_catalog"); // ignore APEX ORCLCDB system tables changes - debeziumProperties.setProperty("database.history.store.only.captured.tables.ddl", "true"); + debeziumProperties.setProperty( + "schema.history.internal.store.only.captured.tables.ddl", "true"); return OracleSource.builder() .hostname(ORACLE_CONTAINER.getHost()) .port(ORACLE_CONTAINER.getOraclePort()) diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactoryTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactoryTest.java new file mode 100644 index 00000000000..d978d3ce12e --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactoryTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.oracle.source.config; + +import org.apache.flink.cdc.connectors.base.options.StartupOptions; + +import io.debezium.config.CommonConnectorConfig; +import io.debezium.connector.oracle.OracleConnectorConfig; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link OracleSourceConfigFactory}. */ +class OracleSourceConfigFactoryTest { + + /** + * Debezium 2.0 renamed {@code database.server.name} to {@code topic.prefix}. Without it the + * topic naming strategy cannot be built, which breaks every fetch task. + */ + @Test + void testTopicPrefixIsConfigured() { + OracleSourceConfigFactory factory = new OracleSourceConfigFactory(); + factory.hostname("localhost"); + factory.port(1521); + factory.username("dbzuser"); + factory.password("dbz"); + factory.databaseList("ORCLCDB"); + factory.schemaList("DEBEZIUM"); + factory.tableList("ORCLCDB.DEBEZIUM.PRODUCTS"); + factory.startupOptions(StartupOptions.initial()); + + OracleConnectorConfig connectorConfig = + new OracleConnectorConfig(factory.create(0).getDbzConfiguration()); + + assertThat(connectorConfig.getLogicalName()).isEqualTo("oracle_logminer"); + assertThat( + connectorConfig.getTopicNamingStrategy( + CommonConnectorConfig.TOPIC_NAMING_STRATEGY)) + .isNotNull(); + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/source/reader/OracleSourceReaderTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/source/reader/OracleSourceReaderTest.java index 06069fd6185..3108897cd66 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/source/reader/OracleSourceReaderTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/source/reader/OracleSourceReaderTest.java @@ -66,6 +66,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.function.Supplier; import static org.apache.flink.core.io.InputStatus.MORE_AVAILABLE; @@ -267,8 +268,20 @@ private OracleSourceConfig getConfig(String[] captureTables) { Arrays.stream(captureTables) .map(tableName -> ORACLE_SCHEMA + "." + tableName) .toArray(String[]::new); + // Every other Oracle test in this repository pins log.mining.strategy to online_catalog; + // this one was the sole outlier and relied on Debezium's redo_log_catalog default. That + // strategy mines with DICT_FROM_REDO_LOGS + DDL_DICT_TRACKING, which requires the data + // dictionary that DBMS_LOGMNR_D.BUILD wrote into the redo logs to still be reachable from + // every log added to the session. The incremental snapshot backfill opens a mining session + // per split over a very narrow SCN range, and under Debezium 2.x that combination fails + // with "ORA-01291: missing log file". online_catalog needs no dictionary in the logs and + // is what this connector is documented and otherwise tested with. + Properties debeziumProperties = new Properties(); + debeziumProperties.setProperty("log.mining.strategy", "online_catalog"); + return (OracleSourceConfig) new OracleSourceConfigFactory() + .debeziumProperties(debeziumProperties) .startupOptions(StartupOptions.initial()) .databaseList(ORACLE_DATABASE) .tableList(captureTableIds) diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/table/OracleConnectorITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/table/OracleConnectorITCase.java index 54da75b5ced..57f60cbe38b 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/table/OracleConnectorITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/test/java/org/apache/flink/cdc/connectors/oracle/table/OracleConnectorITCase.java @@ -825,7 +825,7 @@ void testAllDataTypes(boolean parallelismSnapshot) throws Throwable { + "2022-10-30T12:34:56.130, " + "2022-10-30T12:34:56.125500, " + "2022-10-30T12:34:56.125457, " - + "2022-10-30T01:34:56.00789-11:00, " + + "2022-10-30T01:34:56.007890-11:00, " + "2022-10-29T17:34:56.007890Z, " + "-110451600000000, " + "-93784560000, " diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/PostgresObjectUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/PostgresObjectUtils.java index f5132bffdd7..383ab667e02 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/PostgresObjectUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/PostgresObjectUtils.java @@ -25,7 +25,7 @@ import io.debezium.connector.postgresql.spi.SlotState; import io.debezium.relational.TableId; import io.debezium.relational.history.TableChanges; -import io.debezium.schema.TopicSelector; +import io.debezium.spi.topic.TopicNamingStrategy; import io.debezium.util.Clock; import io.debezium.util.Metronome; import org.slf4j.Logger; @@ -40,6 +40,9 @@ * A factory for creating various Debezium objects * *

    It is a hack to access package-private constructor in debezium. + * + *

    This is a Flink CDC owned class (not a Debezium fork), placed in the {@code io.debezium} + * package for access to package-private Debezium members. */ public class PostgresObjectUtils { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresObjectUtils.class); @@ -49,7 +52,7 @@ public static RelationAwarePostgresSchema newSchema( PostgresConnection connection, PostgresConnectorConfig config, TypeRegistry typeRegistry, - TopicSelector topicSelector, + TopicNamingStrategy topicSelector, PostgresValueConverter valueConverter) throws SQLException { RelationAwarePostgresSchema schema = @@ -67,7 +70,7 @@ public static RelationAwarePostgresSchema newSchema( PostgresConnection connection, PostgresConnectorConfig config, TypeRegistry typeRegistry, - TopicSelector topicSelector, + TopicNamingStrategy topicSelector, PostgresValueConverter valueConverter, Collection tableChanges) throws SQLException { @@ -85,7 +88,7 @@ public static RelationAwarePostgresSchema newSchema( public static PostgresTaskContext newTaskContext( PostgresConnectorConfig connectorConfig, PostgresSchema schema, - TopicSelector topicSelector) { + TopicNamingStrategy topicSelector) { return new PostgresTaskContext(connectorConfig, schema, topicSelector); } @@ -127,7 +130,10 @@ public static ReplicationConnection createReplicationConnection( while (retryCount <= maxRetries) { try { LOGGER.info("Creating a new replication connection for {}", taskContext); - return taskContext.createReplicationConnection(doSnapshot, postgresConnection); + // Debezium 2.1 dropped the doSnapshot parameter from + // PostgresTaskContext#createReplicationConnection (it was an unused/dead flag). The + // doSnapshot parameter is kept on this wrapper for caller compatibility. + return taskContext.createReplicationConnection(postgresConnection); } catch (SQLException ex) { retryCount++; if (retryCount > maxRetries) { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java index e309a3b29a4..3e128e629d9 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java @@ -11,14 +11,15 @@ import io.debezium.connector.postgresql.connection.PostgresConnection; import io.debezium.connector.postgresql.connection.PostgresReplicationConnection; import io.debezium.connector.postgresql.connection.ReplicationConnection; +import io.debezium.connector.postgresql.connection.ReplicationMessage; import io.debezium.connector.postgresql.connection.ReplicationMessage.Operation; import io.debezium.connector.postgresql.connection.ReplicationStream; import io.debezium.connector.postgresql.connection.WalPositionLocator; -import io.debezium.connector.postgresql.spi.Snapshotter; import io.debezium.heartbeat.Heartbeat; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.source.spi.StreamingChangeEventSource; import io.debezium.relational.TableId; +import io.debezium.snapshot.SnapshotterService; import io.debezium.util.Clock; import io.debezium.util.DelayStrategy; import io.debezium.util.ElapsedTimeStrategy; @@ -35,9 +36,15 @@ import java.util.concurrent.atomic.AtomicReference; /** - * Copied from Debezium 1.9.8.final + * Copied from Debezium project(2.7.4.Final). to support ending the replication stream. * - *

    Line 150~151 : set the ending lsn for the replication connection. + *

    Change 1: the effective offset is initialised at the top of {@code execute} — Debezium's + * coordinator calls {@code init(offsetContext)} before {@code execute}, but Flink CDC drives this + * source directly and never calls {@code init}. + * + *

    Change 2: pass the streaming stopping LSN to {@link + * io.debezium.connector.postgresql.connection.PostgresReplicationConnection#setEndingPos} so the + * replication stream ends at the high watermark instead of hanging on keep-alive messages. */ public class PostgresStreamingChangeEventSource implements StreamingChangeEventSource { @@ -66,7 +73,7 @@ public class PostgresStreamingChangeEventSource private final PostgresTaskContext taskContext; private final ReplicationConnection replicationConnection; private final AtomicReference replicationStream = new AtomicReference<>(); - private final Snapshotter snapshotter; + private final SnapshotterService snapshotterService; private final DelayStrategy pauseNoMessage; private final ElapsedTimeStrategy connectionProbeTimer; @@ -84,10 +91,11 @@ public class PostgresStreamingChangeEventSource private long numberOfEventsSinceLastEventSentOrWalGrowingWarning = 0; private Lsn lastCompletelyProcessedLsn; + private PostgresOffsetContext effectiveOffset; public PostgresStreamingChangeEventSource( PostgresConnectorConfig connectorConfig, - Snapshotter snapshotter, + SnapshotterService snapshotterService, PostgresConnection connection, PostgresEventDispatcher dispatcher, ErrorHandler errorHandler, @@ -101,10 +109,9 @@ public PostgresStreamingChangeEventSource( this.errorHandler = errorHandler; this.clock = clock; this.schema = schema; - pauseNoMessage = - DelayStrategy.constant(taskContext.getConfig().getPollInterval().toMillis()); + pauseNoMessage = DelayStrategy.constant(taskContext.getConfig().getPollInterval()); this.taskContext = taskContext; - this.snapshotter = snapshotter; + this.snapshotterService = snapshotterService; this.replicationConnection = replicationConnection; this.connectionProbeTimer = ElapsedTimeStrategy.constant( @@ -112,8 +119,17 @@ public PostgresStreamingChangeEventSource( } @Override - public void init() { + public void init(PostgresOffsetContext offsetContext) { + + this.effectiveOffset = + offsetContext == null + ? PostgresOffsetContext.initialContext(connectorConfig, connection, clock) + : offsetContext; // refresh the schema so we have a latest view of the DB tables + initSchema(); + } + + private void initSchema() { try { taskContext.refreshSchema(connection, true); } catch (SQLException e) { @@ -127,36 +143,39 @@ public void execute( PostgresPartition partition, PostgresOffsetContext offsetContext) throws InterruptedException { - if (!snapshotter.shouldStream()) { - LOGGER.info("Streaming is not enabled in correct configuration"); - return; + // Debezium's ChangeEventSourceCoordinator calls init(offsetContext) before execute(...) to + // set the effective offset. Flink CDC drives this source directly and never calls init, so + // fall back to the offset context it passes in. + if (this.effectiveOffset == null) { + init(offsetContext); } lsnFlushingAllowed = false; - // replication slot could exist at the time of starting Debezium so we will stream from the + // replication slot could exist at the time of starting Debezium, so we will stream from the // position in the slot // instead of the last position in the database boolean hasStartLsnStoredInContext = offsetContext != null; - if (!hasStartLsnStoredInContext) { - offsetContext = - PostgresOffsetContext.initialContext(connectorConfig, connection, clock); - } - try { final WalPositionLocator walPosition; ((PostgresReplicationConnection) replicationConnection) - .setEndingPos(offsetContext.getStreamingStoppingLsn()); + .setEndingPos(this.effectiveOffset.getStreamingStoppingLsn()); if (hasStartLsnStoredInContext) { // start streaming from the last recorded position in the offset final Lsn lsn = - offsetContext.lastCompletelyProcessedLsn() != null - ? offsetContext.lastCompletelyProcessedLsn() - : offsetContext.lsn(); + this.effectiveOffset.hasCompletelyProcessedPosition() + ? this.effectiveOffset.lastCompletelyProcessedLsn() + : this.effectiveOffset.lsn(); + final Operation lastProcessedMessageType = + this.effectiveOffset.lastProcessedMessageType(); LOGGER.info("Retrieved latest position from stored offset '{}'", lsn); - walPosition = new WalPositionLocator(offsetContext.lastCommitLsn(), lsn); + walPosition = + new WalPositionLocator( + this.effectiveOffset.lastCommitLsn(), + lsn, + lastProcessedMessageType); replicationStream.compareAndSet( null, replicationConnection.startStreaming(lsn, walPosition)); } else { @@ -176,21 +195,22 @@ public void execute( connectorConfig.getLogicalName(), KEEP_ALIVE_THREAD_NAME)); - init(); + initSchema(); // If we need to do a pre-snapshot streaming catch up, we should allow the snapshot // transaction to persist // but normally we want to start streaming without any open transactions. - if (!isInPreSnapshotCatchUpStreaming(offsetContext)) { + if (!isInPreSnapshotCatchUpStreaming(this.effectiveOffset)) { connection.commit(); } this.lastCompletelyProcessedLsn = replicationStream.get().startLsn(); - if (walPosition.searchingEnabled()) { - searchWalPosition(context, stream, walPosition); + if (walPosition.searchingEnabled() + && this.effectiveOffset.hasCompletelyProcessedPosition()) { + searchWalPosition(context, partition, this.effectiveOffset, stream, walPosition); try { - if (!isInPreSnapshotCatchUpStreaming(offsetContext)) { + if (!isInPreSnapshotCatchUpStreaming(this.effectiveOffset)) { connection.commit(); } } catch (Exception e) { @@ -209,7 +229,7 @@ public void execute( connectorConfig.getLogicalName(), KEEP_ALIVE_THREAD_NAME)); } - processMessages(context, partition, offsetContext, stream); + processMessages(context, partition, this.effectiveOffset, stream); } catch (Throwable e) { errorHandler.setProducerThrowable(e); } finally { @@ -255,100 +275,9 @@ private void processMessages( boolean receivedMessage = stream.readPending( - message -> { - final Lsn lsn = stream.lastReceivedLsn(); - - if (message.isLastEventForLsn()) { - lastCompletelyProcessedLsn = lsn; - } - - // Tx BEGIN/END event - if (message.isTransactionalMessage()) { - if (!connectorConfig.shouldProvideTransactionMetadata()) { - LOGGER.trace("Received transactional message {}", message); - // Don't skip on BEGIN message as it would flush LSN for the - // whole transaction - // too early - if (message.getOperation() == Operation.COMMIT) { - commitMessage(partition, offsetContext, lsn); - } - return; - } - - offsetContext.updateWalPosition( - lsn, - lastCompletelyProcessedLsn, - message.getCommitTime(), - toLong(message.getTransactionId()), - taskContext.getSlotXmin(connection), - null); - if (message.getOperation() == Operation.BEGIN) { - dispatcher.dispatchTransactionStartedEvent( - partition, - toString(message.getTransactionId()), - offsetContext); - } else if (message.getOperation() == Operation.COMMIT) { - commitMessage(partition, offsetContext, lsn); - dispatcher.dispatchTransactionCommittedEvent( - partition, offsetContext); - } - maybeWarnAboutGrowingWalBacklog(true); - } else if (message.getOperation() == Operation.MESSAGE) { - offsetContext.updateWalPosition( - lsn, - lastCompletelyProcessedLsn, - message.getCommitTime(), - toLong(message.getTransactionId()), - taskContext.getSlotXmin(connection)); - - // non-transactional message that will not be followed by a - // COMMIT message - if (message.isLastEventForLsn()) { - commitMessage(partition, offsetContext, lsn); - } - - dispatcher.dispatchLogicalDecodingMessage( - partition, - offsetContext, - clock.currentTimeAsInstant().toEpochMilli(), - (LogicalDecodingMessage) message); - - maybeWarnAboutGrowingWalBacklog(true); - } - // DML event - else { - TableId tableId = null; - if (message.getOperation() != Operation.NOOP) { - tableId = PostgresSchema.parse(message.getTable()); - Objects.requireNonNull(tableId); - } - - offsetContext.updateWalPosition( - lsn, - lastCompletelyProcessedLsn, - message.getCommitTime(), - toLong(message.getTransactionId()), - taskContext.getSlotXmin(connection), - tableId); - - boolean dispatched = - message.getOperation() != Operation.NOOP - && dispatcher.dispatchDataChangeEvent( - partition, - tableId, - new PostgresChangeRecordEmitter( - partition, - offsetContext, - clock, - connectorConfig, - schema, - connection, - tableId, - message)); - - maybeWarnAboutGrowingWalBacklog(dispatched); - } - }); + message -> + processReplicationMessages( + partition, offsetContext, stream, message)); probeConnectionIfNeeded(); @@ -356,9 +285,8 @@ private void processMessages( noMessageIterations = 0; lsnFlushingAllowed = true; } else { - if (offsetContext.hasCompletelyProcessedPosition()) { - dispatcher.dispatchHeartbeatEvent(partition, offsetContext); - } + dispatcher.dispatchHeartbeatEventAlsoToIncrementalSnapshot( + partition, offsetContext); noMessageIterations++; if (noMessageIterations >= THROTTLE_NO_MESSAGE_BEFORE_PAUSE) { noMessageIterations = 0; @@ -375,11 +303,127 @@ private void processMessages( // for the snapshot, this block must not commit during catch up streaming. connection.commit(); } + + if (context.isPaused()) { + LOGGER.info("Streaming will now pause"); + context.streamingPaused(); + context.waitSnapshotCompletion(); + LOGGER.info("Streaming resumed"); + } + } + } + + private void processReplicationMessages( + PostgresPartition partition, + PostgresOffsetContext offsetContext, + ReplicationStream stream, + ReplicationMessage message) + throws SQLException, InterruptedException { + + final Lsn lsn = stream.lastReceivedLsn(); + LOGGER.trace("Processing replication message {}", message); + if (message.isLastEventForLsn()) { + lastCompletelyProcessedLsn = lsn; + } + + // Tx BEGIN/END event + if (message.isTransactionalMessage()) { + + offsetContext.updateWalPosition( + lsn, + lastCompletelyProcessedLsn, + message.getCommitTime(), + toLong(message.getTransactionId()), + taskContext.getSlotXmin(connection), + null, + message.getOperation()); + + if (!connectorConfig.shouldProvideTransactionMetadata()) { + LOGGER.trace("Received transactional message {}", message); + // Don't skip on BEGIN message as it would flush LSN for the whole transaction + // too early + if (message.getOperation() == Operation.COMMIT) { + commitMessage(partition, offsetContext, lsn); + dispatcher.dispatchTransactionCommittedEvent( + partition, offsetContext, message.getCommitTime()); + } + return; + } + + if (message.getOperation() == Operation.BEGIN) { + dispatcher.dispatchTransactionStartedEvent( + partition, + toString(message.getTransactionId()), + offsetContext, + message.getCommitTime()); + } else if (message.getOperation() == Operation.COMMIT) { + commitMessage(partition, offsetContext, lsn); + dispatcher.dispatchTransactionCommittedEvent( + partition, offsetContext, message.getCommitTime()); + } + maybeWarnAboutGrowingWalBacklog(true); + } else if (message.getOperation() == Operation.MESSAGE) { + offsetContext.updateWalPosition( + lsn, + lastCompletelyProcessedLsn, + message.getCommitTime(), + toLong(message.getTransactionId()), + taskContext.getSlotXmin(connection), + message.getOperation()); + + // non-transactional message that will not be followed by a COMMIT message + if (message.isLastEventForLsn()) { + commitMessage(partition, offsetContext, lsn); + } + + dispatcher.dispatchLogicalDecodingMessage( + partition, + offsetContext, + clock.currentTimeAsInstant().toEpochMilli(), + (LogicalDecodingMessage) message); + + maybeWarnAboutGrowingWalBacklog(true); + } + // DML event + else { + TableId tableId = null; + if (message.getOperation() != Operation.NOOP) { + tableId = PostgresSchema.parse(message.getTable()); + Objects.requireNonNull(tableId); + } + + offsetContext.updateWalPosition( + lsn, + lastCompletelyProcessedLsn, + message.getCommitTime(), + toLong(message.getTransactionId()), + taskContext.getSlotXmin(connection), + tableId, + message.getOperation()); + + boolean dispatched = + message.getOperation() != Operation.NOOP + && dispatcher.dispatchDataChangeEvent( + partition, + tableId, + new PostgresChangeRecordEmitter( + partition, + offsetContext, + clock, + connectorConfig, + schema, + connection, + tableId, + message)); + + maybeWarnAboutGrowingWalBacklog(dispatched); } } private void searchWalPosition( ChangeEventSourceContext context, + PostgresPartition partition, + PostgresOffsetContext offsetContext, final ReplicationStream stream, final WalPositionLocator walPosition) throws SQLException, InterruptedException { @@ -399,6 +443,7 @@ private void searchWalPosition( if (receivedMessage) { noMessageIterations = 0; } else { + dispatcher.dispatchHeartbeatEvent(partition, offsetContext); noMessageIterations++; if (noMessageIterations >= THROTTLE_NO_MESSAGE_BEFORE_PAUSE) { noMessageIterations = 0; @@ -465,7 +510,7 @@ private void maybeWarnAboutGrowingWalBacklog(boolean dispatched) { } @Override - public void commitOffset(Map offset) { + public void commitOffset(Map partition, Map offset) { try { ReplicationStream replicationStream = this.replicationStream.get(); final Lsn commitLsn = @@ -478,6 +523,10 @@ public void commitOffset(Map offset) { .LAST_COMPLETELY_PROCESSED_LSN_KEY)); final Lsn lsn = (commitLsn != null) ? commitLsn : changeLsn; + LOGGER.debug( + "Received offset commit request on commit LSN '{}' and change LSN '{}'", + commitLsn, + changeLsn); if (replicationStream != null && lsn != null) { if (!lsnFlushingAllowed) { LOGGER.info( @@ -500,6 +549,11 @@ public void commitOffset(Map offset) { } } + @Override + public PostgresOffsetContext getOffsetContext() { + return effectiveOffset; + } + /** * Returns whether the current streaming phase is running a catch up streaming phase that runs * before a snapshot. This is useful for transaction management. @@ -524,7 +578,7 @@ private String toString(OptionalLong l) { } @FunctionalInterface - public static interface PgConnectionSupplier { + public interface PgConnectionSupplier { BaseConnection get() throws SQLException; } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/Utils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/Utils.java index 3e780afaae1..a3714ede16b 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/Utils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/Utils.java @@ -31,7 +31,12 @@ import java.util.HashMap; import java.util.Map; -/** A utility class for accessing various Debezium package-private methods. */ +/** + * A utility class for accessing various Debezium package-private methods. * + * + *

    This is a Flink CDC owned class (not a Debezium fork), placed in the {@code io.debezium} + * package for access to package-private Debezium members. + */ public final class Utils { private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/Lsn.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/Lsn.java index cc08c519692..a769355f370 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/Lsn.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/Lsn.java @@ -11,13 +11,17 @@ import java.nio.ByteBuffer; /** - * Copied from Debezium 1.9.8.final without changes due to the NoSuchMethodError, caused by the fact - * that current Debezium release java version is 11, so we need to compile this file by java 8 - * compiler. More info. - * Abstraction of PostgreSQL log sequence number, adapted from {@link LogSequenceNumber}. + * Abstraction of PostgreSQL log sequence number, adapted from {@link + * org.postgresql.replication.LogSequenceNumber}. * - *

    Line 32: add NO_STOPPING_LSN + *

    Copied from Debezium project(2.7.4.Final).. + * + *

    Change 1: add {@code NO_STOPPING_LSN} and {@code isNonStopping()} — the sentinel used by Flink + * CDC's unbounded stream splits as the "never stop" high watermark. + * + *

    Change 2: {@code valueOf(ByteBuffer)} resets the buffer position through {@code + * java.nio.Buffer} so the class links on Java 8 runtimes (newer JDKs give {@code + * ByteBuffer.position(int)} a covariant return type). */ public class Lsn implements Comparable { @@ -27,7 +31,6 @@ public class Lsn implements Comparable { */ public static final Lsn INVALID_LSN = new Lsn(0); - /** The max lsn for the wal file. */ public static final Lsn NO_STOPPING_LSN = Lsn.valueOf("FFFFFFFF/FFFFFFFF"); private final long value; @@ -85,6 +88,8 @@ public static Lsn valueOf(String strValue) { final ByteBuffer buf = ByteBuffer.allocate(8); buf.putInt(logicalXlog); buf.putInt(segment); + // Cast through Buffer: compiled against newer JDKs, ByteBuffer.position(int) has a + // covariant ByteBuffer return type that does not exist on Java 8 runtimes. ((java.nio.Buffer) buf).position(0); final long value = buf.getLong(); @@ -113,6 +118,8 @@ public LogSequenceNumber asLogSequenceNumber() { public String asString() { final ByteBuffer buf = ByteBuffer.allocate(8); buf.putLong(value); + // Cast through Buffer: compiled against newer JDKs, ByteBuffer.position(int) has a + // covariant ByteBuffer return type that does not exist on Java 8 runtimes. ((java.nio.Buffer) buf).position(0); final int logicalXlog = buf.getInt(); @@ -143,15 +150,16 @@ public boolean isValid() { return this != INVALID_LSN; } - public boolean isNonStopping() { - return this == NO_STOPPING_LSN; - } - @Override public String toString() { return "LSN{" + asString() + '}'; } + /** Whether this is the {@link #NO_STOPPING_LSN} sentinel. */ + public boolean isNonStopping() { + return this == NO_STOPPING_LSN; + } + @Override public int compareTo(Lsn o) { if (value == o.value) { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnection.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnection.java index 610bd436005..3e6e7c9dc45 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnection.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnection.java @@ -9,10 +9,11 @@ import com.zaxxer.hikari.pool.HikariProxyConnection; import io.debezium.DebeziumException; import io.debezium.annotation.VisibleForTesting; +import io.debezium.config.CommonConnectorConfig; import io.debezium.config.Configuration; import io.debezium.connector.postgresql.PgOid; import io.debezium.connector.postgresql.PostgresConnectorConfig; -import io.debezium.connector.postgresql.PostgresSchema; +import io.debezium.connector.postgresql.PostgresOffsetContext; import io.debezium.connector.postgresql.PostgresType; import io.debezium.connector.postgresql.PostgresValueConverter; import io.debezium.connector.postgresql.TypeRegistry; @@ -20,14 +21,20 @@ import io.debezium.data.SpecialValueDecimal; import io.debezium.jdbc.JdbcConfiguration; import io.debezium.jdbc.JdbcConnection; +import io.debezium.pipeline.source.snapshot.incremental.ChunkQueryBuilder; +import io.debezium.pipeline.source.snapshot.incremental.RowValueConstructorChunkQueryBuilder; +import io.debezium.pipeline.spi.OffsetContext; +import io.debezium.pipeline.spi.Partition; import io.debezium.relational.Column; import io.debezium.relational.ColumnEditor; +import io.debezium.relational.RelationalDatabaseConnectorConfig; import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.relational.Tables; -import io.debezium.schema.DatabaseSchema; +import io.debezium.spi.schema.DataCollectionId; import io.debezium.util.Clock; import io.debezium.util.Metronome; +import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.ConnectException; import org.postgresql.core.BaseConnection; import org.postgresql.jdbc.PgConnection; @@ -41,39 +48,41 @@ import java.nio.charset.Charset; import java.sql.Connection; import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Types; import java.time.Duration; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * {@link JdbcConnection} connection extension used for connecting to Postgres instances. * - * @author Horia Chiorean - *

    Copied from Debezium 1.9.8-Final with the following modifications: - *

      - *
    • Constructor PostgresConnection( Configuration config, PostgresValueConverterBuilder - * valueConverterBuilder, ConnectionFactory factory) to allow passing a custom - * ConnectionFactory - *
    • override connection() to return a unwrapped PgConnection (otherwise, it will complain - * about HikariProxyConnection cannot be cast to class org.postgresql.core.BaseConnection) - *
    • override isTableUniqueIndexIncluded: Copied DBZ-5398 from Debezium 2.0.0.Final to fix - * https://github.com/ververica/flink-cdc-connectors/issues/2710. Remove this comment - * after bumping debezium version to 2.0.0.Final. - *
    • FLINK-38965: Modified doReadTableColumn to filter out columns from other tables that - * might be returned due to PostgreSQL LIKE wildcard matching. The underscore '_' matches - * any single character, and '%' matches any sequence of characters. For example, when - * querying table 'user_sink', the LIKE pattern may also match 'userbsink' (due to '_'); - * when querying table 'user%data' (where % is a literal character in the table name), the - * LIKE pattern may also match 'user_test_data' (due to '%'). See also: - * https://github.com/debezium/debezium/blob/main/debezium-core/src/main/java/io/debezium/jdbc/JdbcConnection.java#L1327 - *
    + *

    Copied from Debezium project(2.7.4.Final). with the following modifications: + * + *

    Change 1: add the {@code (config, valueConverterBuilder, connectionUsage, factory)} + * constructor so Flink CDC can pass a pooled (HikariCP) {@code ConnectionFactory}; the 3-arg + * constructor delegates to it with the default {@code FACTORY}. + * + *

    Change 2: override {@code connection()} to unwrap {@code HikariProxyConnection} to the + * underlying {@code PgConnection} — the replication code casts to {@code + * org.postgresql.core.BaseConnection}. + * + *

    Change 3 (FLINK-38965): {@code doReadTableColumn} filters out columns of other tables that + * JDBC metadata may return because the table name is used as a LIKE pattern ('_' matches any + * character, '%' any sequence). + * + *

    (The DBZ-5398 {@code isTableUniqueIndexIncluded} patch carried by earlier versions of this + * fork is upstream since Debezium 2.0.0 and is no longer a modification.) */ public class PostgresConnection extends JdbcConnection { @@ -137,15 +146,9 @@ public PostgresConnection( } /** - * Creates a Postgres connection using the supplied configuration. If necessary this connection - * is able to resolve data type mappings. Such a connection requires a {@link - * PostgresValueConverter}, and will provide its own {@link TypeRegistry}. Usually only one such - * connection per connector is needed. - * - * @param config {@link Configuration} instance, may not be null. - * @param valueConverterBuilder supplies a configured {@link PostgresValueConverter} for a given - * {@link TypeRegistry} - * @param connectionUsage a symbolic name of the connection to be tracked in monitoring tools + * Creates a Postgres connection using the supplied configuration and connection factory. See + * {@link #PostgresConnection(JdbcConfiguration, PostgresValueConverterBuilder, String)}; this + * variant lets Flink CDC supply a pooled connection factory. */ public PostgresConnection( JdbcConfiguration config, @@ -156,7 +159,6 @@ public PostgresConnection( addDefaultSettings(config, connectionUsage), factory, PostgresConnection::validateServerVersion, - null, "\"", "\""); @@ -169,7 +171,8 @@ public PostgresConnection( final PostgresValueConverter valueConverter = valueConverterBuilder.build(this.typeRegistry); this.defaultValueConverter = - new PostgresDefaultValueConverter(valueConverter, this.getTimestampUtils()); + new PostgresDefaultValueConverter( + valueConverter, this.getTimestampUtils(), typeRegistry); } } @@ -186,9 +189,9 @@ public PostgresConnection( addDefaultSettings(config.getJdbcConfig(), connectionUsage), FACTORY, PostgresConnection::validateServerVersion, - null, "\"", "\""); + if (Objects.isNull(typeRegistry)) { this.typeRegistry = null; this.defaultValueConverter = null; @@ -197,7 +200,8 @@ public PostgresConnection( final PostgresValueConverter valueConverter = PostgresValueConverter.of(config, this.getDatabaseCharset(), typeRegistry); this.defaultValueConverter = - new PostgresDefaultValueConverter(valueConverter, this.getTimestampUtils()); + new PostgresDefaultValueConverter( + valueConverter, this.getTimestampUtils(), typeRegistry); } } @@ -212,7 +216,7 @@ public PostgresConnection(JdbcConfiguration config, String connectionUsage) { this(config, null, connectionUsage); } - /** Return an unwrapped PgConnection instead of HikariProxyConnection */ + /** Return an unwrapped PgConnection instead of HikariProxyConnection. */ @Override public synchronized Connection connection() throws SQLException { Connection conn = connection(true); @@ -254,7 +258,8 @@ public String connectionString() { * @throws SQLException if there is a problem obtaining the replica identity information for the * given table */ - public ServerInfo.ReplicaIdentity readReplicaIdentityInfo(TableId tableId) throws SQLException { + @VisibleForTesting + public ReplicaIdentityInfo readReplicaIdentityInfo(TableId tableId) throws SQLException { String statement = "SELECT relreplident FROM pg_catalog.pg_class c " + "LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace=n.oid " @@ -279,7 +284,78 @@ public ServerInfo.ReplicaIdentity readReplicaIdentityInfo(TableId tableId) throw tableId); } }); - return ServerInfo.ReplicaIdentity.parseFromDB(replIdentity.toString()); + return new ReplicaIdentityInfo( + ReplicaIdentityInfo.ReplicaIdentity.parseFromDB(replIdentity.toString())); + } + + /** + * This query retrieves information about the INDEX as long as replica identity is configure + * USING INDEX + * + * @param tableId the identifier of the table + * @return Index name linked to replica identity; never null + * @throws SQLException if there is a problem obtaining the replica identity and index + * information for the given table + */ + @VisibleForTesting + public String readIndexOfReplicaIdentity(TableId tableId) throws SQLException { + String statement = + "with rel_index as (" + + "select split_part(indexrelid::regclass::text, '.', 1) as index_schema, split_part(indexrelid::regclass::text, '.', 2) as index_name " + + "from pg_catalog.pg_index " + + "where indisreplident " + + ") " + + "SELECT i.index_name " + + "FROM pg_catalog.pg_class c " + + " LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace=n.oid " + + " LEFT join rel_index i on n.nspname = i.index_schema " + + "WHERE n.nspname=? and c.relname=?"; + String schema = + tableId.schema() != null && tableId.schema().length() > 0 + ? tableId.schema() + : "public"; + StringBuilder indexName = new StringBuilder(); + prepareQuery( + statement, + stmt -> { + stmt.setString(1, schema); + stmt.setString(2, tableId.table()); + }, + rs -> { + if (rs.next()) { + indexName.append(rs.getString(1)); + } else { + LOGGER.warn( + "Cannot determine index linked to REPLICA IDENTITY for table '{}'", + tableId); + } + }); + return indexName.toString(); + } + + /** + * Update REPLICA IDENTITY status of a table. This in turn determines how much information is + * available for UPDATE and DELETE operations for logical replication. + * + * @param tableId the identifier of the table + * @param replicaIdentityValue Replica Identity value + */ + public void setReplicaIdentityForTable( + TableId tableId, ReplicaIdentityInfo replicaIdentityValue) { + try { + LOGGER.debug("Updating Replica Identity '{}'", tableId.table()); + execute( + String.format( + "ALTER TABLE %s REPLICA IDENTITY %s;", tableId, replicaIdentityValue)); + } catch (SQLException e) { + + if (e.getSQLState().equals("42501")) { + LOGGER.error( + "Replica identity could not be updated because of lack of privileges", e); + } else { + LOGGER.error("Unexpected error while attempting to alter Replica Identity", e); + } + } } /** @@ -423,26 +499,58 @@ private Lsn parseConfirmedFlushLsn( try { confirmedFlushedLsn = tryParseLsn(slotName, pluginName, database, rs, "confirmed_flush_lsn"); - } catch (SQLException e) { - LOGGER.info("unable to find confirmed_flushed_lsn, falling back to restart_lsn"); - try { - confirmedFlushedLsn = - tryParseLsn(slotName, pluginName, database, rs, "restart_lsn"); - } catch (SQLException e2) { - throw new ConnectException( - "Neither confirmed_flush_lsn nor restart_lsn could be found"); + if (confirmedFlushedLsn == null) { + LOGGER.info("Failed to obtain valid replication slot, confirmed flush lsn is null"); + if (!hasIdleTransactions()) { + confirmedFlushedLsn = + tryFallbackToRestartLsn(slotName, pluginName, database, rs); + } } + } catch (SQLException e) { + confirmedFlushedLsn = tryFallbackToRestartLsn(slotName, pluginName, database, rs); } return confirmedFlushedLsn; } + private boolean hasIdleTransactions() throws SQLException { + return queryAndMap( + "select * from pg_stat_activity where state like 'idle in transaction' AND application_name != '" + + CONNECTION_GENERAL + + "' AND pid <> pg_backend_pid()", + rs -> { + if (rs.next()) { + LOGGER.debug( + "Found at least one idle transaction with pid " + + rs.getInt("pid") + + " for application" + + rs.getString("application_name")); + return true; + } else { + return false; + } + }); + } + + private Lsn tryFallbackToRestartLsn( + String slotName, String pluginName, String database, ResultSet rs) { + Lsn confirmedFlushedLsn; + LOGGER.info("Unable to find confirmed_flushed_lsn, falling back to restart_lsn"); + try { + confirmedFlushedLsn = tryParseLsn(slotName, pluginName, database, rs, "restart_lsn"); + } catch (SQLException e) { + throw new DebeziumException( + "Neither confirmed_flush_lsn nor restart_lsn could be found", e); + } + return confirmedFlushedLsn; + } + private Lsn parseRestartLsn(String slotName, String pluginName, String database, ResultSet rs) { Lsn restartLsn = null; try { restartLsn = tryParseLsn(slotName, pluginName, database, rs, "restart_lsn"); } catch (SQLException e) { - throw new ConnectException("restart_lsn could be found"); + throw new DebeziumException("restart_lsn could be found"); } return restartLsn; @@ -460,7 +568,7 @@ private Lsn tryParseLsn( try { lsn = Lsn.valueOf(lsnStr); } catch (Exception e) { - throw new ConnectException( + throw new DebeziumException( "Value " + column + " in the pg_replication_slots table for slot = '" @@ -472,7 +580,7 @@ private Lsn tryParseLsn( + "' is not valid. This is an abnormal situation and the database status should be checked."); } if (!lsn.isValid()) { - throw new ConnectException("Invalid LSN returned from database"); + throw new DebeziumException("Invalid LSN returned from database"); } return lsn; } @@ -635,6 +743,9 @@ public ServerInfo serverInfo() throws SQLException { } }); } + + serverInfo.withMajorVersion(connection().getMetaData().getDatabaseMajorVersion()); + return serverInfo; } @@ -667,7 +778,7 @@ private static void validateServerVersion(Statement statement) throws SQLExcepti @Override public String quotedColumnIdString(String columnName) { if (columnName.contains("\"")) { - columnName = columnName.replaceAll("\"", "\"\""); + columnName = columnName.replace("\"", "\"\""); } return super.quotedColumnIdString(columnName); @@ -779,14 +890,12 @@ public TypeRegistry getTypeRegistry() { } @Override - public > Object getColumnValue( - ResultSet rs, int columnIndex, Column column, Table table, T schema) + public Object getColumnValue(ResultSet rs, int columnIndex, Column column, Table table) throws SQLException { try { final ResultSetMetaData metaData = rs.getMetaData(); final String columnTypeName = metaData.getColumnTypeName(columnIndex); - final PostgresType type = - ((PostgresSchema) schema).getTypeRegistry().get(columnTypeName); + final PostgresType type = getTypeRegistry().get(columnTypeName); LOGGER.trace("Type of incoming data is: {}", type.getOid()); LOGGER.trace("ColumnTypeName is: {}", columnTypeName); @@ -824,8 +933,7 @@ public > Object getColumnValue( ? value.get() : new SpecialValueDecimal(rs.getBigDecimal(columnIndex)); case PgOid.TIME: - // To handle time 24:00:00 supported by TIME columns, read the column as a - // string. + // To handle time 24:00:00 supported by TIME columns, read the column as a string. case PgOid.TIMETZ: // In order to guarantee that we resolve TIMETZ columns with proper microsecond // precision, @@ -843,7 +951,7 @@ public > Object getColumnValue( } } catch (SQLException e) { // not a known type - return super.getColumnValue(rs, columnIndex, column, table, schema); + return super.getColumnValue(rs, columnIndex, column, table); } } @@ -877,6 +985,79 @@ public Set getAllTableIds(String catalogName) throws SQLException { return readTableNames(catalogName, null, null, new String[] {"TABLE", "PARTITIONED TABLE"}); } + @Override + public ChunkQueryBuilder chunkQueryBuilder( + RelationalDatabaseConnectorConfig connectorConfig) { + // PostgreSQL definitely must use row value constructors in order to yield optimal results. + // See DBZ-5071. + return new RowValueConstructorChunkQueryBuilder<>(connectorConfig, this); + } + + @Override + public Optional nullsSortLast() { + // "By default, null values sort as if larger than any non-null value" + // https://www.postgresql.org/docs/16/queries-order.html + return Optional.of(true); + } + + @Override + public Map reselectColumns( + Table table, + List columns, + List keyColumns, + List keyValues, + Struct source) + throws SQLException { + final String query = + String.format( + "SELECT %s FROM %s WHERE %s", + columns.stream() + .map(this::quotedColumnIdString) + .collect(Collectors.joining(",")), + quotedTableIdString(table.id()), + keyColumns.stream() + .map(key -> key + "=?::" + table.columnWithName(key).typeName()) + .collect(Collectors.joining(" AND "))); + return reselectColumns(query, table.id(), columns, keyValues); + } + + @Override + public void setQueryColumnValue( + PreparedStatement statement, Column column, int pos, Object value) throws SQLException { + final PostgresType resolvedType = typeRegistry.get(column.nativeType()); + + if (resolvedType != null && resolvedType.isEnumType()) { + // ENUMs require explicit casting so the comparison operators can correctly work + statement.setObject(pos, value, Types.OTHER); + } else { + super.setQueryColumnValue(statement, column, pos, value); + } + } + + @Override + public TableId createTableId(String databaseName, String schemaName, String tableName) { + return new TableId(null, schemaName, tableName); + } + + public boolean validateLogPosition( + Partition partition, OffsetContext offset, CommonConnectorConfig config) { + + final Lsn storedLsn = ((PostgresOffsetContext) offset).lastCommitLsn(); + final String slotName = ((PostgresConnectorConfig) config).slotName(); + final String postgresPluginName = + ((PostgresConnectorConfig) config).plugin().getPostgresPluginName(); + + try { + SlotState slotState = getReplicationSlotState(slotName, postgresPluginName); + if (slotState == null) { + return false; + } + return storedLsn == null || slotState.slotRestartLsn().compareTo(storedLsn) < 0; + } catch (SQLException e) { + throw new DebeziumException("Unable to get last available log position", e); + } + } + @FunctionalInterface public interface PostgresValueConverterBuilder { PostgresValueConverter build(TypeRegistry registry); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnectionUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnectionUtils.java index bd45a9d6552..0efd69b7baa 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnectionUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnectionUtils.java @@ -33,6 +33,9 @@ /** * A utility class for accessing various Debezium PostgresConnection private-package classes/methods + * + *

    This is a Flink CDC owned class (not a Debezium fork), placed in the {@code io.debezium} + * package for access to package-private Debezium members. */ public class PostgresConnectionUtils { private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresReplicationConnection.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresReplicationConnection.java index 0c607045e7d..b72e3c4261e 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresReplicationConnection.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/io/debezium/connector/postgresql/connection/PostgresReplicationConnection.java @@ -9,6 +9,7 @@ import io.debezium.DebeziumException; import io.debezium.connector.postgresql.PostgresConnectorConfig; import io.debezium.connector.postgresql.PostgresSchema; +import io.debezium.connector.postgresql.ReplicaIdentityMapper; import io.debezium.connector.postgresql.TypeRegistry; import io.debezium.connector.postgresql.spi.SlotCreationResult; import io.debezium.jdbc.JdbcConfiguration; @@ -49,18 +50,28 @@ import static java.lang.Math.toIntExact; /** - * Copied from Debezium 1.9.8.Final + * Implementation of a {@link ReplicationConnection} for Postgresql. Note that replication + * connections in PG cannot execute regular statements but only a limited number of + * replication-related commands. * - *

    The {@link ReplicationConnection} created from {@code createReplicationStream} will hang when - * the wal logs only contain the keepAliveMessage. Support to set an ending Lsn to stop hanging. + *

    Copied from Debezium project(2.7.4.Final). to support ending the replication stream. * - *

    Line 83, 711~713 : add endingPos and its setter. + *

    Change 1: add the {@code endingPos} field and its {@code setEndingPos} setter. * - *

    Line 571~576, 595~600: ReplicationStream from {@code createReplicationStream} will stop when - * endingPos reached. + *

    Change 2: in {@code read}/{@code readPending}, emit a {@code NoopMessage} and stop once {@code + * reachEnd(lsn)} reports the ending position has been passed — the raw {@link + * org.postgresql.replication.PGReplicationStream} otherwise hangs when the WAL only contains + * keep-alive messages. + * + *

    Change 3: {@code validateSlotIsInExpectedState} is overridden to a no-op. Flink CDC manages + * the streaming start position through its own incremental-snapshot offsets, so Debezium 2.1's + * {@code pg_replication_slot_advance} seek (absent in 2.0.1) must not run; advancing to a + * Flink-managed offset can raise "invalid target WAL LSN". */ public class PostgresReplicationConnection extends JdbcConnection implements ReplicationConnection { + private static final String SQL_STATE_INSUFFICIENT_PRIVILEGE = "42501"; + private static Logger LOGGER = LoggerFactory.getLogger(PostgresReplicationConnection.class); private final String slotName; @@ -77,10 +88,11 @@ public class PostgresReplicationConnection extends JdbcConnection implements Rep private final Properties streamParams; private Lsn defaultStartingPos; + private Lsn endingPos; private SlotCreationResult slotCreationInfo; private boolean hasInitedSlot; - private Lsn endingPos; + private Optional replicaIdentityMapper; /** * Creates a new replication connection with the given params. @@ -98,12 +110,10 @@ public class PostgresReplicationConnection extends JdbcConnection implements Rep * closed * @param statusUpdateInterval the interval at which the replication connection should * periodically send status - * @param doSnapshot whether the connector is doing snapshot * @param jdbcConnection general PostgreSQL JDBC connection * @param typeRegistry registry with PostgreSQL types * @param streamParams additional parameters to pass to the replication stream * @param schema the schema; must not be null - *

    updates to the server */ private PostgresReplicationConnection( PostgresConnectorConfig config, @@ -113,19 +123,12 @@ private PostgresReplicationConnection( PostgresConnectorConfig.AutoCreateMode publicationAutocreateMode, PostgresConnectorConfig.LogicalDecoder plugin, boolean dropSlotOnClose, - boolean doSnapshot, Duration statusUpdateInterval, PostgresConnection jdbcConnection, TypeRegistry typeRegistry, Properties streamParams, PostgresSchema schema) { - super( - addDefaultSettings(config.getJdbcConfig()), - PostgresConnection.FACTORY, - null, - null, - "\"", - "\""); + super(addDefaultSettings(config.getJdbcConfig()), PostgresConnection.FACTORY, "\"", "\""); this.connectorConfig = config; this.slotName = slotName; @@ -142,6 +145,7 @@ private PostgresReplicationConnection( this.streamParams = streamParams; this.slotCreationInfo = null; this.hasInitedSlot = false; + this.replicaIdentityMapper = config.replicaIdentityMapper(); } private static JdbcConfiguration addDefaultSettings(JdbcConfiguration configuration) { @@ -167,8 +171,6 @@ private ServerInfo.ReplicationSlot getSlotInfo() throws SQLException, Interrupte } protected void initPublication() { - String createPublicationStmt; - String tableFilterString = null; if (PostgresConnectorConfig.LogicalDecoder.PGOUTPUT.equals(plugin)) { LOGGER.info("Initializing PgOutput logical decoder publication"); try { @@ -178,52 +180,61 @@ protected void initPublication() { String selectPublication = String.format( - "SELECT COUNT(1) FROM pg_publication WHERE pubname = '%s'", + "SELECT puballtables FROM pg_publication WHERE pubname = '%s'", publicationName); try (Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(selectPublication)) { - if (rs.next()) { - Long count = rs.getLong(1); + if (!rs.next()) { // Close eagerly as the transaction might stay running - if (count == 0L) { - LOGGER.info( - "Creating new publication '{}' for plugin '{}'", - publicationName, - plugin); - switch (publicationAutocreateMode) { - case DISABLED: - throw new ConnectException( - "Publication autocreation is disabled, please create one and restart the connector."); - case ALL_TABLES: - createPublicationStmt = + LOGGER.info( + "Creating new publication '{}' for plugin '{}'", + publicationName, + plugin); + switch (publicationAutocreateMode) { + case DISABLED: + throw new ConnectException( + "Publication autocreation is disabled, please create one and restart the connector."); + case ALL_TABLES: + String createPublicationStmt = + String.format( + "CREATE PUBLICATION %s FOR ALL TABLES;", + publicationName); + LOGGER.info( + "Creating Publication with statement '{}'", + createPublicationStmt); + // Publication doesn't exist, create it. + stmt.execute(createPublicationStmt); + break; + case FILTERED: + createOrUpdatePublicationModeFilterted(stmt, false); + break; + } + } else { + switch (publicationAutocreateMode) { + case FILTERED: + // Checking that publication can be altered + Boolean allTables = rs.getBoolean(1); + if (allTables) { + throw new DebeziumException( String.format( - "CREATE PUBLICATION %s FOR ALL TABLES;", - publicationName); - LOGGER.info( - "Creating Publication with statement '{}'", - createPublicationStmt); - // Publication doesn't exist, create it. - stmt.execute(createPublicationStmt); - break; - case FILTERED: - createOrUpdatePublicationModeFilterted( - tableFilterString, stmt, false); - break; - } - } else { - switch (publicationAutocreateMode) { - case FILTERED: - createOrUpdatePublicationModeFilterted( - tableFilterString, stmt, true); - break; - default: - LOGGER.trace( - "A logical publication named '{}' for plugin '{}' and database '{}' is already active on the server " - + "and will be used by the plugin", - publicationName, - plugin, - database()); - } + "A logical publication for all tables named '%s' for plugin '%s' and database '%s' " + + "is already active on the server and can not be altered. " + + "If you need to exclude some tables or include only specific subset, " + + "please recreate the publication with necessary configuration " + + "or let plugin recreate it by dropping existing publication. " + + "Otherwise please change the 'publication.autocreate.mode' property to 'all_tables'.", + publicationName, plugin, database())); + } else { + createOrUpdatePublicationModeFilterted(stmt, true); + } + break; + default: + LOGGER.trace( + "A logical publication named '{}' for plugin '{}' and database '{}' is already active on the server " + + "and will be used by the plugin", + publicationName, + plugin, + database()); } } } @@ -235,8 +246,8 @@ protected void initPublication() { } } - private void createOrUpdatePublicationModeFilterted( - String tableFilterString, Statement stmt, boolean isUpdate) { + private void createOrUpdatePublicationModeFilterted(Statement stmt, boolean isUpdate) { + String tableFilterString = null; String createOrUpdatePublicationStmt; try { Set tablesToCapture = determineCapturedTables(); @@ -273,6 +284,74 @@ private void createOrUpdatePublicationModeFilterted( } } + /** + * Check all tables captured by the connector, contained in {@link Set} from {@link + * PostgresReplicationConnection#determineCapturedTables()}. Updating Replica Identity in + * PostgreSQL database based on {@link PostgresConnectorConfig#REPLICA_IDENTITY_AUTOSET_VALUES} + * configuration parameter for each {@link TableId} + * + * @throws Exception + */ + private void initReplicaIdentity() { + + if (this.replicaIdentityMapper.isPresent()) { + LOGGER.info("Updating Replica Identity"); + Set tablesCaptured; + try { + tablesCaptured = determineCapturedTables(); + } catch (Exception e) { + throw new DebeziumException("Unable to get Captured tables", e); + } + tablesCaptured.forEach( + tableId -> { + try { + Optional newReplicaIdentity = + this.replicaIdentityMapper.get().findReplicaIdentity(tableId); + + if (newReplicaIdentity.isPresent()) { + ReplicaIdentityInfo currentReplicaIdentity = null; + try { + currentReplicaIdentity = + jdbcConnection.readReplicaIdentityInfo(tableId); + if (currentReplicaIdentity.getReplicaIdentity() + == ReplicaIdentityInfo.ReplicaIdentity.INDEX) { + currentReplicaIdentity.setIndexName( + jdbcConnection.readIndexOfReplicaIdentity(tableId)); + } + } catch (SQLException e) { + LOGGER.error( + "Cannot determine REPLICA IDENTITY information for table {}", + tableId); + } + if (currentReplicaIdentity != null + && !currentReplicaIdentity + .toString() + .equals(newReplicaIdentity.get().toString())) { + jdbcConnection.setReplicaIdentityForTable( + tableId, newReplicaIdentity.get()); + LOGGER.info( + "Replica identity set to {} for table '{}'", + newReplicaIdentity.get(), + tableId); + } else { + LOGGER.info( + "Replica identity for table '{}' is already {}", + tableId, + currentReplicaIdentity); + } + } else { + LOGGER.debug( + "Replica identity for table '{}' will not be updated because Replica Identity is not defined on REPLICA_IDENTITY_AUTOSET_VALUES property", + tableId); + } + } catch (Exception e) { + LOGGER.error( + "Unable to update Replica Identity for table {}", tableId, e); + } + }); + } + } + private Set determineCapturedTables() throws Exception { Set allTableIds = jdbcConnection.getAllTableIds(connectorConfig.databaseName()); @@ -357,9 +436,9 @@ private boolean useTemporarySlot() throws SQLException { * creating a replication connection and starting to stream involves a few steps: 1. we create * the connection and ensure that a. the slot exists b. the slot isn't currently being used 2. * we query to get our potential start position in the slot (lsn) 3. we try and start streaming, - * depending on our options (such as in wal2json) this may fail, which can result in the - * connection being killed and we need to start the process over if we are using a temporary - * slot 4. actually start the streamer + * depending on our options this may fail, which can result in the connection being killed and + * we need to start the process over if we are using a temporary slot 4. actually start the + * streamer * *

    This method takes care of all of these and this method queries for a default starting * position If you know where you are starting from you should call {@link #startStreaming(Lsn, @@ -394,6 +473,9 @@ public ReplicationStream startStreaming(Lsn offset, WalPositionLocator walPositi int tryCount = 0; while (true) { try { + if (connectorConfig.slotSeekToKnownOffsetOnStart()) { + validateSlotIsInExpectedState(walPosition); + } return createReplicationStream(lsn, walPosition); } catch (Exception e) { String message = "Failed to start replication stream at " + lsn; @@ -408,7 +490,8 @@ public ReplicationStream startStreaming(Lsn offset, WalPositionLocator walPositi message + ", waiting for {} ms and retrying, attempt number {} over {}", delay, tryCount, - maxRetries); + maxRetries, + e); final Metronome metronome = Metronome.sleeper(delay, Clock.SYSTEM); metronome.pause(); } @@ -416,11 +499,24 @@ public ReplicationStream startStreaming(Lsn offset, WalPositionLocator walPositi } } + protected void validateSlotIsInExpectedState(WalPositionLocator walPosition) + throws SQLException { + // Flink CDC drives the streaming start position through its own incremental-snapshot + // offset tracking (see the split start LSN passed to startStreaming and the endingPos + // watermark), so it must NOT let Debezium seek/advance the replication slot on startup. + // Debezium 2.1 added this pg_replication_slot_advance() seek (absent in 2.0.1); advancing + // to a Flink-managed offset can raise "invalid target WAL LSN". Keep it a no-op to + // preserve the pre-2.1 behavior. + } + @Override public void initConnection() throws SQLException, InterruptedException { // See https://www.postgresql.org/docs/current/logical-replication-quick-setup.html // For pgoutput specifically, the publication must be created before the slot. initPublication(); + + initReplicaIdentity(); + if (!hasInitedSlot) { initReplicationSlot(); } @@ -495,13 +591,7 @@ private ReplicationStream createReplicationStream( try { try { - s = - startPgReplicationStream( - startLsn, - plugin.forceRds() - ? messageDecoder::optionsWithoutMetadata - : messageDecoder::optionsWithMetadata); - messageDecoder.setContainsMetadata(plugin.forceRds() ? false : true); + s = startPgReplicationStream(startLsn, messageDecoder::defaultOptions); } catch (PSQLException e) { LOGGER.debug( "Could not register for streaming, retrying without optional options", e); @@ -512,29 +602,10 @@ private ReplicationStream createReplicationStream( initReplicationSlot(); } - s = - startPgReplicationStream( - startLsn, - plugin.forceRds() - ? messageDecoder::optionsWithoutMetadata - : messageDecoder::optionsWithMetadata); - messageDecoder.setContainsMetadata(plugin.forceRds() ? false : true); + s = startPgReplicationStream(startLsn, messageDecoder::defaultOptions); } } catch (PSQLException e) { - if (e.getMessage().matches("(?s)ERROR: option .* is unknown.*")) { - // It is possible we are connecting to an old wal2json plug-in - LOGGER.warn( - "Could not register for streaming with metadata in messages, falling back to messages without metadata"); - - // re-init the slot after a failed start of slot, as this - // may have closed the slot - if (useTemporarySlot()) { - initReplicationSlot(); - } - - s = startPgReplicationStream(startLsn, messageDecoder::optionsWithoutMetadata); - messageDecoder.setContainsMetadata(false); - } else if (e.getMessage() + if (e.getMessage() .matches("(?s)ERROR: requested WAL segment .* has already been removed.*")) { LOGGER.error("Cannot rewind to last processed WAL position", e); throw new ConnectException( @@ -613,6 +684,15 @@ public boolean readPending(ReplicationMessageProcessor processor) return true; } + private boolean reachEnd(Lsn receivedLsn) { + if (receivedLsn == null) { + return false; + } + return endingPos != null + && (!endingPos.isNonStopping()) + && endingPos.compareTo(receivedLsn) < 0; + } + private void deserializeMessages( ByteBuffer buffer, ReplicationMessageProcessor processor) throws SQLException, InterruptedException { @@ -629,7 +709,9 @@ public void close() throws SQLException { @Override public void flushLsn(Lsn lsn) throws SQLException { - doFlushLsn(lsn); + if (connectorConfig.isFlushLsnOnSource()) { + doFlushLsn(lsn); + } } private void doFlushLsn(Lsn lsn) throws SQLException { @@ -696,22 +778,9 @@ private void processWarnings(final boolean forced) throws SQLException { public Lsn startLsn() { return startLsn; } - - private boolean reachEnd(Lsn receivedLsn) { - if (receivedLsn == null) { - return false; - } - return endingPos != null - && (!endingPos.isNonStopping()) - && endingPos.compareTo(receivedLsn) < 0; - } }; } - public void setEndingPos(Lsn endingPos) { - this.endingPos = endingPos; - } - private PGReplicationStream startPgReplicationStream( final Lsn lsn, BiFunction< @@ -789,7 +858,10 @@ public synchronized void close(boolean dropSlot) { } } - @Override + public void setEndingPos(Lsn endingPos) { + this.endingPos = endingPos; + } + public void reconnect() throws SQLException { close(false); // Don't re-execute initial commands on reconnection @@ -808,7 +880,6 @@ protected static class ReplicationConnectionBuilder implements Builder { PostgresConnectorConfig.LogicalDecoder.DECODERBUFS; private boolean dropSlotOnClose = DEFAULT_DROP_SLOT_ON_CLOSE; private Duration statusUpdateIntervalVal; - private boolean doSnapshot; private TypeRegistry typeRegistry; private PostgresSchema schema; private Properties slotStreamParams = new Properties(); @@ -888,12 +959,6 @@ public ReplicationConnectionBuilder statusUpdateInterval( return this; } - @Override - public Builder doSnapshot(boolean doSnapshot) { - this.doSnapshot = doSnapshot; - return this; - } - @Override public Builder jdbcMetadataConnection(PostgresConnection jdbcConnection) { this.jdbcConnection = jdbcConnection; @@ -911,7 +976,6 @@ public ReplicationConnection build() { publicationAutocreateMode, plugin, dropSlotOnClose, - doSnapshot, statusUpdateIntervalVal, jdbcConnection, typeRegistry, diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/PostgreSQLSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/PostgreSQLSource.java index 73938c450a7..c9d5f3df4a0 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/PostgreSQLSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/PostgreSQLSource.java @@ -157,7 +157,8 @@ public DebeziumSourceFunction build() { // database server/cluster being monitored. The logical name should be unique across // all other connectors, since it is used as a prefix for all Kafka topic names coming // from this connector. Only alphanumeric characters and underscores should be used. - props.setProperty("database.server.name", "postgres_cdc_source"); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", "postgres_cdc_source"); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.dbname", checkNotNull(database)); props.setProperty("database.user", checkNotNull(username)); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/PostgresDialect.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/PostgresDialect.java index 4743247899a..6f33832bfff 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/PostgresDialect.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/PostgresDialect.java @@ -34,12 +34,13 @@ import org.apache.flink.cdc.connectors.postgres.source.utils.CustomPostgresSchema; import org.apache.flink.cdc.connectors.postgres.source.utils.TableDiscoveryUtils; import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.TemporaryClassLoaderContext; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.postgresql.PostgresConnectorConfig; import io.debezium.connector.postgresql.PostgresObjectUtils; import io.debezium.connector.postgresql.PostgresSchema; import io.debezium.connector.postgresql.PostgresTaskContext; -import io.debezium.connector.postgresql.PostgresTopicSelector; import io.debezium.connector.postgresql.connection.PostgresConnection; import io.debezium.connector.postgresql.connection.PostgresConnectionUtils; import io.debezium.connector.postgresql.connection.PostgresReplicationConnection; @@ -47,7 +48,7 @@ import io.debezium.relational.TableId; import io.debezium.relational.Tables; import io.debezium.relational.history.TableChanges.TableChange; -import io.debezium.schema.TopicSelector; +import io.debezium.spi.topic.TopicNamingStrategy; import javax.annotation.Nullable; @@ -105,7 +106,17 @@ public PostgresReplicationConnection openPostgresReplicationConnection( PostgresConnection jdbcConnection) { try { PostgresConnectorConfig pgConnectorConfig = sourceConfig.getDbzConnectorConfig(); - TopicSelector topicSelector = PostgresTopicSelector.create(pgConnectorConfig); + TopicNamingStrategy topicSelector; + // Debezium 2.0 resolves classes through the thread context class loader; pin it to the + // loader that loaded Debezium so a stale/closed Flink user class loader cannot break + // it. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(CommonConnectorConfig.class.getClassLoader())) { + topicSelector = + pgConnectorConfig.getTopicNamingStrategy( + io.debezium.connector.postgresql.PostgresConnectorConfig + .TOPIC_NAMING_STRATEGY); + } PostgresConnection.PostgresValueConverterBuilder valueConverterBuilder = newPostgresValueConverterBuilder(pgConnectorConfig); PostgresSchema schema = diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/config/PostgresSourceConfig.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/config/PostgresSourceConfig.java index 8891c976d5e..2b18415deeb 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/config/PostgresSourceConfig.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/config/PostgresSourceConfig.java @@ -19,6 +19,7 @@ import org.apache.flink.cdc.connectors.base.config.JdbcSourceConfig; import org.apache.flink.cdc.connectors.base.options.StartupOptions; +import org.apache.flink.util.TemporaryClassLoaderContext; import io.debezium.config.Configuration; import io.debezium.connector.postgresql.PostgresConnectorConfig; @@ -152,7 +153,14 @@ public String getJdbcUrl() { @Override public PostgresConnectorConfig getDbzConnectorConfig() { - return new PostgresConnectorConfig(getDbzConfiguration()); + // Debezium 2.0 resolves classes through the thread context class loader + // (io.debezium.config.Instantiator, and the ServiceLoader based SPIs). On a Flink + // thread that loader may be a user code class loader from a previous job attempt, + // which is already closed. Pin it to the loader that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(PostgresConnectorConfig.class.getClassLoader())) { + return new PostgresConnectorConfig(getDbzConfiguration()); + } } /** Returns whether to include database in the generated Table ID. */ diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/config/PostgresSourceConfigFactory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/config/PostgresSourceConfigFactory.java index 9731cf1cbcf..e84955a45e5 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/config/PostgresSourceConfigFactory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/config/PostgresSourceConfigFactory.java @@ -71,7 +71,8 @@ public PostgresSourceConfig create(int subtaskId) { // database server/cluster being monitored. The logical name should be unique across // all other connectors, since it is used as a prefix for all Kafka topic names coming // from this connector. Only alphanumeric characters and underscores should be used. - props.setProperty("database.server.name", "postgres_cdc_source"); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", "postgres_cdc_source"); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.dbname", checkNotNull(database)); props.setProperty("database.user", checkNotNull(username)); @@ -83,10 +84,11 @@ public PostgresSourceConfig create(int subtaskId) { props.setProperty("slot.name", checkNotNull(slotName)); // database history props.setProperty( - "database.history", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); - props.setProperty("database.history.instance.name", UUID.randomUUID() + "_" + subtaskId); - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); - props.setProperty("database.history.refer.ddl", String.valueOf(true)); + "schema.history.internal", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); + props.setProperty( + "schema.history.internal.instance.name", UUID.randomUUID() + "_" + subtaskId); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.prefer.ddl", String.valueOf(true)); // we have to enable heartbeat for PG to make sure DebeziumChangeConsumer#handleBatch // is invoked after job restart // Enable TCP keep-alive probe to verify that the database connection is still alive diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/CDCPostgresDispatcher.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/CDCPostgresDispatcher.java index ce9183af0ec..a12049aba11 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/CDCPostgresDispatcher.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/CDCPostgresDispatcher.java @@ -36,8 +36,8 @@ import io.debezium.relational.TableId; import io.debezium.schema.DataCollectionFilters; import io.debezium.schema.DatabaseSchema; -import io.debezium.schema.TopicSelector; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.topic.TopicNamingStrategy; import org.apache.kafka.connect.source.SourceRecord; import java.util.Map; @@ -50,7 +50,7 @@ public class CDCPostgresDispatcher extends PostgresEventDispatcher public CDCPostgresDispatcher( PostgresConnectorConfig connectorConfig, - TopicSelector topicSelector, + TopicNamingStrategy topicSelector, DatabaseSchema schema, ChangeEventQueue queue, DataCollectionFilters.DataCollectionFilter filter, @@ -58,6 +58,10 @@ public CDCPostgresDispatcher( EventMetadataProvider metadataProvider, HeartbeatFactory heartbeatFactory, SchemaNameAdjuster schemaNameAdjuster) { + // Debezium 2.0's PostgresEventDispatcher constructor inserts an InconsistentSchemaHandler + // parameter and takes a built Heartbeat instead of a HeartbeatFactory. + // Debezium 2.3 appends a trailing SignalProcessor parameter; Flink CDC uses its own + // snapshot mechanism and no Debezium signals, so it is passed as null. super( connectorConfig, topicSelector, @@ -65,10 +69,12 @@ public CDCPostgresDispatcher( queue, filter, changeEventCreator, + null, metadataProvider, - heartbeatFactory, - schemaNameAdjuster); - this.topic = topicSelector.getPrimaryTopic(); + heartbeatFactory.createHeartbeat(), + schemaNameAdjuster, + null); + this.topic = connectorConfig.getLogicalName(); this.queue = queue; } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresScanFetchTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresScanFetchTask.java index d2d47d709fd..e234754ca0c 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresScanFetchTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresScanFetchTask.java @@ -36,13 +36,17 @@ import io.debezium.connector.postgresql.connection.ReplicationConnection; import io.debezium.connector.postgresql.spi.SlotState; import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.notification.NotificationService; +import io.debezium.pipeline.signal.actions.snapshotting.SnapshotConfiguration; import io.debezium.pipeline.source.AbstractSnapshotChangeEventSource; +import io.debezium.pipeline.source.SnapshottingTask; import io.debezium.pipeline.source.spi.SnapshotProgressListener; import io.debezium.pipeline.spi.SnapshotResult; import io.debezium.relational.RelationalSnapshotChangeEventSource; import io.debezium.relational.SnapshotChangeRecordEmitter; import io.debezium.relational.Table; import io.debezium.relational.TableId; +import io.debezium.schema.SchemaFactory; import io.debezium.util.Clock; import io.debezium.util.ColumnUtils; import io.debezium.util.Strings; @@ -53,6 +57,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -111,7 +116,11 @@ protected void executeDataSnapshot(Context context) throws Exception { new StoppableChangeEventSourceContext(); SnapshotResult snapshotResult = snapshotSplitReadTask.execute( - changeEventSourceContext, ctx.getPartition(), ctx.getOffsetContext()); + changeEventSourceContext, + ctx.getPartition(), + ctx.getOffsetContext(), + snapshotSplitReadTask.getSnapshottingTask( + ctx.getPartition(), ctx.getOffsetContext())); if (!snapshotResult.isCompletedOrSkipped()) { taskRunning = false; @@ -134,7 +143,7 @@ protected void executeBackfillTask(Context context, StreamSplit backfillStreamSp final PostgresStreamFetchTask.StreamSplitReadTask backfillReadTask = new PostgresStreamFetchTask.StreamSplitReadTask( ctx.getDbzConnectorConfig(), - ctx.getSnapShotter(), + ctx.getSnapshotterService(), ctx.getConnection(), ctx.getEventDispatcher(), ctx.getWaterMarkDispatcher(), @@ -232,7 +241,14 @@ public PostgresSnapshotSplitReadTask( PostgresEventDispatcher eventDispatcher, SnapshotProgressListener snapshotProgressListener, SnapshotSplit snapshotSplit) { - super(connectorConfig, snapshotProgressListener); + super( + connectorConfig, + snapshotProgressListener, + new NotificationService<>( + Collections.emptyList(), + connectorConfig, + SchemaFactory.get(), + notification -> {})); this.jdbcConnection = jdbcConnection; this.sourceConfig = sourceConfig; this.snapshotProgressListener = snapshotProgressListener; @@ -343,7 +359,11 @@ private void createDataEventsForTable( snapshotContext.offset.event(table.id(), clock.currentTime()); SnapshotChangeRecordEmitter emitter = new SnapshotChangeRecordEmitter<>( - snapshotContext.partition, snapshotContext.offset, row, clock); + snapshotContext.partition, + snapshotContext.offset, + row, + clock, + sourceConfig.getDbzConnectorConfig()); eventDispatcher.dispatchSnapshotEvent( snapshotContext.partition, table.id(), emitter, snapshotReceiver); } @@ -363,22 +383,35 @@ private Threads.Timer getTableScanLogTimer() { } @Override - protected SnapshottingTask getSnapshottingTask( + public SnapshottingTask getSnapshottingTask( PostgresPartition partition, PostgresOffsetContext previousOffset) { - return new SnapshottingTask(false, true); + return new SnapshottingTask( + false, true, Collections.emptyList(), Collections.emptyMap(), false); + } + + @Override + public SnapshottingTask getBlockingSnapshottingTask( + PostgresPartition partition, + PostgresOffsetContext previousOffset, + SnapshotConfiguration snapshotConfiguration) { + // Debezium 2.6 made this abstract. Flink CDC drives its own split snapshot and never + // runs Debezium's signal-based blocking snapshot, so it behaves like the regular one. + return getSnapshottingTask(partition, previousOffset); } @Override - protected PostgresSnapshotContext prepare(PostgresPartition partition) throws Exception { - return new PostgresSnapshotContext(partition); + protected PostgresSnapshotContext prepare(PostgresPartition partition, boolean onDemand) + throws Exception { + return new PostgresSnapshotContext(partition, onDemand); } private static class PostgresSnapshotContext extends RelationalSnapshotChangeEventSource.RelationalSnapshotContext< PostgresPartition, PostgresOffsetContext> { - public PostgresSnapshotContext(PostgresPartition partition) throws SQLException { - super(partition, ""); + public PostgresSnapshotContext(PostgresPartition partition, boolean onDemand) + throws SQLException { + super(partition, "", onDemand); } } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresSourceFetchTaskContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresSourceFetchTaskContext.java index ff94ecfcc04..a154cf5bd1b 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresSourceFetchTaskContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresSourceFetchTaskContext.java @@ -33,11 +33,13 @@ import org.apache.flink.cdc.connectors.postgres.source.schema.RelationAwarePostgresSchema; import org.apache.flink.cdc.connectors.postgres.source.utils.ChunkUtils; import org.apache.flink.cdc.connectors.postgres.source.utils.PostgresSourceRecordUtils; +import org.apache.flink.cdc.debezium.internal.SnapshotterServiceFactory; import org.apache.flink.table.types.logical.RowType; import io.debezium.DebeziumException; import io.debezium.config.Configuration; import io.debezium.connector.base.ChangeEventQueue; +import io.debezium.connector.postgresql.PostgresConnector; import io.debezium.connector.postgresql.PostgresConnectorConfig; import io.debezium.connector.postgresql.PostgresErrorHandler; import io.debezium.connector.postgresql.PostgresEventDispatcher; @@ -46,10 +48,8 @@ import io.debezium.connector.postgresql.PostgresPartition; import io.debezium.connector.postgresql.PostgresSchema; import io.debezium.connector.postgresql.PostgresTaskContext; -import io.debezium.connector.postgresql.PostgresTopicSelector; import io.debezium.connector.postgresql.connection.PostgresConnection; import io.debezium.connector.postgresql.connection.ReplicationConnection; -import io.debezium.connector.postgresql.spi.Snapshotter; import io.debezium.data.Envelope; import io.debezium.heartbeat.Heartbeat; import io.debezium.heartbeat.HeartbeatFactory; @@ -63,7 +63,8 @@ import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.relational.Tables; -import io.debezium.schema.TopicSelector; +import io.debezium.snapshot.SnapshotterService; +import io.debezium.spi.topic.TopicNamingStrategy; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.RetriableException; import org.apache.kafka.connect.source.SourceRecord; @@ -77,7 +78,6 @@ import static io.debezium.connector.postgresql.PostgresConnectorConfig.DROP_SLOT_ON_STOP; import static io.debezium.connector.postgresql.PostgresConnectorConfig.PLUGIN_NAME; import static io.debezium.connector.postgresql.PostgresConnectorConfig.SLOT_NAME; -import static io.debezium.connector.postgresql.PostgresConnectorConfig.SNAPSHOT_MODE; import static io.debezium.connector.postgresql.PostgresObjectUtils.createReplicationConnection; import static io.debezium.connector.postgresql.PostgresObjectUtils.newPostgresValueConverterBuilder; @@ -99,7 +99,7 @@ public class PostgresSourceFetchTaskContext extends JdbcSourceFetchTaskContext { private CDCPostgresDispatcher postgresDispatcher; private EventMetadataProvider metadataProvider; private SnapshotChangeEventSourceMetrics snapshotChangeEventSourceMetrics; - private Snapshotter snapShotter; + private SnapshotterService snapshotterService; public PostgresSourceFetchTaskContext( JdbcSourceConfig sourceConfig, PostgresDialect dataSourceDialect) { @@ -169,10 +169,10 @@ public void configure(SourceSplitBase sourceSplitBase) { .build()); } setDbzConnectorConfig(dbzConfig); - PostgresConnectorConfig.SnapshotMode snapshotMode = - PostgresConnectorConfig.SnapshotMode.parse( - dbzConfig.getConfig().getString(SNAPSHOT_MODE)); - this.snapShotter = snapshotMode.getSnapshotter(dbzConfig.getConfig()); + // Debezium 2.6 replaced the Postgres-specific Snapshotter SPI with the core + // SnapshotterService, which is resolved from the connector config's service registry. + this.snapshotterService = + SnapshotterServiceFactory.create(dbzConfig, PostgresConnector.class); PostgresConnection.PostgresValueConverterBuilder valueConverterBuilder = newPostgresValueConverterBuilder(dbzConfig); @@ -180,7 +180,10 @@ public void configure(SourceSplitBase sourceSplitBase) { new PostgresConnection( dbzConfig.getJdbcConfig(), valueConverterBuilder, CONNECTION_NAME); - TopicSelector topicSelector = PostgresTopicSelector.create(dbzConfig); + TopicNamingStrategy topicSelector = + dbzConfig.getTopicNamingStrategy( + io.debezium.connector.postgresql.PostgresConnectorConfig + .TOPIC_NAMING_STRATEGY); try { this.schema = @@ -198,7 +201,8 @@ public void configure(SourceSplitBase sourceSplitBase) { this.offsetContext = loadStartingOffsetState( new PostgresOffsetContext.Loader(dbzConfig), sourceSplitBase); - this.partition = new PostgresPartition(dbzConfig.getLogicalName()); + this.partition = + new PostgresPartition(dbzConfig.getLogicalName(), dbzConfig.databaseName()); this.taskContext = PostgresObjectUtils.newTaskContext(dbzConfig, schema, topicSelector); if (replicationConnection == null) { @@ -206,7 +210,9 @@ public void configure(SourceSplitBase sourceSplitBase) { createReplicationConnection( this.taskContext, jdbcConnection, - this.snapShotter.shouldSnapshot(), + this.snapshotterService + .getSnapshotter() + .shouldSnapshotData(false, false), dbzConfig); } @@ -224,7 +230,7 @@ public void configure(SourceSplitBase sourceSplitBase) { // .buffering() .build(); - this.errorHandler = new PostgresErrorHandler(getDbzConnectorConfig(), queue); + this.errorHandler = new PostgresErrorHandler(getDbzConnectorConfig(), queue, null); this.metadataProvider = PostgresObjectUtils.newEventMetadataProvider(); PostgresConnectorConfig finalDbzConfig = dbzConfig; @@ -373,8 +379,8 @@ public ReplicationConnection getReplicationConnection() { return snapshotChangeEventSourceMetrics; } - public Snapshotter getSnapShotter() { - return snapShotter; + public SnapshotterService getSnapshotterService() { + return snapshotterService; } public String getSlotName() { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresStreamFetchTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresStreamFetchTask.java index a3d28a12c3f..6e669f49ef6 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresStreamFetchTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/PostgresStreamFetchTask.java @@ -37,10 +37,10 @@ import io.debezium.connector.postgresql.connection.Lsn; import io.debezium.connector.postgresql.connection.PostgresConnection; import io.debezium.connector.postgresql.connection.ReplicationConnection; -import io.debezium.connector.postgresql.spi.Snapshotter; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.source.spi.ChangeEventSource; import io.debezium.relational.TableId; +import io.debezium.snapshot.SnapshotterService; import io.debezium.util.Clock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,7 +85,7 @@ public void execute(Context context) throws Exception { streamSplitReadTask = new StreamSplitReadTask( sourceFetchContext.getDbzConnectorConfig(), - sourceFetchContext.getSnapShotter(), + sourceFetchContext.getSnapshotterService(), sourceFetchContext.getConnection(), sourceFetchContext.getEventDispatcher(), sourceFetchContext.getWaterMarkDispatcher(), @@ -159,7 +159,7 @@ public void commitCurrentOffset(@Nullable Offset offsetToCommit) { "Committing offset {} for {}", Lsn.valueOf(lastCommitLsn), streamSplitReadTask.streamSplit); - streamSplitReadTask.commitOffset(offsets); + streamSplitReadTask.commitOffset(null, offsets); } } } @@ -181,7 +181,7 @@ public static class StreamSplitReadTask extends PostgresStreamingChangeEventSour public StreamSplitReadTask( PostgresConnectorConfig connectorConfig, - Snapshotter snapshotter, + SnapshotterService snapshotterService, PostgresConnection connection, PostgresEventDispatcher eventDispatcher, WatermarkDispatcher watermarkDispatcher, @@ -194,7 +194,7 @@ public StreamSplitReadTask( super( connectorConfig, - snapshotter, + snapshotterService, connection, eventDispatcher, errorHandler, diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/StoppableChangeEventSourceContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/StoppableChangeEventSourceContext.java index 79f9a8a4462..e9dec822f3a 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/StoppableChangeEventSourceContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/fetch/StoppableChangeEventSourceContext.java @@ -36,4 +36,24 @@ public void stopChangeEventSource() { public boolean isRunning() { return isRunning; } + + // The following methods are only used by Debezium's signal-based blocking snapshot, + // which Flink CDC does not use, so they are no-ops here. + + @Override + public boolean isPaused() { + return false; + } + + @Override + public void resumeStreaming() throws InterruptedException {} + + @Override + public void waitSnapshotCompletion() throws InterruptedException {} + + @Override + public void streamingPaused() {} + + @Override + public void waitStreamingPaused() throws InterruptedException {} } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/offset/PostgresOffsetUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/offset/PostgresOffsetUtils.java index b516388cd82..b8d10272f4a 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/offset/PostgresOffsetUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/offset/PostgresOffsetUtils.java @@ -19,28 +19,90 @@ import org.apache.flink.cdc.connectors.base.source.meta.offset.Offset; +import io.debezium.connector.AbstractSourceInfo; import io.debezium.connector.postgresql.PostgresOffsetContext; +import io.debezium.connector.postgresql.SourceInfo; +import io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotContext; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Objects; +import java.util.Set; /** Utils for handling {@link PostgresOffset}. */ public class PostgresOffsetUtils { + + /** + * {@code TransactionContext.OFFSET_TRANSACTION_ID}, which is not public in every Debezium + * version this connector has been built against. + */ + private static final String OFFSET_TRANSACTION_ID = "transaction_id"; + + /** Keys that {@link PostgresOffsetContext.Loader#load} reads as a {@code Boolean}. */ + private static final Set BOOLEAN_KEYS = + new HashSet<>( + Arrays.asList( + AbstractSourceInfo.SNAPSHOT_KEY, SourceInfo.LAST_SNAPSHOT_RECORD_KEY)); + + /** + * Keys that are read as a {@code String} even though their value may look numeric. Debezium 2.1 + * added {@code messageType} to the Postgres offset, which is where the previous "every value is + * a long" assumption broke; {@code transaction_id} is a numeric Postgres txid that must still + * be kept as a String. + */ + private static final Set STRING_KEYS = + new HashSet<>(Arrays.asList(SourceInfo.MSG_TYPE_KEY, OFFSET_TRANSACTION_ID)); + + /** + * The incremental snapshot context stores serialized keys and collection names under this + * prefix and always reads them back as Strings. + */ + private static final String INCREMENTAL_SNAPSHOT_PREFIX = + AbstractIncrementalSnapshotContext.INCREMENTAL_SNAPSHOT_KEY; + + /** + * The one key under {@link #INCREMENTAL_SNAPSHOT_PREFIX} that is read back as a number, used + * only by the read-only incremental snapshot context. + */ + private static final String INCREMENTAL_SNAPSHOT_SIGNAL_OFFSET = + INCREMENTAL_SNAPSHOT_PREFIX + "_signal_offset"; + public static PostgresOffsetContext getPostgresOffsetContext( PostgresOffsetContext.Loader loader, Offset offset) { Map offsetStrMap = Objects.requireNonNull(offset, "offset is null for the sourceSplitBase") .getOffset(); - // all the keys happen to be long type for PostgresOffsetContext.Loader.load + // The offset was flattened to strings by PostgresOffset; restore the types that + // PostgresOffsetContext.Loader.load expects. Numeric-looking values default to Long + // (lsn, txId, ts_usec, per-table event counts, ...), everything else stays a String. Map offsetMap = new HashMap<>(); - for (String key : offsetStrMap.keySet()) { - String value = offsetStrMap.get(key); - if (value != null) { - offsetMap.put(key, Long.parseLong(value)); + for (Map.Entry entry : offsetStrMap.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (value == null) { + continue; + } + if (BOOLEAN_KEYS.contains(key)) { + offsetMap.put(key, Boolean.parseBoolean(value)); + } else if (STRING_KEYS.contains(key) + || (key.startsWith(INCREMENTAL_SNAPSHOT_PREFIX) + && !INCREMENTAL_SNAPSHOT_SIGNAL_OFFSET.equals(key))) { + offsetMap.put(key, value); + } else { + offsetMap.put(key, toLongOrKeepString(value)); } } return loader.load(offsetMap); } + + private static Object toLongOrKeepString(String value) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + return value; + } + } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/schema/RelationAwarePostgresSchema.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/schema/RelationAwarePostgresSchema.java index e07cbe443a9..930bf1d7e4f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/schema/RelationAwarePostgresSchema.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/schema/RelationAwarePostgresSchema.java @@ -24,7 +24,7 @@ import io.debezium.connector.postgresql.connection.PostgresDefaultValueConverter; import io.debezium.relational.Table; import io.debezium.relational.TableId; -import io.debezium.schema.TopicSelector; +import io.debezium.spi.topic.TopicNamingStrategy; /** * Extends PostgresSchema to dispatch Relation messages as schema change events via the event queue @@ -38,9 +38,11 @@ public RelationAwarePostgresSchema( PostgresConnectorConfig config, TypeRegistry typeRegistry, PostgresDefaultValueConverter defaultValueConverter, - TopicSelector topicSelector, + TopicNamingStrategy topicSelector, PostgresValueConverter valueConverter) { - super(config, typeRegistry, defaultValueConverter, topicSelector, valueConverter); + // Debezium 2.0 dropped the TypeRegistry parameter from the PostgresSchema constructor + // (it is now derived internally from the value converter). + super(config, defaultValueConverter, topicSelector, valueConverter); } public void setDispatcher(SchemaDispatcher dispatcher) { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/utils/CustomPostgresSchema.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/utils/CustomPostgresSchema.java index 714baf5dde7..3ef20ac7237 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/utils/CustomPostgresSchema.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/main/java/org/apache/flink/cdc/connectors/postgres/source/utils/CustomPostgresSchema.java @@ -104,7 +104,8 @@ private List readTableSchema(List tableIds) throws SQLExce final PostgresOffsetContext offsetContext = PostgresOffsetContext.initialContext(dbzConfig, jdbcConnection, Clock.SYSTEM); - PostgresPartition partition = new PostgresPartition(dbzConfig.getLogicalName()); + PostgresPartition partition = + new PostgresPartition(dbzConfig.getLogicalName(), dbzConfig.databaseName()); Tables tables = new Tables(); try { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/source/fetch/IncrementalSourceStreamFetcherTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/source/fetch/IncrementalSourceStreamFetcherTest.java index 419c2be88ae..70bf50eb9a3 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/source/fetch/IncrementalSourceStreamFetcherTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/source/fetch/IncrementalSourceStreamFetcherTest.java @@ -171,6 +171,8 @@ void testSchemaChangeRecordInQueue() throws Exception { + "primary key: [Id]\n" + "default charset: null\n" + "comment: null\n" + + "attributes: {\n" + + "}\n" + "}"); assertThat(allRecords.get(1).toString()).contains("before_ddl"); assertThat(allRecords.get(2).toString()) @@ -185,6 +187,8 @@ void testSchemaChangeRecordInQueue() throws Exception { + "primary key: [Id]\n" + "default charset: null\n" + "comment: null\n" + + "attributes: {\n" + + "}\n" + "}"); assertThat(allRecords.get(3).toString()).contains("after_ddl"); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/source/offset/PostgresOffsetUtilsTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/source/offset/PostgresOffsetUtilsTest.java new file mode 100644 index 00000000000..78d7e006351 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/source/offset/PostgresOffsetUtilsTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.postgres.source.offset; + +import io.debezium.connector.postgresql.PostgresConnectorConfig; +import io.debezium.connector.postgresql.PostgresOffsetContext; +import io.debezium.connector.postgresql.SourceInfo; +import io.debezium.connector.postgresql.connection.Lsn; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link PostgresOffsetUtils}. */ +class PostgresOffsetUtilsTest { + + /** + * Debezium 2.1 added {@code messageType} to the Postgres offset. Its value is an operation name + * such as {@code INSERT}, so the offset map is no longer "all longs" and must not be parsed as + * such - otherwise every stream split fails with {@code NumberFormatException}. + */ + @Test + void testOffsetWithNonNumericValuesIsRestored() { + Map offsetMap = new HashMap<>(); + offsetMap.put(SourceInfo.LSN_KEY, "23456789"); + offsetMap.put(SourceInfo.TXID_KEY, "755"); + offsetMap.put(SourceInfo.TIMESTAMP_USEC_KEY, "1690000000000000"); + offsetMap.put(SourceInfo.MSG_TYPE_KEY, "INSERT"); + offsetMap.put("snapshot", "false"); + offsetMap.put(SourceInfo.LAST_SNAPSHOT_RECORD_KEY, "false"); + + PostgresOffsetContext offsetContext = + PostgresOffsetUtils.getPostgresOffsetContext( + new PostgresOffsetContext.Loader( + new PostgresConnectorConfig( + io.debezium.config.Configuration.create() + .with("topic.prefix", "test_server") + .with("database.hostname", "localhost") + .with("database.user", "postgres") + .with("database.password", "postgres") + .with("database.dbname", "postgres") + .with("plugin.name", "pgoutput") + .with("slot.name", "flink") + .build())), + new PostgresOffset(offsetMap)); + + Map restored = offsetContext.getOffset(); + assertThat(restored.get(SourceInfo.LSN_KEY)).isEqualTo(Lsn.valueOf(23456789L).asLong()); + assertThat(restored.get(SourceInfo.TXID_KEY)).isEqualTo(755L); + } +} diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/testutils/TestHelper.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/testutils/TestHelper.java index e3bcca7c615..145e5596f90 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/testutils/TestHelper.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc/src/test/java/org/apache/flink/cdc/connectors/postgres/testutils/TestHelper.java @@ -20,7 +20,6 @@ import io.debezium.config.Configuration; import io.debezium.connector.postgresql.PostgresConnectorConfig; import io.debezium.jdbc.JdbcConfiguration; -import io.debezium.relational.RelationalDatabaseConnectorConfig; /** A helper class for testing. */ public class TestHelper { @@ -49,7 +48,7 @@ public static Configuration.Builder defaultConfig() { JdbcConfiguration jdbcConfiguration = defaultJdbcConfig(); Configuration.Builder builder = Configuration.create(); jdbcConfiguration.forEach((field, value) -> builder.with("database." + field, value)); - builder.with(RelationalDatabaseConnectorConfig.SERVER_NAME, TEST_SERVER) + builder.with(io.debezium.config.CommonConnectorConfig.TOPIC_PREFIX, TEST_SERVER) .with(PostgresConnectorConfig.DROP_SLOT_ON_STOP, true) .with(PostgresConnectorConfig.STATUS_UPDATE_INTERVAL_MS, 100) .with(PostgresConnectorConfig.PLUGIN_NAME, "decoderbufs") diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/io/debezium/connector/sqlserver/SqlServerOffsetContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/io/debezium/connector/sqlserver/SqlServerOffsetContext.java index 02547786c76..b301cde791e 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/io/debezium/connector/sqlserver/SqlServerOffsetContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/io/debezium/connector/sqlserver/SqlServerOffsetContext.java @@ -1,35 +1,47 @@ /* - * Copyright Debezium Authors. + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package io.debezium.connector.sqlserver; import io.debezium.connector.SnapshotRecord; +import io.debezium.pipeline.CommonOffsetContext; import io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotContext; import io.debezium.pipeline.source.snapshot.incremental.SignalBasedIncrementalSnapshotContext; import io.debezium.pipeline.spi.OffsetContext; import io.debezium.pipeline.txmetadata.TransactionContext; import io.debezium.relational.TableId; -import io.debezium.schema.DataCollectionId; +import io.debezium.spi.schema.DataCollectionId; import io.debezium.util.Collect; import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.Struct; import java.time.Instant; import java.util.Map; /** - * Copied from Debezium project(1.9.8.final) to modify the - * io.debezium.connector.sqlserver.SqlServerOffsetContext.Loader#load(java.util.Map) method. The - * original method is not able to get the eventSerialNo from the offset map which stores as string. + * Copied from Debezium project(2.7.4.Final).. + * + *

    Change 1: {@code Loader.load} accepts {@code event_serial_no} stored as either a String or a + * Long — offsets restored from Flink state carry it as a String. */ -public class SqlServerOffsetContext implements OffsetContext { +public class SqlServerOffsetContext extends CommonOffsetContext { private static final String SNAPSHOT_COMPLETED_KEY = "snapshot_completed"; private final Schema sourceInfoSchema; - private final SourceInfo sourceInfo; private boolean snapshotCompleted; private final TransactionContext transactionContext; private final IncrementalSnapshotContext incrementalSnapshotContext; @@ -45,7 +57,7 @@ public SqlServerOffsetContext( long eventSerialNo, TransactionContext transactionContext, IncrementalSnapshotContext incrementalSnapshotContext) { - sourceInfo = new SourceInfo(connectorConfig); + super(new SourceInfo(connectorConfig)); sourceInfo.setCommitLsn(position.getCommitLsn()); sourceInfo.setChangeLsn(position.getInTxLsn()); @@ -107,11 +119,6 @@ public Schema getSourceInfoSchema() { return sourceInfoSchema; } - @Override - public Struct getSourceInfo() { - return sourceInfo.struct(); - } - public TxLogPosition getChangePosition() { return TxLogPosition.valueOf(sourceInfo.getCommitLsn(), sourceInfo.getChangeLsn()); } @@ -151,11 +158,6 @@ public void preSnapshotCompletion() { snapshotCompleted = true; } - @Override - public void postSnapshotCompletion() { - sourceInfo.setSnapshot(SnapshotRecord.FALSE); - } - public static class Loader implements OffsetContext.Loader { private final SqlServerConnectorConfig connectorConfig; @@ -172,10 +174,10 @@ public SqlServerOffsetContext load(Map offset) { boolean snapshotCompleted = Boolean.TRUE.equals(offset.get(SNAPSHOT_COMPLETED_KEY)); // only introduced in 0.10.Beta1, so it might be not present when upgrading from earlier - // versions - // if value is String, convert it to Long + // versions. + // The eventSerialNo may be stored as a String in the offset map (e.g. when restored + // from Flink state); handle both String and Long representations. long eventSerialNo; - Object eventSerialNoObj = offset.get(SourceInfo.EVENT_SERIAL_NO_KEY); if (eventSerialNoObj != null) { if (eventSerialNoObj instanceof String) { @@ -212,11 +214,6 @@ public String toString() { + "]"; } - @Override - public void markLastSnapshotRecord() { - sourceInfo.setSnapshot(SnapshotRecord.LAST); - } - @Override public void event(DataCollectionId tableId, Instant timestamp) { sourceInfo.setSourceTime(timestamp); @@ -228,11 +225,6 @@ public TransactionContext getTransactionContext() { return transactionContext; } - @Override - public void incrementalSnapshotEvents() { - sourceInfo.setSnapshot(SnapshotRecord.INCREMENTAL); - } - @Override public IncrementalSnapshotContext getIncrementalSnapshotContext() { return incrementalSnapshotContext; diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/io/debezium/connector/sqlserver/SqlServerStreamingChangeEventSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/io/debezium/connector/sqlserver/SqlServerStreamingChangeEventSource.java index f7c62fb3d58..09e0875eab3 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/io/debezium/connector/sqlserver/SqlServerStreamingChangeEventSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/io/debezium/connector/sqlserver/SqlServerStreamingChangeEventSource.java @@ -1,35 +1,53 @@ /* - * Copyright Debezium Authors. + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.debezium.connector.sqlserver; -import io.debezium.connector.sqlserver.SqlServerConnectorConfig.SnapshotMode; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.notification.Notification; +import io.debezium.pipeline.notification.NotificationService; import io.debezium.pipeline.source.spi.StreamingChangeEventSource; import io.debezium.relational.ChangeTable; import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.schema.SchemaChangeEvent.SchemaChangeEventType; +import io.debezium.snapshot.SnapshotterService; import io.debezium.util.Clock; import io.debezium.util.ElapsedTimeStrategy; import io.debezium.util.Metronome; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.ResultSet; import java.sql.SQLException; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.PriorityQueue; import java.util.Queue; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; @@ -37,35 +55,18 @@ import java.util.stream.Collectors; /** - * Copied from Debezium project(1.9.8.final) to add method {@link - * SqlServerStreamingChangeEventSource#afterHandleLsn(SqlServerPartition, Lsn)}. Also implemented - * {@link SqlServerStreamingChangeEventSource#execute(ChangeEventSourceContext, SqlServerPartition, - * SqlServerOffsetContext)}. A {@link StreamingChangeEventSource} based on SQL Server change data - * capture functionality. A main loop polls database DDL change and change data tables and turns - * them into change events. - * - *

    The connector uses CDC functionality of SQL Server that is implemented as as a process that - * monitors source table and write changes from the table into the change table. + * Copied from Debezium project(2.7.4.Final).. * - *

    The main loop keeps a pointer to the LSN of changes that were already processed. It queries - * all change tables and get result set of changes. It always finds the smallest LSN across all - * tables and the change is converted into the event message and sent downstream. The process - * repeats until all result sets are empty. The LSN is marked and the procedure repeats. + *

    Change 1: add the {@code afterHandleLsn} hook, called after each iteration commits its + * position, so Flink CDC's bounded stream task can stop at the high watermark. * - *

    The schema changes detection follows the procedure recommended by SQL Server CDC - * documentation. The database operator should create one more capture process (and table) when a - * table schema is updated. The code detects presence of two change tables for a single source - * table. It decides which table is the new one depending on LSNs stored in them. The loop streams - * changes from the older table till there are events in new table with the LSN larger than in the - * old one. Then the change table is switched and streaming is executed from the new one. + *

    Change 2: {@code processErrorFromChangeTableQuery} translates SQL Server error 313 (the CDC + * query window is invalid because the data for the saved LSN was already purged) into an {@code + * IllegalStateException} that names the affected capture instances instead of retrying forever. */ public class SqlServerStreamingChangeEventSource implements StreamingChangeEventSource { - /** - * SQL Server Msg 313 is misleading for CDC table-valued functions. In this path it usually - * means the requested LSN interval fell behind the cleanup window. - */ private static final int INVALID_CDC_LSN_RANGE_ERROR_CODE = 313; private static final Pattern MISSING_CDC_FUNCTION_CHANGES_ERROR = @@ -94,11 +95,19 @@ public class SqlServerStreamingChangeEventSource private final Clock clock; private final SqlServerDatabaseSchema schema; private final Duration pollInterval; + private final SnapshotterService snapshotterService; private final SqlServerConnectorConfig connectorConfig; private final ElapsedTimeStrategy pauseBetweenCommits; private final Map streamingExecutionContexts; + private final Map> changeTablesWithKnownStopLsn = + new HashMap<>(); + + private boolean checkAgent; + private SqlServerOffsetContext effectiveOffset; + private final NotificationService + notificationService; public SqlServerStreamingChangeEventSource( SqlServerConnectorConfig connectorConfig, @@ -107,7 +116,9 @@ public SqlServerStreamingChangeEventSource( EventDispatcher dispatcher, ErrorHandler errorHandler, Clock clock, - SqlServerDatabaseSchema schema) { + SqlServerDatabaseSchema schema, + NotificationService notificationService, + SnapshotterService snapshotterService) { this.connectorConfig = connectorConfig; this.dataConnection = dataConnection; this.metadataConnection = metadataConnection; @@ -115,7 +126,9 @@ public SqlServerStreamingChangeEventSource( this.errorHandler = errorHandler; this.clock = clock; this.schema = schema; + this.notificationService = notificationService; this.pollInterval = connectorConfig.getPollInterval(); + this.snapshotterService = snapshotterService; final Duration intervalBetweenCommitsBasedOnPoll = this.pollInterval.multipliedBy(INTERVAL_BETWEEN_COMMITS_BASED_ON_POLL_FACTOR); this.pauseBetweenCommits = @@ -126,8 +139,17 @@ public SqlServerStreamingChangeEventSource( > 0 ? DEFAULT_INTERVAL_BETWEEN_COMMITS.toMillis() : intervalBetweenCommitsBasedOnPoll.toMillis()); - this.pauseBetweenCommits.hasElapsed(); this.streamingExecutionContexts = new HashMap<>(); + this.checkAgent = true; + } + + @Override + public void init(SqlServerOffsetContext offsetContext) throws InterruptedException { + this.effectiveOffset = + offsetContext == null + ? new SqlServerOffsetContext( + connectorConfig, TxLogPosition.NULL, false, false) + : offsetContext; } @Override @@ -136,6 +158,10 @@ public void execute( SqlServerPartition partition, SqlServerOffsetContext offsetContext) throws InterruptedException { + // Upstream throws UnsupportedOperationException here: SQL Server is Debezium's only + // multi-partition connector, so its streaming loop lives in the coordinator, which drives + // executeIteration() once per partition. Flink CDC runs the streaming source directly, + // without that coordinator, so the fork keeps the single-partition loop. final Metronome metronome = Metronome.sleeper(pollInterval, clock); LOGGER.info("Starting streaming"); @@ -155,22 +181,26 @@ public void execute( public boolean executeIteration( ChangeEventSourceContext context, SqlServerPartition partition, - SqlServerOffsetContext offsetContext) - throws InterruptedException { - if (connectorConfig.getSnapshotMode().equals(SnapshotMode.INITIAL_ONLY)) { - LOGGER.info("Streaming is not enabled in current configuration"); + SqlServerOffsetContext offsetContext) { + + if (!snapshotterService.getSnapshotter().shouldStream()) { + LOGGER.info( + "Streaming is disabled for snapshot mode {}", + snapshotterService.getSnapshotter().name()); return false; } final String databaseName = partition.getDatabaseName(); + this.effectiveOffset = offsetContext; + try { final SqlServerStreamingExecutionContext streamingExecutionContext = streamingExecutionContexts.getOrDefault( partition, new SqlServerStreamingExecutionContext( new PriorityQueue<>( - (x, y) -> x.getStopLsn().compareTo(y.getStopLsn())), + Comparator.comparing(SqlServerChangeTable::getStopLsn)), new AtomicReference<>(), offsetContext.getChangePosition(), new AtomicBoolean(false), @@ -214,14 +244,31 @@ public boolean executeIteration( // Shouldn't happen if the agent is running, but it is better to guard against such // situation if (!toLsn.isAvailable()) { - LOGGER.warn( - "No maximum LSN recorded in the database; please ensure that the SQL Server Agent is running"); + if (checkAgent) { + try { + if (!dataConnection.isAgentRunning(databaseName)) { + LOGGER.error( + "No maximum LSN recorded in the database; SQL Server Agent is not running"); + } + } catch (SQLException e) { + LOGGER.warn( + "No maximum LSN recorded in the database; this may happen if there are no changes recorded in the change table yet or " + + "low activity database where the cdc clean up job periodically clears entries from the cdc tables. " + + "Otherwise, this may be an indication that the SQL Server Agent is not running. " + + "You should follow the documentation on how to configure SQL Server Agent running status query."); + LOGGER.warn("Cannot query the status of the SQL Server Agent", e); + } + checkAgent = false; + } return false; + } else if (!checkAgent) { + checkAgent = true; } // There is no change in the database if (toLsn.compareTo(lastProcessedPosition.getCommitLsn()) <= 0 && streamingExecutionContext.getShouldIncreaseFromLsn()) { LOGGER.debug("No change in the database"); + dispatcher.dispatchHeartbeatEvent(partition, offsetContext); return false; } @@ -248,9 +295,11 @@ public boolean executeIteration( schemaChangeCheckpoints.add(table); } } + collectChangeTablesWithKnownStopLsn(partition, tables); } if (tablesSlot.get() == null) { tablesSlot.set(getChangeTablesToQuery(partition, offsetContext, toLsn)); + collectChangeTablesWithKnownStopLsn(partition, tablesSlot.get()); } try { dataConnection.getChangesForTables( @@ -268,9 +317,7 @@ public boolean executeIteration( for (int i = 0; i < tableCount; i++) { changeTables[i] = new SqlServerChangeTablePointer( - tables[i], - resultSets[i], - connectorConfig.getSourceTimestampMode()); + tables[i], resultSets[i]); changeTables[i].next(); } @@ -425,17 +472,19 @@ public boolean executeIteration( ? tableWithSmallestLsn.getData() : null; + final ResultSet resultSet = tableWithSmallestLsn.getResultSet(); offsetContext.setChangePosition( tableWithSmallestLsn.getChangePosition(), eventCount); offsetContext.event( tableWithSmallestLsn .getChangeTable() .getSourceTableId(), - connectorConfig - .getSourceTimestampMode() + resultSet .getTimestamp( - clock, - tableWithSmallestLsn.getResultSet())); + resultSet + .getMetaData() + .getColumnCount()) + .toInstant()); dispatcher.dispatchDataChangeEvent( partition, @@ -446,7 +495,8 @@ public boolean executeIteration( operation, data, dataNext, - clock)); + clock, + connectorConfig)); tableWithSmallestLsn.next(); } }); @@ -469,6 +519,25 @@ public boolean executeIteration( return true; } + @Override + public SqlServerOffsetContext getOffsetContext() { + return effectiveOffset; + } + + private void collectChangeTablesWithKnownStopLsn( + SqlServerPartition partition, SqlServerChangeTable[] tables) { + for (SqlServerChangeTable table : tables) { + if (table.getStopLsn().isAvailable()) { + synchronized (changeTablesWithKnownStopLsn) { + LOGGER.info("The stop lsn of {} change table became known", table); + changeTablesWithKnownStopLsn + .computeIfAbsent(partition, x -> new HashSet<>()) + .add(table); + } + } + } + } + private void commitTransaction() throws SQLException { // When reading from read-only Always On replica the default and only transaction isolation // is snapshot. This means that CDC metadata are not visible for long-running transactions. @@ -497,12 +566,14 @@ private void migrateTable( } dispatcher.dispatchSchemaChangeEvent( partition, + offsetContext, newTable.getSourceTableId(), new SqlServerSchemaChangeEventEmitter( partition, offsetContext, newTable, tableSchema, + schema, SchemaChangeEventType.ALTER)); newTable.setSourceTable(tableSchema); } @@ -580,7 +651,7 @@ private SqlServerChangeTable[] getChangeTablesToQuery( return true; } else { LOGGER.info( - "CDC is enabled for table {} but the table is not whitelisted by connector", + "CDC is enabled for table {} but the table is not on connector's table include list", changeTable); return false; } @@ -589,7 +660,7 @@ private SqlServerChangeTable[] getChangeTablesToQuery( if (includeListChangeTables.isEmpty()) { LOGGER.warn( - "No whitelisted table has enabled CDC, whitelisted table list does not contain any table with CDC enabled or no table match the white/blacklist filter(s)"); + "No table on connector's include list has enabled CDC, tables on include list do not contain any table with CDC enabled or no table match the include/exclude filter(s)"); } final List tables = new ArrayList<>(); @@ -624,12 +695,14 @@ private SqlServerChangeTable[] getChangeTablesToQuery( offsetContext.event(currentTable.getSourceTableId(), Instant.now()); dispatcher.dispatchSchemaChangeEvent( partition, + offsetContext, currentTable.getSourceTableId(), new SqlServerSchemaChangeEventEmitter( partition, offsetContext, currentTable, dataConnection.getTableSchemaFromTable(databaseName, currentTable), + schema, SchemaChangeEventType.CREATE)); } @@ -669,7 +742,57 @@ private Lsn getToLsn( databaseName, fromLsn, maxTransactionsPerIteration); } - /** expose control to the user to stop the connector. */ + @Override + public void commitOffset(Map sourcePartition, Map offset) { + Lsn commitLsn = Lsn.valueOf((String) offset.get("commit_lsn")); + synchronized (changeTablesWithKnownStopLsn) { + Optional optionalPartition = + changeTablesWithKnownStopLsn.keySet().stream() + .filter(p -> p.getSourcePartition().equals(sourcePartition)) + .findFirst(); + + if (optionalPartition.isEmpty()) { + return; + } + + SqlServerPartition partition = optionalPartition.get(); + Set partitionTables = changeTablesWithKnownStopLsn.get(partition); + + List changeTablesToCompleteReadingFrom = + partitionTables.stream() + .filter(t -> t.getStopLsn().compareTo(commitLsn) < 0) + .collect(Collectors.toList()); + + for (SqlServerChangeTable table : changeTablesToCompleteReadingFrom) { + Map additionalData = new HashMap<>(); + additionalData.put("connector_name", connectorConfig.getLogicalName()); + additionalData.put("capture_instance", table.getCaptureInstance()); + additionalData.put("start_lsn", table.getStartLsn().toString()); + additionalData.put("stop_lsn", table.getStopLsn().toString()); + additionalData.put("commit_lsn", commitLsn.toString()); + additionalData.putAll(partition.getSourcePartition()); + + notificationService.notify( + Notification.Builder.builder() + .withId(UUID.randomUUID().toString()) + .withAggregateType("Capture Instance") + .withType("COMPLETED") + .withAdditionalData(additionalData) + .withTimestamp(clock.currentTimeInMillis()) + .build()); + + partitionTables.remove(table); + + LOGGER.info( + "Complete reading from change table {} as the committed change lsn ({}) is greater than the table's stop lsn ({})", + table, + offset, + table.getStopLsn().toString()); + } + } + } + + /** Expose control to the subclass to stop the connector. */ protected void afterHandleLsn(SqlServerPartition partition, Lsn toLsn) { // do nothing } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/SqlServerSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/SqlServerSource.java index 4c357c43e6d..de62467d873 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/SqlServerSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/SqlServerSource.java @@ -124,13 +124,22 @@ public DebeziumSourceFunction build() { // all other connectors, since it is used as a prefix for all Kafka topic names // emanating from this connector. Only alphanumeric characters and underscores should be // used. - props.setProperty("database.server.name", DATABASE_SERVER_NAME); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", DATABASE_SERVER_NAME); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); props.setProperty("database.port", String.valueOf(port)); - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); props.setProperty("database.dbname", checkNotNull(database)); + // Debezium 2.0 replaced "database.dbname" with the multi-database + // "database.names"; it is a required option and validation fails when it is empty. + props.setProperty("database.names", checkNotNull(database)); + // The mssql-jdbc driver bundled with Debezium 2.x defaults "encrypt" to true, + // while the 9.x driver used by Debezium 1.9 defaulted it to false. Keep the + // historical Flink CDC behavior; users can opt into TLS with + // "debezium.database.encrypt". + props.setProperty("database.encrypt", "false"); if (tableList != null) { props.setProperty("table.include.list", String.join(",", tableList)); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/SqlServerValidator.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/SqlServerValidator.java index e37310b6814..c6f029f8abb 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/SqlServerValidator.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/SqlServerValidator.java @@ -86,9 +86,31 @@ public static Connection openConnection(Properties properties) throws SQLExcepti String dbname = properties.getProperty("database.dbname"); String userName = properties.getProperty("database.user"); String userpwd = properties.getProperty("database.password"); - return DriverManager.getConnection( - "jdbc:sqlserver://" + hostname + ":" + port + ";" + "databaseName=" + dbname, - userName, - userpwd); + StringBuilder url = + new StringBuilder("jdbc:sqlserver://") + .append(hostname) + .append(":") + .append(port) + .append(";databaseName=") + .append(dbname); + // This validator opens its own connection rather than reusing Debezium's, so it also + // has to carry the connection settings Debezium is given. mssql-jdbc 10.2 -- the + // version Debezium 2.0 pulls in, replacing 9.4 -- flipped the "encrypt" default from + // false to true, so without these the validator fails the TLS handshake against a + // server with a self-signed certificate even though the connector itself connects + // fine, and every non-incremental SqlServer source dies in open() before the engine + // ever starts. + appendIfPresent(url, properties, "database.encrypt", "encrypt"); + appendIfPresent( + url, properties, "database.trustServerCertificate", "trustServerCertificate"); + return DriverManager.getConnection(url.toString(), userName, userpwd); + } + + private static void appendIfPresent( + StringBuilder url, Properties properties, String propertyName, String urlName) { + String value = properties.getProperty(propertyName); + if (value != null && !value.isEmpty()) { + url.append(";").append(urlName).append("=").append(value); + } } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfig.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfig.java index 5ff78c307c7..16880befb7c 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfig.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfig.java @@ -19,6 +19,7 @@ import org.apache.flink.cdc.connectors.base.config.JdbcSourceConfig; import org.apache.flink.cdc.connectors.base.options.StartupOptions; +import org.apache.flink.util.TemporaryClassLoaderContext; import io.debezium.config.Configuration; import io.debezium.connector.sqlserver.SqlServerConnectorConfig; @@ -90,6 +91,13 @@ public SqlServerSourceConfig( @Override public SqlServerConnectorConfig getDbzConnectorConfig() { - return new SqlServerConnectorConfig(getDbzConfiguration()); + // Debezium 2.0 resolves classes through the thread context class loader + // (io.debezium.config.Instantiator, and the ServiceLoader based SPIs). On a Flink + // thread that loader may be a user code class loader from a previous job attempt, + // which is already closed. Pin it to the loader that loaded Debezium. + try (TemporaryClassLoaderContext ignored = + TemporaryClassLoaderContext.of(SqlServerConnectorConfig.class.getClassLoader())) { + return new SqlServerConnectorConfig(getDbzConfiguration()); + } } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfigFactory.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfigFactory.java index f485019ac71..029d246169a 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfigFactory.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfigFactory.java @@ -58,8 +58,17 @@ public SqlServerSourceConfig create(int subtask) { // set database history impl to flink database history props.setProperty( - "database.history", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); - props.setProperty("database.history.instance.name", UUID.randomUUID() + "_" + subtask); + "schema.history.internal", EmbeddedFlinkDatabaseHistory.class.getCanonicalName()); + props.setProperty( + "schema.history.internal.instance.name", UUID.randomUUID() + "_" + subtask); + + // SQL Server is Debezium's only multi-partition connector, and for those Metrics tags + // the JMX ObjectName with "task.id" as well as the topic prefix. Kafka Connect supplies + // that property; Flink never does, so Metrics#metricName passes null to + // Sanitizer.jmxSanitize and every SQL Server task dies with a NullPointerException + // before it reads a row. The subtask index is the natural analogue of a Connect task id + // and keeps the name unique within the TaskManager JVM. + props.setProperty("task.id", String.valueOf(subtask)); // hard code server name, because we don't need to distinguish it, docs: // Logical name that identifies and provides a namespace for the SQL Server database @@ -67,13 +76,21 @@ public SqlServerSourceConfig create(int subtask) { // all other connectors, since it is used as a prefix for all Kafka topic names // emanating from this connector. Only alphanumeric characters and underscores should be // used. - props.setProperty("database.server.name", DATABASE_SERVER_NAME); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", DATABASE_SERVER_NAME); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); props.setProperty("database.port", String.valueOf(port)); - props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true)); + props.setProperty("schema.history.internal.skip.unparseable.ddl", String.valueOf(true)); props.setProperty("database.dbname", checkNotNull(databaseList.get(0))); + // Debezium 2.0 replaced "database.dbname" with the multi-database "database.names"; + // it is a required option and validation fails when it is empty. + props.setProperty("database.names", checkNotNull(databaseList.get(0))); + // The mssql-jdbc driver bundled with Debezium 2.x defaults "encrypt" to true, while the + // 9.x driver used by Debezium 1.9 defaulted it to false. Keep the historical Flink CDC + // behavior; users can opt into TLS with "debezium.database.encrypt". + props.setProperty("database.encrypt", "false"); if (tableList != null) { props.setProperty("table.include.list", String.join(",", tableList)); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerScanFetchTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerScanFetchTask.java index a2c266cb237..b482bec1fd9 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerScanFetchTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerScanFetchTask.java @@ -30,7 +30,10 @@ import io.debezium.connector.sqlserver.SqlServerPartition; import io.debezium.heartbeat.Heartbeat; import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.notification.NotificationService; +import io.debezium.pipeline.signal.actions.snapshotting.SnapshotConfiguration; import io.debezium.pipeline.source.AbstractSnapshotChangeEventSource; +import io.debezium.pipeline.source.SnapshottingTask; import io.debezium.pipeline.source.spi.ChangeEventSource; import io.debezium.pipeline.source.spi.SnapshotProgressListener; import io.debezium.pipeline.spi.ChangeRecordEmitter; @@ -39,6 +42,7 @@ import io.debezium.relational.SnapshotChangeRecordEmitter; import io.debezium.relational.Table; import io.debezium.relational.TableId; +import io.debezium.schema.SchemaFactory; import io.debezium.util.Clock; import io.debezium.util.ColumnUtils; import io.debezium.util.Strings; @@ -51,6 +55,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Duration; +import java.util.Collections; import static org.apache.flink.cdc.connectors.sqlserver.source.utils.SqlServerUtils.buildSplitScanQuery; import static org.apache.flink.cdc.connectors.sqlserver.source.utils.SqlServerUtils.readTableSplitDataStatement; @@ -83,7 +88,10 @@ protected void executeDataSnapshot(Context context) throws Exception { snapshotSplitReadTask.execute( changeEventSourceContext, sourceFetchContext.getPartition(), - sourceFetchContext.getOffsetContext()); + sourceFetchContext.getOffsetContext(), + snapshotSplitReadTask.getSnapshottingTask( + sourceFetchContext.getPartition(), + sourceFetchContext.getOffsetContext())); // execute stream read task if (!snapshotResult.isCompletedOrSkipped()) { taskRunning = false; @@ -169,7 +177,14 @@ public SqlServerSnapshotSplitReadTask( EventDispatcher eventDispatcher, EventDispatcher.SnapshotReceiver snapshotReceiver, SnapshotSplit snapshotSplit) { - super(connectorConfig, snapshotProgressListener); + super( + connectorConfig, + snapshotProgressListener, + new NotificationService<>( + Collections.emptyList(), + connectorConfig, + SchemaFactory.get(), + notification -> {})); this.offsetContext = previousOffset; this.connectorConfig = connectorConfig; this.databaseSchema = databaseSchema; @@ -185,12 +200,12 @@ public SqlServerSnapshotSplitReadTask( public SnapshotResult execute( ChangeEventSourceContext context, SqlServerPartition partition, - SqlServerOffsetContext previousOffset) + SqlServerOffsetContext previousOffset, + SnapshottingTask snapshottingTask) throws InterruptedException { - SnapshottingTask snapshottingTask = getSnapshottingTask(partition, previousOffset); final SqlServerSnapshotContext ctx; try { - ctx = prepare(partition); + ctx = prepare(partition, false); } catch (Exception e) { LOG.error("Failed to initialize snapshot context.", e); throw new RuntimeException(e); @@ -222,14 +237,26 @@ protected SnapshotResult doExecute( } @Override - protected SnapshottingTask getSnapshottingTask( + public SnapshottingTask getSnapshottingTask( SqlServerPartition partition, SqlServerOffsetContext previousOffset) { - return new SnapshottingTask(false, true); + return new SnapshottingTask( + false, true, Collections.emptyList(), Collections.emptyMap(), false); } @Override - protected SqlServerSnapshotContext prepare(SqlServerPartition partition) throws Exception { - return new SqlServerSnapshotContext(partition); + public SnapshottingTask getBlockingSnapshottingTask( + SqlServerPartition partition, + SqlServerOffsetContext previousOffset, + SnapshotConfiguration snapshotConfiguration) { + // Debezium 2.6 made this abstract. Flink CDC drives its own split snapshot and never + // runs Debezium's signal-based blocking snapshot, so it behaves like the regular one. + return getSnapshottingTask(partition, previousOffset); + } + + @Override + protected SqlServerSnapshotContext prepare(SqlServerPartition partition, boolean onDemand) + throws Exception { + return new SqlServerSnapshotContext(partition, onDemand); } private void createDataEvents(SqlServerSnapshotContext snapshotContext, TableId tableId) @@ -283,8 +310,7 @@ private void createDataEventsForTable( while (rs.next()) { rows++; - final Object[] row = - jdbcConnection.rowToArray(table, databaseSchema, rs, columnArray); + final Object[] row = jdbcConnection.rowToArray(table, rs, columnArray); if (logTimer.expired()) { long stop = clock.currentTimeInMillis(); LOG.info( @@ -316,7 +342,7 @@ protected ChangeRecordEmitter getChangeRecordEmitter( SqlServerSnapshotContext snapshotContext, TableId tableId, Object[] row) { snapshotContext.offset.event(tableId, clock.currentTime()); return new SnapshotChangeRecordEmitter<>( - snapshotContext.partition, snapshotContext.offset, row, clock); + snapshotContext.partition, snapshotContext.offset, row, clock, connectorConfig); } private Threads.Timer getTableScanLogTimer() { @@ -327,8 +353,9 @@ private static class SqlServerSnapshotContext extends RelationalSnapshotChangeEventSource.RelationalSnapshotContext< SqlServerPartition, SqlServerOffsetContext> { - public SqlServerSnapshotContext(SqlServerPartition partition) throws SQLException { - super(partition, ""); + public SqlServerSnapshotContext(SqlServerPartition partition, boolean onDemand) + throws SQLException { + super(partition, "", onDemand); } } } @@ -348,5 +375,25 @@ public void finished() { public boolean isRunning() { return taskRunning; } + + // The following methods are only used by Debezium's signal-based blocking snapshot, + // which Flink CDC does not use, so they are no-ops here. + + @Override + public boolean isPaused() { + return false; + } + + @Override + public void resumeStreaming() throws InterruptedException {} + + @Override + public void waitSnapshotCompletion() throws InterruptedException {} + + @Override + public void streamingPaused() {} + + @Override + public void waitStreamingPaused() throws InterruptedException {} } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerSourceFetchTaskContext.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerSourceFetchTaskContext.java index 07390f460f4..5d13a3c68c4 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerSourceFetchTaskContext.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerSourceFetchTaskContext.java @@ -31,6 +31,7 @@ import org.apache.flink.cdc.connectors.sqlserver.source.utils.SqlServerUtils; import org.apache.flink.table.types.logical.RowType; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.base.ChangeEventQueue; import io.debezium.connector.base.ChangeEventQueue.Builder; import io.debezium.connector.sqlserver.SourceInfo; @@ -42,7 +43,6 @@ import io.debezium.connector.sqlserver.SqlServerOffsetContext.Loader; import io.debezium.connector.sqlserver.SqlServerPartition; import io.debezium.connector.sqlserver.SqlServerTaskContext; -import io.debezium.connector.sqlserver.SqlServerTopicSelector; import io.debezium.data.Envelope.FieldName; import io.debezium.pipeline.DataChangeEvent; import io.debezium.pipeline.ErrorHandler; @@ -56,10 +56,10 @@ import io.debezium.relational.Table; import io.debezium.relational.TableId; import io.debezium.relational.Tables.TableFilter; -import io.debezium.schema.DataCollectionId; -import io.debezium.schema.TopicSelector; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.schema.DataCollectionId; +import io.debezium.spi.topic.TopicNamingStrategy; import io.debezium.util.Collect; -import io.debezium.util.SchemaNameAdjuster; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; @@ -93,7 +93,7 @@ public class SqlServerSourceFetchTaskContext extends JdbcSourceFetchTaskContext private SqlServerErrorHandler errorHandler; private ChangeEventQueue queue; private SqlServerTaskContext taskContext; - private TopicSelector topicSelector; + private TopicNamingStrategy topicSelector; private EventDispatcher.SnapshotReceiver snapshotReceiver; private SnapshotChangeEventSourceMetrics snapshotChangeEventSourceMetrics; private StreamingChangeEventSourceMetrics streamingChangeEventSourceMetrics; @@ -113,7 +113,8 @@ public SqlServerSourceFetchTaskContext( public void configure(SourceSplitBase sourceSplitBase) { // initial stateful objects final SqlServerConnectorConfig connectorConfig = getDbzConnectorConfig(); - this.topicSelector = SqlServerTopicSelector.defaultSelector(connectorConfig); + this.topicSelector = + connectorConfig.getTopicNamingStrategy(CommonConnectorConfig.TOPIC_NAMING_STRATEGY); EmbeddedFlinkDatabaseHistory.registerHistory( sourceConfig .getDbzConfiguration() @@ -126,7 +127,7 @@ public void configure(SourceSplitBase sourceSplitBase) { String serverName = connectorConfig.getLogicalName(); String dbName = connectorConfig.getJdbcConfig().getDatabase(); - this.partition = new SqlServerPartition(serverName, dbName, false); + this.partition = new SqlServerPartition(serverName, dbName); validateAndLoadDatabaseHistory(offsetContext, databaseSchema); @@ -168,7 +169,7 @@ public void configure(SourceSplitBase sourceSplitBase) { this.streamingChangeEventSourceMetrics = changeEventSourceMetricsFactory.getStreamingMetrics( taskContext, queue, metadataProvider); - this.errorHandler = new SqlServerErrorHandler(connectorConfig, queue); + this.errorHandler = new SqlServerErrorHandler(connectorConfig, queue, null); } /** Loads the connector's persistent offset (if present) via the given loader. */ diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerStreamFetchTask.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerStreamFetchTask.java index dcf9b836632..acd8720a355 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerStreamFetchTask.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/reader/fetch/SqlServerStreamFetchTask.java @@ -25,10 +25,12 @@ import org.apache.flink.cdc.connectors.base.source.reader.external.FetchTask; import org.apache.flink.cdc.connectors.sqlserver.source.offset.LsnOffset; import org.apache.flink.cdc.connectors.sqlserver.source.reader.fetch.SqlServerScanFetchTask.SqlServerSnapshotSplitChangeEventSourceContext; +import org.apache.flink.cdc.debezium.internal.SnapshotterServiceFactory; import io.debezium.DebeziumException; import io.debezium.connector.sqlserver.Lsn; import io.debezium.connector.sqlserver.SqlServerConnection; +import io.debezium.connector.sqlserver.SqlServerConnector; import io.debezium.connector.sqlserver.SqlServerConnectorConfig; import io.debezium.connector.sqlserver.SqlServerDatabaseSchema; import io.debezium.connector.sqlserver.SqlServerOffsetContext; @@ -36,12 +38,16 @@ import io.debezium.connector.sqlserver.SqlServerStreamingChangeEventSource; import io.debezium.pipeline.ErrorHandler; import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.notification.NotificationService; import io.debezium.pipeline.source.spi.ChangeEventSource; import io.debezium.relational.TableId; +import io.debezium.schema.SchemaFactory; import io.debezium.util.Clock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; + import static org.apache.flink.cdc.connectors.sqlserver.source.offset.LsnOffset.NO_STOPPING_OFFSET; /** The task to work for fetching data of SqlServer table stream split . */ @@ -123,7 +129,13 @@ public StreamSplitReadTask( eventDispatcher, errorHandler, Clock.system(), - schema); + schema, + new NotificationService<>( + Collections.emptyList(), + connectorConfig, + SchemaFactory.get(), + notification -> {}), + SnapshotterServiceFactory.create(connectorConfig, SqlServerConnector.class)); this.lsnSplit = lsnSplit; this.watermarkDispatcher = watermarkDispatcher; this.errorHandler = errorHandler; @@ -178,5 +190,25 @@ private class RedoLogSplitChangeEventSourceContext public boolean isRunning() { return taskRunning; } + + // The following methods are only used by Debezium's signal-based blocking snapshot, + // which Flink CDC does not use, so they are no-ops here. + + @Override + public boolean isPaused() { + return false; + } + + @Override + public void resumeStreaming() throws InterruptedException {} + + @Override + public void waitSnapshotCompletion() throws InterruptedException {} + + @Override + public void streamingPaused() {} + + @Override + public void waitStreamingPaused() throws InterruptedException {} } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/utils/SqlServerConnectionUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/utils/SqlServerConnectionUtils.java index 400318a415d..503c262eeae 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/utils/SqlServerConnectionUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/utils/SqlServerConnectionUtils.java @@ -17,11 +17,9 @@ package org.apache.flink.cdc.connectors.sqlserver.source.utils; -import io.debezium.config.Configuration; import io.debezium.connector.sqlserver.SqlServerConnection; import io.debezium.connector.sqlserver.SqlServerConnectorConfig; import io.debezium.connector.sqlserver.SqlServerValueConverters; -import io.debezium.jdbc.JdbcConfiguration; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.RelationalDatabaseConnectorConfig; import io.debezium.relational.RelationalTableFilters; @@ -39,18 +37,18 @@ public class SqlServerConnectionUtils { public static SqlServerConnection createSqlServerConnection( RelationalDatabaseConnectorConfig connectorConfig) { - Configuration dbzConnectorConfig = connectorConfig.getJdbcConfig(); - final SqlServerValueConverters valueConverters = new SqlServerValueConverters( connectorConfig.getDecimalMode(), connectorConfig.getTemporalPrecisionMode(), connectorConfig.binaryHandlingMode()); + // Debezium 2.0 removed the sourceTimestampMode and classloader-supplier constructor + // arguments; the constructor is now (config, valueConverters, skippedOperations, + // useSingleDatabase). Debezium 2.4 takes the SqlServerConnectorConfig itself as the + // first argument (2.1 through 2.3 took a SqlServerJdbcConfiguration). return new SqlServerConnection( - JdbcConfiguration.adapt(dbzConnectorConfig), - ((SqlServerConnectorConfig) connectorConfig).getSourceTimestampMode(), + (SqlServerConnectorConfig) connectorConfig, valueConverters, - SqlServerConnectionUtils.class::getClassLoader, connectorConfig.getSkippedOperations(), false); } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/utils/SqlServerUtils.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/utils/SqlServerUtils.java index deca1e7b3b7..d2cbb0d99f9 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/utils/SqlServerUtils.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/main/java/org/apache/flink/cdc/connectors/sqlserver/source/utils/SqlServerUtils.java @@ -24,19 +24,19 @@ import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.FlinkRuntimeException; +import io.debezium.config.CommonConnectorConfig; import io.debezium.connector.sqlserver.Lsn; import io.debezium.connector.sqlserver.SourceInfo; import io.debezium.connector.sqlserver.SqlServerConnection; import io.debezium.connector.sqlserver.SqlServerConnectorConfig; import io.debezium.connector.sqlserver.SqlServerDatabaseSchema; -import io.debezium.connector.sqlserver.SqlServerTopicSelector; import io.debezium.connector.sqlserver.SqlServerValueConverters; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.Column; import io.debezium.relational.Table; import io.debezium.relational.TableId; -import io.debezium.schema.TopicSelector; -import io.debezium.util.SchemaNameAdjuster; +import io.debezium.schema.SchemaNameAdjuster; +import io.debezium.spi.topic.TopicNamingStrategy; import org.apache.kafka.connect.source.SourceRecord; import javax.annotation.Nullable; @@ -229,8 +229,8 @@ public static PreparedStatement readTableSplitDataStatement( public static SqlServerDatabaseSchema createSqlServerDatabaseSchema( SqlServerConnectorConfig connectorConfig, SqlServerConnection connection) { - TopicSelector topicSelector = - SqlServerTopicSelector.defaultSelector(connectorConfig); + TopicNamingStrategy topicSelector = + connectorConfig.getTopicNamingStrategy(CommonConnectorConfig.TOPIC_NAMING_STRATEGY); SchemaNameAdjuster schemaNameAdjuster = SchemaNameAdjuster.create(); SqlServerValueConverters valueConverters = new SqlServerValueConverters( @@ -350,10 +350,32 @@ private static String buildSelectWithRowLimits( return sql.toString(); } + /** + * Builds SQL Server's fully qualified name for the table, including the database. + * + *

    Debezium 2.0 replaced the single-database {@code database.dbname} option with the + * multi-database {@code database.names}, and {@code SqlServerConnection} only appends {@code + * ;databaseName=...} to the JDBC URL when it is in single-database mode. The connection is + * therefore no longer bound to the captured database, so an unqualified {@code + * [schema].[table]} resolves against whichever database the session happens to point at and + * fails with "Invalid object name". Emitting the three-part name makes every query independent + * of the session's current database context. + */ private static String quoteSchemaAndTable(TableId tableId) { StringBuilder quoted = new StringBuilder(); - if (tableId.schema() != null && !tableId.schema().isEmpty()) { + boolean hasCatalog = tableId.catalog() != null && !tableId.catalog().isEmpty(); + boolean hasSchema = tableId.schema() != null && !tableId.schema().isEmpty(); + + if (hasCatalog) { + quoted.append(quote(tableId.catalog())).append("."); + // A three-part name must keep the schema position, even when it is empty, otherwise + // the database name would be read as the schema: [db]..[table], not [db].[table]. + if (hasSchema) { + quoted.append(quote(tableId.schema())); + } + quoted.append("."); + } else if (hasSchema) { quoted.append(quote(tableId.schema())).append("."); } @@ -370,7 +392,7 @@ public static String quote(String name) { } public static String quote(TableId tableId) { - return String.format("%s.%s", quote(tableId.schema()), quote(tableId.table())); + return quoteSchemaAndTable(tableId); } private static void addPrimaryKeyColumnsToCondition( diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/test/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfigFactoryTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/test/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfigFactoryTest.java index 36b1b5f6b38..c52cb04ec2f 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/test/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfigFactoryTest.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/test/java/org/apache/flink/cdc/connectors/sqlserver/source/config/SqlServerSourceConfigFactoryTest.java @@ -19,9 +19,13 @@ import org.apache.flink.cdc.connectors.base.options.StartupOptions; +import io.debezium.config.CommonConnectorConfig; +import io.debezium.connector.sqlserver.SqlServerConnectorConfig; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Properties; + /** Unit tests for {@link SqlServerSourceConfigFactory}. */ class SqlServerSourceConfigFactoryTest { @@ -46,4 +50,55 @@ void testTimestampStartupOption() { Assertions.assertThat(sourceConfig.getDbzProperties().getProperty("snapshot.mode")) .isEqualTo("schema_only"); } + + /** + * Debezium 2.0 renamed {@code database.server.name} to {@code topic.prefix} and replaced {@code + * database.dbname} with the multi-database {@code database.names}; both are required, and + * without them the topic naming strategy cannot even be instantiated. + */ + @Test + void testTopicPrefixAndDatabaseNamesAreConfigured() { + SqlServerConnectorConfig connectorConfig = + new SqlServerConnectorConfig(newFactory().create(0).getDbzConfiguration()); + + Assertions.assertThat(connectorConfig.getLogicalName()) + .isEqualTo("sqlserver_transaction_log_source"); + Assertions.assertThat(connectorConfig.getDatabaseNames()).containsExactly("inventory"); + Assertions.assertThat( + connectorConfig.getTopicNamingStrategy( + CommonConnectorConfig.TOPIC_NAMING_STRATEGY)) + .isNotNull(); + } + + /** + * The mssql-jdbc driver shipped with Debezium 2.x defaults {@code encrypt} to true, while the + * one shipped with Debezium 1.9 defaulted it to false. Flink CDC keeps the historical default + * but users can still opt into TLS. + */ + @Test + void testEncryptDefaultsToFalseAndCanBeOverridden() { + Assertions.assertThat( + newFactory().create(0).getDbzConfiguration().getString("database.encrypt")) + .isEqualTo("false"); + + Properties dbzProperties = new Properties(); + dbzProperties.setProperty("database.encrypt", "true"); + SqlServerSourceConfigFactory factory = newFactory(); + factory.debeziumProperties(dbzProperties); + Assertions.assertThat(factory.create(0).getDbzConfiguration().getString("database.encrypt")) + .isEqualTo("true"); + } + + private static SqlServerSourceConfigFactory newFactory() { + SqlServerSourceConfigFactory factory = new SqlServerSourceConfigFactory(); + factory.hostname("localhost") + .port(1433) + .databaseList("inventory") + .tableList("inventory.dbo.products") + .username("flinkuser") + .password("flinkpw") + .serverTimeZone("UTC"); + factory.startupOptions(StartupOptions.initial()); + return factory; + } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/test/java/org/apache/flink/cdc/connectors/sqlserver/table/SqlServerConnectorITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/test/java/org/apache/flink/cdc/connectors/sqlserver/table/SqlServerConnectorITCase.java index 48de0dc42e5..8c660fe77f0 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/test/java/org/apache/flink/cdc/connectors/sqlserver/table/SqlServerConnectorITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc/src/test/java/org/apache/flink/cdc/connectors/sqlserver/table/SqlServerConnectorITCase.java @@ -460,8 +460,12 @@ void testAllTypes(boolean parallelismSnapshot) throws Throwable { String expectedSnapshotRecord = "+I(0,cc ,vcc,tc,cč ,vcč,tč,1.123,2,3.323,4.323,5,6,true,22,333,4444,55555,2018-07-13,10:23:45.680,10:23:45.678,2018-07-13T11:23:45.340,2018-07-13T01:23:45.456Z,2018-07-13T13:23:45.780,2018-07-13T14:24,b)"; + // val_time is time(4) holding 10:23:45.6789. Until Debezium 2.7 the streamed + // update carried .679 while the snapshot carried .678, because the change table was + // read through a lossy java.sql.Time (millisecond, rounded) path; 2.7 reads it at + // full precision, so both now truncate to .678 and finally agree. String expectedUpdatedRecord = - "+U(0,cc ,vcc,tc,cč ,vcč,tč,1.123,2,3.323,4.323,5,6,true,22,333,8888,55555,2018-07-13,10:23:45.680,10:23:45.679,2018-07-13T11:23:45.340,2018-07-13T01:23:45.456Z,2018-07-13T13:23:45.780,2018-07-13T14:24,b)"; + "+U(0,cc ,vcc,tc,cč ,vcč,tč,1.123,2,3.323,4.323,5,6,true,22,333,8888,55555,2018-07-13,10:23:45.680,10:23:45.678,2018-07-13T11:23:45.340,2018-07-13T01:23:45.456Z,2018-07-13T13:23:45.780,2018-07-13T14:24,b)"; List actual = TestValuesTableFactory.getRawResultsAsStrings("sink"); Assertions.assertThat(actual) .containsExactlyInAnyOrder(expectedSnapshotRecord, expectedUpdatedRecord); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc/pom.xml b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc/pom.xml index 53a6006bf3f..4a10d529ca8 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc/pom.xml +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc/pom.xml @@ -78,6 +78,20 @@ limitations under the License. mysql-connector-java 8.0.26 test + + + + com.google.protobuf + protobuf-java + + diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc/src/main/java/org/apache/flink/cdc/connectors/vitess/VitessSource.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc/src/main/java/org/apache/flink/cdc/connectors/vitess/VitessSource.java index 7730d3fcb8e..5cd583c887d 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc/src/main/java/org/apache/flink/cdc/connectors/vitess/VitessSource.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc/src/main/java/org/apache/flink/cdc/connectors/vitess/VitessSource.java @@ -263,7 +263,8 @@ public DebeziumSourceFunction build() { // Vtgate server/cluster being monitored. The logical name should be unique across // all other connectors, since it is used as a prefix for all Kafka topic names coming // from this connector. Only alphanumeric characters and underscores should be used. - props.setProperty("database.server.name", "vitess_cdc_source"); + // Debezium 2.0 renamed "database.server.name" to "topic.prefix". + props.setProperty("topic.prefix", "vitess_cdc_source"); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.port", String.valueOf(port)); props.setProperty("vitess.keyspace", checkNotNull(keyspace)); diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc/pom.xml b/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc/pom.xml index eb8aef15fde..e2bf8191161 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc/pom.xml +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc/pom.xml @@ -55,8 +55,17 @@ limitations under the License. io.debezium:debezium-api io.debezium:debezium-embedded io.debezium:debezium-core + + io.debezium:debezium-storage-kafka io.debezium:debezium-ddl-parser io.debezium:debezium-connector-mysql + io.debezium:debezium-connector-binlog org.apache.flink:flink-cdc-common org.apache.flink:flink-connector-debezium org.apache.flink:flink-connector-mysql-cdc diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc/pom.xml b/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc/pom.xml index 912d8db662d..859b7bbbcda 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc/pom.xml +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc/pom.xml @@ -61,8 +61,17 @@ limitations under the License. io.debezium:debezium-api io.debezium:debezium-embedded io.debezium:debezium-core + + io.debezium:debezium-storage-kafka io.debezium:debezium-ddl-parser io.debezium:debezium-connector-mysql + io.debezium:debezium-connector-binlog org.apache.flink:flink-cdc-common org.apache.flink:flink-connector-debezium org.apache.flink:flink-connector-mysql-cdc diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc/pom.xml b/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc/pom.xml index 696af55cb86..89a52cea502 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc/pom.xml +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc/pom.xml @@ -55,6 +55,14 @@ limitations under the License. io.debezium:debezium-api io.debezium:debezium-embedded io.debezium:debezium-core + + io.debezium:debezium-storage-kafka io.debezium:debezium-ddl-parser io.debezium:debezium-connector-oracle org.apache.flink:flink-connector-debezium diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc/pom.xml b/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc/pom.xml index 21d96a72c0a..7fc5b3c6b42 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc/pom.xml +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc/pom.xml @@ -55,6 +55,14 @@ limitations under the License. io.debezium:debezium-api io.debezium:debezium-embedded io.debezium:debezium-core + + io.debezium:debezium-storage-kafka io.debezium:debezium-connector-sqlserver org.apache.flink:flink-connector-debezium org.apache.flink:flink-cdc-base diff --git a/pom.xml b/pom.xml index 6bc39e7a65f..adf9b61d2ec 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,8 @@ limitations under the License. 33.4.0-jre-20.0 ${flink.1.x.shaded.guava.version} 17.0 - 1.9.8.Final + 2.7.4.Final + 1.3.9 3.2.0 2.2.0 1.21.4 @@ -292,6 +293,16 @@ limitations under the License. ${flink.version} provided + + + com.google.code.findbugs + jsr305 + ${jsr305.version} + provided + + org.slf4j slf4j-api @@ -610,6 +621,21 @@ limitations under the License. false false true + + + + *:*