diff --git a/.github/workflows/flink_cdc_base.yml b/.github/workflows/flink_cdc_base.yml index 982ea81e3e3..4bac2ac1bc9 100644 --- a/.github/workflows/flink_cdc_base.yml +++ b/.github/workflows/flink_cdc_base.yml @@ -90,7 +90,7 @@ jobs: maven-version: 3.8.6 - name: Compile and test - timeout-minutes: 90 + timeout-minutes: 120 run: | . .github/workflows/utils.sh jvm_timezone=$(random_timezone) diff --git a/docs/content.zh/docs/connectors/pipeline-connectors/kafka.md b/docs/content.zh/docs/connectors/pipeline-connectors/kafka.md index bc655a5b487..75cf55d0de3 100644 --- a/docs/content.zh/docs/connectors/pipeline-connectors/kafka.md +++ b/docs/content.zh/docs/connectors/pipeline-connectors/kafka.md @@ -26,12 +26,74 @@ under the License. # Kafka Pipeline 连接器 -Kafka Pipeline 连接器可以用作 Pipeline 的 *Data Sink*，将数据写入[Kafka](https://kafka.apache.org)。 本文档介绍如何设置 Kafka Pipeline 连接器。 +Kafka Pipeline 连接器可以用作 Pipeline 的 *Data Source* 或 *Data Sink*。作为 Source 时，它消费 Debezium JSON 或 Canal JSON Changelog，并将推断出的表结构变化转换为 Pipeline Schema 事件。 ## 连接器的功能 * 自动建表 * 表结构变更同步 * 数据实时同步 +* 消费 Debezium JSON 或 Canal JSON Changelog + +Kafka Source +---------------- + +下面的 Pipeline 从 Kafka 多分区消费 Debezium JSON，并写入 StarRocks： + +```yaml +source: + type: kafka + name: Kafka Debezium Source + topic: inventory.customers + group-id: flink-cdc-kafka-source + scan.startup.mode: group-offsets + properties.bootstrap.servers: localhost:9092 + +transform: + - source-table: inventory.\.* + primary-keys: id + +sink: + type: starrocks + name: StarRocks Sink + jdbc-url: jdbc:mysql://localhost:9030 + load-url: localhost:8030 + username: root + password: "" + +pipeline: + name: Kafka to StarRocks Pipeline + parallelism: 4 + schema.change.behavior: lenient +``` + +Kafka Source 配置项： + +* `topic`：单个 Topic 或逗号分隔的 Topic 列表。 +* `topic-pattern`：用于动态发现 Topic 的正则表达式；`topic` 与 `topic-pattern` 必须且只能配置一个。 +* `group-id`（未配置 `properties.group.id` 时必填）：Kafka Consumer Group。 +* `scan.startup.mode`：`group-offsets`（默认）、`earliest-offset`、`latest-offset`、`timestamp` 或 `specific-offsets`。 +* `scan.startup.timestamp-millis`：当 `scan.startup.mode` 为 `timestamp` 时必填。 +* `scan.startup.specific-offsets`：当 `scan.startup.mode` 为 `specific-offsets` 时必填。仅配置一个 `topic` 时可写 `partition:0,offset:42;partition:1,offset:300`；多 Topic 或 `topic-pattern` 时每条必须带 topic，例如 `topic:dbz.customers,partition:0,offset:42`。未列出的分区从 earliest offset 开始。 +* `tables`：可选，按 Debezium `source.db`/`source.table` 或 Canal `database`/`table` 做包含过滤，语法与 MySQL source 的 `tables` 相同，例如 `inventory.customers` 或 `inventory.\\.*`。 +* `tables.exclude`：可选，排除匹配的表，可单独使用，也可与 `tables` 同时使用。 +* `value.format`：`debezium-json`（默认）或 `canal-json`。 +* `properties.bootstrap.servers`（必填）及 `properties.*`：Kafka Consumer 参数。 + +Debezium JSON 不推断主键，请在 Pipeline 的 `transform` 中通过 `primary-keys` 指定。Canal JSON 会使用消息里的 `pkNames` 作为主键，仍可用 transform 覆盖。写入 StarRocks 前表必须具备主键。 + +Debezium JSON 的 value 必须同时包含 `schema` 与 `payload`。不包含 schema 的 value 无法可靠识别字段类型变化，因此会被拒绝。 + +Canal JSON 用 `mysqlType` 推断列类型、用 `pkNames` 作为主键。只消费 `INSERT`/`UPDATE`/`DELETE`；`isDdl=true` 以及其他 `type` 会被跳过。Canal 的 `UPDATE` 往往只在 `old` 里放变更列，Source 会用 `old` 覆盖 `data` 拼出完整 before。 + +Transform 中的主键用于下游分区和 upsert 语义，但无法恢复 Kafka 中已经丢失的顺序；生产端仍需保证相同逻辑主键的变更进入同一个 Kafka 分区。 + +### Schema Evolution 与多分区 + +Kafka 只保证分区内有序。Source 会先在每个 Source subtask 内对其负责分区的 schema 做单调扩宽，再由 distributed schema coordinator 跨 subtask 合并。新 schema 出现后到达的旧格式消息会被转换到当前最宽 schema，不会触发类型回退。 + +Source 支持首次发现表时建表、增加 nullable 字段，以及把 schema 做成单调超集：源端删列或改名时会保留旧列（NOT NULL 会改为 nullable）并加入新列名，不会发出 Drop/Rename。历史行的新列为 null，之后行的旧列为 null。另外支持 `INT → BIGINT`、`INT → STRING`、Decimal 精度扩大等兼容扩宽。Kafka Connect 的 `string` 一律映射为 `STRING`（MySQL 的 `CHAR`/`VARCHAR`/`TEXT` 在 Debezium JSON 中都是 `string`）。从旧 offset 重放并跨过 `INT → STRING` 时，历史整型值会被转成字符串，而不会失败。类型缩窄、不兼容类型变化会明确失败。并行 Kafka Source 应配置 `schema.change.behavior: lenient`。 + +从旧 offset 重刷时，空目标表会按历史顺序执行 `CREATE → ADD/ALTER`。已有 StarRocks 表必须是历史 schema 的兼容超集；重复的 Create/Add/Alter 会按幂等方式处理，目标表额外字段必须 nullable 或有默认值。StarRocks 主键表可以通过 upsert 覆盖旧记录；duplicate-key 表全量重刷前应清表或改写新表。 如何创建 Pipeline ---------------- diff --git a/docs/content/docs/connectors/pipeline-connectors/kafka.md b/docs/content/docs/connectors/pipeline-connectors/kafka.md index 2a5cc4ed538..a69a0dc655f 100644 --- a/docs/content/docs/connectors/pipeline-connectors/kafka.md +++ b/docs/content/docs/connectors/pipeline-connectors/kafka.md @@ -26,10 +26,80 @@ under the License. # Kafka Pipeline Connector -The Kafka Pipeline connector can be used as the *Data Sink* of the pipeline, and write data to [Kafka](https://kafka.apache.org). This document describes how to set up the Kafka Pipeline connector. +The Kafka Pipeline connector can be used as a *Data Source* or *Data Sink* of the pipeline. As a source, it consumes Debezium JSON or Canal JSON changelog records and converts inferred schema changes into pipeline schema events. ## What can the connector do? * Data synchronization +* Consume Debezium JSON or Canal JSON changelog records +* Infer create table, add column, and compatible column type widening events + +Kafka Source +---------------- + +The following pipeline consumes Debezium JSON from multiple Kafka partitions and writes it to StarRocks: + +```yaml +source: + type: kafka + name: Kafka Debezium Source + topic: inventory.customers + group-id: flink-cdc-kafka-source + scan.startup.mode: group-offsets + properties.bootstrap.servers: localhost:9092 + +transform: + - source-table: inventory.\.* + primary-keys: id + +sink: + type: starrocks + name: StarRocks Sink + jdbc-url: jdbc:mysql://localhost:9030 + load-url: localhost:8030 + username: root + password: "" + +pipeline: + name: Kafka to StarRocks Pipeline + parallelism: 4 + schema.change.behavior: lenient +``` + +Kafka source options: + +* `topic`: one topic or a comma-separated topic list. +* `topic-pattern`: a regular expression for discovering topics. Configure exactly one of `topic` and `topic-pattern`. +* `group-id` (required unless `properties.group.id` is set): Kafka consumer group. +* `scan.startup.mode`: `group-offsets` (default), `earliest-offset`, `latest-offset`, `timestamp`, or `specific-offsets`. +* `scan.startup.timestamp-millis`: required when `scan.startup.mode` is `timestamp`. +* `scan.startup.specific-offsets`: required when `scan.startup.mode` is `specific-offsets`. Use `partition:0,offset:42;partition:1,offset:300` when exactly one `topic` is configured, or include a topic in each entry such as `topic:dbz.customers,partition:0,offset:42`. Partitions that are not listed start from the earliest offset. +* `tables`: optional inclusion patterns matched against Debezium `source.db`/`source.table` or Canal `database`/`table` (same selector syntax as the MySQL source, for example `inventory.customers` or `inventory.\\.*`). +* `tables.exclude`: optional exclusion patterns. Can be used alone or together with `tables`. +* `value.format`: `debezium-json` (default) or `canal-json`. +* `properties.bootstrap.servers` (required) and `properties.*`: Kafka consumer properties. + +Primary keys are not inferred from Debezium JSON. Assign them with a pipeline `transform` `primary-keys` option. Canal JSON uses `pkNames` as the table primary key when present; transform can still override them. A StarRocks sink still requires a primary key before it can create a table. + +Debezium JSON values must include the `schema` and `payload` fields. Values without an embedded schema cannot reliably describe column type changes and are rejected. + +Canal JSON values use `mysqlType` for column types and `pkNames` for primary keys. `INSERT`/`UPDATE`/`DELETE` are consumed; `isDdl=true` and other `type` values are skipped. Canal `UPDATE` records often put only changed columns in `old`: the source builds a full before image by overlaying `old` onto `data`. + +The transform primary key determines downstream partitioning and upsert semantics, but it cannot restore ordering already lost in Kafka. Producers must still send changes for the same logical primary key to the same Kafka partition. + +### Schema evolution and multiple partitions + +Kafka only guarantees ordering within a partition. The source therefore merges schemas monotonically across the partitions assigned to each source subtask and the distributed schema coordinator merges them again across subtasks. Old records arriving after a newer schema are converted to the widest known schema instead of reverting it. + +The source supports: + +* creating a table from the first record seen for a table; +* adding nullable columns; +* keeping a monotonic column superset: dropped or renamed source columns are retained (NOT NULL columns become nullable) and new names are added. Historical rows have nulls in new columns; later rows have nulls in old columns. The source does not emit drop or rename events; +* compatible type widening, such as `INT` to `BIGINT`, `INT` to `STRING`, or increasing decimal precision. Kafka Connect `string` is always mapped to `STRING` (MySQL `CHAR`/`VARCHAR`/`TEXT` all become `string` in Debezium JSON). Replaying from an old offset that spans an `INT` → `STRING` change converts historical integer values to strings instead of failing. + +Narrowing types and incompatible type changes fail explicitly. Use `schema.change.behavior: lenient` for a parallel Kafka source. + +When replaying from an old offset, an empty target table follows the historical `CREATE → ADD/ALTER` sequence. An existing StarRocks table must be a compatible superset. Replayed create/add/alter operations are idempotent; extra target columns must be nullable or have defaults. Replaying rows is safe for primary-key tables through upserts. Duplicate-key tables should be cleared or replaced before a full replay. How to create Pipeline ---------------- diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/sink/SupportsParallelMetadataSource.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/sink/SupportsParallelMetadataSource.java new file mode 100644 index 00000000000..30a11145238 --- /dev/null +++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/sink/SupportsParallelMetadataSource.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.common.sink; + +import org.apache.flink.cdc.common.annotation.Internal; + +/** + * Internal capability for sinks whose topology must adapt to sources that emit metadata from + * parallel subtasks. + */ +@Internal +public interface SupportsParallelMetadataSource { + + void setParallelMetadataSource(boolean parallelMetadataSource); +} diff --git a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposer.java b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposer.java index 4d843ac873c..28606597eb6 100644 --- a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposer.java +++ b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposer.java @@ -29,6 +29,7 @@ import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior; import org.apache.flink.cdc.common.sink.DataSink; import org.apache.flink.cdc.common.sink.DefaultDataChangeEventHashFunctionProvider; +import org.apache.flink.cdc.common.sink.SupportsParallelMetadataSource; import org.apache.flink.cdc.common.sink.TableIdHashFunctionProvider; import org.apache.flink.cdc.common.source.DataSource; import org.apache.flink.cdc.composer.PipelineComposer; @@ -172,6 +173,10 @@ private void translate(StreamExecutionEnvironment env, PipelineDef pipelineDef) resolveHashFunctionProvider(pipelineDefConfig, sinkDefinedHashFunctionProvider); boolean isParallelMetadataSource = dataSource.isParallelMetadataSource(); + if (dataSink instanceof SupportsParallelMetadataSource) { + ((SupportsParallelMetadataSource) dataSink) + .setParallelMetadataSource(isParallelMetadataSource); + } // O ---> Source DataStream stream = diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonDeserializationSchema.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonDeserializationSchema.java new file mode 100644 index 00000000000..5acbecab70d --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonDeserializationSchema.java @@ -0,0 +1,479 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.json.canal; + +import org.apache.flink.cdc.common.data.RecordData; +import org.apache.flink.cdc.common.event.AddColumnEvent; +import org.apache.flink.cdc.common.event.AlterColumnTypeEvent; +import org.apache.flink.cdc.common.event.CreateTableEvent; +import org.apache.flink.cdc.common.event.DataChangeEvent; +import org.apache.flink.cdc.common.event.Event; +import org.apache.flink.cdc.common.event.TableId; +import org.apache.flink.cdc.common.schema.Column; +import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.schema.Selectors; +import org.apache.flink.cdc.common.types.DataType; +import org.apache.flink.cdc.common.types.DataTypeRoot; +import org.apache.flink.cdc.common.types.DataTypes; +import org.apache.flink.cdc.common.types.DecimalType; +import org.apache.flink.util.Collector; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.kafka.clients.consumer.ConsumerRecord; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalMeta.IS_DDL; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalMeta.MYSQL_TYPE; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalMeta.TS; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.DATA; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.DATABASE; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.OLD; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.PK_NAMES; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.TABLE; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.TYPE; + +/** + * Deserialization schema from Canal JSON to Flink CDC pipeline internal data structure {@link + * Event}. + */ +public class CanalJsonDeserializationSchema implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String tables; + private final String tablesExclude; + + private transient ObjectMapper mapper; + private transient CanalJsonSchemaParser schemaParser; + private transient CanalJsonRecordDataConverter recordConverter; + private transient Selectors includeSelectors; + private transient Selectors excludeSelectors; + private transient Map globalTableSchemas; + private transient Map partitionTableSchemas; + + public CanalJsonDeserializationSchema() { + this(null, null); + } + + public CanalJsonDeserializationSchema(String tables, String tablesExclude) { + this.tables = tables; + this.tablesExclude = tablesExclude; + } + + public void open() { + initialize(); + } + + public void deserialize(ConsumerRecord record, Collector out) + throws IOException { + initialize(); + JsonNode root = mapper.readTree(record.value()); + JsonNode typeNode = root.path(TYPE.getFieldName()); + JsonNode databaseNode = root.path(DATABASE.getFieldName()); + JsonNode tableNode = root.path(TABLE.getFieldName()); + if (typeNode.isMissingNode() + || typeNode.isNull() + || databaseNode.isMissingNode() + || databaseNode.isNull() + || tableNode.isMissingNode() + || tableNode.isNull() + || root.path(IS_DDL.getFieldName()).asBoolean(false)) { + return; + } + + CanalJsonStruct.CanalOperation operation = + CanalJsonStruct.CanalOperation.fromFieldName(typeNode.asText()); + if (operation == null) { + return; + } + TableId tableId = TableId.tableId(databaseNode.asText(), tableNode.asText()); + if (!acceptTable(tableId)) { + return; + } + + JsonNode data = root.get(DATA.getFieldName()); + JsonNode old = root.get(OLD.getFieldName()); + JsonNode sampleRow = firstObject(data); + if (sampleRow == null) { + sampleRow = firstObject(old); + } + List primaryKeys = schemaParser.parsePrimaryKeys(root.get(PK_NAMES.getFieldName())); + Schema incomingSchema = + schemaParser.parseSchema( + root.get(MYSQL_TYPE.getFieldName()), sampleRow, primaryKeys); + if (incomingSchema.getColumnCount() == 0) { + throw failure(record, "Canal value does not contain mysqlType or row fields."); + } + + PartitionTableKey partitionTableKey = + new PartitionTableKey(record.topic(), record.partition(), tableId); + Schema partitionSchema = partitionTableSchemas.get(partitionTableKey); + if (partitionSchema != null) { + validatePartitionEvolution(record, partitionSchema, incomingSchema); + } + + TableSchemaState state = globalTableSchemas.get(tableId); + List schemaEvents = new ArrayList<>(); + if (state == null) { + state = new TableSchemaState(incomingSchema); + globalTableSchemas.put(tableId, state); + schemaEvents.add(new CreateTableEvent(tableId, incomingSchema)); + } else { + if (!primaryKeys.isEmpty() && !primaryKeys.equals(state.schema.primaryKeys())) { + throw failure( + record, + "Incompatible primary key change for table '" + + tableId + + "': " + + state.schema.primaryKeys() + + " versus " + + primaryKeys + + "."); + } + evolveGlobalSchema(record, tableId, state, incomingSchema, schemaEvents); + } + partitionTableSchemas.put(partitionTableKey, incomingSchema); + for (Event schemaEvent : schemaEvents) { + out.collect(schemaEvent); + } + + if (data == null || !data.isArray()) { + return; + } + Map meta = new LinkedHashMap<>(); + meta.put("topic", record.topic()); + meta.put("partition", String.valueOf(record.partition())); + meta.put("offset", String.valueOf(record.offset())); + JsonNode tsNode = root.get(TS.getFieldName()); + if (tsNode != null && !tsNode.isNull() && !tsNode.isMissingNode()) { + meta.put("ts", tsNode.asText()); + } + for (int i = 0; i < data.size(); i++) { + JsonNode afterRow = data.get(i); + JsonNode oldRow = old != null && old.isArray() && i < old.size() ? old.get(i) : null; + emitDataChange(tableId, operation, afterRow, oldRow, state.schema, meta, out); + } + } + + private void emitDataChange( + TableId tableId, + CanalJsonStruct.CanalOperation operation, + JsonNode afterRow, + JsonNode oldRow, + Schema schema, + Map meta, + Collector out) { + switch (operation) { + case INSERT: + out.collect( + DataChangeEvent.insertEvent( + tableId, + require(recordConverter.convertRecord(afterRow, schema), "data"), + meta)); + break; + case UPDATE: + out.collect( + DataChangeEvent.updateEvent( + tableId, + require( + recordConverter.convertUpdateBefore( + afterRow, oldRow, schema), + "old"), + require(recordConverter.convertRecord(afterRow, schema), "data"), + meta)); + break; + case DELETE: + out.collect( + DataChangeEvent.deleteEvent( + tableId, + require(recordConverter.convertRecord(afterRow, schema), "data"), + meta)); + break; + default: + throw new IllegalStateException("Unexpected Canal operation " + operation); + } + } + + private void initialize() { + if (mapper != null) { + return; + } + mapper = new ObjectMapper(); + schemaParser = new CanalJsonSchemaParser(); + recordConverter = new CanalJsonRecordDataConverter(); + globalTableSchemas = new HashMap<>(); + partitionTableSchemas = new HashMap<>(); + if (tables != null) { + includeSelectors = new Selectors.SelectorsBuilder().includeTables(tables).build(); + } + if (tablesExclude != null) { + excludeSelectors = + new Selectors.SelectorsBuilder().includeTables(tablesExclude).build(); + } + } + + private boolean acceptTable(TableId tableId) { + if (includeSelectors != null && !includeSelectors.isMatch(tableId)) { + return false; + } + return excludeSelectors == null || !excludeSelectors.isMatch(tableId); + } + + private JsonNode firstObject(JsonNode array) { + if (array == null || !array.isArray() || array.size() == 0) { + return null; + } + JsonNode first = array.get(0); + return first != null && first.isObject() ? first : null; + } + + private void validatePartitionEvolution( + ConsumerRecord record, Schema previous, Schema incoming) { + Map incomingColumns = columnsByName(incoming); + for (Column previousColumn : previous.getColumns()) { + Column incomingColumn = incomingColumns.get(previousColumn.getName()); + if (incomingColumn == null) { + continue; + } + DataType merged = mergeType(previousColumn.getType(), incomingColumn.getType()); + if (merged == null || !merged.equals(incomingColumn.getType())) { + throw failure( + record, + "Incompatible or narrowing type change for column '" + + previousColumn.getName() + + "' within Kafka partition: " + + previousColumn.getType() + + " -> " + + incomingColumn.getType() + + "."); + } + } + } + + private void evolveGlobalSchema( + ConsumerRecord record, + TableId tableId, + TableSchemaState state, + Schema incoming, + List events) { + List widestColumns = new ArrayList<>(state.schema.getColumns()); + List additions = new ArrayList<>(); + Map alteredTypes = new LinkedHashMap<>(); + Map oldTypes = new LinkedHashMap<>(); + Map currentPositions = new HashMap<>(); + for (int i = 0; i < widestColumns.size(); i++) { + currentPositions.put(widestColumns.get(i).getName(), i); + } + Map incomingColumns = columnsByName(incoming); + for (int i = 0; i < widestColumns.size(); i++) { + Column currentColumn = widestColumns.get(i); + if (!incomingColumns.containsKey(currentColumn.getName()) + && !currentColumn.getType().isNullable()) { + DataType nullableType = currentColumn.getType().nullable(); + widestColumns.set(i, Column.physicalColumn(currentColumn.getName(), nullableType)); + alteredTypes.put(currentColumn.getName(), nullableType); + oldTypes.put(currentColumn.getName(), currentColumn.getType()); + } + } + for (Column incomingColumn : incoming.getColumns()) { + Integer position = currentPositions.get(incomingColumn.getName()); + if (position == null) { + Column nullableColumn = + Column.physicalColumn( + incomingColumn.getName(), incomingColumn.getType().nullable()); + currentPositions.put(nullableColumn.getName(), widestColumns.size()); + widestColumns.add(nullableColumn); + additions.add(AddColumnEvent.last(nullableColumn)); + continue; + } + Column currentColumn = widestColumns.get(position); + DataType merged = mergeType(currentColumn.getType(), incomingColumn.getType()); + if (merged == null) { + DataType reverseMerged = + mergeType(incomingColumn.getType(), currentColumn.getType()); + if (reverseMerged != null && reverseMerged.equals(currentColumn.getType())) { + continue; + } + throw failure( + record, + "Incompatible type change for column '" + + incomingColumn.getName() + + "': " + + currentColumn.getType() + + " versus " + + incomingColumn.getType() + + "."); + } + if (!merged.equals(currentColumn.getType())) { + widestColumns.set(position, Column.physicalColumn(currentColumn.getName(), merged)); + alteredTypes.put(currentColumn.getName(), merged); + oldTypes.put(currentColumn.getName(), currentColumn.getType()); + } + } + if (!additions.isEmpty()) { + events.add(new AddColumnEvent(tableId, additions)); + } + if (!alteredTypes.isEmpty()) { + events.add(new AlterColumnTypeEvent(tableId, alteredTypes, oldTypes)); + } + if (!additions.isEmpty() || !alteredTypes.isEmpty()) { + state.schema = state.schema.copy(widestColumns); + } + } + + private DataType mergeType(DataType current, DataType incoming) { + boolean nullable = current.isNullable() || incoming.isNullable(); + DataType currentNullable = current.copy(nullable); + DataType incomingNullable = incoming.copy(nullable); + if (currentNullable.equals(incomingNullable)) { + return currentNullable; + } + if (incoming.is(DataTypeRoot.VARCHAR)) { + return DataTypes.STRING().copy(nullable); + } + int currentRank = integerRank(current.getTypeRoot()); + int incomingRank = integerRank(incoming.getTypeRoot()); + if (currentRank > 0 && incomingRank > currentRank) { + return incomingNullable; + } + if (current.is(DataTypeRoot.FLOAT) && incoming.is(DataTypeRoot.DOUBLE)) { + return incomingNullable; + } + if (current.is(DataTypeRoot.VARCHAR) + && incoming.is(DataTypeRoot.VARCHAR) + && DataTypes.getLength(incoming).orElse(0) + > DataTypes.getLength(current).orElse(0)) { + return DataTypes.VARCHAR(DataTypes.getLength(incoming).getAsInt()).copy(nullable); + } + if (current.is(DataTypeRoot.VARBINARY) + && incoming.is(DataTypeRoot.VARBINARY) + && DataTypes.getLength(incoming).orElse(0) + > DataTypes.getLength(current).orElse(0)) { + return DataTypes.VARBINARY(DataTypes.getLength(incoming).getAsInt()).copy(nullable); + } + if (current.getTypeRoot() == incoming.getTypeRoot() + && DataTypes.getPrecision(current).isPresent() + && DataTypes.getPrecision(incoming).isPresent() + && DataTypes.getPrecision(incoming).getAsInt() + > DataTypes.getPrecision(current).getAsInt()) { + return incomingNullable; + } + if (current.is(DataTypeRoot.DECIMAL) && incoming.is(DataTypeRoot.DECIMAL)) { + DecimalType left = (DecimalType) current; + DecimalType right = (DecimalType) incoming; + int scale = Math.max(left.getScale(), right.getScale()); + int integerDigits = + Math.max( + left.getPrecision() - left.getScale(), + right.getPrecision() - right.getScale()); + int precision = integerDigits + scale; + if (precision <= 38 && (precision > left.getPrecision() || scale > left.getScale())) { + return DataTypes.DECIMAL(precision, scale).copy(nullable); + } + } + return null; + } + + private int integerRank(DataTypeRoot root) { + switch (root) { + case TINYINT: + return 1; + case SMALLINT: + return 2; + case INTEGER: + return 3; + case BIGINT: + return 4; + default: + return 0; + } + } + + private Map columnsByName(Schema schema) { + Map result = new HashMap<>(); + for (Column column : schema.getColumns()) { + result.put(column.getName(), column); + } + return result; + } + + private RecordData require(RecordData record, String name) { + return Objects.requireNonNull(record, "Canal operation requires non-null " + name + "."); + } + + private IllegalArgumentException failure( + ConsumerRecord record, String message) { + return new IllegalArgumentException( + message + + " Kafka position " + + record.topic() + + "-" + + record.partition() + + "@" + + record.offset()); + } + + private static class TableSchemaState { + private Schema schema; + + private TableSchemaState(Schema schema) { + this.schema = schema; + } + } + + private static class PartitionTableKey { + private final String topic; + private final int partition; + private final TableId tableId; + + private PartitionTableKey(String topic, int partition, TableId tableId) { + this.topic = topic; + this.partition = partition; + this.tableId = tableId; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof PartitionTableKey)) { + return false; + } + PartitionTableKey that = (PartitionTableKey) object; + return partition == that.partition + && Objects.equals(topic, that.topic) + && Objects.equals(tableId, that.tableId); + } + + @Override + public int hashCode() { + return Objects.hash(topic, partition, tableId); + } + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonRecordDataConverter.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonRecordDataConverter.java new file mode 100644 index 00000000000..2fe4749847e --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonRecordDataConverter.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.json.canal; + +import org.apache.flink.cdc.common.data.DateData; +import org.apache.flink.cdc.common.data.DecimalData; +import org.apache.flink.cdc.common.data.GenericRecordData; +import org.apache.flink.cdc.common.data.RecordData; +import org.apache.flink.cdc.common.data.TimeData; +import org.apache.flink.cdc.common.data.TimestampData; +import org.apache.flink.cdc.common.data.binary.BinaryStringData; +import org.apache.flink.cdc.common.schema.Column; +import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.types.DataType; +import org.apache.flink.cdc.common.types.DecimalType; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.Base64; + +/** Converts a Canal JSON row into CDC {@link RecordData}. */ +class CanalJsonRecordDataConverter { + + private static final DateTimeFormatter CANAL_TIMESTAMP = + new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss") + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .optionalEnd() + .toFormatter(); + + RecordData convertRecord(JsonNode row, Schema targetSchema) { + if (row == null || row.isNull()) { + return null; + } + GenericRecordData result = new GenericRecordData(targetSchema.getColumnCount()); + for (int i = 0; i < targetSchema.getColumnCount(); i++) { + Column column = targetSchema.getColumns().get(i); + result.setField(i, convertValue(row.get(column.getName()), column.getType())); + } + return result; + } + + RecordData convertUpdateBefore(JsonNode after, JsonNode old, Schema targetSchema) { + if (after == null || after.isNull()) { + return null; + } + GenericRecordData result = new GenericRecordData(targetSchema.getColumnCount()); + for (int i = 0; i < targetSchema.getColumnCount(); i++) { + Column column = targetSchema.getColumns().get(i); + JsonNode node = after.get(column.getName()); + if (old != null && old.has(column.getName())) { + node = old.get(column.getName()); + } + result.setField(i, convertValue(node, column.getType())); + } + return result; + } + + private Object convertValue(JsonNode node, DataType targetType) { + if (node == null || node.isNull()) { + return null; + } + switch (targetType.getTypeRoot()) { + case TINYINT: + return (byte) Integer.parseInt(node.asText()); + case SMALLINT: + return (short) Integer.parseInt(node.asText()); + case INTEGER: + return node.isNumber() ? node.asInt() : Integer.parseInt(node.asText()); + case BIGINT: + return node.isNumber() ? node.asLong() : Long.parseLong(node.asText()); + case FLOAT: + return node.isNumber() ? (float) node.asDouble() : Float.parseFloat(node.asText()); + case DOUBLE: + return node.isNumber() ? node.asDouble() : Double.parseDouble(node.asText()); + case BOOLEAN: + if (node.isBoolean()) { + return node.asBoolean(); + } + String booleanText = node.asText(); + return "1".equals(booleanText) || Boolean.parseBoolean(booleanText); + case CHAR: + case VARCHAR: + return BinaryStringData.fromString(node.asText()); + case BINARY: + case VARBINARY: + return decodeBinary(node); + case DECIMAL: + DecimalType decimalType = (DecimalType) targetType; + return DecimalData.fromBigDecimal( + new BigDecimal(node.asText()), + decimalType.getPrecision(), + decimalType.getScale()); + case DATE: + return DateData.fromLocalDate(LocalDate.parse(node.asText())); + case TIME_WITHOUT_TIME_ZONE: + return TimeData.fromLocalTime(LocalTime.parse(node.asText())); + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return TimestampData.fromLocalDateTime(parseTimestamp(node.asText())); + default: + throw new IllegalArgumentException( + "Unsupported target type " + targetType.asSummaryString() + "."); + } + } + + private byte[] decodeBinary(JsonNode node) { + if (node.isBinary()) { + try { + return node.binaryValue(); + } catch (Exception e) { + throw new IllegalArgumentException("Cannot decode Canal binary value.", e); + } + } + try { + return Base64.getDecoder().decode(node.asText().getBytes(StandardCharsets.UTF_8)); + } catch (IllegalArgumentException ignored) { + return node.asText().getBytes(StandardCharsets.UTF_8); + } + } + + private LocalDateTime parseTimestamp(String value) { + if (value.indexOf('T') >= 0) { + return LocalDateTime.parse(value); + } + return LocalDateTime.parse(value, CANAL_TIMESTAMP); + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonSchemaParser.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonSchemaParser.java new file mode 100644 index 00000000000..1579645b0ae --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonSchemaParser.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.json.canal; + +import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.types.DataType; +import org.apache.flink.cdc.common.types.DataTypes; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Parses Canal {@code mysqlType} / row fields into a CDC {@link Schema}. */ +class CanalJsonSchemaParser { + + private static final Pattern MYSQL_TYPE_PATTERN = + Pattern.compile( + "^([A-Z]+)(?:\\s+UNSIGNED)?(?:\\s+ZEROFILL)?(?:\\((\\d+)(?:,\\s*(\\d+))?\\))?(?:\\s+UNSIGNED)?(?:\\s+ZEROFILL)?.*$"); + + Schema parseSchema(JsonNode mysqlType, JsonNode sampleRow, List primaryKeys) { + Set primaryKeySet = new HashSet<>(primaryKeys); + Schema.Builder builder = Schema.newBuilder(); + if (mysqlType != null && mysqlType.isObject() && mysqlType.size() > 0) { + Iterator> fields = mysqlType.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + DataType type = parseMysqlType(field.getValue().asText()); + builder.physicalColumn( + field.getKey(), + primaryKeySet.contains(field.getKey()) ? type.notNull() : type.nullable()); + } + } else if (sampleRow != null && sampleRow.isObject()) { + Iterator names = sampleRow.fieldNames(); + while (names.hasNext()) { + String name = names.next(); + DataType type = DataTypes.STRING(); + builder.physicalColumn( + name, primaryKeySet.contains(name) ? type.notNull() : type.nullable()); + } + } + if (!primaryKeys.isEmpty()) { + builder.primaryKey(primaryKeys); + } + return builder.build(); + } + + List parsePrimaryKeys(JsonNode pkNames) { + List result = new ArrayList<>(); + if (pkNames == null || !pkNames.isArray()) { + return result; + } + for (JsonNode name : pkNames) { + if (name != null && !name.isNull() && !name.asText().isEmpty()) { + result.add(name.asText()); + } + } + return result; + } + + DataType parseMysqlType(String mysqlType) { + if (mysqlType == null || mysqlType.trim().isEmpty()) { + return DataTypes.STRING(); + } + String normalized = mysqlType.trim().toUpperCase(); + Matcher matcher = MYSQL_TYPE_PATTERN.matcher(normalized); + String typeName = normalized; + Integer length = null; + Integer scale = null; + if (matcher.matches()) { + typeName = matcher.group(1); + if (matcher.group(2) != null) { + length = Integer.parseInt(matcher.group(2)); + } + if (matcher.group(3) != null) { + scale = Integer.parseInt(matcher.group(3)); + } + } + boolean unsigned = normalized.contains("UNSIGNED") || normalized.equals("SERIAL"); + switch (typeName) { + case "BIT": + if (length == null || length <= 1) { + return DataTypes.BOOLEAN(); + } + return DataTypes.VARBINARY((length + 7) / 8); + case "BOOL": + case "BOOLEAN": + return DataTypes.BOOLEAN(); + case "TINYINT": + return unsigned ? DataTypes.SMALLINT() : DataTypes.TINYINT(); + case "SMALLINT": + return unsigned ? DataTypes.INT() : DataTypes.SMALLINT(); + case "MEDIUMINT": + case "INT": + case "INTEGER": + case "YEAR": + return unsigned ? DataTypes.BIGINT() : DataTypes.INT(); + case "BIGINT": + case "SERIAL": + return unsigned ? DataTypes.DECIMAL(20, 0) : DataTypes.BIGINT(); + case "FLOAT": + return DataTypes.FLOAT(); + case "REAL": + case "DOUBLE": + return DataTypes.DOUBLE(); + case "DECIMAL": + case "NUMERIC": + case "FIXED": + int precision = length == null ? 10 : Math.min(38, length); + int decimalScale = scale == null ? 0 : Math.min(scale, precision); + return DataTypes.DECIMAL(precision, decimalScale); + case "DATE": + return DataTypes.DATE(); + case "TIME": + return length == null ? DataTypes.TIME(0) : DataTypes.TIME(Math.min(length, 9)); + case "DATETIME": + case "TIMESTAMP": + return length == null + ? DataTypes.TIMESTAMP(0) + : DataTypes.TIMESTAMP(Math.min(length, 9)); + case "BINARY": + case "VARBINARY": + case "TINYBLOB": + case "BLOB": + case "MEDIUMBLOB": + case "LONGBLOB": + return DataTypes.BYTES(); + case "CHAR": + case "VARCHAR": + case "TINYTEXT": + case "TEXT": + case "MEDIUMTEXT": + case "LONGTEXT": + case "JSON": + case "ENUM": + case "SET": + default: + return DataTypes.STRING(); + } + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonSerializationSchema.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonSerializationSchema.java index a9a0209a666..a944e42967f 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonSerializationSchema.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonSerializationSchema.java @@ -27,6 +27,7 @@ import org.apache.flink.cdc.common.types.utils.DataTypeUtils; import org.apache.flink.cdc.common.utils.SchemaUtils; import org.apache.flink.cdc.connectors.kafka.json.TableSchemaInfo; +import org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalOperation; import org.apache.flink.formats.common.TimestampFormat; import org.apache.flink.formats.json.JsonFormatOptions; import org.apache.flink.formats.json.JsonRowDataSerializationSchema; @@ -44,6 +45,12 @@ import java.util.Map; import static java.lang.String.format; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.DATA; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.DATABASE; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.OLD; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.PK_NAMES; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.TABLE; +import static org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonStruct.CanalStruct.TYPE; import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType; /** @@ -56,10 +63,6 @@ public class CanalJsonSerializationSchema implements SerializationSchema private static final long serialVersionUID = 1L; - private static final StringData OP_INSERT = StringData.fromString("INSERT"); - private static final StringData OP_DELETE = StringData.fromString("DELETE"); - private static final StringData OP_UPDATE = StringData.fromString("UPDATE"); - private transient GenericRowData reuseGenericRowData; /** The serializer to serialize Canal JSON data. */ @@ -139,11 +142,13 @@ public byte[] serialize(Event event) { DataChangeEvent dataChangeEvent = (DataChangeEvent) event; reuseGenericRowData.setField( - 3, StringData.fromString(dataChangeEvent.tableId().getSchemaName())); + DATABASE.getPosition(), + StringData.fromString(dataChangeEvent.tableId().getSchemaName())); reuseGenericRowData.setField( - 4, StringData.fromString(dataChangeEvent.tableId().getTableName())); + TABLE.getPosition(), + StringData.fromString(dataChangeEvent.tableId().getTableName())); reuseGenericRowData.setField( - 5, + PK_NAMES.getPosition(), new GenericArrayData( jsonSerializers .get(dataChangeEvent.tableId()) @@ -155,9 +160,9 @@ public byte[] serialize(Event event) { try { switch (dataChangeEvent.op()) { case INSERT: - reuseGenericRowData.setField(0, null); + reuseGenericRowData.setField(OLD.getPosition(), null); reuseGenericRowData.setField( - 1, + DATA.getPosition(), new GenericArrayData( new RowData[] { jsonSerializers @@ -165,15 +170,16 @@ public byte[] serialize(Event event) { .getRowDataFromRecordData( dataChangeEvent.after(), false) })); - reuseGenericRowData.setField(2, OP_INSERT); + reuseGenericRowData.setField( + TYPE.getPosition(), toStringData(CanalOperation.INSERT)); return jsonSerializers .get(dataChangeEvent.tableId()) .getSerializationSchema() .serialize(reuseGenericRowData); case DELETE: - reuseGenericRowData.setField(0, null); + reuseGenericRowData.setField(OLD.getPosition(), null); reuseGenericRowData.setField( - 1, + DATA.getPosition(), new GenericArrayData( new RowData[] { jsonSerializers @@ -181,7 +187,8 @@ public byte[] serialize(Event event) { .getRowDataFromRecordData( dataChangeEvent.before(), false) })); - reuseGenericRowData.setField(2, OP_DELETE); + reuseGenericRowData.setField( + TYPE.getPosition(), toStringData(CanalOperation.DELETE)); return jsonSerializers .get(dataChangeEvent.tableId()) .getSerializationSchema() @@ -189,7 +196,7 @@ public byte[] serialize(Event event) { case UPDATE: case REPLACE: reuseGenericRowData.setField( - 0, + OLD.getPosition(), new GenericArrayData( new RowData[] { jsonSerializers @@ -198,7 +205,7 @@ public byte[] serialize(Event event) { dataChangeEvent.before(), false) })); reuseGenericRowData.setField( - 1, + DATA.getPosition(), new GenericArrayData( new RowData[] { jsonSerializers @@ -206,7 +213,8 @@ public byte[] serialize(Event event) { .getRowDataFromRecordData( dataChangeEvent.after(), false) })); - reuseGenericRowData.setField(2, OP_UPDATE); + reuseGenericRowData.setField( + TYPE.getPosition(), toStringData(CanalOperation.UPDATE)); return jsonSerializers .get(dataChangeEvent.tableId()) .getSerializationSchema() @@ -222,6 +230,10 @@ public byte[] serialize(Event event) { } } + private static StringData toStringData(CanalOperation operation) { + return StringData.fromString(operation.getFieldName()); + } + /** * Refer to Canal @@ -230,12 +242,16 @@ public byte[] serialize(Event event) { private static RowType createJsonRowType(DataType databaseSchema) { return (RowType) DataTypes.ROW( - DataTypes.FIELD("old", DataTypes.ARRAY(databaseSchema)), - DataTypes.FIELD("data", DataTypes.ARRAY(databaseSchema)), - DataTypes.FIELD("type", DataTypes.STRING()), - DataTypes.FIELD("database", DataTypes.STRING()), - DataTypes.FIELD("table", DataTypes.STRING()), - DataTypes.FIELD("pkNames", DataTypes.ARRAY(DataTypes.STRING()))) + DataTypes.FIELD( + OLD.getFieldName(), DataTypes.ARRAY(databaseSchema)), + DataTypes.FIELD( + DATA.getFieldName(), DataTypes.ARRAY(databaseSchema)), + DataTypes.FIELD(TYPE.getFieldName(), DataTypes.STRING()), + DataTypes.FIELD(DATABASE.getFieldName(), DataTypes.STRING()), + DataTypes.FIELD(TABLE.getFieldName(), DataTypes.STRING()), + DataTypes.FIELD( + PK_NAMES.getFieldName(), + DataTypes.ARRAY(DataTypes.STRING()))) .getLogicalType(); } } diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonStruct.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonStruct.java new file mode 100644 index 00000000000..caa8b550e73 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/canal/CanalJsonStruct.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.json.canal; + +/** Canal JSON struct. */ +public class CanalJsonStruct { + + enum CanalStruct { + OLD(0, "old"), + DATA(1, "data"), + TYPE(2, "type"), + DATABASE(3, "database"), + TABLE(4, "table"), + PK_NAMES(5, "pkNames"); + + private final int position; + private final String fieldName; + + CanalStruct(int position, String fieldName) { + this.position = position; + this.fieldName = fieldName; + } + + public int getPosition() { + return position; + } + + public String getFieldName() { + return fieldName; + } + } + + enum CanalMeta { + MYSQL_TYPE(0, "mysqlType"), + IS_DDL(1, "isDdl"), + TS(2, "ts"); + + private final int position; + private final String fieldName; + + CanalMeta(int position, String fieldName) { + this.position = position; + this.fieldName = fieldName; + } + + public int getPosition() { + return position; + } + + public String getFieldName() { + return fieldName; + } + } + + enum CanalOperation { + INSERT("INSERT"), + UPDATE("UPDATE"), + DELETE("DELETE"); + + private final String fieldName; + + CanalOperation(String fieldName) { + this.fieldName = fieldName; + } + + public String getFieldName() { + return fieldName; + } + + static CanalOperation fromFieldName(String fieldName) { + for (CanalOperation operation : values()) { + if (operation.fieldName.equalsIgnoreCase(fieldName)) { + return operation; + } + } + return null; + } + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonDeserializationSchema.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonDeserializationSchema.java new file mode 100644 index 00000000000..bfeee9787b8 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonDeserializationSchema.java @@ -0,0 +1,432 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.json.debezium; + +import org.apache.flink.cdc.common.data.RecordData; +import org.apache.flink.cdc.common.event.AddColumnEvent; +import org.apache.flink.cdc.common.event.AlterColumnTypeEvent; +import org.apache.flink.cdc.common.event.CreateTableEvent; +import org.apache.flink.cdc.common.event.DataChangeEvent; +import org.apache.flink.cdc.common.event.Event; +import org.apache.flink.cdc.common.event.TableId; +import org.apache.flink.cdc.common.schema.Column; +import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.schema.Selectors; +import org.apache.flink.cdc.common.types.DataType; +import org.apache.flink.cdc.common.types.DataTypeRoot; +import org.apache.flink.cdc.common.types.DataTypes; +import org.apache.flink.cdc.common.types.DecimalType; +import org.apache.flink.util.Collector; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.kafka.clients.consumer.ConsumerRecord; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonStruct.DebeziumPayload.AFTER; +import static org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonStruct.DebeziumPayload.BEFORE; +import static org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonStruct.DebeziumPayload.OPERATION; +import static org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonStruct.DebeziumPayload.SOURCE; +import static org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonStruct.DebeziumSource.DATABASE; +import static org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonStruct.DebeziumSource.TABLE; +import static org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonStruct.DebeziumStruct.PAYLOAD; +import static org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonStruct.DebeziumStruct.SCHEMA; + +/** + * Deserialization schema from Debezium JSON to Flink CDC pipeline internal data structure {@link + * Event}. + */ +public class DebeziumJsonDeserializationSchema implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String tables; + private final String tablesExclude; + + private transient ObjectMapper mapper; + private transient DebeziumJsonSchemaParser schemaParser; + private transient DebeziumJsonRecordDataConverter recordConverter; + private transient Selectors includeSelectors; + private transient Selectors excludeSelectors; + private transient Map globalTableSchemas; + private transient Map partitionTableSchemas; + + public DebeziumJsonDeserializationSchema() { + this(null, null); + } + + public DebeziumJsonDeserializationSchema(String tables, String tablesExclude) { + this.tables = tables; + this.tablesExclude = tablesExclude; + } + + public void open() { + initialize(); + } + + public void deserialize(ConsumerRecord record, Collector out) + throws IOException { + initialize(); + JsonNode root = mapper.readTree(record.value()); + JsonNode payload = root.path(PAYLOAD.getFieldName()); + JsonNode source = payload.path(SOURCE.getFieldName()); + JsonNode opNode = payload.path(OPERATION.getFieldName()); + if (payload.isMissingNode() + || payload.isNull() + || opNode.isMissingNode() + || opNode.isNull() + || source.path(DATABASE.getFieldName()).isMissingNode() + || source.path(TABLE.getFieldName()).isMissingNode()) { + return; + } + + DebeziumJsonStruct.DebeziumOperation operation = + DebeziumJsonStruct.DebeziumOperation.fromCode(opNode.asText()); + if (operation == null) { + return; + } + TableId tableId = + TableId.tableId( + source.path(DATABASE.getFieldName()).asText(), + source.path(TABLE.getFieldName()).asText()); + if (!acceptTable(tableId)) { + return; + } + JsonNode rowSchemaNode = + schemaParser.findFieldSchema( + root.path(SCHEMA.getFieldName()), AFTER.getFieldName()); + if (rowSchemaNode == null) { + rowSchemaNode = + schemaParser.findFieldSchema( + root.path(SCHEMA.getFieldName()), BEFORE.getFieldName()); + } + if (rowSchemaNode == null || !rowSchemaNode.path("fields").isArray()) { + throw failure(record, "Debezium value does not contain a before/after row schema."); + } + + Schema incomingSchema = schemaParser.parseSchema(rowSchemaNode); + PartitionTableKey partitionTableKey = + new PartitionTableKey(record.topic(), record.partition(), tableId); + Schema partitionSchema = partitionTableSchemas.get(partitionTableKey); + if (partitionSchema != null) { + validatePartitionEvolution(record, partitionSchema, incomingSchema); + } + + TableSchemaState state = globalTableSchemas.get(tableId); + List schemaEvents = new ArrayList<>(); + if (state == null) { + state = new TableSchemaState(incomingSchema); + globalTableSchemas.put(tableId, state); + schemaEvents.add(new CreateTableEvent(tableId, incomingSchema)); + } else { + evolveGlobalSchema(record, tableId, state, incomingSchema, schemaEvents); + } + partitionTableSchemas.put(partitionTableKey, incomingSchema); + for (Event schemaEvent : schemaEvents) { + out.collect(schemaEvent); + } + + Map meta = new LinkedHashMap<>(); + meta.put("topic", record.topic()); + meta.put("partition", String.valueOf(record.partition())); + meta.put("offset", String.valueOf(record.offset())); + RecordData before = + recordConverter.convertRecord(payload.get(BEFORE.getFieldName()), state.schema); + RecordData after = + recordConverter.convertRecord(payload.get(AFTER.getFieldName()), state.schema); + switch (operation) { + case READ: + case CREATE: + out.collect(DataChangeEvent.insertEvent(tableId, require(after, "after"), meta)); + break; + case UPDATE: + out.collect( + DataChangeEvent.updateEvent( + tableId, require(before, "before"), require(after, "after"), meta)); + break; + case DELETE: + out.collect(DataChangeEvent.deleteEvent(tableId, require(before, "before"), meta)); + break; + default: + throw new IllegalStateException("Unexpected Debezium operation " + operation); + } + } + + private void initialize() { + if (mapper != null) { + return; + } + mapper = new ObjectMapper(); + schemaParser = new DebeziumJsonSchemaParser(); + recordConverter = new DebeziumJsonRecordDataConverter(); + globalTableSchemas = new HashMap<>(); + partitionTableSchemas = new HashMap<>(); + if (tables != null) { + includeSelectors = new Selectors.SelectorsBuilder().includeTables(tables).build(); + } + if (tablesExclude != null) { + excludeSelectors = + new Selectors.SelectorsBuilder().includeTables(tablesExclude).build(); + } + } + + private boolean acceptTable(TableId tableId) { + if (includeSelectors != null && !includeSelectors.isMatch(tableId)) { + return false; + } + return excludeSelectors == null || !excludeSelectors.isMatch(tableId); + } + + private void validatePartitionEvolution( + ConsumerRecord record, Schema previous, Schema incoming) { + Map incomingColumns = columnsByName(incoming); + for (Column previousColumn : previous.getColumns()) { + Column incomingColumn = incomingColumns.get(previousColumn.getName()); + if (incomingColumn == null) { + // Dropped or renamed source columns stay in the widest schema. New names are + // added later; missing values are coerced to null. + continue; + } + DataType merged = mergeType(previousColumn.getType(), incomingColumn.getType()); + if (merged == null || !merged.equals(incomingColumn.getType())) { + throw failure( + record, + "Incompatible or narrowing type change for column '" + + previousColumn.getName() + + "' within Kafka partition: " + + previousColumn.getType() + + " -> " + + incomingColumn.getType() + + "."); + } + } + } + + private void evolveGlobalSchema( + ConsumerRecord record, + TableId tableId, + TableSchemaState state, + Schema incoming, + List events) { + List widestColumns = new ArrayList<>(state.schema.getColumns()); + List additions = new ArrayList<>(); + Map alteredTypes = new LinkedHashMap<>(); + Map oldTypes = new LinkedHashMap<>(); + Map currentPositions = new HashMap<>(); + for (int i = 0; i < widestColumns.size(); i++) { + currentPositions.put(widestColumns.get(i).getName(), i); + } + Map incomingColumns = columnsByName(incoming); + for (int i = 0; i < widestColumns.size(); i++) { + Column currentColumn = widestColumns.get(i); + if (!incomingColumns.containsKey(currentColumn.getName()) + && !currentColumn.getType().isNullable()) { + DataType nullableType = currentColumn.getType().nullable(); + widestColumns.set(i, Column.physicalColumn(currentColumn.getName(), nullableType)); + alteredTypes.put(currentColumn.getName(), nullableType); + oldTypes.put(currentColumn.getName(), currentColumn.getType()); + } + } + for (Column incomingColumn : incoming.getColumns()) { + Integer position = currentPositions.get(incomingColumn.getName()); + if (position == null) { + Column nullableColumn = + Column.physicalColumn( + incomingColumn.getName(), incomingColumn.getType().nullable()); + currentPositions.put(nullableColumn.getName(), widestColumns.size()); + widestColumns.add(nullableColumn); + additions.add(AddColumnEvent.last(nullableColumn)); + continue; + } + Column currentColumn = widestColumns.get(position); + DataType merged = mergeType(currentColumn.getType(), incomingColumn.getType()); + if (merged == null) { + DataType reverseMerged = + mergeType(incomingColumn.getType(), currentColumn.getType()); + if (reverseMerged != null && reverseMerged.equals(currentColumn.getType())) { + continue; + } + throw failure( + record, + "Incompatible type change for column '" + + incomingColumn.getName() + + "': " + + currentColumn.getType() + + " versus " + + incomingColumn.getType() + + "."); + } + if (!merged.equals(currentColumn.getType())) { + widestColumns.set(position, Column.physicalColumn(currentColumn.getName(), merged)); + alteredTypes.put(currentColumn.getName(), merged); + oldTypes.put(currentColumn.getName(), currentColumn.getType()); + } + } + if (!additions.isEmpty()) { + events.add(new AddColumnEvent(tableId, additions)); + } + if (!alteredTypes.isEmpty()) { + events.add(new AlterColumnTypeEvent(tableId, alteredTypes, oldTypes)); + } + if (!additions.isEmpty() || !alteredTypes.isEmpty()) { + state.schema = state.schema.copy(widestColumns); + } + } + + private DataType mergeType(DataType current, DataType incoming) { + boolean nullable = current.isNullable() || incoming.isNullable(); + DataType currentNullable = current.copy(nullable); + DataType incomingNullable = incoming.copy(nullable); + if (currentNullable.equals(incomingNullable)) { + return currentNullable; + } + // STRING is the universal widening target used by SchemaMergingUtils. Replay of an + // INT → STRING change (MySQL ALTER to VARCHAR) must follow the same rule. + if (incoming.is(DataTypeRoot.VARCHAR)) { + return DataTypes.STRING().copy(nullable); + } + int currentRank = integerRank(current.getTypeRoot()); + int incomingRank = integerRank(incoming.getTypeRoot()); + if (currentRank > 0 && incomingRank > currentRank) { + return incomingNullable; + } + if (current.is(DataTypeRoot.FLOAT) && incoming.is(DataTypeRoot.DOUBLE)) { + return incomingNullable; + } + if (current.is(DataTypeRoot.VARCHAR) + && incoming.is(DataTypeRoot.VARCHAR) + && DataTypes.getLength(incoming).orElse(0) + > DataTypes.getLength(current).orElse(0)) { + return DataTypes.VARCHAR(DataTypes.getLength(incoming).getAsInt()).copy(nullable); + } + if (current.is(DataTypeRoot.VARBINARY) + && incoming.is(DataTypeRoot.VARBINARY) + && DataTypes.getLength(incoming).orElse(0) + > DataTypes.getLength(current).orElse(0)) { + return DataTypes.VARBINARY(DataTypes.getLength(incoming).getAsInt()).copy(nullable); + } + if (current.getTypeRoot() == incoming.getTypeRoot() + && DataTypes.getPrecision(current).isPresent() + && DataTypes.getPrecision(incoming).isPresent() + && DataTypes.getPrecision(incoming).getAsInt() + > DataTypes.getPrecision(current).getAsInt()) { + return incomingNullable; + } + if (current.is(DataTypeRoot.DECIMAL) && incoming.is(DataTypeRoot.DECIMAL)) { + DecimalType left = (DecimalType) current; + DecimalType right = (DecimalType) incoming; + int scale = Math.max(left.getScale(), right.getScale()); + int integerDigits = + Math.max( + left.getPrecision() - left.getScale(), + right.getPrecision() - right.getScale()); + int precision = integerDigits + scale; + if (precision <= 38 && (precision > left.getPrecision() || scale > left.getScale())) { + return DataTypes.DECIMAL(precision, scale).copy(nullable); + } + } + return null; + } + + private int integerRank(DataTypeRoot root) { + switch (root) { + case TINYINT: + return 1; + case SMALLINT: + return 2; + case INTEGER: + return 3; + case BIGINT: + return 4; + default: + return 0; + } + } + + private Map columnsByName(Schema schema) { + Map result = new HashMap<>(); + for (Column column : schema.getColumns()) { + result.put(column.getName(), column); + } + return result; + } + + private RecordData require(RecordData record, String name) { + return Objects.requireNonNull(record, "Debezium operation requires non-null " + name + "."); + } + + private IllegalArgumentException failure( + ConsumerRecord record, String message) { + return new IllegalArgumentException( + message + + " Kafka position " + + record.topic() + + "-" + + record.partition() + + "@" + + record.offset()); + } + + private static class TableSchemaState { + private Schema schema; + + private TableSchemaState(Schema schema) { + this.schema = schema; + } + } + + private static class PartitionTableKey { + private final String topic; + private final int partition; + private final TableId tableId; + + private PartitionTableKey(String topic, int partition, TableId tableId) { + this.topic = topic; + this.partition = partition; + this.tableId = tableId; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof PartitionTableKey)) { + return false; + } + PartitionTableKey that = (PartitionTableKey) object; + return partition == that.partition + && Objects.equals(topic, that.topic) + && Objects.equals(tableId, that.tableId); + } + + @Override + public int hashCode() { + return Objects.hash(topic, partition, tableId); + } + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonRecordDataConverter.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonRecordDataConverter.java new file mode 100644 index 00000000000..873f198476c --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonRecordDataConverter.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.json.debezium; + +import org.apache.flink.cdc.common.data.DateData; +import org.apache.flink.cdc.common.data.DecimalData; +import org.apache.flink.cdc.common.data.GenericRecordData; +import org.apache.flink.cdc.common.data.RecordData; +import org.apache.flink.cdc.common.data.TimeData; +import org.apache.flink.cdc.common.data.TimestampData; +import org.apache.flink.cdc.common.data.ZonedTimestampData; +import org.apache.flink.cdc.common.data.binary.BinaryStringData; +import org.apache.flink.cdc.common.schema.Column; +import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.types.DataType; +import org.apache.flink.cdc.common.types.DataTypes; +import org.apache.flink.cdc.common.types.DecimalType; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Base64; + +/** Converts a Debezium JSON row into CDC {@link RecordData}. */ +class DebeziumJsonRecordDataConverter { + + RecordData convertRecord(JsonNode row, Schema targetSchema) { + if (row == null || row.isNull()) { + return null; + } + GenericRecordData result = new GenericRecordData(targetSchema.getColumnCount()); + for (int i = 0; i < targetSchema.getColumnCount(); i++) { + Column column = targetSchema.getColumns().get(i); + result.setField(i, convertValue(row.get(column.getName()), column.getType())); + } + return result; + } + + private Object convertValue(JsonNode node, DataType targetType) { + if (node == null || node.isNull()) { + return null; + } + switch (targetType.getTypeRoot()) { + case TINYINT: + return (byte) node.asInt(); + case SMALLINT: + return (short) node.asInt(); + case INTEGER: + return node.asInt(); + case BIGINT: + return node.asLong(); + case FLOAT: + return (float) node.asDouble(); + case DOUBLE: + return node.asDouble(); + case BOOLEAN: + return node.asBoolean(); + case CHAR: + case VARCHAR: + return BinaryStringData.fromString(node.asText()); + case BINARY: + case VARBINARY: + return node.isBinary() + ? binaryValue(node) + : Base64.getDecoder() + .decode(node.asText().getBytes(StandardCharsets.UTF_8)); + case DECIMAL: + DecimalType decimalType = (DecimalType) targetType; + return DecimalData.fromBigDecimal( + decimalValue(node, decimalType.getScale()), + decimalType.getPrecision(), + decimalType.getScale()); + case DATE: + return node.isIntegralNumber() + ? DateData.fromEpochDay(node.asInt()) + : DateData.fromIsoLocalDateString(node.asText()); + case TIME_WITHOUT_TIME_ZONE: + return node.isIntegralNumber() + ? TimeData.fromNanoOfDay( + normalizeTimeToNanos( + node.asLong(), + DataTypes.getPrecision(targetType).orElse(3))) + : TimeData.fromIsoLocalTimeString(node.asText()); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return node.isIntegralNumber() + ? TimestampData.fromLocalDateTime( + LocalDateTime.ofInstant( + Instant.ofEpochMilli( + normalizeTimestampToMillis( + node.asLong(), + DataTypes.getPrecision(targetType) + .orElse(3))), + ZoneOffset.UTC)) + : TimestampData.fromLocalDateTime(LocalDateTime.parse(node.asText())); + case TIMESTAMP_WITH_TIME_ZONE: + return ZonedTimestampData.fromZonedDateTime(ZonedDateTime.parse(node.asText())); + default: + throw new IllegalArgumentException( + "Unsupported target type " + targetType.asSummaryString() + "."); + } + } + + private byte[] binaryValue(JsonNode node) { + try { + return node.binaryValue(); + } catch (IOException e) { + throw new IllegalArgumentException("Cannot decode Debezium binary value.", e); + } + } + + private BigDecimal decimalValue(JsonNode node, int scale) { + if (node.isNumber()) { + return node.decimalValue(); + } + try { + return new BigDecimal(node.asText()); + } catch (NumberFormatException ignored) { + byte[] unscaled = Base64.getDecoder().decode(node.asText()); + return new BigDecimal(new BigInteger(unscaled), scale); + } + } + + private long normalizeTimeToNanos(long value, int precision) { + if (precision <= 3) { + return value * 1_000_000L; + } + if (precision <= 6) { + return value * 1_000L; + } + return value; + } + + private long normalizeTimestampToMillis(long value, int precision) { + if (precision > 6) { + return value / 1_000_000L; + } + if (precision > 3) { + return value / 1_000L; + } + return value; + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonSchemaParser.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonSchemaParser.java new file mode 100644 index 00000000000..e3601651a4e --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonSchemaParser.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.json.debezium; + +import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.types.DataType; +import org.apache.flink.cdc.common.types.DataTypes; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; + +import java.util.Optional; + +/** Parses a Debezium / Kafka Connect field schema into a CDC {@link Schema}. */ +class DebeziumJsonSchemaParser { + + Schema parseSchema(JsonNode rowSchema) { + Schema.Builder builder = Schema.newBuilder(); + for (JsonNode field : rowSchema.path("fields")) { + builder.physicalColumn( + requiredText(field, "field", "Debezium row schema field"), parseType(field)); + } + return builder.build(); + } + + JsonNode findFieldSchema(JsonNode envelopeSchema, String fieldName) { + for (JsonNode field : envelopeSchema.path("fields")) { + if (fieldName.equals(field.path("field").asText())) { + return field; + } + } + return null; + } + + private DataType parseType(JsonNode schema) { + String logicalName = schema.path("name").asText(""); + DataType type; + switch (logicalName) { + case "io.debezium.time.Date": + type = DataTypes.DATE(); + break; + case "io.debezium.time.Time": + type = DataTypes.TIME(3); + break; + case "io.debezium.time.MicroTime": + type = DataTypes.TIME(6); + break; + case "io.debezium.time.NanoTime": + type = DataTypes.TIME(9); + break; + case "io.debezium.time.Timestamp": + type = DataTypes.TIMESTAMP(3); + break; + case "io.debezium.time.MicroTimestamp": + type = DataTypes.TIMESTAMP(6); + break; + case "io.debezium.time.NanoTimestamp": + type = DataTypes.TIMESTAMP(9); + break; + case "io.debezium.time.ZonedTimestamp": + type = DataTypes.TIMESTAMP_TZ(9); + break; + case "io.debezium.time.Year": + type = DataTypes.INT(); + break; + case "io.debezium.data.Bits": + type = + DataTypes.VARBINARY( + positiveParameter(schema, "length").orElse(Integer.MAX_VALUE)); + break; + case "io.debezium.data.Enum": + case "io.debezium.data.Json": + type = DataTypes.STRING(); + break; + case "org.apache.kafka.connect.data.Decimal": + int scale = schema.path("parameters").path("scale").asInt(0); + int precision = + schema.path("parameters").path("connect.decimal.precision").asInt(38); + type = DataTypes.DECIMAL(Math.min(38, precision), Math.min(scale, precision)); + break; + default: + type = parsePrimitiveType(schema); + } + return schema.path("optional").asBoolean(true) ? type.nullable() : type.notNull(); + } + + private DataType parsePrimitiveType(JsonNode schema) { + String type = schema.path("type").asText(); + switch (type) { + case "int8": + return DataTypes.TINYINT(); + case "int16": + return DataTypes.SMALLINT(); + case "int32": + return DataTypes.INT(); + case "int64": + return DataTypes.BIGINT(); + case "float": + case "float32": + return DataTypes.FLOAT(); + case "double": + case "float64": + return DataTypes.DOUBLE(); + case "boolean": + return DataTypes.BOOLEAN(); + case "bytes": + return DataTypes.BYTES(); + case "string": + // Kafka Connect has no VARCHAR; MySQL CHAR/VARCHAR/TEXT all become string. + return DataTypes.STRING(); + default: + throw new IllegalArgumentException( + "Unsupported Debezium schema type '" + type + "'."); + } + } + + private Optional positiveParameter(JsonNode schema, String name) { + JsonNode value = schema.path("parameters").path(name); + if (value.isMissingNode() || value.isNull()) { + return Optional.empty(); + } + try { + int parsed = Integer.parseInt(value.asText()); + return parsed > 0 ? Optional.of(parsed) : Optional.empty(); + } catch (NumberFormatException ignored) { + return Optional.empty(); + } + } + + private String requiredText(JsonNode node, String field, String description) { + JsonNode value = node.get(field); + if (value == null || value.isNull() || value.asText().isEmpty()) { + throw new IllegalArgumentException(description + " is missing '" + field + "'."); + } + return value.asText(); + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonStruct.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonStruct.java index e1c314b9b5a..74cc742d1f8 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonStruct.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/json/debezium/DebeziumJsonStruct.java @@ -84,4 +84,30 @@ public String getFieldName() { return fieldName; } } + + enum DebeziumOperation { + READ("r"), + CREATE("c"), + UPDATE("u"), + DELETE("d"); + + private final String code; + + DebeziumOperation(String code) { + this.code = code; + } + + public String getCode() { + return code; + } + + static DebeziumOperation fromCode(String code) { + for (DebeziumOperation operation : values()) { + if (operation.code.equals(code)) { + return operation; + } + } + return null; + } + } } diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSource.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSource.java new file mode 100644 index 00000000000..34e9624be25 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSource.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.flink.cdc.common.annotation.Internal; +import org.apache.flink.cdc.common.event.Event; +import org.apache.flink.cdc.common.event.TableId; +import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.source.DataSource; +import org.apache.flink.cdc.common.source.EventSourceProvider; +import org.apache.flink.cdc.common.source.FlinkSourceProvider; +import org.apache.flink.cdc.common.source.MetadataAccessor; +import org.apache.flink.cdc.connectors.kafka.json.JsonSerializationType; +import org.apache.flink.connector.kafka.source.KafkaSource; +import org.apache.flink.connector.kafka.source.KafkaSourceBuilder; +import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer; + +import org.apache.kafka.common.TopicPartition; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Pattern; + +/** A pipeline {@link DataSource} backed by Flink's {@link KafkaSource}. */ +@Internal +public class KafkaDataSource implements DataSource { + + private final List topics; + private final @Nullable String topicPattern; + private final Properties properties; + private final String startupMode; + private final @Nullable Long startupTimestampMillis; + private final Map specificOffsets; + private final @Nullable String tables; + private final @Nullable String tablesExclude; + private final JsonSerializationType valueFormat; + + KafkaDataSource( + List topics, + @Nullable String topicPattern, + Properties properties, + String startupMode, + @Nullable Long startupTimestampMillis, + Map specificOffsets, + @Nullable String tables, + @Nullable String tablesExclude, + JsonSerializationType valueFormat) { + this.topics = topics; + this.topicPattern = topicPattern; + this.properties = properties; + this.startupMode = startupMode; + this.startupTimestampMillis = startupTimestampMillis; + this.specificOffsets = specificOffsets; + this.tables = tables; + this.tablesExclude = tablesExclude; + this.valueFormat = valueFormat; + } + + @Override + public EventSourceProvider getEventSourceProvider() { + KafkaSourceBuilder builder = + KafkaSource.builder() + .setProperties(properties) + .setDeserializer( + new PipelineKafkaRecordDeserializationSchema( + valueFormat, tables, tablesExclude)); + if (topicPattern == null) { + builder.setTopics(topics); + } else { + builder.setTopicPattern(Pattern.compile(topicPattern)); + } + switch (startupMode.toLowerCase(Locale.ROOT)) { + case "earliest-offset": + builder.setStartingOffsets(OffsetsInitializer.earliest()); + break; + case "latest-offset": + builder.setStartingOffsets(OffsetsInitializer.latest()); + break; + case "group-offsets": + builder.setStartingOffsets(OffsetsInitializer.committedOffsets()); + break; + case "timestamp": + builder.setStartingOffsets(OffsetsInitializer.timestamp(startupTimestampMillis)); + break; + case "specific-offsets": + builder.setStartingOffsets(OffsetsInitializer.offsets(specificOffsets)); + break; + default: + throw new IllegalArgumentException( + "Unsupported scan.startup.mode '" + + startupMode + + "'. Supported values are earliest-offset, latest-offset, " + + "group-offsets, timestamp, and specific-offsets."); + } + return FlinkSourceProvider.of(builder.build()); + } + + @Override + public MetadataAccessor getMetadataAccessor() { + return new MetadataAccessor() { + private UnsupportedOperationException unsupported() { + return new UnsupportedOperationException( + "Kafka source discovers table metadata from consumed records."); + } + + @Override + public List listNamespaces() { + throw unsupported(); + } + + @Override + public List listSchemas(@Nullable String namespace) { + throw unsupported(); + } + + @Override + public List listTables( + @Nullable String namespace, @Nullable String schemaName) { + throw unsupported(); + } + + @Override + public Schema getTableSchema(TableId tableId) { + throw unsupported(); + } + }; + } + + @Override + public boolean isParallelMetadataSource() { + return true; + } + + List getTopics() { + return topics; + } + + @Nullable + String getTopicPattern() { + return topicPattern; + } + + Properties getProperties() { + return properties; + } + + String getStartupMode() { + return startupMode; + } + + @Nullable + Long getStartupTimestampMillis() { + return startupTimestampMillis; + } + + Map getSpecificOffsets() { + return Collections.unmodifiableMap(specificOffsets); + } + + @Nullable + String getTables() { + return tables; + } + + @Nullable + String getTablesExclude() { + return tablesExclude; + } + + JsonSerializationType getValueFormat() { + return valueFormat; + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceFactory.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceFactory.java new file mode 100644 index 00000000000..0d61473700c --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceFactory.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.flink.cdc.common.annotation.Internal; +import org.apache.flink.cdc.common.configuration.ConfigOption; +import org.apache.flink.cdc.common.configuration.Configuration; +import org.apache.flink.cdc.common.factories.DataSourceFactory; +import org.apache.flink.cdc.common.factories.Factory; +import org.apache.flink.cdc.common.factories.FactoryHelper; +import org.apache.flink.cdc.common.source.DataSource; +import org.apache.flink.table.api.ValidationException; + +import org.apache.kafka.common.TopicPartition; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +/** A {@link Factory} for creating Kafka pipeline sources. */ +@Internal +public class KafkaDataSourceFactory implements DataSourceFactory { + + public static final String IDENTIFIER = "kafka"; + + @Override + public DataSource createDataSource(Context context) { + FactoryHelper.createFactoryHelper(this, context) + .validateExcept(KafkaDataSourceOptions.PROPERTIES_PREFIX); + Configuration configuration = context.getFactoryConfiguration(); + boolean hasTopics = configuration.getOptional(KafkaDataSourceOptions.TOPIC).isPresent(); + boolean hasTopicPattern = + configuration.getOptional(KafkaDataSourceOptions.TOPIC_PATTERN).isPresent(); + if (hasTopics == hasTopicPattern) { + throw new ValidationException( + "Exactly one of options 'topic' and 'topic-pattern' must be configured."); + } + List topics = Collections.emptyList(); + String topicPattern = null; + if (hasTopics) { + topics = + Arrays.stream(configuration.get(KafkaDataSourceOptions.TOPIC).split(",")) + .map(String::trim) + .filter(topic -> !topic.isEmpty()) + .collect(Collectors.toList()); + if (topics.isEmpty()) { + throw new ValidationException("Option 'topic' must contain at least one topic."); + } + } else { + topicPattern = configuration.get(KafkaDataSourceOptions.TOPIC_PATTERN).trim(); + if (topicPattern.isEmpty()) { + throw new ValidationException("Option 'topic-pattern' must not be empty."); + } + } + + Properties properties = new Properties(); + for (Map.Entry entry : configuration.toMap().entrySet()) { + if (entry.getKey().startsWith(KafkaDataSourceOptions.PROPERTIES_PREFIX)) { + properties.setProperty( + entry.getKey().substring(KafkaDataSourceOptions.PROPERTIES_PREFIX.length()), + entry.getValue()); + } + } + configuration + .getOptional(KafkaDataSourceOptions.GROUP_ID) + .ifPresent(groupId -> properties.setProperty("group.id", groupId)); + if (!properties.containsKey("bootstrap.servers")) { + throw new ValidationException( + "Kafka bootstrap servers must be configured with 'properties.bootstrap.servers'."); + } + if (!properties.containsKey("group.id")) { + throw new ValidationException( + "Kafka consumer group must be configured with 'group-id' or 'properties.group.id'."); + } + + String startupMode = + configuration + .get(KafkaDataSourceOptions.SCAN_STARTUP_MODE) + .toLowerCase(Locale.ROOT); + Long startupTimestampMillis = + configuration + .getOptional(KafkaDataSourceOptions.SCAN_STARTUP_TIMESTAMP_MILLIS) + .orElse(null); + String specificOffsetsSpec = + configuration + .getOptional(KafkaDataSourceOptions.SCAN_STARTUP_SPECIFIC_OFFSETS) + .orElse(null); + Map specificOffsets = Collections.emptyMap(); + switch (startupMode) { + case "earliest-offset": + case "latest-offset": + case "group-offsets": + if (startupTimestampMillis != null) { + throw new ValidationException( + "Option 'scan.startup.timestamp-millis' is only supported when 'scan.startup.mode' is 'timestamp'."); + } + if (specificOffsetsSpec != null) { + throw new ValidationException( + "Option 'scan.startup.specific-offsets' is only supported when 'scan.startup.mode' is 'specific-offsets'."); + } + break; + case "timestamp": + if (startupTimestampMillis == null) { + throw new ValidationException( + "Option 'scan.startup.timestamp-millis' is required when 'scan.startup.mode' is 'timestamp'."); + } + if (specificOffsetsSpec != null) { + throw new ValidationException( + "Option 'scan.startup.specific-offsets' is only supported when 'scan.startup.mode' is 'specific-offsets'."); + } + break; + case "specific-offsets": + if (specificOffsetsSpec == null || specificOffsetsSpec.trim().isEmpty()) { + throw new ValidationException( + "Option 'scan.startup.specific-offsets' is required when 'scan.startup.mode' is 'specific-offsets'."); + } + if (startupTimestampMillis != null) { + throw new ValidationException( + "Option 'scan.startup.timestamp-millis' is only supported when 'scan.startup.mode' is 'timestamp'."); + } + String defaultTopic = topics.size() == 1 ? topics.get(0) : null; + try { + specificOffsets = KafkaStartupOffsets.parse(specificOffsetsSpec, defaultTopic); + } catch (IllegalArgumentException e) { + throw new ValidationException(e.getMessage(), e); + } + break; + default: + throw new ValidationException( + "Unsupported scan.startup.mode '" + + startupMode + + "'. Supported values are earliest-offset, latest-offset, " + + "group-offsets, timestamp, and specific-offsets."); + } + + return new KafkaDataSource( + topics, + topicPattern, + properties, + startupMode, + startupTimestampMillis, + specificOffsets, + blankToNull(configuration.getOptional(KafkaDataSourceOptions.TABLES).orElse(null)), + blankToNull( + configuration + .getOptional(KafkaDataSourceOptions.TABLES_EXCLUDE) + .orElse(null)), + configuration.get(KafkaDataSourceOptions.VALUE_FORMAT)); + } + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + return Collections.emptySet(); + } + + @Override + public Set> optionalOptions() { + return new HashSet<>( + Arrays.asList( + KafkaDataSourceOptions.TOPIC, + KafkaDataSourceOptions.TOPIC_PATTERN, + KafkaDataSourceOptions.GROUP_ID, + KafkaDataSourceOptions.SCAN_STARTUP_MODE, + KafkaDataSourceOptions.SCAN_STARTUP_TIMESTAMP_MILLIS, + KafkaDataSourceOptions.SCAN_STARTUP_SPECIFIC_OFFSETS, + KafkaDataSourceOptions.TABLES, + KafkaDataSourceOptions.TABLES_EXCLUDE, + KafkaDataSourceOptions.VALUE_FORMAT)); + } + + private static String blankToNull(String value) { + if (value == null || value.trim().isEmpty()) { + return null; + } + return value.trim(); + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceOptions.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceOptions.java new file mode 100644 index 00000000000..9c88c2a4332 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceOptions.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.flink.cdc.common.configuration.ConfigOption; +import org.apache.flink.cdc.connectors.kafka.json.JsonSerializationType; + +import static org.apache.flink.cdc.common.configuration.ConfigOptions.key; + +/** Options for the Kafka pipeline source. */ +public class KafkaDataSourceOptions { + + public static final String PROPERTIES_PREFIX = "properties."; + + public static final ConfigOption TOPIC = + key("topic") + .stringType() + .noDefaultValue() + .withDescription("Comma-separated Kafka topics to consume."); + + public static final ConfigOption TOPIC_PATTERN = + key("topic-pattern") + .stringType() + .noDefaultValue() + .withDescription("Regular expression matching Kafka topics to consume."); + + public static final ConfigOption GROUP_ID = + key("group-id") + .stringType() + .noDefaultValue() + .withDescription("Kafka consumer group id."); + + public static final ConfigOption SCAN_STARTUP_MODE = + key("scan.startup.mode") + .stringType() + .defaultValue("group-offsets") + .withDescription( + "Startup mode. Supported values are earliest-offset, latest-offset, " + + "group-offsets, timestamp, and specific-offsets."); + + public static final ConfigOption SCAN_STARTUP_TIMESTAMP_MILLIS = + key("scan.startup.timestamp-millis") + .longType() + .noDefaultValue() + .withDescription( + "Optional timestamp used in case of \"timestamp\" startup mode."); + + public static final ConfigOption SCAN_STARTUP_SPECIFIC_OFFSETS = + key("scan.startup.specific-offsets") + .stringType() + .noDefaultValue() + .withDescription( + "Partition offsets used in case of \"specific-offsets\" startup mode. " + + "Use 'partition:0,offset:42;partition:1,offset:300' when exactly one topic " + + "is configured, or include a topic in each entry such as " + + "'topic:dbz.customers,partition:0,offset:42'. Unspecified partitions " + + "start from the earliest offset."); + + public static final ConfigOption TABLES = + key("tables") + .stringType() + .noDefaultValue() + .withDescription( + "Optional table inclusion patterns matched against Debezium source.db " + + "and source.table. Regular expressions are supported. The dot (.) is " + + "treated as a delimiter for database and table names. " + + "eg. inventory.customers, inventory.user_table_[0-9]+"); + + public static final ConfigOption TABLES_EXCLUDE = + key("tables.exclude") + .stringType() + .noDefaultValue() + .withDescription( + "Optional table exclusion patterns matched against Debezium source.db " + + "and source.table. Regular expressions are supported. Can be used " + + "alone or together with 'tables'."); + + public static final ConfigOption VALUE_FORMAT = + key("value.format") + .enumType(JsonSerializationType.class) + .defaultValue(JsonSerializationType.DEBEZIUM_JSON) + .withDescription( + "Value format of Kafka records. Supported values are debezium-json " + + "and canal-json. Default is debezium-json."); + + private KafkaDataSourceOptions() {} +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaStartupOffsets.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaStartupOffsets.java new file mode 100644 index 00000000000..69568454908 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/KafkaStartupOffsets.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.kafka.common.TopicPartition; + +import javax.annotation.Nullable; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Parses {@code scan.startup.specific-offsets} into Kafka topic-partition offsets. + * + *

Supported entries are {@code partition:0,offset:42} when a default topic is provided, or + * {@code topic:dbz.customers,partition:0,offset:42}. Multiple entries are separated by {@code ;}. + */ +class KafkaStartupOffsets { + + private KafkaStartupOffsets() {} + + static Map parse(String spec, @Nullable String defaultTopic) { + if (spec == null || spec.trim().isEmpty()) { + throw new IllegalArgumentException( + "Option 'scan.startup.specific-offsets' must not be empty."); + } + Map result = new LinkedHashMap<>(); + for (String rawEntry : spec.split(";")) { + String entry = rawEntry.trim(); + if (entry.isEmpty()) { + continue; + } + String topic = defaultTopic; + Integer partition = null; + Long offset = null; + for (String rawPart : entry.split(",")) { + String part = rawPart.trim(); + int colon = part.indexOf(':'); + if (colon <= 0 || colon == part.length() - 1) { + throw new IllegalArgumentException( + "Invalid specific-offsets entry '" + entry + "'."); + } + String key = part.substring(0, colon).trim(); + String value = part.substring(colon + 1).trim(); + switch (key) { + case "topic": + topic = value; + break; + case "partition": + partition = parseInteger(value, "partition", entry); + break; + case "offset": + offset = parseLong(value, "offset", entry); + break; + default: + throw new IllegalArgumentException( + "Unknown key '" + + key + + "' in specific-offsets entry '" + + entry + + "'."); + } + } + if (topic == null || topic.isEmpty()) { + throw new IllegalArgumentException( + "Each specific-offsets entry must include 'topic' unless exactly one topic is configured."); + } + if (partition == null || offset == null) { + throw new IllegalArgumentException( + "Each specific-offsets entry must include 'partition' and 'offset': '" + + entry + + "'."); + } + result.put(new TopicPartition(topic, partition), offset); + } + if (result.isEmpty()) { + throw new IllegalArgumentException( + "Option 'scan.startup.specific-offsets' must contain at least one entry."); + } + return result; + } + + private static int parseInteger(String value, String name, String entry) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Invalid " + + name + + " '" + + value + + "' in specific-offsets entry '" + + entry + + "'.", + e); + } + } + + private static long parseLong(String value, String name, String entry) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Invalid " + + name + + " '" + + value + + "' in specific-offsets entry '" + + entry + + "'.", + e); + } + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchema.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchema.java new file mode 100644 index 00000000000..e2d4a42f0b8 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchema.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.cdc.common.event.Event; +import org.apache.flink.cdc.connectors.kafka.json.JsonSerializationType; +import org.apache.flink.cdc.connectors.kafka.json.canal.CanalJsonDeserializationSchema; +import org.apache.flink.cdc.connectors.kafka.json.debezium.DebeziumJsonDeserializationSchema; +import org.apache.flink.connector.kafka.source.reader.deserializer.KafkaRecordDeserializationSchema; +import org.apache.flink.util.Collector; + +import org.apache.kafka.clients.consumer.ConsumerRecord; + +import java.io.IOException; + +/** + * A {@link KafkaRecordDeserializationSchema} to deserialize Kafka records into pipeline {@link + * Event}s. + * + *

Tombstone records are skipped. Value bytes are parsed by the configured {@code value.format}. + */ +public class PipelineKafkaRecordDeserializationSchema + implements KafkaRecordDeserializationSchema { + + private static final long serialVersionUID = 1L; + + private final JsonSerializationType valueFormat; + private final DebeziumJsonDeserializationSchema debeziumDeserialization; + private final CanalJsonDeserializationSchema canalDeserialization; + + public PipelineKafkaRecordDeserializationSchema() { + this(JsonSerializationType.DEBEZIUM_JSON, null, null); + } + + public PipelineKafkaRecordDeserializationSchema(String tables, String tablesExclude) { + this(JsonSerializationType.DEBEZIUM_JSON, tables, tablesExclude); + } + + public PipelineKafkaRecordDeserializationSchema( + JsonSerializationType valueFormat, String tables, String tablesExclude) { + this.valueFormat = valueFormat == null ? JsonSerializationType.DEBEZIUM_JSON : valueFormat; + switch (this.valueFormat) { + case CANAL_JSON: + this.canalDeserialization = + new CanalJsonDeserializationSchema(tables, tablesExclude); + this.debeziumDeserialization = null; + break; + case DEBEZIUM_JSON: + default: + this.debeziumDeserialization = + new DebeziumJsonDeserializationSchema(tables, tablesExclude); + this.canalDeserialization = null; + } + } + + @Override + public void open(DeserializationSchema.InitializationContext context) { + if (debeziumDeserialization != null) { + debeziumDeserialization.open(); + } else { + canalDeserialization.open(); + } + } + + @Override + public void deserialize(ConsumerRecord record, Collector out) + throws IOException { + if (record.value() == null) { + return; + } + if (valueFormat == JsonSerializationType.CANAL_JSON) { + canalDeserialization.deserialize(record, out); + } else { + debeziumDeserialization.deserialize(record, out); + } + } + + @Override + public TypeInformation getProducedType() { + return TypeInformation.of(Event.class); + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory index fa13cd60099..232fd44f61e 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory @@ -14,3 +14,4 @@ # limitations under the License. org.apache.flink.cdc.connectors.kafka.sink.KafkaDataSinkFactory +org.apache.flink.cdc.connectors.kafka.source.KafkaDataSourceFactory diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceFactoryTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceFactoryTest.java new file mode 100644 index 00000000000..e24a8190e93 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/KafkaDataSourceFactoryTest.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.flink.cdc.common.configuration.Configuration; +import org.apache.flink.cdc.common.factories.DataSourceFactory; +import org.apache.flink.cdc.common.factories.FactoryHelper; +import org.apache.flink.cdc.common.source.DataSource; +import org.apache.flink.cdc.composer.utils.FactoryDiscoveryUtils; +import org.apache.flink.cdc.connectors.kafka.json.JsonSerializationType; +import org.apache.flink.table.api.ValidationException; + +import org.apache.kafka.common.TopicPartition; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** Tests for {@link KafkaDataSourceFactory}. */ +class KafkaDataSourceFactoryTest { + + @Test + void testFactoryDiscoveryAndOptions() { + DataSourceFactory factory = + FactoryDiscoveryUtils.getFactoryByIdentifier("kafka", DataSourceFactory.class); + Map options = new HashMap<>(); + options.put("topic", "orders-a, orders-b"); + options.put("group-id", "pipeline-group"); + options.put("properties.bootstrap.servers", "localhost:9092"); + options.put("properties.client.id", "pipeline-client"); + Configuration configuration = Configuration.fromMap(options); + + DataSource source = + factory.createDataSource( + new FactoryHelper.DefaultContext( + configuration, + configuration, + Thread.currentThread().getContextClassLoader())); + + Assertions.assertThat(source).isInstanceOf(KafkaDataSource.class); + KafkaDataSource kafkaSource = (KafkaDataSource) source; + Assertions.assertThat(kafkaSource.getTopics()).containsExactly("orders-a", "orders-b"); + Assertions.assertThat(kafkaSource.getProperties()) + .containsEntry("group.id", "pipeline-group") + .containsEntry("client.id", "pipeline-client"); + Assertions.assertThat(source.isParallelMetadataSource()).isTrue(); + } + + @Test + void testTopicPattern() { + Map options = validConnectionOptions(); + options.put("topic-pattern", "orders-.*"); + + KafkaDataSource source = (KafkaDataSource) createSource(options); + + Assertions.assertThat(source.getTopics()).isEmpty(); + Assertions.assertThat(source.getTopicPattern()).isEqualTo("orders-.*"); + } + + @Test + void testTopicAndTopicPatternAreMutuallyExclusive() { + Map options = validConnectionOptions(); + options.put("topic", "orders"); + options.put("topic-pattern", "orders-.*"); + + Assertions.assertThatThrownBy(() -> createSource(options)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("Exactly one of options 'topic' and 'topic-pattern'"); + + options.remove("topic"); + options.remove("topic-pattern"); + Assertions.assertThatThrownBy(() -> createSource(options)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("Exactly one of options 'topic' and 'topic-pattern'"); + } + + @Test + void testTimestampStartupRequiresMillis() { + Map options = validConnectionOptions(); + options.put("topic", "orders"); + options.put("scan.startup.mode", "timestamp"); + + Assertions.assertThatThrownBy(() -> createSource(options)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("scan.startup.timestamp-millis"); + + options.put("scan.startup.timestamp-millis", "1710000000000"); + KafkaDataSource source = (KafkaDataSource) createSource(options); + Assertions.assertThat(source.getStartupMode()).isEqualTo("timestamp"); + Assertions.assertThat(source.getStartupTimestampMillis()).isEqualTo(1710000000000L); + } + + @Test + void testSpecificOffsetsStartup() { + Map options = validConnectionOptions(); + options.put("topic", "orders"); + options.put("scan.startup.mode", "specific-offsets"); + + Assertions.assertThatThrownBy(() -> createSource(options)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("scan.startup.specific-offsets"); + + options.put("scan.startup.specific-offsets", "partition:0,offset:42"); + KafkaDataSource source = (KafkaDataSource) createSource(options); + Assertions.assertThat(source.getStartupMode()).isEqualTo("specific-offsets"); + Assertions.assertThat(source.getSpecificOffsets()) + .containsEntry(new TopicPartition("orders", 0), 42L); + } + + @Test + void testSpecificOffsetsWithoutTopicFailsForMultipleTopics() { + Map options = validConnectionOptions(); + options.put("topic", "orders-a,orders-b"); + options.put("scan.startup.mode", "specific-offsets"); + options.put("scan.startup.specific-offsets", "partition:0,offset:42"); + + Assertions.assertThatThrownBy(() -> createSource(options)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("must include 'topic'"); + } + + @Test + void testSpecificOffsetsWithoutTopicFailsForTopicPattern() { + Map options = validConnectionOptions(); + options.put("topic-pattern", "orders-.*"); + options.put("scan.startup.mode", "specific-offsets"); + options.put("scan.startup.specific-offsets", "partition:0,offset:42"); + + Assertions.assertThatThrownBy(() -> createSource(options)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("must include 'topic'"); + } + + @Test + void testInvalidSpecificOffsetsFails() { + Map options = validConnectionOptions(); + options.put("topic", "orders"); + options.put("scan.startup.mode", "specific-offsets"); + options.put("scan.startup.specific-offsets", "partition:0,offset:abc"); + + Assertions.assertThatThrownBy(() -> createSource(options)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("offset"); + } + + @Test + void testTablesOptionsArePreserved() { + Map options = validConnectionOptions(); + options.put("topic", "orders"); + options.put("tables", "inventory.customers"); + options.put("tables.exclude", "inventory.orders"); + + KafkaDataSource source = (KafkaDataSource) createSource(options); + Assertions.assertThat(source.getTables()).isEqualTo("inventory.customers"); + Assertions.assertThat(source.getTablesExclude()).isEqualTo("inventory.orders"); + Assertions.assertThat(source.getValueFormat()) + .isEqualTo(JsonSerializationType.DEBEZIUM_JSON); + } + + @Test + void testValueFormatCanalJson() { + Map options = validConnectionOptions(); + options.put("topic", "orders"); + options.put("value.format", "canal-json"); + + KafkaDataSource source = (KafkaDataSource) createSource(options); + Assertions.assertThat(source.getValueFormat()).isEqualTo(JsonSerializationType.CANAL_JSON); + } + + @Test + void testInvalidValueFormatFails() { + Map options = validConnectionOptions(); + options.put("topic", "orders"); + options.put("value.format", "avro"); + + Assertions.assertThatThrownBy(() -> createSource(options)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("value.format") + .hasMessageContaining("avro"); + } + + @Test + void testRequiresConnectionAndGroupProperties() { + KafkaDataSourceFactory factory = new KafkaDataSourceFactory(); + Configuration configuration = + Configuration.fromMap(Collections.singletonMap("topic", "orders")); + + Assertions.assertThatThrownBy( + () -> + factory.createDataSource( + new FactoryHelper.DefaultContext( + configuration, + configuration, + Thread.currentThread().getContextClassLoader()))) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("properties.bootstrap.servers"); + } + + private static Map validConnectionOptions() { + Map options = new HashMap<>(); + options.put("group-id", "pipeline-group"); + options.put("properties.bootstrap.servers", "localhost:9092"); + return options; + } + + private static DataSource createSource(Map options) { + Configuration configuration = Configuration.fromMap(options); + return new KafkaDataSourceFactory() + .createDataSource( + new FactoryHelper.DefaultContext( + configuration, + configuration, + Thread.currentThread().getContextClassLoader())); + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/KafkaStartupOffsetsTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/KafkaStartupOffsetsTest.java new file mode 100644 index 00000000000..2e070d9c627 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/KafkaStartupOffsetsTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.kafka.common.TopicPartition; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +/** Tests for {@link KafkaStartupOffsets}. */ +class KafkaStartupOffsetsTest { + + @Test + void testParseSingleTopicEntries() { + Map offsets = + KafkaStartupOffsets.parse("partition:0,offset:42;partition:1,offset:300", "orders"); + + Assertions.assertThat(offsets) + .containsEntry(new TopicPartition("orders", 0), 42L) + .containsEntry(new TopicPartition("orders", 1), 300L) + .hasSize(2); + } + + @Test + void testParseMultiTopicEntries() { + Map offsets = + KafkaStartupOffsets.parse( + "topic:dbz.customers,partition:0,offset:42;topic:dbz.orders,partition:1,offset:10", + null); + + Assertions.assertThat(offsets) + .containsEntry(new TopicPartition("dbz.customers", 0), 42L) + .containsEntry(new TopicPartition("dbz.orders", 1), 10L) + .hasSize(2); + } + + @Test + void testExplicitTopicOverridesDefault() { + Map offsets = + KafkaStartupOffsets.parse("topic:other,partition:2,offset:7", "orders"); + + Assertions.assertThat(offsets) + .containsExactly(Assertions.entry(new TopicPartition("other", 2), 7L)); + } + + @Test + void testMissingTopicWithoutDefaultFails() { + Assertions.assertThatThrownBy( + () -> KafkaStartupOffsets.parse("partition:0,offset:42", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must include 'topic'"); + } + + @Test + void testMissingPartitionOrOffsetFails() { + Assertions.assertThatThrownBy(() -> KafkaStartupOffsets.parse("partition:0", "orders")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("partition") + .hasMessageContaining("offset"); + } + + @Test + void testNonNumericOffsetFails() { + Assertions.assertThatThrownBy( + () -> KafkaStartupOffsets.parse("partition:0,offset:abc", "orders")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("offset") + .hasMessageContaining("abc"); + } + + @Test + void testUnknownKeyFails() { + Assertions.assertThatThrownBy( + () -> KafkaStartupOffsets.parse("partition:0,offset:1,foo:bar", "orders")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unknown key"); + } + + @Test + void testEmptySpecFails() { + Assertions.assertThatThrownBy(() -> KafkaStartupOffsets.parse(" ; ; ", "orders")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("at least one entry"); + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchemaCanalTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchemaCanalTest.java new file mode 100644 index 00000000000..99cf4b96c21 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchemaCanalTest.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.flink.cdc.common.event.AddColumnEvent; +import org.apache.flink.cdc.common.event.AlterColumnTypeEvent; +import org.apache.flink.cdc.common.event.CreateTableEvent; +import org.apache.flink.cdc.common.event.DataChangeEvent; +import org.apache.flink.cdc.common.event.Event; +import org.apache.flink.cdc.common.event.OperationType; +import org.apache.flink.cdc.common.types.DataTypeRoot; +import org.apache.flink.cdc.common.types.DataTypes; +import org.apache.flink.cdc.connectors.kafka.json.JsonSerializationType; +import org.apache.flink.util.Collector; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** Canal JSON tests for {@link PipelineKafkaRecordDeserializationSchema}. */ +class PipelineKafkaRecordDeserializationSchemaCanalTest { + + @Test + void testInsertUpdateDeleteAndPrimaryKeys() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema( + JsonSerializationType.CANAL_JSON, null, null); + TestCollector collector = new TestCollector(); + + deserializer.deserialize( + record( + 1, + canal( + "INSERT", + "inventory", + "products", + "[\"id\"]", + "{\"id\":\"INTEGER\",\"name\":\"VARCHAR(255)\",\"weight\":\"FLOAT\"}", + "[{\"id\":\"111\",\"name\":\"scooter\",\"weight\":\"5.18\"}]", + "null")), + collector); + deserializer.deserialize( + record( + 2, + canal( + "UPDATE", + "inventory", + "products", + "[\"id\"]", + "{\"id\":\"INTEGER\",\"name\":\"VARCHAR(255)\",\"weight\":\"FLOAT\"}", + "[{\"id\":\"111\",\"name\":\"scooter\",\"weight\":\"5.18\"}]", + "[{\"weight\":\"5.15\"}]")), + collector); + deserializer.deserialize( + record( + 3, + canal( + "DELETE", + "inventory", + "products", + "[\"id\"]", + "{\"id\":\"INTEGER\",\"name\":\"VARCHAR(255)\",\"weight\":\"FLOAT\"}", + "[{\"id\":\"111\",\"name\":\"scooter\",\"weight\":\"5.18\"}]", + "null")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly( + "CreateTableEvent", + "DataChangeEvent", + "DataChangeEvent", + "DataChangeEvent"); + CreateTableEvent createTable = (CreateTableEvent) collector.events.get(0); + Assertions.assertThat(createTable.getSchema().primaryKeys()).containsExactly("id"); + Assertions.assertThat( + createTable + .getSchema() + .getColumn("name") + .orElseThrow(AssertionError::new) + .getType()) + .isEqualTo(DataTypes.STRING().nullable()); + Assertions.assertThat( + collector.events.subList(1, 4).stream() + .map(event -> ((DataChangeEvent) event).op())) + .containsExactly(OperationType.INSERT, OperationType.UPDATE, OperationType.DELETE); + + DataChangeEvent insert = (DataChangeEvent) collector.events.get(1); + Assertions.assertThat(insert.tableId().toString()).isEqualTo("inventory.products"); + Assertions.assertThat(insert.after().getInt(0)).isEqualTo(111); + Assertions.assertThat(insert.after().getFloat(2)).isEqualTo(5.18f); + + DataChangeEvent update = (DataChangeEvent) collector.events.get(2); + Assertions.assertThat(update.before().getInt(0)).isEqualTo(111); + Assertions.assertThat(update.before().getString(1).toString()).isEqualTo("scooter"); + Assertions.assertThat(update.before().getFloat(2)).isEqualTo(5.15f); + Assertions.assertThat(update.after().getFloat(2)).isEqualTo(5.18f); + } + + @Test + void testMultipleRowsInOneRecord() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema( + JsonSerializationType.CANAL_JSON, null, null); + TestCollector collector = new TestCollector(); + deserializer.deserialize( + record( + 1, + canal( + "INSERT", + "inventory", + "products", + "[\"id\"]", + "{\"id\":\"INTEGER\",\"name\":\"VARCHAR(255)\"}", + "[{\"id\":\"1\",\"name\":\"a\"},{\"id\":\"2\",\"name\":\"b\"}]", + "null")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent", "DataChangeEvent"); + Assertions.assertThat(((DataChangeEvent) collector.events.get(1)).after().getInt(0)) + .isEqualTo(1); + Assertions.assertThat(((DataChangeEvent) collector.events.get(2)).after().getInt(0)) + .isEqualTo(2); + } + + @Test + void testSkipDdlAndUnknownType() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema( + JsonSerializationType.CANAL_JSON, null, null); + TestCollector collector = new TestCollector(); + deserializer.deserialize( + record( + 1, + bytes( + "{\"data\":null,\"database\":\"inventory\",\"isDdl\":true,\"table\":\"products\",\"type\":\"CREATE\"}")), + collector); + deserializer.deserialize( + record( + 2, + bytes( + "{\"data\":[],\"database\":\"inventory\",\"table\":\"products\",\"type\":\"QUERY\"}")), + collector); + + Assertions.assertThat(collector.events).isEmpty(); + } + + @Test + void testTablesFilter() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema( + JsonSerializationType.CANAL_JSON, "inventory.products", "inventory.orders"); + TestCollector collector = new TestCollector(); + deserializer.deserialize( + record( + 1, + canal( + "INSERT", + "inventory", + "products", + "[\"id\"]", + "{\"id\":\"INTEGER\"}", + "[{\"id\":\"1\"}]", + "null")), + collector); + deserializer.deserialize( + record( + 2, + canal( + "INSERT", + "inventory", + "orders", + "[\"id\"]", + "{\"id\":\"INTEGER\"}", + "[{\"id\":\"2\"}]", + "null")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent"); + Assertions.assertThat(((DataChangeEvent) collector.events.get(1)).tableId().toString()) + .isEqualTo("inventory.products"); + } + + @Test + void testAddColumnAndTypeWidening() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema( + JsonSerializationType.CANAL_JSON, null, null); + TestCollector collector = new TestCollector(); + deserializer.deserialize( + record( + 1, + canal( + "INSERT", + "inventory", + "products", + "[\"id\"]", + "{\"id\":\"INTEGER\",\"name\":\"VARCHAR(255)\"}", + "[{\"id\":\"1\",\"name\":\"a\"}]", + "null")), + collector); + deserializer.deserialize( + record( + 2, + canal( + "INSERT", + "inventory", + "products", + "[\"id\"]", + "{\"id\":\"BIGINT\",\"name\":\"VARCHAR(255)\",\"email\":\"VARCHAR(255)\"}", + "[{\"id\":\"2\",\"name\":\"b\",\"email\":\"b@example.com\"}]", + "null")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly( + "CreateTableEvent", + "DataChangeEvent", + "AddColumnEvent", + "AlterColumnTypeEvent", + "DataChangeEvent"); + AddColumnEvent addColumn = (AddColumnEvent) collector.events.get(2); + Assertions.assertThat(addColumn.getAddedColumns()) + .extracting(column -> column.getAddColumn().getName()) + .containsExactly("email"); + AlterColumnTypeEvent alter = (AlterColumnTypeEvent) collector.events.get(3); + Assertions.assertThat(alter.getTypeMapping().get("id").getTypeRoot()) + .isEqualTo(DataTypeRoot.BIGINT); + } + + private static ConsumerRecord record(long offset, byte[] value) { + return new ConsumerRecord<>("canal.inventory.products", 0, offset, null, value); + } + + private static byte[] canal( + String type, + String database, + String table, + String pkNames, + String mysqlType, + String data, + String old) { + return bytes( + "{\"data\":" + + data + + ",\"database\":\"" + + database + + "\",\"isDdl\":false,\"mysqlType\":" + + mysqlType + + ",\"old\":" + + old + + ",\"pkNames\":" + + pkNames + + ",\"table\":\"" + + table + + "\",\"ts\":1589373560798,\"type\":\"" + + type + + "\"}"); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static class TestCollector implements Collector { + private final List events = new ArrayList<>(); + + @Override + public void collect(Event event) { + events.add(event); + } + + @Override + public void close() {} + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchemaTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchemaTest.java new file mode 100644 index 00000000000..cbb85e2ff35 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/test/java/org/apache/flink/cdc/connectors/kafka/source/PipelineKafkaRecordDeserializationSchemaTest.java @@ -0,0 +1,610 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.kafka.source; + +import org.apache.flink.cdc.common.data.RecordData; +import org.apache.flink.cdc.common.event.AddColumnEvent; +import org.apache.flink.cdc.common.event.AlterColumnTypeEvent; +import org.apache.flink.cdc.common.event.CreateTableEvent; +import org.apache.flink.cdc.common.event.DataChangeEvent; +import org.apache.flink.cdc.common.event.Event; +import org.apache.flink.cdc.common.event.OperationType; +import org.apache.flink.cdc.common.types.DataTypeRoot; +import org.apache.flink.cdc.common.types.DataTypes; +import org.apache.flink.util.Collector; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** Tests for {@link PipelineKafkaRecordDeserializationSchema}. */ +class PipelineKafkaRecordDeserializationSchemaTest { + + private static final byte[] KEY = + bytes( + "{\"schema\":{\"type\":\"struct\",\"fields\":[" + + field("int32", "id", false) + + "]},\"payload\":{\"id\":1}}"); + + @Test + void testDeserializeOperationsMetadataAndIgnoredRecords() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String fields = field("int32", "id", false) + "," + field("string", "name", true); + + deserializer.deserialize( + record(3, 11, KEY, value(fields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record(3, 12, KEY, value(fields, "u", row(1, "Alice"), row(1, "Bob"))), collector); + deserializer.deserialize( + record(3, 13, KEY, value(fields, "d", row(1, "Bob"), "null")), collector); + deserializer.deserialize(record(3, 14, KEY, null), collector); + deserializer.deserialize( + record( + 3, + 15, + KEY, + bytes("{\"schema\":{},\"payload\":{\"source\":{},\"ts_ms\":1}}")), + collector); + + Assertions.assertThat(collector.events) + .hasSize(4) + .element(0) + .isInstanceOf(CreateTableEvent.class); + Assertions.assertThat( + ((CreateTableEvent) collector.events.get(0)).getSchema().primaryKeys()) + .isEmpty(); + Assertions.assertThat( + collector.events.subList(1, 4).stream() + .map(event -> ((DataChangeEvent) event).op())) + .containsExactly(OperationType.INSERT, OperationType.UPDATE, OperationType.DELETE); + DataChangeEvent insert = (DataChangeEvent) collector.events.get(1); + Assertions.assertThat(insert.tableId().toString()).isEqualTo("inventory.customers"); + Assertions.assertThat(insert.meta()) + .containsEntry("topic", "dbserver.inventory.customers") + .containsEntry("partition", "3") + .containsEntry("offset", "11"); + Assertions.assertThat(insert.after().getInt(0)).isEqualTo(1); + Assertions.assertThat(insert.after().getString(1).toString()).isEqualTo("Alice"); + } + + @Test + void testInterleavedOldNewOldSchemasUseWidestSchema() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String oldFields = field("int32", "id", false) + "," + field("string", "name", true); + String newFields = + field("int64", "id", false) + + "," + + field("string", "name", true) + + "," + + field("string", "email", true); + + deserializer.deserialize( + record(0, 1, KEY, value(oldFields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record( + 1, + 1, + KEY, + value( + newFields, + "c", + "null", + "{\"id\":2147483648,\"name\":\"Bob\",\"email\":\"b@example.com\"}")), + collector); + deserializer.deserialize( + record(0, 2, KEY, value(oldFields, "c", "null", row(2, "Carol"))), collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly( + "CreateTableEvent", + "DataChangeEvent", + "AddColumnEvent", + "AlterColumnTypeEvent", + "DataChangeEvent", + "DataChangeEvent"); + AlterColumnTypeEvent alter = (AlterColumnTypeEvent) collector.events.get(3); + Assertions.assertThat(alter.getTypeMapping().get("id").getTypeRoot()) + .isEqualTo(DataTypeRoot.BIGINT); + DataChangeEvent oldAfterWidening = (DataChangeEvent) collector.events.get(5); + RecordData converted = oldAfterWidening.after(); + Assertions.assertThat(converted.getArity()).isEqualTo(3); + Assertions.assertThat(converted.getLong(0)).isEqualTo(2L); + Assertions.assertThat(converted.isNullAt(2)).isTrue(); + } + + @Test + void testNewSchemaFirstThenOldSchemaOnAnotherPartition() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String oldFields = field("int32", "id", false) + "," + field("string", "name", true); + String newFields = + field("int64", "id", false) + + "," + + field("string", "name", true) + + "," + + field("string", "email", true); + + deserializer.deserialize( + record( + 1, + 1, + KEY, + value( + newFields, + "c", + "null", + "{\"id\":2147483648,\"name\":\"Bob\",\"email\":\"b@example.com\"}")), + collector); + deserializer.deserialize( + record(0, 1, KEY, value(oldFields, "c", "null", row(2, "Carol"))), collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent", "DataChangeEvent"); + RecordData converted = ((DataChangeEvent) collector.events.get(2)).after(); + Assertions.assertThat(converted.getArity()).isEqualTo(3); + Assertions.assertThat(converted.getLong(0)).isEqualTo(2L); + Assertions.assertThat(converted.isNullAt(2)).isTrue(); + } + + @Test + void testIntToStringWideningOnSamePartition() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String intFields = field("int32", "id", false) + "," + field("int32", "age", true); + String stringFields = field("int32", "id", false) + "," + field("string", "age", true); + + deserializer.deserialize( + record(0, 1, KEY, value(intFields, "c", "null", "{\"id\":1,\"age\":18}")), + collector); + deserializer.deserialize( + record(0, 2, KEY, value(stringFields, "c", "null", "{\"id\":2,\"age\":\"hello\"}")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly( + "CreateTableEvent", + "DataChangeEvent", + "AlterColumnTypeEvent", + "DataChangeEvent"); + CreateTableEvent createTable = (CreateTableEvent) collector.events.get(0); + Assertions.assertThat( + createTable + .getSchema() + .getColumn("age") + .orElseThrow(AssertionError::new) + .getType() + .getTypeRoot()) + .isEqualTo(DataTypeRoot.INTEGER); + AlterColumnTypeEvent alter = (AlterColumnTypeEvent) collector.events.get(2); + Assertions.assertThat(alter.getTypeMapping().get("age")) + .isEqualTo(DataTypes.STRING().nullable()); + DataChangeEvent intRecord = (DataChangeEvent) collector.events.get(1); + Assertions.assertThat(intRecord.after().getInt(1)).isEqualTo(18); + DataChangeEvent stringRecord = (DataChangeEvent) collector.events.get(3); + Assertions.assertThat(stringRecord.after().getString(1).toString()).isEqualTo("hello"); + } + + @Test + void testStringThenHistoricalIntFromAnotherPartition() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String intFields = field("int32", "id", false) + "," + field("int32", "age", true); + String stringFields = field("int32", "id", false) + "," + field("string", "age", true); + + deserializer.deserialize( + record(1, 1, KEY, value(stringFields, "c", "null", "{\"id\":1,\"age\":\"hello\"}")), + collector); + deserializer.deserialize( + record(0, 1, KEY, value(intFields, "c", "null", "{\"id\":2,\"age\":19}")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent", "DataChangeEvent"); + DataChangeEvent historical = (DataChangeEvent) collector.events.get(2); + Assertions.assertThat(historical.after().getString(1).toString()).isEqualTo("19"); + } + + @Test + void testStringToIntWithinPartitionFailsClearly() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String stringFields = field("int32", "id", false) + "," + field("string", "age", true); + String intFields = field("int32", "id", false) + "," + field("int32", "age", true); + deserializer.deserialize( + record(0, 1, KEY, value(stringFields, "c", "null", "{\"id\":1,\"age\":\"hello\"}")), + collector); + + Assertions.assertThatThrownBy( + () -> + deserializer.deserialize( + record( + 0, + 2, + KEY, + value( + intFields, + "c", + "null", + "{\"id\":2,\"age\":19}")), + collector)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Incompatible or narrowing type change") + .hasMessageContaining("age") + .hasMessageContaining("@2"); + } + + @Test + void testDroppedColumnStaysInSchemaAndReadsAsNull() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String fields = field("int32", "id", false) + "," + field("string", "name", true); + deserializer.deserialize( + record(0, 1, KEY, value(fields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record(0, 2, KEY, value(field("int32", "id", false), "c", "null", "{\"id\":2}")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent", "DataChangeEvent"); + CreateTableEvent createTable = (CreateTableEvent) collector.events.get(0); + Assertions.assertThat(createTable.getSchema().getColumnNames()) + .containsExactly("id", "name"); + DataChangeEvent dropped = (DataChangeEvent) collector.events.get(2); + Assertions.assertThat(dropped.after().getArity()).isEqualTo(2); + Assertions.assertThat(dropped.after().getInt(0)).isEqualTo(2); + Assertions.assertThat(dropped.after().isNullAt(1)).isTrue(); + } + + @Test + void testRenamedColumnKeepsOldNameAndAddsNewColumn() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String oldFields = field("int32", "id", false) + "," + field("string", "name", true); + String renamedFields = + field("int32", "id", false) + "," + field("string", "full_name", true); + + deserializer.deserialize( + record(0, 1, KEY, value(oldFields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record( + 0, + 2, + KEY, + value(renamedFields, "c", "null", "{\"id\":2,\"full_name\":\"Bob\"}")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly( + "CreateTableEvent", "DataChangeEvent", "AddColumnEvent", "DataChangeEvent"); + AddColumnEvent addColumn = (AddColumnEvent) collector.events.get(2); + Assertions.assertThat(addColumn.getAddedColumns()) + .extracting(column -> column.getAddColumn().getName()) + .containsExactly("full_name"); + DataChangeEvent beforeRename = (DataChangeEvent) collector.events.get(1); + Assertions.assertThat(beforeRename.after().getArity()).isEqualTo(2); + Assertions.assertThat(beforeRename.after().getString(1).toString()).isEqualTo("Alice"); + DataChangeEvent afterRename = (DataChangeEvent) collector.events.get(3); + Assertions.assertThat(afterRename.after().getArity()).isEqualTo(3); + Assertions.assertThat(afterRename.after().isNullAt(1)).isTrue(); + Assertions.assertThat(afterRename.after().getString(2).toString()).isEqualTo("Bob"); + } + + @Test + void testNonNullableDroppedColumnBecomesNullable() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String fields = field("int32", "id", false) + "," + field("string", "name", false); + deserializer.deserialize( + record(0, 1, KEY, value(fields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record(0, 2, KEY, value(field("int32", "id", false), "c", "null", "{\"id\":2}")), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly( + "CreateTableEvent", + "DataChangeEvent", + "AlterColumnTypeEvent", + "DataChangeEvent"); + AlterColumnTypeEvent alter = (AlterColumnTypeEvent) collector.events.get(2); + Assertions.assertThat(alter.getTypeMapping().get("name")) + .isEqualTo(DataTypes.STRING().nullable()); + DataChangeEvent dropped = (DataChangeEvent) collector.events.get(3); + Assertions.assertThat(dropped.after().isNullAt(1)).isTrue(); + } + + @Test + void testHistoricalNameAfterRenameFromAnotherPartition() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String oldFields = field("int32", "id", false) + "," + field("string", "name", true); + String renamedFields = + field("int32", "id", false) + "," + field("string", "full_name", true); + + deserializer.deserialize( + record(0, 1, KEY, value(oldFields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record( + 0, + 2, + KEY, + value(renamedFields, "c", "null", "{\"id\":2,\"full_name\":\"Bob\"}")), + collector); + deserializer.deserialize( + record(1, 1, KEY, value(oldFields, "c", "null", row(3, "Carol"))), collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly( + "CreateTableEvent", + "DataChangeEvent", + "AddColumnEvent", + "DataChangeEvent", + "DataChangeEvent"); + DataChangeEvent historical = (DataChangeEvent) collector.events.get(4); + Assertions.assertThat(historical.after().getArity()).isEqualTo(3); + Assertions.assertThat(historical.after().getString(1).toString()).isEqualTo("Carol"); + Assertions.assertThat(historical.after().isNullAt(2)).isTrue(); + } + + @Test + void testKafkaConnectFloatingTypesAndStringMapsToString() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + String fields = + field("int32", "id", false) + + "," + + field("float", "score", true) + + "," + + field("double", "ratio", true) + + "," + + field("string", "name", true); + + deserializer.deserialize( + record( + 0, + 1, + KEY, + value( + fields, + "c", + "null", + "{\"id\":1,\"score\":1.5,\"ratio\":2.5,\"name\":\"Alice\"}")), + collector); + deserializer.deserialize( + record( + 0, + 2, + KEY, + value( + fields, + "c", + "null", + "{\"id\":2,\"score\":3.5,\"ratio\":4.5,\"name\":\"Bob\"}")), + collector); + + CreateTableEvent createTable = (CreateTableEvent) collector.events.get(0); + Assertions.assertThat( + createTable + .getSchema() + .getColumn("name") + .orElseThrow(AssertionError::new) + .getType()) + .isEqualTo(DataTypes.STRING().nullable()); + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent", "DataChangeEvent"); + DataChangeEvent firstRecord = (DataChangeEvent) collector.events.get(1); + Assertions.assertThat(firstRecord.after().getFloat(1)).isEqualTo(1.5f); + Assertions.assertThat(firstRecord.after().getDouble(2)).isEqualTo(2.5d); + } + + @Test + void testTablesIncludeKeepsMatchingTableOnly() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema("inventory.customers", null); + TestCollector collector = new TestCollector(); + String fields = field("int32", "id", false) + "," + field("string", "name", true); + + deserializer.deserialize( + record(0, 1, KEY, value(fields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record(0, 2, KEY, value("inventory", "orders", fields, "c", "null", row(2, "Bob"))), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent"); + Assertions.assertThat(((CreateTableEvent) collector.events.get(0)).tableId().toString()) + .isEqualTo("inventory.customers"); + Assertions.assertThat(((DataChangeEvent) collector.events.get(1)).tableId().toString()) + .isEqualTo("inventory.customers"); + } + + @Test + void testTablesExcludeDropsMatchingTable() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(null, "inventory.orders"); + TestCollector collector = new TestCollector(); + String fields = field("int32", "id", false) + "," + field("string", "name", true); + + deserializer.deserialize( + record(0, 1, KEY, value(fields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record(0, 2, KEY, value("inventory", "orders", fields, "c", "null", row(2, "Bob"))), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent"); + Assertions.assertThat(((DataChangeEvent) collector.events.get(1)).tableId().toString()) + .isEqualTo("inventory.customers"); + } + + @Test + void testTablesIncludeAndExclude() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema("inventory.\\.*", "inventory.orders"); + TestCollector collector = new TestCollector(); + String fields = field("int32", "id", false) + "," + field("string", "name", true); + + deserializer.deserialize( + record(0, 1, KEY, value(fields, "c", "null", row(1, "Alice"))), collector); + deserializer.deserialize( + record(0, 2, KEY, value("inventory", "orders", fields, "c", "null", row(2, "Bob"))), + collector); + deserializer.deserialize( + record( + 0, + 3, + KEY, + value("other", "customers", fields, "c", "null", row(3, "Carol"))), + collector); + + Assertions.assertThat(collector.events) + .extracting(event -> event.getClass().getSimpleName()) + .containsExactly("CreateTableEvent", "DataChangeEvent"); + Assertions.assertThat(((DataChangeEvent) collector.events.get(1)).tableId().toString()) + .isEqualTo("inventory.customers"); + } + + @Test + void testDebeziumColumnLengthParameterDoesNotCreateVarchar() throws Exception { + PipelineKafkaRecordDeserializationSchema deserializer = + new PipelineKafkaRecordDeserializationSchema(); + TestCollector collector = new TestCollector(); + + String fields = field("int32", "id", false) + "," + stringFieldWithLength("name", 32); + deserializer.deserialize( + record(0, 1, KEY, value(fields, "c", "null", "{\"id\":1,\"name\":\"Alice\"}")), + collector); + + CreateTableEvent createTable = (CreateTableEvent) collector.events.get(0); + Assertions.assertThat( + createTable + .getSchema() + .getColumn("name") + .orElseThrow(AssertionError::new) + .getType()) + .isEqualTo(DataTypes.STRING().nullable()); + } + + private static ConsumerRecord record( + int partition, long offset, byte[] key, byte[] value) { + return new ConsumerRecord<>("dbserver.inventory.customers", partition, offset, key, value); + } + + private static byte[] value(String fields, String operation, String before, String after) { + return value("inventory", "customers", fields, operation, before, after); + } + + private static byte[] value( + String db, String table, String fields, String operation, String before, String after) { + String rowSchema = + "{\"type\":\"struct\",\"fields\":[" + + fields + + "],\"optional\":true,\"name\":\"" + + db + + "." + + table + + ".Value\"}"; + return bytes( + "{\"schema\":{\"type\":\"struct\",\"fields\":[" + + withField(rowSchema, "before") + + "," + + withField(rowSchema, "after") + + "]},\"payload\":{\"before\":" + + before + + ",\"after\":" + + after + + ",\"source\":{\"db\":\"" + + db + + "\",\"table\":\"" + + table + + "\"},\"op\":\"" + + operation + + "\"}}"); + } + + private static String withField(String schema, String field) { + return schema.substring(0, schema.length() - 1) + ",\"field\":\"" + field + "\"}"; + } + + private static String field(String type, String name, boolean optional) { + return "{\"type\":\"" + + type + + "\",\"optional\":" + + optional + + ",\"field\":\"" + + name + + "\"}"; + } + + private static String stringFieldWithLength(String name, int length) { + return "{\"type\":\"string\",\"optional\":true,\"parameters\":{" + + "\"__debezium.source.column.length\":\"" + + length + + "\"},\"field\":\"" + + name + + "\"}"; + } + + private static String row(long id, String name) { + return "{\"id\":" + id + ",\"name\":\"" + name + "\"}"; + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static class TestCollector implements Collector { + private final List events = new ArrayList<>(); + + @Override + public void collect(Event event) { + events.add(event); + } + + @Override + public void close() {} + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonDataSink.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonDataSink.java index f48879605b1..d3ad71bc65e 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonDataSink.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonDataSink.java @@ -25,6 +25,7 @@ import org.apache.flink.cdc.common.sink.EventSinkProvider; import org.apache.flink.cdc.common.sink.FlinkSinkProvider; import org.apache.flink.cdc.common.sink.MetadataApplier; +import org.apache.flink.cdc.common.sink.SupportsParallelMetadataSource; import org.apache.flink.cdc.connectors.paimon.sink.v2.PaimonEventSink; import org.apache.flink.cdc.connectors.paimon.sink.v2.PaimonRecordSerializer; @@ -36,7 +37,7 @@ import java.util.Map; /** A {@link DataSink} for Paimon connector that supports schema evolution. */ -public class PaimonDataSink implements DataSink, Serializable { +public class PaimonDataSink implements DataSink, SupportsParallelMetadataSource, Serializable { // options for creating Paimon catalog. private final Options options; @@ -54,6 +55,8 @@ public class PaimonDataSink implements DataSink, Serializable { public final String schemaOperatorUid; + private boolean parallelMetadataSource; + public PaimonDataSink( Options options, Map tableOptions, @@ -74,7 +77,18 @@ public PaimonDataSink( @Override public EventSinkProvider getEventSinkProvider() { return FlinkSinkProvider.of( - new PaimonEventSink(options, commitUser, serializer, schemaOperatorUid, zoneId)); + new PaimonEventSink( + options, + commitUser, + serializer, + schemaOperatorUid, + zoneId, + parallelMetadataSource)); + } + + @Override + public void setParallelMetadataSource(boolean parallelMetadataSource) { + this.parallelMetadataSource = parallelMetadataSource; } @Override diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/PaimonEventSink.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/PaimonEventSink.java index d350ca6068c..f152e2e3742 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/PaimonEventSink.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/PaimonEventSink.java @@ -17,12 +17,16 @@ package org.apache.flink.cdc.connectors.paimon.sink.v2; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.cdc.common.event.Event; import org.apache.flink.cdc.connectors.paimon.sink.v2.bucket.BucketAssignOperator; import org.apache.flink.cdc.connectors.paimon.sink.v2.bucket.BucketWrapper; import org.apache.flink.cdc.connectors.paimon.sink.v2.bucket.BucketWrapperChangeEvent; import org.apache.flink.cdc.connectors.paimon.sink.v2.bucket.BucketWrapperEventTypeInfo; import org.apache.flink.cdc.connectors.paimon.sink.v2.bucket.FlushEventAlignmentOperator; +import org.apache.flink.cdc.connectors.paimon.sink.v2.bucket.FlushReplicateOperator; +import org.apache.flink.cdc.runtime.typeutils.EventTypeInfo; import org.apache.flink.core.io.SimpleVersionedSerializer; import org.apache.flink.streaming.api.connector.sink2.SupportsPreWriteTopology; import org.apache.flink.streaming.api.datastream.DataStream; @@ -41,19 +45,43 @@ public class PaimonEventSink extends PaimonSink implements SupportsPreWri public final ZoneId zoneId; + private final boolean parallelMetadataSource; + public PaimonEventSink( Options catalogOptions, String commitUser, PaimonRecordSerializer serializer, String schemaOperatorUid, ZoneId zoneId) { + this(catalogOptions, commitUser, serializer, schemaOperatorUid, zoneId, false); + } + + public PaimonEventSink( + Options catalogOptions, + String commitUser, + PaimonRecordSerializer serializer, + String schemaOperatorUid, + ZoneId zoneId, + boolean parallelMetadataSource) { super(catalogOptions, commitUser, serializer); this.schemaOperatorUid = schemaOperatorUid; this.zoneId = zoneId; + this.parallelMetadataSource = parallelMetadataSource; } @Override public DataStream addPreWriteTopology(DataStream dataStream) { + if (parallelMetadataSource) { + dataStream = + dataStream + .transform( + "ReplicateFlush", + new TupleTypeInfo<>(Types.INT, new EventTypeInfo()), + new FlushReplicateOperator()) + .partitionCustom(Math::floorMod, FlushReplicateOperator::targetSubtask) + .map(FlushReplicateOperator::unwrap) + .returns(new EventTypeInfo()); + } // Shuffle by key hash => Assign bucket => Shuffle by bucket. return dataStream .transform( @@ -79,7 +107,7 @@ public DataStream addPreWriteTopology(DataStream dataStream) { .transform( "FlushEventAlignment", new BucketWrapperEventTypeInfo(), - new FlushEventAlignmentOperator()); + new FlushEventAlignmentOperator(parallelMetadataSource)); } @Override diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushEventAlignmentOperator.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushEventAlignmentOperator.java index 35b803b9604..4c51ebbd8eb 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushEventAlignmentOperator.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushEventAlignmentOperator.java @@ -34,6 +34,8 @@ public class FlushEventAlignmentOperator extends AbstractStreamOperatorAdapter implements OneInputStreamOperator { + private final boolean decodeReplicatedSource; + private transient int totalTasksNumber; /** @@ -45,8 +47,13 @@ public class FlushEventAlignmentOperator extends AbstractStreamOperatorAdapter streamRecord) { output.collect( new StreamRecord<>( new FlushEvent( - sourceSubTaskId, + decodeReplicatedSource + ? Math.floorDiv(sourceSubTaskId, totalTasksNumber) + : sourceSubTaskId, bucketWrapperFlushEvent.getTableIds(), bucketWrapperFlushEvent.getSchemaChangeEventType()))); sourceTaskIdToAssignBucketSubTaskIds.remove(sourceSubTaskId); diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushReplicateOperator.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushReplicateOperator.java new file mode 100644 index 00000000000..15ef5309fbe --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushReplicateOperator.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.paimon.sink.v2.bucket; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.cdc.common.event.Event; +import org.apache.flink.cdc.common.event.FlushEvent; +import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter; +import org.apache.flink.cdc.runtime.serializer.event.EventSerializer; +import org.apache.flink.streaming.api.operators.ChainingStrategy; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; + +/** + * Replicates each distributed {@link FlushEvent} to every bucket assigner. + * + *

The source partition and emitting schema subtask are encoded into an internal alignment key. + * This keeps concurrent flushes independent even when schema subtasks process broadcasts in + * different orders. + */ +public class FlushReplicateOperator extends AbstractStreamOperatorAdapter> + implements OneInputStreamOperator> { + + private transient int parallelism; + private transient int subtaskId; + + public FlushReplicateOperator() { + this.chainingStrategy = ChainingStrategy.ALWAYS; + } + + @Override + public void open() throws Exception { + super.open(); + this.parallelism = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks(); + this.subtaskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + } + + @Override + public void processElement(StreamRecord streamRecord) { + Event event = streamRecord.getValue(); + if (event instanceof FlushEvent) { + FlushEvent flushEvent = (FlushEvent) event; + int alignmentKey = + Math.addExact( + Math.multiplyExact(flushEvent.getSourceSubTaskId(), parallelism), + subtaskId); + FlushEvent replicatedFlush = + new FlushEvent( + alignmentKey, + flushEvent.getTableIds(), + flushEvent.getSchemaChangeEventType()); + for (int target = 0; target < parallelism; target++) { + Event payload = + target == subtaskId + ? replicatedFlush + : EventSerializer.INSTANCE.copy(replicatedFlush); + output.collect(new StreamRecord<>(Tuple2.of(target, payload))); + } + } else { + output.collect(new StreamRecord<>(Tuple2.of(subtaskId, event))); + } + } + + public static Integer targetSubtask(Tuple2 tuple) { + return tuple.f0; + } + + public static Event unwrap(Tuple2 tuple) { + return tuple.f1; + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonDataSinkFactoryTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonDataSinkFactoryTest.java index dcbbdd68241..742853dc565 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonDataSinkFactoryTest.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonDataSinkFactoryTest.java @@ -205,5 +205,14 @@ void testSpecifyingCommitUser(String commitUserKey) { .asString() .hasSize(39) // 3 ("yux") + 36 (Random UUID) .startsWith("yux"); + + PaimonDataSink paimonDataSink = (PaimonDataSink) dataSink; + Assertions.assertThat(((FlinkSinkProvider) paimonDataSink.getEventSinkProvider()).getSink()) + .extracting("parallelMetadataSource") + .isEqualTo(false); + paimonDataSink.setParallelMetadataSource(true); + Assertions.assertThat(((FlinkSinkProvider) paimonDataSink.getEventSinkProvider()).getSink()) + .extracting("parallelMetadataSource") + .isEqualTo(true); } } diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushReplicateAndAlignmentTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushReplicateAndAlignmentTest.java new file mode 100644 index 00000000000..507b858cab6 --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/FlushReplicateAndAlignmentTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.paimon.sink.v2.bucket; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.cdc.common.event.Event; +import org.apache.flink.cdc.common.event.FlushEvent; +import org.apache.flink.cdc.common.event.SchemaChangeEventType; +import org.apache.flink.cdc.common.event.TableId; +import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; +import org.apache.flink.util.OutputTag; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +/** Tests for distributed flush replication and the existing per-source alignment. */ +class FlushReplicateAndAlignmentTest { + + private static final TableId CUSTOMERS = TableId.tableId("inventory", "customers"); + private static final TableId ORDERS = TableId.tableId("inventory", "orders"); + + @Test + void testEverySchemaSubtaskReplicatesWithDistinctAlignmentKey() throws Exception { + FlushReplicateOperator first = replicateOperator(2, 0); + CollectingOutput> firstOutput = collectTo(first); + first.processElement(new StreamRecord<>(flushEvent(1, ORDERS))); + + Assertions.assertThat(firstOutput.records) + .extracting(record -> record.getValue().f0) + .containsExactly(0, 1); + Assertions.assertThat(firstOutput.records) + .extracting(record -> ((FlushEvent) record.getValue().f1).getSourceSubTaskId()) + .containsOnly(2); + + FlushReplicateOperator second = replicateOperator(2, 1); + CollectingOutput> secondOutput = collectTo(second); + second.processElement(new StreamRecord<>(flushEvent(0, CUSTOMERS))); + + Assertions.assertThat(secondOutput.records) + .extracting(record -> record.getValue().f0) + .containsExactly(0, 1); + Assertions.assertThat(secondOutput.records) + .extracting(record -> ((FlushEvent) record.getValue().f1).getSourceSubTaskId()) + .containsOnly(1); + } + + @Test + void testConcurrentCreatesFromDifferentSourcesBothAlign() throws Exception { + FlushEventAlignmentOperator operator = alignmentOperator(2, true); + CollectingOutput output = collectTo(operator); + + // Schema subtask 0 handles source 1 first (key 2), while schema subtask 1 handles source 0 + // first (key 1). Each original flush remains an independent alignment round. + operator.processElement(flushRecord(2, 0, ORDERS)); + operator.processElement(flushRecord(1, 1, CUSTOMERS)); + Assertions.assertThat(output.records).isEmpty(); + + operator.processElement(flushRecord(2, 1, ORDERS)); + operator.processElement(flushRecord(1, 0, CUSTOMERS)); + + List sources = + output.records.stream() + .map(record -> ((FlushEvent) record.getValue()).getSourceSubTaskId()) + .collect(Collectors.toList()); + Assertions.assertThat(sources).containsExactlyInAnyOrder(0, 1); + } + + @Test + void testAlignmentKeepsIndependentRoundsForSameSource() throws Exception { + FlushEventAlignmentOperator operator = alignmentOperator(2, false); + CollectingOutput output = collectTo(operator); + + operator.processElement(flushRecord(0, 0, CUSTOMERS)); + operator.processElement(flushRecord(0, 1, CUSTOMERS)); + operator.processElement(flushRecord(0, 0, CUSTOMERS)); + operator.processElement(flushRecord(0, 1, CUSTOMERS)); + + Assertions.assertThat(output.records).hasSize(2); + } + + private static FlushReplicateOperator replicateOperator(int parallelism, int subtaskId) + throws Exception { + FlushReplicateOperator operator = new FlushReplicateOperator(); + setField(operator, "parallelism", parallelism); + setField(operator, "subtaskId", subtaskId); + return operator; + } + + private static FlushEventAlignmentOperator alignmentOperator( + int parallelism, boolean decodeReplicatedSource) throws Exception { + FlushEventAlignmentOperator operator = + new FlushEventAlignmentOperator(decodeReplicatedSource); + setField(operator, "totalTasksNumber", parallelism); + setField(operator, "currentSubTaskId", 0); + setField(operator, "sourceTaskIdToAssignBucketSubTaskIds", new HashMap<>()); + return operator; + } + + private static CollectingOutput collectTo(Object operator) throws Exception { + CollectingOutput output = new CollectingOutput<>(); + setField(operator, "output", output); + return output; + } + + private static FlushEvent flushEvent(int sourceSubTaskId, TableId tableId) { + return new FlushEvent( + sourceSubTaskId, + Collections.singletonList(tableId), + SchemaChangeEventType.CREATE_TABLE); + } + + private static StreamRecord flushRecord( + int sourceSubTaskId, int assignerId, TableId tableId) { + return new StreamRecord<>( + new BucketWrapperFlushEvent( + 0, + sourceSubTaskId, + assignerId, + Collections.singletonList(tableId), + SchemaChangeEventType.CREATE_TABLE)); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Class current = target.getClass(); + while (current != null) { + try { + Field field = current.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + return; + } catch (NoSuchFieldException e) { + current = current.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } + + private static class CollectingOutput implements Output> { + private final List> records = new ArrayList<>(); + + public void emitWatermark(org.apache.flink.runtime.event.WatermarkEvent watermark) {} + + @Override + public void emitWatermark(Watermark mark) {} + + @Override + public void emitWatermarkStatus(WatermarkStatus watermarkStatus) {} + + @Override + public void collect(OutputTag outputTag, StreamRecord streamRecord) {} + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) {} + + @Override + public void emitRecordAttributes(RecordAttributes recordAttributes) {} + + @Override + public void collect(StreamRecord record) { + records.add(record); + } + + @Override + public void close() {} + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/main/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksEnrichedCatalog.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/main/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksEnrichedCatalog.java index 70f1f50f7eb..3cf66c23caa 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/main/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksEnrichedCatalog.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/main/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksEnrichedCatalog.java @@ -106,7 +106,8 @@ public void renameColumn( } } - public void alterColumnType(String databaseName, String tableName, StarRocksColumn column) + public void alterColumnType( + String databaseName, String tableName, StarRocksColumn column, long timeoutSecond) throws StarRocksCatalogException { checkTableArgument(databaseName, tableName); Preconditions.checkArgument( @@ -115,7 +116,7 @@ public void alterColumnType(String databaseName, String tableName, StarRocksColu String alterSql = buildAlterColumnTypeSql(databaseName, tableName, buildColumnStmt(column)); try { long startTimeMillis = System.currentTimeMillis(); - executeUpdateStatement(alterSql); + executeAlter(databaseName, tableName, alterSql, timeoutSecond); LOG.info( "Success to alter table {}.{} modify column type, duration: {}ms, sql: {}", databaseName, @@ -158,14 +159,27 @@ private String buildAlterColumnTypeSql( "ALTER TABLE `%s`.`%s` MODIFY COLUMN %s", databaseName, tableName, columnStmt); } - private void executeUpdateStatement(String sql) throws StarRocksCatalogException { + protected void executeAlter( + String databaseName, String tableName, String alterSql, long timeoutSecond) { + invokeParent( + "executeAlter", + new Class[] {String.class, String.class, String.class, long.class}, + databaseName, + tableName, + alterSql, + timeoutSecond); + } + + protected void executeUpdateStatement(String sql) throws StarRocksCatalogException { + invokeParent("executeUpdateStatement", new Class[] {String.class}, sql); + } + + private void invokeParent(String methodName, Class[] parameterTypes, Object... args) { try { - Method m = - getClass() - .getSuperclass() - .getDeclaredMethod("executeUpdateStatement", String.class); - m.setAccessible(true); - m.invoke(this, sql); + Method method = + getClass().getSuperclass().getDeclaredMethod(methodName, parameterTypes); + method.setAccessible(true); + method.invoke(this, args); } catch (InvocationTargetException | NoSuchMethodException | IllegalAccessException e) { throw new RuntimeException(e); } diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/main/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplier.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/main/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplier.java index 7f020844106..8cc2114c792 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/main/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplier.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/main/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplier.java @@ -42,8 +42,11 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import static org.apache.flink.cdc.connectors.starrocks.sink.StarRocksUtils.toStarRocksDataType; @@ -134,6 +137,16 @@ private void applyCreateTable(CreateTableEvent createTableEvent) throws SchemaEv } try { + Optional existingTable = + catalog.getTable( + starRocksTable.getDatabaseName(), starRocksTable.getTableName()); + if (existingTable.isPresent()) { + validateExistingTable(createTableEvent, existingTable.get(), starRocksTable); + LOG.info( + "Table already exists with a compatible schema, event: {}", + createTableEvent); + return; + } catalog.createTable(starRocksTable, true); LOG.info("Successful to create table, event: {}", createTableEvent); } catch (StarRocksCatalogException e) { @@ -144,6 +157,7 @@ private void applyCreateTable(CreateTableEvent createTableEvent) throws SchemaEv private void applyAddColumn(AddColumnEvent addColumnEvent) throws SchemaEvolveException { List addColumns = new ArrayList<>(); + StarRocksTable existingTable = getRequiredTable(addColumnEvent); for (AddColumnEvent.ColumnWithPosition columnWithPosition : addColumnEvent.getAddedColumns()) { // we will ignore position information, and always add the column to the last. @@ -163,7 +177,25 @@ private void applyAddColumn(AddColumnEvent addColumnEvent) throws SchemaEvolveEx StarRocksUtils.convertInvalidTimestampDefaultValue( column.getDefaultValueExpression(), column.getType())); toStarRocksDataType(column, false, builder, tableCreateConfig.getUnicodeCharMaxBytes()); - addColumns.add(builder.build()); + StarRocksColumn targetColumn = builder.build(); + StarRocksColumn existingColumn = existingTable.getColumn(targetColumn.getColumnName()); + if (existingColumn == null) { + addColumns.add(targetColumn); + } else if (isSameOrWider(existingColumn, targetColumn)) { + LOG.info( + "Column {} already exists with a compatible type, skipping replayed add column event.", + targetColumn.getColumnName()); + } else { + throw new SchemaEvolveException( + addColumnEvent, + String.format( + "Existing column %s is incompatible with replayed column %s", + existingColumn, targetColumn), + null); + } + } + if (addColumns.isEmpty()) { + return; } TableId tableId = addColumnEvent.tableId(); @@ -317,6 +349,7 @@ private void applyAlterColumnType(AlterColumnTypeEvent event) throws SchemaEvolv try { TableId tableId = event.tableId(); Map typeMapping = event.getTypeMapping(); + StarRocksTable existingTable = getRequiredTable(event); for (Map.Entry entry : typeMapping.entrySet()) { StarRocksColumn.Builder builder = @@ -326,14 +359,150 @@ private void applyAlterColumnType(AlterColumnTypeEvent event) throws SchemaEvolv false, builder, tableCreateConfig.getUnicodeCharMaxBytes()); + StarRocksColumn targetColumn = builder.build(); + StarRocksColumn existingColumn = existingTable.getColumn(entry.getKey()); + if (existingColumn == null) { + throw new SchemaEvolveException( + event, "Cannot alter non-existing column " + entry.getKey(), null); + } + if (isSameOrWider(existingColumn, targetColumn) + && isSameOrWider(targetColumn, existingColumn)) { + LOG.info( + "Column {} is already at type {}, skipping replayed alter column event.", + entry.getKey(), + existingColumn.getDataType()); + continue; + } + if (!isSameOrWider(targetColumn, existingColumn)) { + throw new SchemaEvolveException( + event, + String.format( + "Cannot safely widen column %s from %s to %s", + entry.getKey(), existingColumn, targetColumn), + null); + } catalog.alterColumnType( - tableId.getSchemaName(), tableId.getTableName(), builder.build()); + tableId.getSchemaName(), + tableId.getTableName(), + targetColumn, + schemaChangeConfig.getTimeoutSecond()); } } catch (Exception e) { + if (e instanceof SchemaEvolveException) { + throw (SchemaEvolveException) e; + } throw new SchemaEvolveException(event, "fail to apply alter column type event", e); } } + private StarRocksTable getRequiredTable(SchemaChangeEvent event) { + TableId tableId = event.tableId(); + try { + return catalog.getTable(tableId.getSchemaName(), tableId.getTableName()) + .orElseThrow( + () -> + new SchemaEvolveException( + event, "Table " + tableId + " does not exist", null)); + } catch (StarRocksCatalogException e) { + throw new SchemaEvolveException(event, "Failed to inspect table " + tableId, e); + } + } + + private void validateExistingTable( + CreateTableEvent event, StarRocksTable actual, StarRocksTable expected) { + List actualKeys = actual.getTableKeys().orElse(new ArrayList<>()); + List expectedKeys = expected.getTableKeys().orElse(new ArrayList<>()); + if (!actualKeys.equals(expectedKeys)) { + throw new SchemaEvolveException( + event, + String.format( + "Existing table primary keys %s differ from inferred primary keys %s", + actualKeys, expectedKeys), + null); + } + + Map expectedColumns = new HashMap<>(); + for (StarRocksColumn expectedColumn : expected.getColumns()) { + expectedColumns.put(expectedColumn.getColumnName(), expectedColumn); + StarRocksColumn actualColumn = actual.getColumn(expectedColumn.getColumnName()); + if (actualColumn == null || !isSameOrWider(actualColumn, expectedColumn)) { + throw new SchemaEvolveException( + event, + String.format( + "Existing column %s is missing or incompatible with inferred column %s", + actualColumn, expectedColumn), + null); + } + } + + for (StarRocksColumn actualColumn : actual.getColumns()) { + if (!expectedColumns.containsKey(actualColumn.getColumnName()) + && !actualColumn.isNullable() + && !actualColumn.getDefaultValue().isPresent()) { + throw new SchemaEvolveException( + event, + String.format( + "Existing extra column %s is non-nullable and has no default value", + actualColumn.getColumnName()), + null); + } + } + } + + private boolean isSameOrWider(StarRocksColumn wider, StarRocksColumn narrower) { + if (narrower.isNullable() && !wider.isNullable()) { + return false; + } + + String widerType = wider.getDataType().toUpperCase(Locale.ROOT); + String narrowerType = narrower.getDataType().toUpperCase(Locale.ROOT); + if (widerType.equals(narrowerType)) { + if ("CHAR".equals(widerType) || "VARCHAR".equals(widerType)) { + return wider.getColumnSize().orElse(0) >= narrower.getColumnSize().orElse(0); + } + if ("DECIMAL".equals(widerType)) { + int widerPrecision = wider.getColumnSize().orElse(0); + int widerScale = wider.getDecimalDigits().orElse(0); + int narrowerPrecision = narrower.getColumnSize().orElse(0); + int narrowerScale = narrower.getDecimalDigits().orElse(0); + return widerScale >= narrowerScale + && widerPrecision - widerScale >= narrowerPrecision - narrowerScale; + } + return true; + } + + if ("VARCHAR".equals(widerType) && "CHAR".equals(narrowerType)) { + return wider.getColumnSize().orElse(0) >= narrower.getColumnSize().orElse(0); + } + if ("VARCHAR".equals(widerType) || "STRING".equals(widerType)) { + return numericTypeRank(narrowerType) >= 0 || "BOOLEAN".equals(narrowerType); + } + int widerRank = numericTypeRank(widerType); + int narrowerRank = numericTypeRank(narrowerType); + return widerRank >= 0 && narrowerRank >= 0 && widerRank >= narrowerRank; + } + + private int numericTypeRank(String type) { + switch (type) { + case "TINYINT": + return 0; + case "SMALLINT": + return 1; + case "INT": + return 2; + case "BIGINT": + return 3; + case "LARGEINT": + return 4; + case "FLOAT": + return 5; + case "DOUBLE": + return 6; + default: + return -1; + } + } + private void applyTruncateTable(TruncateTableEvent truncateTableEvent) { try { catalog.truncateTable( diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/MockStarRocksCatalog.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/MockStarRocksCatalog.java index c12be85a735..4dfc6bdc71a 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/MockStarRocksCatalog.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/MockStarRocksCatalog.java @@ -96,6 +96,71 @@ public void createTable(StarRocksTable table, boolean ignoreIfExists) } } + @Override + public void alterColumnType( + String databaseName, String tableName, StarRocksColumn column, long timeoutSecond) + throws StarRocksCatalogException { + Map dbTables = tables.get(databaseName); + if (dbTables == null) { + throw new StarRocksCatalogException( + String.format("database %s does not exist", databaseName)); + } + StarRocksTable oldTable = dbTables.get(tableName); + if (oldTable == null) { + throw new StarRocksCatalogException( + String.format("table %s.%s does not exist", databaseName, tableName)); + } + List newColumns = new ArrayList<>(); + boolean found = false; + for (StarRocksColumn existing : oldTable.getColumns()) { + if (existing.getColumnName().equals(column.getColumnName())) { + found = true; + newColumns.add( + new StarRocksColumn.Builder() + .setColumnName(column.getColumnName()) + .setOrdinalPosition(newColumns.size()) + .setDataType(column.getDataType()) + .setNullable(column.isNullable()) + .setDefaultValue(column.getDefaultValue().orElse(null)) + .setColumnSize(column.getColumnSize().orElse(null)) + .setDecimalDigits(column.getDecimalDigits().orElse(null)) + .setColumnComment(column.getColumnComment().orElse(null)) + .build()); + } else { + newColumns.add( + new StarRocksColumn.Builder() + .setColumnName(existing.getColumnName()) + .setOrdinalPosition(newColumns.size()) + .setDataType(existing.getDataType()) + .setNullable(existing.isNullable()) + .setDefaultValue(existing.getDefaultValue().orElse(null)) + .setColumnSize(existing.getColumnSize().orElse(null)) + .setDecimalDigits(existing.getDecimalDigits().orElse(null)) + .setColumnComment(existing.getColumnComment().orElse(null)) + .build()); + } + } + if (!found) { + throw new StarRocksCatalogException( + String.format( + "column %s does not exist in %s.%s", + column.getColumnName(), databaseName, tableName)); + } + dbTables.put( + tableName, + new StarRocksTable.Builder() + .setDatabaseName(oldTable.getDatabaseName()) + .setTableName(oldTable.getTableName()) + .setTableType(oldTable.getTableType()) + .setColumns(newColumns) + .setTableKeys(oldTable.getTableKeys().orElse(null)) + .setDistributionKeys(oldTable.getDistributionKeys().orElse(null)) + .setNumBuckets(oldTable.getNumBuckets().orElse(null)) + .setComment(oldTable.getComment().orElse(null)) + .setTableProperties(oldTable.getProperties()) + .build()); + } + @Override public void alterAddColumns( String databaseName, diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksEnrichedCatalogTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksEnrichedCatalogTest.java new file mode 100644 index 00000000000..32ecb0ce81b --- /dev/null +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksEnrichedCatalogTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.starrocks.sink; + +import com.starrocks.connector.flink.catalog.StarRocksCatalogException; +import com.starrocks.connector.flink.catalog.StarRocksColumn; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for the async {@code MODIFY COLUMN} path. + * + *

StarRocks schema change is asynchronous. Submitting {@code ALTER} with {@code + * executeUpdateStatement} returns before the job finishes, so a following STRING value such as + * {@code hello} is stream-loaded into an INT column and dropped or stored as null. {@code + * executeAlter} waits until the job is {@code FINISHED}. + */ +class StarRocksEnrichedCatalogTest { + + @Test + void testAlterColumnTypeWaitsUntilJobFinishedBeforeAcceptingStringValue() { + AsyncIntToVarcharCatalog catalog = new AsyncIntToVarcharCatalog(); + StarRocksColumn target = + new StarRocksColumn.Builder() + .setColumnName("age") + .setOrdinalPosition(0) + .setDataType("varchar") + .setColumnSize(1048576) + .setNullable(true) + .build(); + + Assertions.assertThat(catalog.canWriteStringAge()) + .as("STRING values cannot be loaded while age is still INT") + .isFalse(); + + catalog.alterColumnType("inventory", "customers", target, 30); + + Assertions.assertThat(catalog.usedExecuteAlter).isTrue(); + Assertions.assertThat(catalog.usedExecuteUpdate).isFalse(); + Assertions.assertThat(catalog.canWriteStringAge()).isTrue(); + Assertions.assertThat(catalog.lastTimeoutSecond).isEqualTo(30); + } + + /** + * Models StarRocks: {@code age} stays {@code INT} until the alter job finishes. {@code + * executeUpdateStatement} only submits SQL; {@code executeAlter} waits and then flips the type. + */ + private static final class AsyncIntToVarcharCatalog extends StarRocksEnrichedCatalog { + + private boolean alterJobFinished; + private boolean usedExecuteAlter; + private boolean usedExecuteUpdate; + private long lastTimeoutSecond = -1L; + + private AsyncIntToVarcharCatalog() { + super("jdbc:mysql://127.0.0.1:9030", "root", ""); + } + + @Override + protected void executeAlter( + String databaseName, String tableName, String alterSql, long timeoutSecond) { + usedExecuteAlter = true; + lastTimeoutSecond = timeoutSecond; + alterJobFinished = true; + } + + @Override + protected void executeUpdateStatement(String sql) throws StarRocksCatalogException { + usedExecuteUpdate = true; + } + + private boolean canWriteStringAge() { + return alterJobFinished; + } + } +} diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplierTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplierTest.java index 2da34d6cf4f..a3bc6d3229e 100644 --- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplierTest.java +++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplierTest.java @@ -19,13 +19,18 @@ import org.apache.flink.cdc.common.configuration.Configuration; import org.apache.flink.cdc.common.event.AddColumnEvent; +import org.apache.flink.cdc.common.event.AlterColumnTypeEvent; import org.apache.flink.cdc.common.event.CreateTableEvent; import org.apache.flink.cdc.common.event.DropColumnEvent; import org.apache.flink.cdc.common.event.TableId; +import org.apache.flink.cdc.common.exceptions.SchemaEvolveException; import org.apache.flink.cdc.common.schema.Column; import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.types.BigIntType; import org.apache.flink.cdc.common.types.BooleanType; import org.apache.flink.cdc.common.types.DecimalType; +import org.apache.flink.cdc.common.types.DoubleType; +import org.apache.flink.cdc.common.types.FloatType; import org.apache.flink.cdc.common.types.IntType; import org.apache.flink.cdc.common.types.SmallIntType; import org.apache.flink.cdc.common.types.TimeType; @@ -185,6 +190,151 @@ void testAddColumn() throws Exception { Assertions.assertThat(actualTable).isEqualTo(expectTable); } + @Test + void testReplayHistoricalSchemaAgainstWiderExistingTable() { + TableId tableId = TableId.parse("test.replay_tbl"); + Schema currentSchema = + Schema.newBuilder() + .physicalColumn("id", new BigIntType(false)) + .physicalColumn("new_col", new IntType()) + .primaryKey("id") + .build(); + metadataApplier.applySchemaChange(new CreateTableEvent(tableId, currentSchema)); + + Schema historicalSchema = + Schema.newBuilder() + .physicalColumn("id", new IntType(false)) + .primaryKey("id") + .build(); + metadataApplier.applySchemaChange(new CreateTableEvent(tableId, historicalSchema)); + metadataApplier.applySchemaChange( + new AddColumnEvent( + tableId, + Collections.singletonList( + new AddColumnEvent.ColumnWithPosition( + Column.physicalColumn("new_col", new IntType()))))); + metadataApplier.applySchemaChange( + new AlterColumnTypeEvent( + tableId, Collections.singletonMap("id", new BigIntType(false)))); + + StarRocksTable actualTable = + catalog.getTable(tableId.getSchemaName(), tableId.getTableName()).orElse(null); + Assertions.assertThat(actualTable).isNotNull(); + Assertions.assertThat(actualTable.getColumn("id").getDataType()) + .isEqualToIgnoringCase("bigint"); + Assertions.assertThat(actualTable.getColumn("new_col")).isNotNull(); + } + + @Test + void testRejectNarrowingAlterColumnType() { + TableId tableId = TableId.parse("test.narrow_alter_tbl"); + Schema schema = + Schema.newBuilder() + .physicalColumn("id", new BigIntType(false)) + .physicalColumn("number", new DoubleType()) + .primaryKey("id") + .build(); + metadataApplier.applySchemaChange(new CreateTableEvent(tableId, schema)); + + Assertions.assertThatThrownBy( + () -> + metadataApplier.applySchemaChange( + new AlterColumnTypeEvent( + tableId, + Collections.singletonMap( + "id", new IntType(false))))) + .isInstanceOfSatisfying( + SchemaEvolveException.class, + exception -> + Assertions.assertThat(exception.getExceptionMessage()) + .contains("Cannot safely widen")); + + Assertions.assertThatThrownBy( + () -> + metadataApplier.applySchemaChange( + new AlterColumnTypeEvent( + tableId, + Collections.singletonMap( + "number", new FloatType())))) + .isInstanceOfSatisfying( + SchemaEvolveException.class, + exception -> + Assertions.assertThat(exception.getExceptionMessage()) + .contains("Cannot safely widen")); + + StarRocksTable actualTable = + catalog.getTable(tableId.getSchemaName(), tableId.getTableName()).orElse(null); + Assertions.assertThat(actualTable).isNotNull(); + Assertions.assertThat(actualTable.getColumn("id").getDataType()) + .isEqualToIgnoringCase("bigint"); + Assertions.assertThat(actualTable.getColumn("number").getDataType()) + .isEqualToIgnoringCase("double"); + } + + @Test + void testRejectCreateTableWhenExistingTableMissesInferredColumn() { + TableId tableId = TableId.parse("test.narrow_existing_tbl"); + Schema existingSchema = + Schema.newBuilder() + .physicalColumn("id", new IntType(false)) + .physicalColumn("name", new IntType()) + .primaryKey("id") + .build(); + metadataApplier.applySchemaChange(new CreateTableEvent(tableId, existingSchema)); + + Schema firstMessageSchema = + Schema.newBuilder() + .physicalColumn("id", new IntType(false)) + .physicalColumn("name", new IntType()) + .physicalColumn("email", new IntType()) + .primaryKey("id") + .build(); + + Assertions.assertThatThrownBy( + () -> + metadataApplier.applySchemaChange( + new CreateTableEvent(tableId, firstMessageSchema))) + .isInstanceOfSatisfying( + SchemaEvolveException.class, + exception -> + Assertions.assertThat(exception.getExceptionMessage()) + .contains("missing or incompatible") + .contains("email")); + + StarRocksTable actualTable = + catalog.getTable(tableId.getSchemaName(), tableId.getTableName()).orElse(null); + Assertions.assertThat(actualTable).isNotNull(); + Assertions.assertThat(actualTable.getColumn("email")).isNull(); + } + + @Test + void testRejectReplayWhenPrimaryKeysDiffer() { + TableId tableId = TableId.parse("test.incompatible_replay_tbl"); + Schema currentSchema = + Schema.newBuilder() + .physicalColumn("id", new IntType(false)) + .physicalColumn("other_id", new IntType(false)) + .primaryKey("other_id") + .build(); + metadataApplier.applySchemaChange(new CreateTableEvent(tableId, currentSchema)); + + Schema historicalSchema = + Schema.newBuilder() + .physicalColumn("id", new IntType(false)) + .primaryKey("id") + .build(); + + Assertions.assertThatThrownBy( + () -> + metadataApplier.applySchemaChange( + new CreateTableEvent(tableId, historicalSchema))) + .isInstanceOfSatisfying( + SchemaEvolveException.class, + exception -> + Assertions.assertThat(exception.getExceptionMessage()) + .contains("primary keys")); + } + @Test void testDropColumn() throws Exception { TableId tableId = TableId.parse("test.tbl3"); diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/KafkaToPaimonE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/KafkaToPaimonE2eITCase.java new file mode 100644 index 00000000000..c97de2df49e --- /dev/null +++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/KafkaToPaimonE2eITCase.java @@ -0,0 +1,639 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.pipeline.tests; + +import org.apache.flink.cdc.common.test.utils.TestUtils; +import org.apache.flink.cdc.connectors.kafka.sink.KafkaUtil; +import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment; + +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.MountableFile; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.util.DockerImageVersions.KAFKA; + +/** + * End-to-end tests for multi-partition Kafka Debezium and Canal JSON to Paimon pipelines, covering + * create table, add column, alter column type, DML, cross-partition historical replay, + * rename-as-superset and multi-table sync. + */ +class KafkaToPaimonE2eITCase extends PipelineTestEnvironment { + private static final Logger LOG = LoggerFactory.getLogger(KafkaToPaimonE2eITCase.class); + + private static final Duration PAIMON_TESTCASE_TIMEOUT = Duration.ofMinutes(3); + private static final String DATABASE = "inventory"; + private static final String KAFKA_ALIAS = "kafka"; + + @Container + private static final KafkaContainer KAFKA_CONTAINER = + KafkaUtil.createKafkaContainer(KAFKA, LOG) + .withEmbeddedZookeeper() + .withNetwork(NETWORK) + .withNetworkAliases(KAFKA_ALIAS); + + private AdminClient admin; + private KafkaProducer producer; + private String topic; + private String table; + private String warehouse; + private EventFormat eventFormat; + + @BeforeAll + public static void initializeContainers() { + LOG.info("Starting containers..."); + Startables.deepStart(Stream.of(KAFKA_CONTAINER)).join(); + LOG.info("Containers are started."); + } + + @BeforeEach + public void before() throws Exception { + super.before(); + topic = "kafka-customers-" + UUID.randomUUID(); + table = "customers_" + UUID.randomUUID().toString().replace("-", "").substring(0, 8); + warehouse = sharedVolume.toString() + "/paimon_" + UUID.randomUUID(); + jobManager.copyFileToContainer( + MountableFile.forHostPath( + TestUtils.getResource(getPaimonSQLConnectorResourceName())), + sharedVolume.toString() + "/" + getPaimonSQLConnectorResourceName()); + jobManager.copyFileToContainer( + MountableFile.forHostPath(TestUtils.getResource("flink-shade-hadoop.jar")), + sharedVolume.toString() + "/flink-shade-hadoop.jar"); + Properties properties = kafkaProperties(); + admin = AdminClient.create(properties); + admin.createTopics(Collections.singletonList(new NewTopic(topic, 2, (short) 1))) + .all() + .get(); + properties.setProperty("key.serializer", ByteArraySerializer.class.getName()); + properties.setProperty("value.serializer", ByteArraySerializer.class.getName()); + producer = new KafkaProducer<>(properties); + } + + @AfterEach + public void after() { + if (producer != null) { + producer.close(); + } + if (admin != null) { + admin.deleteTopics(Collections.singletonList(topic)); + admin.close(); + } + super.after(); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testCreateTableAddColumnModifyColumnAndDml(EventFormat format) throws Exception { + submitKafkaToPaimonJob(format); + + LOG.info("Create table and snapshot/insert records..."); + send(0, value(createFields(), "c", "null", "{\"id\":1,\"name\":\"alice\",\"age\":18}")); + send(0, value(createFields(), "r", "null", "{\"id\":10,\"name\":\"snapshot\",\"age\":30}")); + validateSinkSchema(Arrays.asList("id, INT", "name, STRING", "age, INT")); + validateSinkResult(Arrays.asList("1, alice, 18", "10, snapshot, 30")); + + LOG.info("Add column..."); + send( + 0, + value( + addColumnFields(), + "c", + "null", + "{\"id\":2,\"name\":\"bob\",\"age\":21,\"email\":\"bob@example.com\"}")); + validateSinkSchema(Arrays.asList("id, INT", "name, STRING", "age, INT", "email, STRING")); + validateSinkResult( + Arrays.asList( + "1, alice, 18, null", + "10, snapshot, 30, null", + "2, bob, 21, bob@example.com")); + + LOG.info("Alter column type INT to BIGINT..."); + send( + 0, + value( + modifyColumnFields(), + "c", + "null", + "{\"id\":3,\"name\":\"charlie\",\"age\":40,\"email\":\"charlie@example.com\"}")); + validateSinkSchema( + Arrays.asList("id, INT", "name, STRING", "age, BIGINT", "email, STRING")); + validateSinkResult( + Arrays.asList( + "1, alice, 18, null", + "10, snapshot, 30, null", + "2, bob, 21, bob@example.com", + "3, charlie, 40, charlie@example.com")); + + LOG.info("Update and delete by primary key..."); + send( + 0, + value( + modifyColumnFields(), + "u", + "{\"id\":1,\"name\":\"alice\",\"age\":18}", + "{\"id\":1,\"name\":\"alice2\",\"age\":18}")); + send( + 0, + value( + modifyColumnFields(), + "d", + "{\"id\":2,\"name\":\"bob\",\"age\":21,\"email\":\"bob@example.com\"}", + "null")); + validateSinkResult( + Arrays.asList( + "1, alice2, 18, null", + "10, snapshot, 30, null", + "3, charlie, 40, charlie@example.com")); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testNewSchemaThenHistoricalSchemaFromAnotherPartition(EventFormat format) + throws Exception { + submitKafkaToPaimonJob(format); + + send( + 1, + value( + newFields(), + "c", + "null", + "{\"id\":2147483648,\"name\":\"new\",\"email\":\"new@example.com\"}")); + validateSinkSchema(Arrays.asList("id, BIGINT", "name, STRING", "email, STRING")); + validateSinkResult(Collections.singletonList("2147483648, new, new@example.com")); + + send(0, value(oldFields(), "c", "null", "{\"id\":2,\"name\":\"old\"}")); + validateSinkResult(Arrays.asList("2, old, null", "2147483648, new, new@example.com")); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testReplayIntToStringAlterFromHistoricalOffset(EventFormat format) throws Exception { + submitKafkaToPaimonJob(format); + + LOG.info("Historical INT records..."); + send(0, value(intAgeFields(), "c", "null", "{\"id\":1,\"name\":\"alice\",\"age\":18}")); + validateSinkSchema(Arrays.asList("id, INT", "name, STRING", "age, INT")); + validateSinkResult(Collections.singletonList("1, alice, 18")); + + LOG.info("ALTER INT to STRING on the same partition..."); + send( + 0, + value( + stringAgeFields(), + "c", + "null", + "{\"id\":2,\"name\":\"bob\",\"age\":\"hello\"}")); + validateSinkSchema(Arrays.asList("id, INT", "name, STRING", "age, STRING")); + validateSinkResult(Arrays.asList("1, alice, 18", "2, bob, hello")); + + LOG.info("Replay remaining historical INT records from another partition..."); + send(1, value(intAgeFields(), "c", "null", "{\"id\":3,\"name\":\"carol\",\"age\":19}")); + validateSinkResult(Arrays.asList("1, alice, 18", "2, bob, hello", "3, carol, 19")); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testSamePartitionRenameKeepsOldColumnAndAddsNew(EventFormat format) throws Exception { + submitKafkaToPaimonJob(format); + + send(0, value(oldFields(), "c", "null", "{\"id\":1,\"name\":\"alice\"}")); + validateSinkSchema(Arrays.asList("id, INT", "name, STRING")); + validateSinkResult(Collections.singletonList("1, alice")); + + LOG.info("Source column name is replaced by full_name on the same partition..."); + send(0, value(renamedFields(), "c", "null", "{\"id\":2,\"full_name\":\"bob\"}")); + validateSinkSchema(Arrays.asList("id, INT", "name, STRING", "full_name, STRING")); + validateSinkResult(Arrays.asList("1, alice, null", "2, null, bob")); + + LOG.info("Historical records that still use name arrive from another partition..."); + send(1, value(oldFields(), "c", "null", "{\"id\":3,\"name\":\"carol\"}")); + validateSinkResult(Arrays.asList("1, alice, null", "2, null, bob", "3, carol, null")); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testMultiTableFromSameTopic(EventFormat format) throws Exception { + String orders = "orders_" + UUID.randomUUID().toString().replace("-", "").substring(0, 8); + submitKafkaToPaimonJob(format); + + send(0, value(table, oldFields(), "c", "null", "{\"id\":1,\"name\":\"alice\"}")); + send(1, value(orders, orderFields(), "c", "null", "{\"id\":1001,\"amount\":19}")); + validateSinkSchema(table, Arrays.asList("id, INT", "name, STRING")); + validateSinkSchema(orders, Arrays.asList("id, INT", "amount, INT")); + validateSinkResult(table, Collections.singletonList("1, alice")); + validateSinkResult(orders, Collections.singletonList("1001, 19")); + + LOG.info("Add column independently on both tables..."); + send( + 0, + value( + table, + oldFields() + "," + stringField("city"), + "c", + "null", + "{\"id\":2,\"name\":\"bob\",\"city\":\"berlin\"}")); + send( + 1, + value( + orders, + orderFields() + "," + stringField("status"), + "c", + "null", + "{\"id\":1002,\"amount\":7,\"status\":\"paid\"}")); + validateSinkSchema(table, Arrays.asList("id, INT", "name, STRING", "city, STRING")); + validateSinkSchema(orders, Arrays.asList("id, INT", "amount, INT", "status, STRING")); + validateSinkResult(table, Arrays.asList("1, alice, null", "2, bob, berlin")); + validateSinkResult(orders, Arrays.asList("1001, 19, null", "1002, 7, paid")); + } + + private void submitKafkaToPaimonJob(EventFormat format) throws Exception { + eventFormat = format; + Path kafkaJar = TestUtils.getResource("kafka-cdc-pipeline-connector.jar"); + Path paimonJar = TestUtils.getResource("paimon-cdc-pipeline-connector.jar"); + Path hadoopJar = TestUtils.getResource("flink-shade-hadoop.jar"); + submitPipelineJob(buildPipelineJob(), kafkaJar, paimonJar, hadoopJar); + waitUntilJobRunning(Duration.ofSeconds(30)); + LOG.info("Pipeline job is running"); + } + + private String buildPipelineJob() { + return String.format( + "source:\n" + + " type: kafka\n" + + " topic: %s\n" + + " group-id: %s\n" + + " scan.startup.mode: earliest-offset\n" + + " value.format: %s\n" + + " properties.bootstrap.servers: %s:9092\n" + + "\n" + + "transform:\n" + + " - source-table: %s.\\.*\n" + + " primary-keys: id\n" + + "\n" + + "sink:\n" + + " type: paimon\n" + + " catalog.properties.warehouse: %s\n" + + " catalog.properties.metastore: filesystem\n" + + " catalog.properties.cache-enabled: false\n" + + "\n" + + "pipeline:\n" + + " parallelism: 2\n" + + " schema.change.behavior: lenient\n", + topic, + UUID.randomUUID(), + eventFormat.optionValue, + KAFKA_ALIAS, + DATABASE, + warehouse); + } + + private void send(int partition, byte[] value) throws Exception { + producer.send(new ProducerRecord<>(topic, partition, null, value)).get(); + producer.flush(); + } + + private void validateSinkResult(List expected) throws InterruptedException { + validateSinkResult(table, expected); + } + + private void validateSinkResult(String tableName, List expected) + throws InterruptedException { + LOG.info("Verifying Paimon {}::{}::{} results...", warehouse, DATABASE, tableName); + long deadline = System.currentTimeMillis() + PAIMON_TESTCASE_TIMEOUT.toMillis(); + List results = Collections.emptyList(); + while (System.currentTimeMillis() < deadline) { + try { + results = fetchPaimonRows("docker/peek-paimon.sql", tableName); + Assertions.assertThat(results).containsExactlyInAnyOrderElementsOf(expected); + LOG.info( + "Successfully verified {} records in {} seconds.", + expected.size(), + (System.currentTimeMillis() - deadline + PAIMON_TESTCASE_TIMEOUT.toMillis()) + / 1000); + return; + } catch (Exception e) { + LOG.warn("Validate failed, waiting for the next loop...", e); + } catch (AssertionError ignored) { + LOG.warn( + "Results mismatch, expected {} records, but got {} actually. Waiting for the next loop...", + expected.size(), + results.size()); + } + Thread.sleep(1000L); + } + Assertions.assertThat(results).containsExactlyInAnyOrderElementsOf(expected); + } + + private void validateSinkSchema(List expected) throws InterruptedException { + validateSinkSchema(table, expected); + } + + private void validateSinkSchema(String tableName, List expected) + throws InterruptedException { + LOG.info("Verifying Paimon {}::{}::{} schema...", warehouse, DATABASE, tableName); + long deadline = System.currentTimeMillis() + PAIMON_TESTCASE_TIMEOUT.toMillis(); + List actual = Collections.emptyList(); + while (System.currentTimeMillis() < deadline) { + try { + actual = + fetchPaimonRows("docker/peek-paimon-schema.sql", tableName).stream() + .map(KafkaToPaimonE2eITCase::normalizeSchemaRow) + .collect(Collectors.toList()); + Assertions.assertThat(actual).containsExactlyElementsOf(expected); + return; + } catch (Exception e) { + LOG.warn("Schema validate failed, waiting for the next loop...", e); + } catch (AssertionError ignored) { + LOG.warn("Schema mismatch.\nExpected: {}\n Actual: {}", expected, actual); + } + Thread.sleep(1000L); + } + Assertions.assertThat(actual).containsExactlyElementsOf(expected); + } + + private List fetchPaimonRows(String sqlResource, String tableName) throws Exception { + String template = + readLines(sqlResource).stream() + .filter(line -> !line.startsWith("--")) + .collect(Collectors.joining("\n")); + String sql = String.format(template, warehouse, DATABASE, tableName); + String containerSqlPath = sharedVolume.toString() + "/peek.sql"; + jobManager.copyFileToContainer(Transferable.of(sql), containerSqlPath); + + org.testcontainers.containers.Container.ExecResult result = + jobManager.execInContainer( + "/opt/flink/bin/sql-client.sh", + "--jar", + sharedVolume.toString() + "/" + getPaimonSQLConnectorResourceName(), + "--jar", + sharedVolume.toString() + "/flink-shade-hadoop.jar", + "-f", + containerSqlPath); + if (result.getExitCode() != 0) { + throw new RuntimeException( + "Failed to execute peek script. Stdout: " + + result.getStdout() + + "; Stderr: " + + result.getStderr()); + } + return Arrays.stream(result.getStdout().split("\n")) + .filter(line -> line.startsWith("|")) + .skip(1) + .map(KafkaToPaimonE2eITCase::extractRow) + .map(row -> String.join(", ", row)) + .collect(Collectors.toList()); + } + + private static String[] extractRow(String row) { + return Arrays.stream(row.split("\\|")) + .map(String::trim) + .filter(col -> !col.isEmpty()) + .map(col -> col.equals("") ? "null" : col) + .toArray(String[]::new); + } + + private static String normalizeSchemaRow(String row) { + String[] parts = row.split(", "); + Assertions.assertThat(parts.length) + .as("Unexpected DESCRIBE row: %s", row) + .isGreaterThanOrEqualTo(2); + return stripIdentifier(parts[0]) + ", " + normalizeType(parts[1]); + } + + private static String stripIdentifier(String value) { + return value.replace("`", ""); + } + + private static String normalizeType(String type) { + String normalized = stripIdentifier(type).replace(" NOT NULL", "").trim(); + if (normalized.equalsIgnoreCase("INTEGER")) { + return "INT"; + } + if (normalized.equalsIgnoreCase("STRING") + || normalized.equalsIgnoreCase("VARCHAR(2147483647)")) { + return "STRING"; + } + return normalized; + } + + private Properties kafkaProperties() { + Properties properties = new Properties(); + properties.setProperty("bootstrap.servers", KAFKA_CONTAINER.getBootstrapServers()); + return properties; + } + + private byte[] value(String fields, String operation, String before, String after) { + return value(table, fields, operation, before, after); + } + + private byte[] value( + String tableName, String fields, String operation, String before, String after) { + if (eventFormat == EventFormat.DEBEZIUM_JSON) { + return debeziumValue(tableName, fields, operation, before, after); + } + if (eventFormat == EventFormat.CANAL_JSON) { + return canalValue(tableName, fields, operation, before, after); + } + throw new IllegalArgumentException("Unsupported event format " + eventFormat); + } + + private byte[] debeziumValue( + String tableName, String fields, String operation, String before, String after) { + String rowSchema = + "{\"type\":\"struct\",\"fields\":[" + + fields + + "],\"optional\":true,\"name\":\"" + + DATABASE + + "." + + tableName + + ".Value\"}"; + return bytes( + "{\"schema\":{\"type\":\"struct\",\"fields\":[" + + withField(rowSchema, "before") + + "," + + withField(rowSchema, "after") + + "]},\"payload\":{\"before\":" + + before + + ",\"after\":" + + after + + ",\"source\":{\"db\":\"" + + DATABASE + + "\",\"table\":\"" + + tableName + + "\"},\"op\":\"" + + operation + + "\"}}"); + } + + private byte[] canalValue( + String tableName, String fields, String operation, String before, String after) { + String data = "d".equals(operation) ? before : after; + String old = "u".equals(operation) ? asArray(before) : "null"; + return bytes( + "{\"data\":" + + asArray(data) + + ",\"database\":\"" + + DATABASE + + "\",\"isDdl\":false,\"mysqlType\":{" + + fields + + "},\"old\":" + + old + + ",\"pkNames\":[\"id\"],\"table\":\"" + + tableName + + "\",\"ts\":1589373560798,\"type\":\"" + + canalOperation(operation) + + "\"}"); + } + + private String createFields() { + return intField("id", false) + "," + stringField("name") + "," + intField("age", true); + } + + private String intAgeFields() { + return createFields(); + } + + private String stringAgeFields() { + return intField("id", false) + "," + stringField("name") + "," + stringField("age"); + } + + private String addColumnFields() { + return createFields() + "," + stringField("email"); + } + + private String modifyColumnFields() { + return intField("id", false) + + "," + + stringField("name") + + "," + + longField("age", true) + + "," + + stringField("email"); + } + + private String oldFields() { + return intField("id", false) + "," + stringField("name"); + } + + private String renamedFields() { + return intField("id", false) + "," + stringField("full_name"); + } + + private String newFields() { + return longField("id", false) + "," + stringField("name") + "," + stringField("email"); + } + + private String orderFields() { + return intField("id", false) + "," + intField("amount", true); + } + + private String intField(String name, boolean optional) { + return field("int32", "INTEGER", name, optional); + } + + private String longField(String name, boolean optional) { + return field("int64", "BIGINT", name, optional); + } + + private String stringField(String name) { + return field("string", "VARCHAR(255)", name, true); + } + + private String field(String debeziumType, String canalType, String name, boolean optional) { + if (eventFormat == EventFormat.DEBEZIUM_JSON) { + return "{\"type\":\"" + + debeziumType + + "\",\"optional\":" + + optional + + ",\"field\":\"" + + name + + "\"}"; + } + if (eventFormat == EventFormat.CANAL_JSON) { + return "\"" + name + "\":\"" + canalType + "\""; + } + throw new IllegalArgumentException("Unsupported event format " + eventFormat); + } + + private static String withField(String schema, String field) { + return schema.substring(0, schema.length() - 1) + ",\"field\":\"" + field + "\"}"; + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static String asArray(String row) { + return "null".equals(row) ? "null" : "[" + row + "]"; + } + + private static String canalOperation(String operation) { + switch (operation) { + case "c": + case "r": + return "INSERT"; + case "u": + return "UPDATE"; + case "d": + return "DELETE"; + default: + throw new IllegalArgumentException("Unsupported operation " + operation); + } + } + + private String getPaimonSQLConnectorResourceName() { + return String.format("paimon-sql-connector-%s.jar", flinkVersion); + } + + private enum EventFormat { + DEBEZIUM_JSON("debezium-json"), + CANAL_JSON("canal-json"); + + private final String optionValue; + + EventFormat(String optionValue) { + this.optionValue = optionValue; + } + } +} diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/KafkaToStarRocksE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/KafkaToStarRocksE2eITCase.java new file mode 100644 index 00000000000..6c8771a1883 --- /dev/null +++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/KafkaToStarRocksE2eITCase.java @@ -0,0 +1,580 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.pipeline.tests; + +import org.apache.flink.cdc.common.test.utils.TestUtils; +import org.apache.flink.cdc.connectors.kafka.sink.KafkaUtil; +import org.apache.flink.cdc.connectors.starrocks.sink.utils.StarRocksContainer; +import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment; + +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.lifecycle.Startables; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.util.DockerImageVersions.KAFKA; + +/** End-to-end tests for multi-partition Kafka Debezium and Canal JSON to StarRocks pipelines. */ +@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*") +class KafkaToStarRocksE2eITCase extends PipelineTestEnvironment { + private static final Logger LOG = LoggerFactory.getLogger(KafkaToStarRocksE2eITCase.class); + + private static final String DATABASE = "inventory"; + private static final String KAFKA_ALIAS = "kafka"; + private static final String STARROCKS_ALIAS = "starrocks"; + + @Container + private static final KafkaContainer KAFKA_CONTAINER = + KafkaUtil.createKafkaContainer(KAFKA, LOG) + .withEmbeddedZookeeper() + .withNetwork(NETWORK) + .withNetworkAliases(KAFKA_ALIAS); + + @Container + private static final StarRocksContainer STARROCKS_CONTAINER = + new StarRocksContainer(NETWORK).withNetworkAliases(STARROCKS_ALIAS); + + private AdminClient admin; + private KafkaProducer producer; + private String topic; + private String table; + private EventFormat eventFormat; + + @BeforeAll + public static void initializeContainers() throws Exception { + LOG.info("Starting containers..."); + Startables.deepStart(Stream.of(KAFKA_CONTAINER, STARROCKS_CONTAINER)).join(); + STARROCKS_CONTAINER.waitForLog( + ".*Enjoy the journey to StarRocks blazing-fast lake-house engine!.*\\s", 1, 240); + waitForStarRocksBackend(); + LOG.info("Containers are started."); + } + + @BeforeEach + public void before() throws Exception { + super.before(); + topic = "kafka-customers-" + UUID.randomUUID(); + table = "customers_" + UUID.randomUUID().toString().replace("-", "").substring(0, 8); + Properties properties = kafkaProperties(); + admin = AdminClient.create(properties); + admin.createTopics(Collections.singletonList(new NewTopic(topic, 2, (short) 1))) + .all() + .get(); + properties.setProperty("key.serializer", ByteArraySerializer.class.getName()); + properties.setProperty("value.serializer", ByteArraySerializer.class.getName()); + producer = new KafkaProducer<>(properties); + } + + @AfterEach + public void after() { + if (producer != null) { + producer.close(); + } + if (admin != null) { + admin.deleteTopics(Collections.singletonList(topic)); + admin.close(); + } + dropTableQuietly(); + super.after(); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testCreateTableAddColumnAndModifyColumnEvents(EventFormat format) throws Exception { + submitKafkaToStarRocksJob(format); + + LOG.info("Test Schema Change - Create Table..."); + send(0, value(createFields(), "{\"id\":1,\"name\":\"alice\",\"age\":18}")); + validateSinkSchema( + Arrays.asList( + "id | int | NO | true | null", + "name | varchar(1048576) | YES | false | null", + "age | int | YES | false | null")); + validateSinkResult(3, Collections.singletonList("1 | alice | 18")); + + LOG.info("Test Schema Change - Add Column..."); + send( + 0, + value( + addColumnFields(), + "{\"id\":2,\"name\":\"bob\",\"age\":21,\"email\":\"bob@example.com\"}")); + validateSinkSchema( + Arrays.asList( + "id | int | NO | true | null", + "name | varchar(1048576) | YES | false | null", + "age | int | YES | false | null", + "email | varchar(1048576) | YES | false | null")); + waitUntilStarRocksSchemaChangeIdle(); + validateSinkResult( + 4, Arrays.asList("1 | alice | 18 | null", "2 | bob | 21 | bob@example.com")); + + LOG.info("Test Schema Change - Alter Column Type..."); + send( + 0, + value( + modifyColumnFields(), + "{\"id\":3,\"name\":\"charlie\",\"age\":40,\"email\":\"charlie@example.com\"}")); + waitUntilStarRocksSchemaChangeIdle(); + validateSinkSchema( + Arrays.asList( + "id | int | NO | true | null", + "name | varchar(1048576) | YES | false | null", + "age | bigint | YES | false | null", + "email | varchar(1048576) | YES | false | null")); + validateSinkResult( + 4, + Arrays.asList( + "1 | alice | 18 | null", + "2 | bob | 21 | bob@example.com", + "3 | charlie | 40 | charlie@example.com")); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testNewSchemaThenHistoricalSchemaFromAnotherPartition(EventFormat format) + throws Exception { + submitKafkaToStarRocksJob(format); + + send( + 1, + value( + newFields(), + "{\"id\":2147483648,\"name\":\"new\",\"email\":\"new@example.com\"}")); + validateSinkSchema( + Arrays.asList( + "id | bigint | NO | true | null", + "name | varchar(1048576) | YES | false | null", + "email | varchar(1048576) | YES | false | null")); + validateSinkResult(3, Collections.singletonList("2147483648 | new | new@example.com")); + + send(0, value(oldFields(), "{\"id\":2,\"name\":\"old\"}")); + validateSinkResult( + 3, Arrays.asList("2 | old | null", "2147483648 | new | new@example.com")); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testReplayIntToStringAlterFromHistoricalOffset(EventFormat format) throws Exception { + submitKafkaToStarRocksJob(format); + + LOG.info("Historical INT records..."); + send(0, value(intAgeFields(), "{\"id\":1,\"name\":\"alice\",\"age\":18}")); + validateSinkSchema( + Arrays.asList( + "id | int | NO | true | null", + "name | varchar(1048576) | YES | false | null", + "age | int | YES | false | null")); + validateSinkResult(3, Collections.singletonList("1 | alice | 18")); + + LOG.info("ALTER INT to STRING on the same partition..."); + send(0, value(stringAgeFields(), "{\"id\":2,\"name\":\"bob\",\"age\":\"hello\"}")); + waitUntilStarRocksSchemaChangeIdle(); + validateSinkSchema( + Arrays.asList( + "id | int | NO | true | null", + "name | varchar(1048576) | YES | false | null", + "age | varchar(1048576) | YES | false | null")); + validateSinkResult(3, Arrays.asList("1 | alice | 18", "2 | bob | hello")); + + LOG.info("Replay remaining historical INT records from another partition..."); + send(1, value(intAgeFields(), "{\"id\":3,\"name\":\"carol\",\"age\":19}")); + validateSinkResult(3, Arrays.asList("1 | alice | 18", "2 | bob | hello", "3 | carol | 19")); + } + + @ParameterizedTest(name = "format: {0}") + @EnumSource(EventFormat.class) + void testSamePartitionRenameKeepsOldColumnAndAddsNew(EventFormat format) throws Exception { + submitKafkaToStarRocksJob(format); + + send(0, value(oldFields(), "{\"id\":1,\"name\":\"alice\"}")); + validateSinkSchema( + Arrays.asList( + "id | int | NO | true | null", + "name | varchar(1048576) | YES | false | null")); + validateSinkResult(2, Collections.singletonList("1 | alice")); + + LOG.info("Source column name is replaced by full_name on the same partition..."); + send(0, value(renamedFields(), "{\"id\":2,\"full_name\":\"bob\"}")); + waitUntilStarRocksSchemaChangeIdle(); + validateSinkSchema( + Arrays.asList( + "id | int | NO | true | null", + "name | varchar(1048576) | YES | false | null", + "full_name | varchar(1048576) | YES | false | null")); + validateSinkResult(3, Arrays.asList("1 | alice | null", "2 | null | bob")); + + LOG.info("Historical records that still use name arrive from another partition..."); + send(1, value(oldFields(), "{\"id\":3,\"name\":\"carol\"}")); + validateSinkResult( + 3, Arrays.asList("1 | alice | null", "2 | null | bob", "3 | carol | null")); + } + + private void submitKafkaToStarRocksJob(EventFormat format) throws Exception { + eventFormat = format; + Path kafkaJar = TestUtils.getResource("kafka-cdc-pipeline-connector.jar"); + Path starRocksJar = TestUtils.getResource("starrocks-cdc-pipeline-connector.jar"); + submitPipelineJob(buildPipelineJob(), kafkaJar, starRocksJar); + waitUntilJobRunning(Duration.ofSeconds(30)); + LOG.info("Pipeline job is running"); + } + + private String buildPipelineJob() { + return String.format( + "source:\n" + + " type: kafka\n" + + " topic: %s\n" + + " group-id: %s\n" + + " scan.startup.mode: earliest-offset\n" + + " value.format: %s\n" + + " properties.bootstrap.servers: %s:9092\n" + + "\n" + + "transform:\n" + + " - source-table: %s.\\.*\n" + + " primary-keys: id\n" + + "\n" + + "sink:\n" + + " type: starrocks\n" + + " jdbc-url: jdbc:mysql://%s:9030\n" + + " load-url: %s:8080\n" + + " username: root\n" + + " password: \"\"\n" + + " table.create.properties.replication_num: 1\n" + + "\n" + + "pipeline:\n" + + " parallelism: 2\n" + + " schema.change.behavior: lenient\n", + topic, + UUID.randomUUID(), + eventFormat.optionValue, + KAFKA_ALIAS, + DATABASE, + STARROCKS_ALIAS, + STARROCKS_ALIAS); + } + + private void send(int partition, byte[] value) throws Exception { + producer.send(new ProducerRecord<>(topic, partition, null, value)).get(); + producer.flush(); + } + + private void validateSinkResult(int columnCount, List expected) throws Exception { + waitAndVerify("SELECT * FROM " + qualifiedTable(), columnCount, expected, true); + } + + private void validateSinkSchema(List expected) throws Exception { + waitAndVerify("DESCRIBE " + qualifiedTable(), 5, expected, false); + } + + private void waitAndVerify( + String sql, int numberOfColumns, List expected, boolean inAnyOrder) + throws Exception { + long deadline = System.currentTimeMillis() + EVENT_WAITING_TIMEOUT.toMillis(); + List actual = Collections.emptyList(); + while (System.currentTimeMillis() < deadline) { + try { + actual = fetchTableContent(sql, numberOfColumns); + if (inAnyOrder) { + if (expected.stream() + .sorted() + .collect(Collectors.toList()) + .equals(actual.stream().sorted().collect(Collectors.toList()))) { + return; + } + } else if (expected.equals(actual)) { + return; + } + LOG.info( + "Executing {} didn't get expected results.\nExpected: {}\n Actual: {}\n Alter: {}", + sql, + expected, + actual, + latestAlterState()); + } catch (SQLException t) { + LOG.info( + "Table {} isn't ready yet. Waiting for the next loop...", qualifiedTable()); + } + Thread.sleep(1000L); + } + Assertions.fail( + String.format( + "Failed to verify content of %s::%s. Actual: %s", DATABASE, sql, actual)); + } + + private List fetchTableContent(String sql, int columnCount) throws Exception { + List results = new ArrayList<>(); + try (Connection conn = STARROCKS_CONTAINER.createConnection(""); + Statement stat = conn.createStatement(); + ResultSet rs = stat.executeQuery(sql)) { + while (rs.next()) { + List columns = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + try { + columns.add(rs.getString(i)); + } catch (SQLException ignored) { + columns.add(null); + } + } + results.add(String.join(" | ", columns)); + } + } + return results; + } + + private void waitUntilStarRocksSchemaChangeIdle() throws Exception { + long deadline = System.currentTimeMillis() + EVENT_WAITING_TIMEOUT.toMillis(); + long idleSince = -1L; + String lastState = "ABSENT"; + while (System.currentTimeMillis() < deadline) { + lastState = latestAlterState(); + if (lastState.startsWith("CANCELLED")) { + Assertions.fail("StarRocks schema change was cancelled: " + lastState); + } + boolean running = + lastState.startsWith("PENDING") + || lastState.startsWith("WAITING_TXN") + || lastState.startsWith("RUNNING"); + if (running) { + idleSince = -1L; + } else if (idleSince < 0) { + idleSince = System.currentTimeMillis(); + } else if (System.currentTimeMillis() - idleSince >= 3000L) { + return; + } + Thread.sleep(1000L); + } + Assertions.fail( + "Timed out waiting for StarRocks schema change to become idle, last state: " + + lastState); + } + + private String latestAlterState() { + try (Connection connection = STARROCKS_CONTAINER.createConnection(""); + Statement statement = connection.createStatement()) { + statement.execute("USE `" + DATABASE + "`"); + try (ResultSet resultSet = + statement.executeQuery( + "SHOW ALTER TABLE COLUMN WHERE TableName = '" + + table + + "' ORDER BY CreateTime DESC LIMIT 1")) { + if (!resultSet.next()) { + return "ABSENT"; + } + String msg = resultSet.getString("Msg"); + return resultSet.getString("State") + (msg == null ? "" : "/" + msg); + } + } catch (Exception e) { + return e.getMessage(); + } + } + + private void dropTableQuietly() { + try (Connection connection = STARROCKS_CONTAINER.createConnection(""); + Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + qualifiedTable()); + } catch (Exception e) { + LOG.info("Failed to drop StarRocks table {}.", qualifiedTable(), e); + } + } + + private String qualifiedTable() { + return "`" + DATABASE + "`.`" + table + "`"; + } + + private static void waitForStarRocksBackend() throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofMinutes(4).toMillis(); + while (System.currentTimeMillis() < deadline) { + try (Connection connection = STARROCKS_CONTAINER.createConnection(""); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SHOW BACKENDS")) { + if (resultSet.next() && resultSet.getBoolean("Alive")) { + return; + } + } catch (Exception e) { + LOG.info("StarRocks backend is not ready yet.", e); + } + Thread.sleep(1000L); + } + throw new RuntimeException("StarRocks backend startup timed out."); + } + + private Properties kafkaProperties() { + Properties properties = new Properties(); + properties.setProperty("bootstrap.servers", KAFKA_CONTAINER.getBootstrapServers()); + return properties; + } + + private byte[] value(String fields, String row) { + if (eventFormat == EventFormat.DEBEZIUM_JSON) { + return debeziumValue(fields, row); + } + if (eventFormat == EventFormat.CANAL_JSON) { + return canalValue(fields, row); + } + throw new IllegalArgumentException("Unsupported event format " + eventFormat); + } + + private byte[] debeziumValue(String fields, String row) { + String rowSchema = + "{\"type\":\"struct\",\"fields\":[" + + fields + + "],\"optional\":true,\"name\":\"" + + DATABASE + + "." + + table + + ".Value\"}"; + return bytes( + "{\"schema\":{\"type\":\"struct\",\"fields\":[" + + withField(rowSchema, "before") + + "," + + withField(rowSchema, "after") + + "]},\"payload\":{\"before\":null,\"after\":" + + row + + ",\"source\":{\"db\":\"" + + DATABASE + + "\",\"table\":\"" + + table + + "\"},\"op\":\"c\"}}"); + } + + private byte[] canalValue(String fields, String row) { + return bytes( + "{\"data\":[" + + row + + "],\"database\":\"" + + DATABASE + + "\",\"isDdl\":false,\"mysqlType\":{" + + fields + + "},\"old\":null,\"pkNames\":[\"id\"],\"table\":\"" + + table + + "\",\"ts\":1589373560798,\"type\":\"INSERT\"}"); + } + + private String createFields() { + return intField("id", false) + "," + stringField("name") + "," + intField("age", true); + } + + private String intAgeFields() { + return createFields(); + } + + private String stringAgeFields() { + return intField("id", false) + "," + stringField("name") + "," + stringField("age"); + } + + private String addColumnFields() { + return createFields() + "," + stringField("email"); + } + + private String modifyColumnFields() { + return intField("id", false) + + "," + + stringField("name") + + "," + + longField("age", true) + + "," + + stringField("email"); + } + + private String oldFields() { + return intField("id", false) + "," + stringField("name"); + } + + private String renamedFields() { + return intField("id", false) + "," + stringField("full_name"); + } + + private String newFields() { + return longField("id", false) + "," + stringField("name") + "," + stringField("email"); + } + + private String intField(String name, boolean optional) { + return field("int32", "INTEGER", name, optional); + } + + private String longField(String name, boolean optional) { + return field("int64", "BIGINT", name, optional); + } + + private String stringField(String name) { + return field("string", "VARCHAR(255)", name, true); + } + + private String field(String debeziumType, String canalType, String name, boolean optional) { + if (eventFormat == EventFormat.DEBEZIUM_JSON) { + return "{\"type\":\"" + + debeziumType + + "\",\"optional\":" + + optional + + ",\"field\":\"" + + name + + "\"}"; + } + if (eventFormat == EventFormat.CANAL_JSON) { + return "\"" + name + "\":\"" + canalType + "\""; + } + throw new IllegalArgumentException("Unsupported event format " + eventFormat); + } + + private static String withField(String schema, String field) { + return schema.substring(0, schema.length() - 1) + ",\"field\":\"" + field + "\"}"; + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private enum EventFormat { + DEBEZIUM_JSON("debezium-json"), + CANAL_JSON("canal-json"); + + private final String optionValue; + + EventFormat(String optionValue) { + this.optionValue = optionValue; + } + } +} diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/docker/peek-paimon-schema.sql b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/docker/peek-paimon-schema.sql new file mode 100644 index 00000000000..6dff30195cb --- /dev/null +++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/docker/peek-paimon-schema.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Format this file with the following arguments: +-- Warehouse Path, Database Name, and Table Name. + +SET 'sql-client.execution.result-mode' = 'tableau'; +SET 'table.display.max-column-width' = '100000'; +SET 'execution.runtime-mode' = 'batch'; + +CREATE CATALOG paimon_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = '%s' +); + +DESCRIBE paimon_catalog.%s.%s; diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinator.java index f9aad19b856..7e193681888 100755 --- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinator.java +++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinator.java @@ -59,9 +59,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; @@ -83,6 +85,10 @@ public class SchemaCoordinator extends SchemaRegistry { Integer, Tuple2>> pendingRequests; + /** Requests arriving while the current schema evolution round is still running. */ + private transient Queue>> + deferredRequests; + /** Tracing sink writers that have flushed successfully. */ protected transient Set flushedSinkWriters; @@ -133,6 +139,7 @@ public void start() throws Exception { super.start(); this.evolvingStatus = new AtomicReference<>(RequestStatus.IDLE); this.pendingRequests = new ConcurrentHashMap<>(); + this.deferredRequests = new ConcurrentLinkedQueue<>(); this.flushedSinkWriters = ConcurrentHashMap.newKeySet(); this.upstreamSchemaTable = HashBasedTable.create(); this.alreadyHandledSchemaChangeEvents = HashMultimap.create(); @@ -216,6 +223,7 @@ protected void handleUnrecoverableError(String taskDescription, Throwable t) { (index, tuple) -> { tuple.f1.completeExceptionally(t); }); + deferredRequests.forEach(tuple -> tuple.f1.completeExceptionally(t)); } // ------------------------- @@ -226,6 +234,14 @@ private void handleSchemaEvolveRequest( SchemaChangeRequest request, CompletableFuture responseFuture) throws Exception { LOG.info("Coordinator received schema change request {}.", request); + if (evolvingStatus.get() == RequestStatus.EVOLVING) { + LOG.info( + "Schema evolution is in progress. Deferring request {} to the next round.", + request); + deferredRequests.add(Tuple2.of(request, responseFuture)); + return; + } + if (!request.isNoOpRequest()) { LOG.info("It's not an align request, will try to deduplicate."); int eventSourcePartitionId = request.getSourceSubTaskId(); @@ -353,10 +369,20 @@ private void startSchemaChange() throws TimeoutException { .ifPresent(schema -> evolvedSchemaView.put(tableId, schema)); } + runInEventLoop( + () -> finishSchemaChange(evolvedSchemaView, successfullyAppliedSchemaChangeEvents), + "Finishing schema change"); + } + + private void finishSchemaChange( + Map evolvedSchemaView, + List successfullyAppliedSchemaChangeEvents) + throws Throwable { List>> futures = new ArrayList<>(pendingRequests.values()); - // Restore coordinator internal states first... + // Restore coordinator internal states first. Since this runs in the coordinator event loop, + // new requests cannot observe IDLE before deferred requests have been promoted. pendingRequests.clear(); LOG.info("Finished schema evolving. Switching from EVOLVING to IDLE."); @@ -364,9 +390,15 @@ private void startSchemaChange() throws TimeoutException { evolvingStatus.compareAndSet(RequestStatus.EVOLVING, RequestStatus.IDLE), "RequestStatus should be EVOLVING when schema evolving finishes."); - // ... and broadcast affected schema changes to mapper and release upstream then. - // Make sure we've cleaned-up internal state before this, or we may receive new requests in - // a dirty state. + try { + processDeferredRequests(); + } catch (Throwable t) { + futures.forEach(tuple -> tuple.f1.completeExceptionally(t)); + throw t; + } + + // Broadcast affected schema changes to mapper and release upstream after internal state has + // been cleaned up. futures.forEach( tuple -> { LOG.info( @@ -380,6 +412,23 @@ private void startSchemaChange() throws TimeoutException { }); } + private void processDeferredRequests() throws Exception { + if (deferredRequests.isEmpty()) { + return; + } + + int deferredRequestCount = deferredRequests.size(); + LOG.info("Processing {} deferred schema change requests.", deferredRequestCount); + for (int i = 0; i < deferredRequestCount; i++) { + Tuple2> deferredRequest = + deferredRequests.peek(); + if (deferredRequest != null) { + handleSchemaEvolveRequest(deferredRequest.f0, deferredRequest.f1); + deferredRequests.poll(); + } + } + } + private Tuple2, List> deduceEvolvedSchemaChanges() { List validSchemaChangeRequests = pendingRequests.values().stream() diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinatorTest.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinatorTest.java new file mode 100644 index 00000000000..d2b3750c9f6 --- /dev/null +++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinatorTest.java @@ -0,0 +1,520 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.runtime.operators.schema.distributed; + +import org.apache.flink.cdc.common.event.AddColumnEvent; +import org.apache.flink.cdc.common.event.CreateTableEvent; +import org.apache.flink.cdc.common.event.SchemaChangeEvent; +import org.apache.flink.cdc.common.event.SchemaChangeEventType; +import org.apache.flink.cdc.common.event.SchemaChangeEventTypeFamily; +import org.apache.flink.cdc.common.event.TableId; +import org.apache.flink.cdc.common.pipeline.RouteMode; +import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior; +import org.apache.flink.cdc.common.schema.Column; +import org.apache.flink.cdc.common.schema.Schema; +import org.apache.flink.cdc.common.types.DataTypes; +import org.apache.flink.cdc.runtime.operators.schema.common.event.FlushSuccessEvent; +import org.apache.flink.cdc.runtime.operators.schema.distributed.event.SchemaChangeRequest; +import org.apache.flink.cdc.runtime.testutils.operators.MockedOperatorCoordinatorContext; +import org.apache.flink.cdc.runtime.testutils.schema.CollectingMetadataApplier; +import org.apache.flink.runtime.jobgraph.OperatorID; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Unit tests for {@link SchemaCoordinator}. */ +class SchemaCoordinatorTest { + + private static final TableId TABLE_ID = TableId.parse("foo.bar"); + private static final int PARALLELISM = 2; + + @Test + void testDefersRequestArrivingDuringSchemaEvolution() throws Exception { + ExecutorService coordinatorExecutor = Executors.newSingleThreadExecutor(); + MockedOperatorCoordinatorContext context = + new MockedOperatorCoordinatorContext( + new OperatorID(), Thread.currentThread().getContextClassLoader()); + CollectingMetadataApplier metadataApplier = + new CollectingMetadataApplier(Duration.ofMillis(300)); + SchemaCoordinator coordinator = + new SchemaCoordinator( + "SchemaCoordinator", + context, + coordinatorExecutor, + metadataApplier, + Collections.emptyList(), + RouteMode.ALL_MATCH, + SchemaChangeBehavior.LENIENT, + Duration.ofSeconds(5)); + + Schema initialSchema = + Schema.newBuilder().physicalColumn("id", DataTypes.INT()).primaryKey("id").build(); + CreateTableEvent createTableEvent = new CreateTableEvent(TABLE_ID, initialSchema); + AddColumnEvent addColumnEvent = + new AddColumnEvent( + TABLE_ID, + Collections.singletonList( + new AddColumnEvent.ColumnWithPosition( + Column.physicalColumn("name", DataTypes.STRING()), + AddColumnEvent.ColumnPosition.LAST, + null))); + + try { + coordinator.start(); + + CompletableFuture createFuture = + coordinator.handleCoordinationRequest( + new SchemaChangeRequest(0, 0, createTableEvent)); + coordinator.handleEventFromOperator(0, 0, new FlushSuccessEvent(0, 0)); + CompletableFuture addColumnFuture = + coordinator.handleCoordinationRequest( + new SchemaChangeRequest(0, 0, addColumnEvent)); + + createFuture.get(5, TimeUnit.SECONDS); + coordinator.handleEventFromOperator(0, 0, new FlushSuccessEvent(0, 0)); + addColumnFuture.get(5, TimeUnit.SECONDS); + + List appliedEvents = metadataApplier.getSchemaChangeEvents(); + assertThat(appliedEvents).containsExactly(createTableEvent, addColumnEvent); + assertThat(context.isJobFailed()).isFalse(); + } finally { + coordinator.close(); + coordinatorExecutor.shutdownNow(); + } + } + + @Test + void testDefersBroadcastRequestsWhenParallelismGreaterThanOne() throws Exception { + ExecutorService coordinatorExecutor = Executors.newSingleThreadExecutor(); + MockedOperatorCoordinatorContext context = mockedContext(PARALLELISM); + CollectingMetadataApplier metadataApplier = + new CollectingMetadataApplier(Duration.ofMillis(300)); + SchemaCoordinator coordinator = coordinator(context, coordinatorExecutor, metadataApplier); + + CreateTableEvent createTableEvent = createTableEvent(); + AddColumnEvent addColumnEvent = addColumnEvent("name"); + + try { + coordinator.start(); + + List> createFutures = + requestBroadcast(coordinator, 0, createTableEvent, PARALLELISM); + flushAll(coordinator, PARALLELISM); + waitUntilApplied(metadataApplier, 1); + + List> addColumnFutures = + requestBroadcast(coordinator, 0, addColumnEvent, PARALLELISM); + + awaitAll(createFutures); + flushAll(coordinator, PARALLELISM); + awaitAll(addColumnFutures); + + assertThat(metadataApplier.getSchemaChangeEvents()) + .containsExactly(createTableEvent, addColumnEvent); + assertThat(context.isJobFailed()).isFalse(); + } finally { + coordinator.close(); + coordinatorExecutor.shutdownNow(); + } + } + + @Test + void testProcessesMultipleDeferredRoundsWhenParallelismGreaterThanOne() throws Exception { + ExecutorService coordinatorExecutor = Executors.newSingleThreadExecutor(); + MockedOperatorCoordinatorContext context = mockedContext(PARALLELISM); + CollectingMetadataApplier metadataApplier = + new CollectingMetadataApplier(Duration.ofMillis(300)); + SchemaCoordinator coordinator = coordinator(context, coordinatorExecutor, metadataApplier); + + CreateTableEvent createTableEvent = createTableEvent(); + AddColumnEvent addNameEvent = addColumnEvent("name"); + AddColumnEvent addEmailEvent = addColumnEvent("email"); + + try { + coordinator.start(); + + List> createFutures = + requestBroadcast(coordinator, 0, createTableEvent, PARALLELISM); + flushAll(coordinator, PARALLELISM); + waitUntilApplied(metadataApplier, 1); + + List> addNameFutures = + requestBroadcast(coordinator, 0, addNameEvent, PARALLELISM); + + awaitAll(createFutures); + flushAll(coordinator, PARALLELISM); + waitUntilApplied(metadataApplier, 2); + + List> addEmailFutures = + requestBroadcast(coordinator, 0, addEmailEvent, PARALLELISM); + + awaitAll(addNameFutures); + flushAll(coordinator, PARALLELISM); + awaitAll(addEmailFutures); + + assertThat(metadataApplier.getSchemaChangeEvents()) + .containsExactly(createTableEvent, addNameEvent, addEmailEvent); + assertThat(context.isJobFailed()).isFalse(); + } finally { + coordinator.close(); + coordinatorExecutor.shutdownNow(); + } + } + + @Test + void testDoesNotLoseRequestArrivingBeforeDeferredRequestsArePromoted() throws Exception { + BlockingSchemaChangeCompletionExecutor coordinatorExecutor = + new BlockingSchemaChangeCompletionExecutor(); + MockedOperatorCoordinatorContext context = mockedContext(PARALLELISM); + BlockingMetadataApplier metadataApplier = new BlockingMetadataApplier(); + SchemaCoordinator coordinator = coordinator(context, coordinatorExecutor, metadataApplier); + + CreateTableEvent createTableEvent = createTableEvent(); + AddColumnEvent deferredEvent = addColumnEvent("name"); + AddColumnEvent concurrentEvent = addColumnEvent("email"); + + try { + coordinator.start(); + + List> createFutures = + requestBroadcast(coordinator, 0, createTableEvent, PARALLELISM); + flushAll(coordinator, PARALLELISM); + metadataApplier.awaitApplying(); + + List> deferredFutures = + requestBroadcast(coordinator, 0, deferredEvent, PARALLELISM); + coordinatorExecutor.awaitQuiescence(); + + coordinatorExecutor.blockNextSchemaThreadSubmission(); + metadataApplier.releaseApplying(); + coordinatorExecutor.awaitBlockedSubmission(); + + CompletableFuture concurrentSubtaskZeroFuture = + coordinator.handleCoordinationRequest( + new SchemaChangeRequest(0, 0, concurrentEvent)); + coordinatorExecutor.awaitQuiescence(); + coordinatorExecutor.releaseBlockedSubmission(); + + awaitAll(createFutures); + flushAll(coordinator, PARALLELISM); + awaitAll(deferredFutures); + + CompletableFuture concurrentSubtaskOneFuture = + coordinator.handleCoordinationRequest( + new SchemaChangeRequest(0, 1, concurrentEvent)); + flushAll(coordinator, PARALLELISM); + concurrentSubtaskZeroFuture.get(5, TimeUnit.SECONDS); + concurrentSubtaskOneFuture.get(5, TimeUnit.SECONDS); + + assertThat(metadataApplier.getSchemaChangeEvents()) + .containsExactly(createTableEvent, deferredEvent, concurrentEvent); + assertThat(context.isJobFailed()).isFalse(); + } finally { + metadataApplier.releaseApplying(); + coordinatorExecutor.releaseBlockedSubmission(); + coordinator.close(); + coordinatorExecutor.shutdownNow(); + } + } + + @Test + void testFailsCurrentAndDeferredRequestsWhenDeferredPromotionFails() throws Exception { + ExecutorService coordinatorExecutor = Executors.newSingleThreadExecutor(); + MockedOperatorCoordinatorContext context = mockedContext(PARALLELISM); + BlockingMetadataApplier metadataApplier = new BlockingMetadataApplier(); + SchemaCoordinator coordinator = coordinator(context, coordinatorExecutor, metadataApplier); + + CreateTableEvent createTableEvent = createTableEvent(); + AddColumnEvent invalidDeferredEvent = + new AddColumnEvent( + TableId.parse("unknown.table"), + Collections.singletonList( + new AddColumnEvent.ColumnWithPosition( + Column.physicalColumn("name", DataTypes.STRING()), + AddColumnEvent.ColumnPosition.LAST, + null))); + + try { + coordinator.start(); + + List> createFutures = + requestBroadcast(coordinator, 0, createTableEvent, PARALLELISM); + flushAll(coordinator, PARALLELISM); + metadataApplier.awaitApplying(); + + List> invalidFutures = + requestBroadcast(coordinator, 0, invalidDeferredEvent, PARALLELISM); + awaitExecutorQuiescence(coordinatorExecutor); + metadataApplier.releaseApplying(); + + for (CompletableFuture future : createFutures) { + assertThatThrownBy(() -> future.get(5, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class); + } + for (CompletableFuture future : invalidFutures) { + assertThatThrownBy(() -> future.get(5, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class); + } + assertThat(context.isJobFailed()).isTrue(); + assertThat(metadataApplier.getSchemaChangeEvents()).containsExactly(createTableEvent); + } finally { + metadataApplier.releaseApplying(); + coordinator.close(); + coordinatorExecutor.shutdownNow(); + } + } + + @Test + void testFailsDeferredRequestsWhenParallelEvolutionFails() throws Exception { + ExecutorService coordinatorExecutor = Executors.newSingleThreadExecutor(); + MockedOperatorCoordinatorContext context = mockedContext(PARALLELISM); + Set enabledEventTypes = + Arrays.stream(SchemaChangeEventTypeFamily.ALL).collect(Collectors.toSet()); + CollectingMetadataApplier metadataApplier = + new CollectingMetadataApplier( + Duration.ofMillis(300), + enabledEventTypes, + Collections.singleton(SchemaChangeEventType.CREATE_TABLE)); + SchemaCoordinator coordinator = coordinator(context, coordinatorExecutor, metadataApplier); + + CreateTableEvent createTableEvent = createTableEvent(); + AddColumnEvent addColumnEvent = addColumnEvent("name"); + + try { + coordinator.start(); + + List> createFutures = + requestBroadcast(coordinator, 0, createTableEvent, PARALLELISM); + flushAll(coordinator, PARALLELISM); + waitUntilApplied(metadataApplier, 1); + + List> addColumnFutures = + requestBroadcast(coordinator, 0, addColumnEvent, PARALLELISM); + // Let the coordinator event loop enqueue the broadcast requests before apply fails. + Thread.sleep(100L); + + for (CompletableFuture future : createFutures) { + assertThatThrownBy(() -> future.get(5, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class); + } + for (CompletableFuture future : addColumnFutures) { + assertThatThrownBy(() -> future.get(5, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class); + } + assertThat(context.isJobFailed()).isTrue(); + assertThat(metadataApplier.getSchemaChangeEvents()).containsExactly(createTableEvent); + } finally { + coordinator.close(); + coordinatorExecutor.shutdownNow(); + } + } + + private static MockedOperatorCoordinatorContext mockedContext(int parallelism) { + return new MockedOperatorCoordinatorContext( + new OperatorID(), parallelism, Thread.currentThread().getContextClassLoader()); + } + + private static SchemaCoordinator coordinator( + MockedOperatorCoordinatorContext context, + ExecutorService coordinatorExecutor, + CollectingMetadataApplier metadataApplier) { + return new SchemaCoordinator( + "SchemaCoordinator", + context, + coordinatorExecutor, + metadataApplier, + Collections.emptyList(), + RouteMode.ALL_MATCH, + SchemaChangeBehavior.LENIENT, + Duration.ofSeconds(5)); + } + + private static CreateTableEvent createTableEvent() { + return new CreateTableEvent( + TABLE_ID, + Schema.newBuilder().physicalColumn("id", DataTypes.INT()).primaryKey("id").build()); + } + + private static AddColumnEvent addColumnEvent(String columnName) { + return new AddColumnEvent( + TABLE_ID, + Collections.singletonList( + new AddColumnEvent.ColumnWithPosition( + Column.physicalColumn(columnName, DataTypes.STRING()), + AddColumnEvent.ColumnPosition.LAST, + null))); + } + + private static List> requestBroadcast( + SchemaCoordinator coordinator, + int sourceSubTaskId, + SchemaChangeEvent event, + int parallelism) { + List> futures = new ArrayList<>(parallelism); + for (int sinkSubTaskId = 0; sinkSubTaskId < parallelism; sinkSubTaskId++) { + futures.add( + coordinator.handleCoordinationRequest( + new SchemaChangeRequest(sourceSubTaskId, sinkSubTaskId, event))); + } + return futures; + } + + private static void flushAll(SchemaCoordinator coordinator, int parallelism) { + for (int sinkSubTaskId = 0; sinkSubTaskId < parallelism; sinkSubTaskId++) { + coordinator.handleEventFromOperator( + sinkSubTaskId, 0, new FlushSuccessEvent(sinkSubTaskId, 0)); + } + } + + private static void awaitAll(List> futures) throws Exception { + for (CompletableFuture future : futures) { + future.get(5, TimeUnit.SECONDS); + } + } + + private static void waitUntilApplied(CollectingMetadataApplier metadataApplier, int count) + throws InterruptedException { + long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(5); + while (System.currentTimeMillis() < deadline) { + if (metadataApplier.getSchemaChangeEvents().size() >= count) { + return; + } + Thread.sleep(10L); + } + assertThat(metadataApplier.getSchemaChangeEvents()) + .as("Timed out waiting for %s applied schema change events", count) + .hasSizeGreaterThanOrEqualTo(count); + } + + private static void awaitExecutorQuiescence(ExecutorService executor) throws Exception { + CompletableFuture quiescence = new CompletableFuture<>(); + executor.execute(() -> quiescence.complete(null)); + quiescence.get(5, TimeUnit.SECONDS); + } + + private static class BlockingMetadataApplier extends CollectingMetadataApplier { + private final CountDownLatch applying = new CountDownLatch(1); + private final CountDownLatch releaseApplying = new CountDownLatch(1); + + private BlockingMetadataApplier() { + super(null); + } + + @Override + public void applySchemaChange(SchemaChangeEvent schemaChangeEvent) { + super.applySchemaChange(schemaChangeEvent); + applying.countDown(); + try { + releaseApplying.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + void awaitApplying() throws InterruptedException { + assertThat(applying.await(5, TimeUnit.SECONDS)).isTrue(); + } + + void releaseApplying() { + releaseApplying.countDown(); + } + } + + private static class BlockingSchemaChangeCompletionExecutor extends AbstractExecutorService { + private final ExecutorService delegate = Executors.newSingleThreadExecutor(); + private final Thread testThread = Thread.currentThread(); + private final AtomicBoolean blockNextSchemaThreadSubmission = new AtomicBoolean(); + private final CountDownLatch blockedSubmission = new CountDownLatch(1); + private final CountDownLatch releaseBlockedSubmission = new CountDownLatch(1); + + @Override + public void execute(Runnable command) { + if (Thread.currentThread() != testThread + && blockNextSchemaThreadSubmission.compareAndSet(true, false)) { + blockedSubmission.countDown(); + try { + releaseBlockedSubmission.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + delegate.execute(command); + } + + void blockNextSchemaThreadSubmission() { + blockNextSchemaThreadSubmission.set(true); + } + + void awaitBlockedSubmission() throws Exception { + assertThat(blockedSubmission.await(5, TimeUnit.SECONDS)).isTrue(); + } + + void releaseBlockedSubmission() { + releaseBlockedSubmission.countDown(); + } + + void awaitQuiescence() throws Exception { + awaitExecutorQuiescence(this); + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + } +} diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/MockedOperatorCoordinatorContext.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/MockedOperatorCoordinatorContext.java index 19ab961eea3..e7a529dff8b 100644 --- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/MockedOperatorCoordinatorContext.java +++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/MockedOperatorCoordinatorContext.java @@ -30,6 +30,11 @@ public MockedOperatorCoordinatorContext( super(operatorID, userCodeClassLoader); } + public MockedOperatorCoordinatorContext( + OperatorID operatorID, int parallelism, ClassLoader userCodeClassLoader) { + super(operatorID, parallelism, userCodeClassLoader); + } + private Throwable failureCause; @Override