[mysql] Support metadata columns for mysql-cdc connector (#496)

Author: wuchong

flink-connector-debezium/src/main/java/com/ververica/cdc/debezium/table/AppendMetadataCollector.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ververica.cdc.debezium.table;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.util.Collector;
+
+import org.apache.kafka.connect.source.SourceRecord;
+
+import java.io.Serializable;
+
+/** Emits a row with physical fields and metadata fields. */
+@Internal
+public final class AppendMetadataCollector implements Collector<RowData>, Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private final MetadataConverter[] metadataConverters;
+
+    public transient SourceRecord inputRecord;
+    public transient Collector<RowData> outputCollector;
+
+    public AppendMetadataCollector(MetadataConverter[] metadataConverters) {
+        this.metadataConverters = metadataConverters;
+    }
+
+    @Override
+    public void collect(RowData physicalRow) {
+        GenericRowData metaRow = new GenericRowData(metadataConverters.length);
+        for (int i = 0; i < metadataConverters.length; i++) {
+            Object meta = metadataConverters[i].read(inputRecord);
+            metaRow.setField(i, meta);
+        }
+        RowData outRow = new JoinedRowData(physicalRow.getRowKind(), physicalRow, metaRow);
+        outputCollector.collect(outRow);
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

flink-connector-debezium/src/main/java/com/ververica/cdc/debezium/table/MetadataConverter.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ververica.cdc.debezium.table;
+
+import org.apache.flink.annotation.Internal;
+
+import org.apache.kafka.connect.source.SourceRecord;
+
+import java.io.Serializable;
+
+/** A converter converts {@link SourceRecord} metadata into Flink internal data structures. */
+@FunctionalInterface
+@Internal
+public interface MetadataConverter extends Serializable {
+    Object read(SourceRecord record);
+}

flink-connector-debezium/src/main/java/com/ververica/cdc/debezium/table/RowDataDebeziumDeserializeSchema.java

@@ -30,8 +30,6 @@
 import org.apache.flink.types.RowKind;
 import org.apache.flink.util.Collector;
 
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
-
 import com.ververica.cdc.debezium.DebeziumDeserializationSchema;
 import com.ververica.cdc.debezium.utils.TemporalConversions;
 import io.debezium.data.Envelope;
@@ -55,13 +53,15 @@
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 /**
  * Deserialization schema from Debezium object to Flink Table/SQL internal data structure {@link
  * RowData}.
  */
 public final class RowDataDebeziumDeserializeSchema
         implements DebeziumDeserializationSchema<RowData> {
-    private static final long serialVersionUID = -4852684966051743776L;
+    private static final long serialVersionUID = 2L;
 
     /** Custom validator to validate the row value. */
     public interface ValueValidator extends Serializable {
@@ -72,26 +72,42 @@ public interface ValueValidator extends Serializable {
     private final TypeInformation<RowData> resultTypeInfo;
 
     /**
-     * Runtime converter that converts {@link JsonNode}s into objects of Flink SQL internal data
-     * structures. *
+     * Runtime converter that converts Kafka {@link SourceRecord}s into {@link RowData} consisted of
+     * physical column values.
      */
-    private final DeserializationRuntimeConverter runtimeConverter;
+    private final DeserializationRuntimeConverter physicalConverter;
+
+    /** Whether the deserializer needs to handle metadata columns. */
+    private final boolean hasMetadata;
+
+    /**
+     * A wrapped output collector which is used to append metadata columns after physical columns.
+     */
+    private final AppendMetadataCollector appendMetadataCollector;
 
     /** Time zone of the database server. */
     private final ZoneId serverTimeZone;
 
     /** Validator to validate the row value. */
     private final ValueValidator validator;
 
-    public RowDataDebeziumDeserializeSchema(
-            RowType rowType,
+    /** Returns a builder to build {@link RowDataDebeziumDeserializeSchema}. */
+    public static Builder newBuilder() {
+        return new Builder();
+    }
+
+    RowDataDebeziumDeserializeSchema(
+            RowType physicalDataType,
+            MetadataConverter[] metadataConverters,
             TypeInformation<RowData> resultTypeInfo,
             ValueValidator validator,
             ZoneId serverTimeZone) {
-        this.runtimeConverter = createConverter(rowType);
-        this.resultTypeInfo = resultTypeInfo;
-        this.validator = validator;
-        this.serverTimeZone = serverTimeZone;
+        this.hasMetadata = checkNotNull(metadataConverters).length > 0;
+        this.appendMetadataCollector = new AppendMetadataCollector(metadataConverters);
+        this.physicalConverter = createConverter(checkNotNull(physicalDataType));
+        this.resultTypeInfo = checkNotNull(resultTypeInfo);
+        this.validator = checkNotNull(validator);
+        this.serverTimeZone = checkNotNull(serverTimeZone);
     }
 
     @Override
@@ -103,42 +119,96 @@ public void deserialize(SourceRecord record, Collector<RowData> out) throws Exce
             GenericRowData insert = extractAfterRow(value, valueSchema);
             validator.validate(insert, RowKind.INSERT);
             insert.setRowKind(RowKind.INSERT);
-            out.collect(insert);
+            emit(record, insert, out);
         } else if (op == Envelope.Operation.DELETE) {
             GenericRowData delete = extractBeforeRow(value, valueSchema);
             validator.validate(delete, RowKind.DELETE);
             delete.setRowKind(RowKind.DELETE);
-            out.collect(delete);
+            emit(record, delete, out);
         } else {
             GenericRowData before = extractBeforeRow(value, valueSchema);
             validator.validate(before, RowKind.UPDATE_BEFORE);
             before.setRowKind(RowKind.UPDATE_BEFORE);
-            out.collect(before);
+            emit(record, before, out);
 
             GenericRowData after = extractAfterRow(value, valueSchema);
             validator.validate(after, RowKind.UPDATE_AFTER);
             after.setRowKind(RowKind.UPDATE_AFTER);
-            out.collect(after);
+            emit(record, after, out);
         }
     }
 
     private GenericRowData extractAfterRow(Struct value, Schema valueSchema) throws Exception {
         Schema afterSchema = valueSchema.field(Envelope.FieldName.AFTER).schema();
         Struct after = value.getStruct(Envelope.FieldName.AFTER);
-        return (GenericRowData) runtimeConverter.convert(after, afterSchema);
+        return (GenericRowData) physicalConverter.convert(after, afterSchema);
     }
 
     private GenericRowData extractBeforeRow(Struct value, Schema valueSchema) throws Exception {
         Schema beforeSchema = valueSchema.field(Envelope.FieldName.BEFORE).schema();
         Struct before = value.getStruct(Envelope.FieldName.BEFORE);
-        return (GenericRowData) runtimeConverter.convert(before, beforeSchema);
+        return (GenericRowData) physicalConverter.convert(before, beforeSchema);
+    }
+
+    private void emit(SourceRecord inRecord, RowData physicalRow, Collector<RowData> collector) {
+        if (!hasMetadata) {
+            collector.collect(physicalRow);
+            return;
+        }
+
+        appendMetadataCollector.inputRecord = inRecord;
+        appendMetadataCollector.outputCollector = collector;
+        appendMetadataCollector.collect(physicalRow);
     }
 
     @Override
     public TypeInformation<RowData> getProducedType() {
         return resultTypeInfo;
     }
 
+    // -------------------------------------------------------------------------------------
+    // Builder
+    // -------------------------------------------------------------------------------------
+
+    /** Builder of {@link RowDataDebeziumDeserializeSchema}. */
+    public static class Builder {
+        private RowType physicalRowType;
+        private TypeInformation<RowData> resultTypeInfo;
+        private MetadataConverter[] metadataConverters = new MetadataConverter[0];
+        private ValueValidator validator = (rowData, rowKind) -> {};
+        private ZoneId serverTimeZone = ZoneId.of("UTC");
+
+        public Builder setPhysicalRowType(RowType physicalRowType) {
+            this.physicalRowType = physicalRowType;
+            return this;
+        }
+
+        public Builder setMetadataConverters(MetadataConverter[] metadataConverters) {
+            this.metadataConverters = metadataConverters;
+            return this;
+        }
+
+        public Builder setResultTypeInfo(TypeInformation<RowData> resultTypeInfo) {
+            this.resultTypeInfo = resultTypeInfo;
+            return this;
+        }
+
+        public Builder setValueValidator(ValueValidator validator) {
+            this.validator = validator;
+            return this;
+        }
+
+        public Builder setServerTimeZone(ZoneId serverTimeZone) {
+            this.serverTimeZone = serverTimeZone;
+            return this;
+        }
+
+        public RowDataDebeziumDeserializeSchema build() {
+            return new RowDataDebeziumDeserializeSchema(
+                    physicalRowType, metadataConverters, resultTypeInfo, validator, serverTimeZone);
+        }
+    }
+
     // -------------------------------------------------------------------------------------
     // Runtime Converters
     // -------------------------------------------------------------------------------------

flink-connector-mysql-cdc/src/main/java/com/ververica/cdc/connectors/mysql/table/MySqlReadableMetadata.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ververica.cdc.connectors.mysql.table;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.DataType;
+
+import com.ververica.cdc.debezium.table.MetadataConverter;
+import io.debezium.connector.AbstractSourceInfo;
+import io.debezium.data.Envelope;
+import org.apache.kafka.connect.data.Struct;
+import org.apache.kafka.connect.source.SourceRecord;
+
+/** Defines the supported metadata columns for {@link MySqlTableSource}. */
+public enum MySqlReadableMetadata {
+    /** Name of the table that contain the row. . */
+    TABLE_NAME(
+            "table_name",
+            DataTypes.STRING().notNull(),
+            new MetadataConverter() {
+                private static final long serialVersionUID = 1L;
+
+                @Override
+                public Object read(SourceRecord record) {
+                    Struct messageStruct = (Struct) record.value();
+                    Struct sourceStruct = messageStruct.getStruct(Envelope.FieldName.SOURCE);
+                    return StringData.fromString(
+                            sourceStruct.getString(AbstractSourceInfo.TABLE_NAME_KEY));
+                }
+            }),
+
+    /** Name of the database that contain the row. */
+    DATABASE_NAME(
+            "database_name",
+            DataTypes.STRING().notNull(),
+            new MetadataConverter() {
+                private static final long serialVersionUID = 1L;
+
+                @Override
+                public Object read(SourceRecord record) {
+                    Struct messageStruct = (Struct) record.value();
+                    Struct sourceStruct = messageStruct.getStruct(Envelope.FieldName.SOURCE);
+                    return StringData.fromString(
+                            sourceStruct.getString(AbstractSourceInfo.DATABASE_NAME_KEY));
+                }
+            }),
+
+    /**
+     * It indicates the time that the change was made in the database. If the record is read from
+     * snapshot of the table instead of the binlog, the value is always 0.
+     */
+    OP_TS(
+            "op_ts",
+            DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).notNull(),
+            new MetadataConverter() {
+                private static final long serialVersionUID = 1L;
+
+                @Override
+                public Object read(SourceRecord record) {
+                    Struct messageStruct = (Struct) record.value();
+                    Struct sourceStruct = messageStruct.getStruct(Envelope.FieldName.SOURCE);
+                    return TimestampData.fromEpochMillis(
+                            (Long) sourceStruct.get(AbstractSourceInfo.TIMESTAMP_KEY));
+                }
+            });
+
+    private final String key;
+
+    private final DataType dataType;
+
+    private final MetadataConverter converter;
+
+    MySqlReadableMetadata(String key, DataType dataType, MetadataConverter converter) {
+        this.key = key;
+        this.dataType = dataType;
+        this.converter = converter;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public DataType getDataType() {
+        return dataType;
+    }
+
+    public MetadataConverter getConverter() {
+        return converter;
+    }
+}