ddl: Fix compatibility with null expression index (#10162)

close #9891

ddl: Fix TiFlash panic when meets expression index with format `((NULL))`
Using `ColumnNothing` in IStorage may bring unexpected behavior when we try to write or read the column. So TiFlash create `ColumnInt8` instead of `ColumnNothing` when `TiDB::ColumnInfo.Tp == Null`.

However, we don't expected the column is used when reading or writing to TiFlash.

* `((null))` is not a normal expression index, this could be created by the user in accident (e.g. executing wrong SQL statement to create the expression index), TiDB should not write or read from it
* Even for normal expression index, TiDB does not use expression when accessing to TiFlash

Signed-off-by: JaySon-Huang <[email protected]>

Author: JaySon-Huang

dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp

@@ -18,6 +18,7 @@
 #include <Common/SyncPoint/SyncPoint.h>
 #include <Core/Defines.h>
 #include <DataTypes/DataTypeMyDateTime.h>
+#include <DataTypes/DataTypesNumber.h>
 #include <Debug/TiFlashTestEnv.h>
 #include <Interpreters/Context.h>
 #include <Storages/DeltaMerge/DeltaMergeDefines.h>
@@ -2473,6 +2474,84 @@ try
 }
 CATCH
 
+TEST_P(DeltaMergeStoreRWTest, DDLAddColumnInvalidExpressionIndex)
+try
+{
+    // const String col_name_to_add = "_v$_idx_name_0";
+    // const DataTypePtr col_type_to_add = DataTypeFactory::instance().getOrSet("Int8");
+
+    // write some rows before DDL
+    size_t num_rows_write = 1;
+    {
+        Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false);
+        store->write(*db_context, db_context->getSettingsRef(), block);
+    }
+
+    // DDL add column for invalid expression index
+    // actual ddl is like: ADD COLUMN `_v$_idx_name_0` Nullable(Int8)
+    {
+        TiDB::TableInfo new_table_info;
+        static const String json_table_info = R"(
+    {"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027}
+            )";
+        new_table_info.deserialize(json_table_info);
+        store->applySchemaChanges(new_table_info);
+    }
+
+    // try read
+    {
+        ColumnDefines cols_to_read;
+        bool has_v_index = false;
+        bool has_id = false;
+        for (const auto & col : store->getTableColumns())
+        {
+            if (col.name == DMTestEnv::pk_name)
+            {
+                cols_to_read.emplace_back(col);
+            }
+            else if (col.name == "id")
+            {
+                cols_to_read.emplace_back(col);
+                has_id = true;
+            }
+            else if (col.name == "_v$_idx_name_0")
+            {
+                // we check the type of this column, but not read from it
+                ASSERT_EQ(col.type->getName(), "Nullable(Int8)") << store->getHeader()->dumpJsonStructure();
+                has_v_index = true;
+            }
+        }
+        ASSERT_TRUE(has_id) << store->getHeader()->dumpJsonStructure();
+        ASSERT_TRUE(has_v_index) << store->getHeader()->dumpJsonStructure();
+
+        auto in = store->read(
+            *db_context,
+            db_context->getSettingsRef(),
+            cols_to_read,
+            {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())},
+            /* num_streams= */ 1,
+            /* start_ts= */ std::numeric_limits<UInt64>::max(),
+            EMPTY_FILTER,
+            std::vector<RuntimeFilterPtr>{},
+            0,
+            TRACING_NAME,
+            /* keep_order= */ false,
+            /* is_fast_scan= */ false,
+            /* expected_block_size= */ 1024)[0];
+        ASSERT_UNORDERED_INPUTSTREAM_COLS_UR(
+            in,
+            Strings({DMTestEnv::pk_name, "id"}),
+            createColumns({
+                createColumn<Int64>(createNumbers<Int64>(0, num_rows_write)),
+                // all "id" are NULL
+                createNullableColumn<Int32>(
+                    std::vector<Int64>(num_rows_write, 0),
+                    std::vector<Int32>(num_rows_write, 1)),
+            }));
+    }
+}
+CATCH
+
 TEST_P(DeltaMergeStoreRWTest, DDLRenameColumn)
 try
 {

dbms/src/TiDB/Decode/TypeMapping.cpp

@@ -204,7 +204,22 @@ DataTypePtr TypeMapping::getDataType(const ColumnInfo & column_info)
 // This does not support the "duration" type.
 DataTypePtr getDataTypeByColumnInfo(const ColumnInfo & column_info)
 {
-    DataTypePtr base = TypeMapping::instance().getDataType(column_info);
+    DataTypePtr base;
+    if (likely(column_info.tp != TiDB::TP::TypeNull))
+    {
+        base = TypeMapping::instance().getDataType(column_info);
+    }
+    else
+    {
+        // Storing a column with `ColumnNothing` is not allowed in `StorageFactory::get/checkAllTypesAreAllowedInTable`
+        // Using `ColumnNothing` in IStorage may bring unexpected behavior when we
+        // try to write or read the column. So we change it to `ColumnInt8`.
+        LOG_WARNING(
+            Logger::get(),
+            "Column type is TiDB::TP::TypeNull, change it to Int8 for compatibility, column_id={}",
+            column_info.id);
+        base = DataTypeFactory::instance().getOrSet("Int8");
+    }
 
     if (!column_info.hasNotNullFlag())
     {

dbms/src/TiDB/Schema/tests/gtest_table_info.cpp

@@ -27,6 +27,7 @@
 #include <TiDB/Decode/TypeMapping.h>
 #include <TiDB/Schema/SchemaSyncer.h>
 #include <TiDB/Schema/TiDB.h>
+#include <TiDB/Schema/TiDBTypes.h>
 #include <gtest/gtest.h>
 
 #include <boost/algorithm/string/replace.hpp>
@@ -126,6 +127,16 @@ try
                         getDataTypeByColumnInfo(ci);
                     }
                 }},
+            // tiflash#9891, expression index like `KEY idx_name ((null))` generate column in this format
+            ParseCase{
+                R"json({"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":242807,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027})json",
+                [](const TableInfo & table_info) {
+                    ASSERT_EQ(table_info.name, "t");
+                    auto col_1 = table_info.getColumnInfo(2);
+                    ASSERT_EQ(col_1.tp, TiDB::TP::TypeNull);
+                    ASSERT_TRUE(col_1.hasMultipleKeyFlag());
+                    ASSERT_TRUE(col_1.hasBinaryFlag());
+                }},
     };
 
     for (const auto & c : cases)
@@ -368,6 +379,14 @@ try
                 R"json({"id":546,"name":{"O":"tcfc7825f","L":"tcfc7825f"},"charset":"utf8mb4","collate":"utf8mb4_general_ci","cols":[{"id":1,"name":{"O":"col_86","L":"col_86"},"offset":0,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":252,"Flag":128,"Flen":65535,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":2,"name":{"O":"col_87","L":"col_87"},"offset":1,"default":"1994-05-0600:00:00","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":12,"Flag":129,"Flen":19,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":3,"name":{"O":"col_88","L":"col_88"},"offset":2,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":16,"Flag":32,"Flen":42,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":4,"name":{"O":"col_89","L":"col_89"},"offset":3,"default":"\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":21,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":5,"name":{"O":"col_90","L":"col_90"},"offset":4,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":1,"Flag":4129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":6,"name":{"O":"col_91","L":"col_91"},"offset":5,"default":"\u0007\u0007","default_bit":"Bwc=","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":16,"Flag":32,"Flen":12,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":7,"name":{"O":"col_92","L":"col_92"},"offset":6,"default":"kY~6to6H4ut*QAPrj@\u0026","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":15,"Flag":129,"Flen":343,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":8,"name":{"O":"col_93","L":"col_93"},"offset":7,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":245,"Flag":128,"Flen":4294967295,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":8,"max_idx_id":0,"max_fk_id":0,"max_cst_id":0,"update_timestamp":452653255976550448,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"compression":"","view":null,"sequence":null,"Lock":null,"version":5,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null},"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null,"revision":1})json", //
                 replace_string_name(R"stmt(CREATE TABLE `db_2`.`t_546`(`col_86` Nullable({StringName}), `col_87` MyDateTime(0), `col_88` Nullable(UInt64), `col_89` {StringName}, `col_90` UInt8, `col_91` Nullable(UInt64), `col_92` {StringName}, `col_93` Nullable({StringName}), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"col_86","O":"col_86"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":128,"Flen":65535,"Tp":252}},{"default":"1994-05-0600:00:00","id":2,"name":{"L":"col_87","O":"col_87"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":19,"Tp":12}},{"id":3,"name":{"L":"col_88","O":"col_88"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":32,"Flen":42,"Tp":16}},{"default":"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000","id":4,"name":{"L":"col_89","O":"col_89"},"offset":3,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":21,"Tp":254}},{"id":5,"name":{"L":"col_90","O":"col_90"},"offset":4,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":4129,"Flen":3,"Tp":1}},{"default":"\\u0007\\u0007","default_bit":"Bwc=","id":6,"name":{"L":"col_91","O":"col_91"},"offset":5,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":32,"Flen":12,"Tp":16}},{"default":"kY~6to6H4ut*QAPrj@&","id":7,"name":{"L":"col_92","O":"col_92"},"offset":6,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":343,"Tp":15}},{"id":8,"name":{"L":"col_93","O":"col_93"},"offset":7,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":128,"Flen":-1,"Tp":245}}],"id":546,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"tcfc7825f","O":"tcfc7825f"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":452653255976550448}', 0))stmt"), //
             },
+            StmtCase{
+                1145, //
+                0,
+                R"json({"id":1939,"db_name":{"O":"customer","L":"customer"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", //
+                R"json({"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027})json", //
+                // Note that the `v$_idx_name_0` column is created as Nullable(Int8) rather than Nullable(Nothing) for compatibility with null expression index (tiflash#9891)
+                R"stmt(CREATE TABLE `db_1939`.`t_1145`(`id` Nullable(Int32), `_v$_idx_name_0` Nullable(Int8), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027}', 0))stmt", //
+            },
     };
     // clang-format on
 

dbms/src/TiDB/tests/gtest_type_mapping.cpp

@@ -18,9 +18,7 @@
 #include <TiDB/Decode/TypeMapping.h>
 #include <TiDB/Schema/TiDB.h>
 
-namespace DB
-{
-namespace tests
+namespace DB::tests
 {
 
 TEST(TypeMappingTest, DataTypeToColumnInfo)
@@ -93,9 +91,18 @@ try
         ASSERT_EQ(data_type->getName(), DataTypeString::getDefaultName());
     }
 
+    {
+        auto str_type = typeFromString("Int8");
+        column_info = reverseGetColumnInfo(NameAndTypePair{name, str_type}, 1, default_field, true);
+        column_info.tp = TiDB::TP::TypeNull; // test for TypeNull
+        ASSERT_EQ(column_info.tp, TiDB::TP::TypeNull);
+        auto data_type = getDataTypeByColumnInfo(column_info);
+        // Use Int8 to compatible "storing" TypeNull in IStorage
+        ASSERT_EQ(data_type->getName(), "Int8");
+    }
+
     // TODO: test decimal, datetime, enum
 }
 CATCH
 
-} // namespace tests
-} // namespace DB
+} // namespace DB::tests

tests/fullstack-test2/ddl/expression_index.test

@@ -0,0 +1,47 @@
+# Copyright 2025 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mysql> drop table if exists test.t;
+
+# test for null expression express tiflash#9891
+mysql> CREATE TABLE test.t (id int, KEY idx_name ((null)));
+mysql> alter table test.t set tiflash replica 1;
+mysql> insert test.t values(0),(1);
+
+func> wait_table test t
+
+mysql> insert into test.t values (2), (3);
+mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t order by id;
++----+
+| id |
++----+
+|  0 |
+|  1 |
+|  2 |
+|  3 |
++----+
+
+mysql> CREATE INDEX idx_n ON test.t ((null));
+mysql> alter table test.t add column c1 int;
+mysql> set session tidb_isolation_read_engines='tiflash'; select id,c1 from test.t order by id;
++------+------+
+| id   | c1   |
++------+------+
+|    0 | NULL |
+|    1 | NULL |
+|    2 | NULL |
+|    3 | NULL |
++------+------+
+
+mysql> drop table if exists test.t;