Skip to content

Commit 40aa2c5

Browse files
ti-chi-botti-chi-bot
authored
*: Fix issue 39999, used wrong column id list for checking partitions (#40003) (#40064)
close #39999
1 parent 048ba8e commit 40aa2c5

File tree

3 files changed

+118
-56
lines changed

3 files changed

+118
-56
lines changed

‎executor/builder.go‎

Lines changed: 38 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -3511,17 +3511,39 @@ func buildIndexRangeForEachPartition(ctx sessionctx.Context, usedPartitions []ta
35113511
return nextRange, nil
35123512
}
35133513

3514-
func keyColumnsIncludeAllPartitionColumns(keyColumns []int, pe *tables.PartitionExpr) bool {
3515-
tmp := make(map[int]struct{}, len(keyColumns))
3516-
for _, offset := range keyColumns {
3517-
tmp[offset] = struct{}{}
3514+
func getPartitionKeyColOffsets(keyColIDs []int64, pt table.PartitionedTable) []int {
3515+
keyColOffsets := make([]int, len(keyColIDs))
3516+
for i, colID := range keyColIDs {
3517+
offset := -1
3518+
for j, col := range pt.Cols() {
3519+
if colID == col.ID {
3520+
offset = j
3521+
break
3522+
}
3523+
}
3524+
if offset == -1 {
3525+
return nil
3526+
}
3527+
keyColOffsets[i] = offset
3528+
}
3529+
3530+
pe, err := pt.(interface {
3531+
PartitionExpr() (*tables.PartitionExpr, error)
3532+
}).PartitionExpr()
3533+
if err != nil {
3534+
return nil
3535+
}
3536+
3537+
offsetMap := make(map[int]struct{})
3538+
for _, offset := range keyColOffsets {
3539+
offsetMap[offset] = struct{}{}
35183540
}
35193541
for _, offset := range pe.ColumnOffset {
3520-
if _, ok := tmp[offset]; !ok {
3521-
return false
3542+
if _, ok := offsetMap[offset]; !ok {
3543+
return nil
35223544
}
35233545
}
3524-
return true
3546+
return keyColOffsets
35253547
}
35263548

35273549
func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table, schema *expression.Schema, partitionInfo *plannercore.PartitionInfo,
@@ -3536,45 +3558,16 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table
35363558
return nil, false, nil, err
35373559
}
35383560

3539-
// check whether can runtime prune.
3540-
type partitionExpr interface {
3541-
PartitionExpr() (*tables.PartitionExpr, error)
3542-
}
3543-
pe, err := tbl.(partitionExpr).PartitionExpr()
3544-
if err != nil {
3545-
return nil, false, nil, err
3546-
}
3547-
35483561
// recalculate key column offsets
35493562
if len(lookUpContent) == 0 {
35503563
return nil, false, nil, nil
35513564
}
35523565
if lookUpContent[0].keyColIDs == nil {
35533566
return nil, false, nil, plannercore.ErrInternal.GenWithStack("cannot get column IDs when dynamic pruning")
35543567
}
3555-
keyColOffsets := make([]int, len(lookUpContent[0].keyColIDs))
3556-
for i, colID := range lookUpContent[0].keyColIDs {
3557-
offset := -1
3558-
for j, col := range partitionTbl.Cols() {
3559-
if colID == col.ID {
3560-
offset = j
3561-
break
3562-
}
3563-
}
3564-
if offset == -1 {
3565-
return nil, false, nil, plannercore.ErrInternal.GenWithStack("invalid column offset when dynamic pruning")
3566-
}
3567-
keyColOffsets[i] = offset
3568-
}
3569-
3570-
offsetMap := make(map[int]bool)
3571-
for _, offset := range keyColOffsets {
3572-
offsetMap[offset] = true
3573-
}
3574-
for _, offset := range pe.ColumnOffset {
3575-
if _, ok := offsetMap[offset]; !ok {
3576-
return condPruneResult, false, nil, nil
3577-
}
3568+
keyColOffsets := getPartitionKeyColOffsets(lookUpContent[0].keyColIDs, partitionTbl)
3569+
if len(keyColOffsets) == 0 {
3570+
return condPruneResult, false, nil, nil
35783571
}
35793572

35803573
locateKey := make([]types.Datum, len(partitionTbl.Cols()))
@@ -4149,12 +4142,6 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
41494142
}
41504143
tbl, _ := builder.is.TableByID(tbInfo.ID)
41514144
pt := tbl.(table.PartitionedTable)
4152-
pe, err := tbl.(interface {
4153-
PartitionExpr() (*tables.PartitionExpr, error)
4154-
}).PartitionExpr()
4155-
if err != nil {
4156-
return nil, err
4157-
}
41584145
partitionInfo := &v.PartitionInfo
41594146
usedPartitionList, err := builder.partitionPruning(pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames)
41604147
if err != nil {
@@ -4165,8 +4152,12 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
41654152
usedPartitions[p.GetPhysicalID()] = p
41664153
}
41674154
var kvRanges []kv.KeyRange
4155+
var keyColOffsets []int
4156+
if len(lookUpContents) > 0 {
4157+
keyColOffsets = getPartitionKeyColOffsets(lookUpContents[0].keyColIDs, pt)
4158+
}
41684159
if v.IsCommonHandle {
4169-
if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) {
4160+
if len(keyColOffsets) > 0 {
41704161
locateKey := make([]types.Datum, e.Schema().Len())
41714162
kvRanges = make([]kv.KeyRange, 0, len(lookUpContents))
41724163
// lookUpContentsByPID groups lookUpContents by pid(partition) so that kv ranges for same partition can be merged.
@@ -4212,7 +4203,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
42124203

42134204
handles, lookUpContents := dedupHandles(lookUpContents)
42144205

4215-
if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) {
4206+
if len(keyColOffsets) > 0 {
42164207
locateKey := make([]types.Datum, e.Schema().Len())
42174208
kvRanges = make([]kv.KeyRange, 0, len(lookUpContents))
42184209
for _, content := range lookUpContents {

‎executor/index_lookup_join_test.go‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -428,17 +428,19 @@ PARTITIONS 1`)
428428

429429
// Why does the t2.prefiller need be at least 2^32 ? If smaller the bug will not appear!?!
430430
tk.MustExec("insert into t2 values ( pow(2,32), 1, 1), ( pow(2,32)+1, 2, 0)")
431+
tk.MustExec(`analyze table t1`)
432+
tk.MustExec(`analyze table t2`)
431433

432434
// Why must it be = 1 and not 2?
433-
tk.MustQuery("explain select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows("" +
434-
"IndexJoin_15 10.00 root inner join, inner:TableReader_14, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)]\n" +
435-
"[├─HashAgg_25(Build) 8.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller]\n" +
436-
"[│ └─TableReader_26 8.00 root data:HashAgg_20]\n" +
437-
"[│ └─HashAgg_20 8.00 cop[tikv] group by:test.t2.prefiller, ]\n" +
438-
"[│ └─Selection_24 10.00 cop[tikv] eq(test.t2.postfiller, 1)]\n" +
439-
"[│ └─TableFullScan_23 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo]\n" +
440-
"[└─TableReader_14(Probe) 8.00 root partition:all data:TableRangeScan_13]\n" +
441-
"[ └─TableRangeScan_13 8.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo"))
435+
tk.MustQuery("explain format='brief' select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows(""+
436+
`IndexJoin 1.25 root inner join, inner:TableReader, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)`,
437+
`├─HashAgg(Build) 1.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller`,
438+
`│ └─TableReader 1.00 root data:HashAgg`,
439+
`│ └─HashAgg 1.00 cop[tikv] group by:test.t2.prefiller, `,
440+
`│ └─Selection 1.00 cop[tikv] eq(test.t2.postfiller, 1)`,
441+
`│ └─TableFullScan 2.00 cop[tikv] table:t2 keep order:false`,
442+
`└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan`,
443+
` └─TableRangeScan 1.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo`))
442444
tk.MustQuery("show warnings").Check(testkit.Rows())
443445
// without fix it fails with: "runtime error: index out of range [0] with length 0"
444446
tk.MustQuery("select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows())

‎executor/partition_table_test.go‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3831,3 +3831,72 @@ func TestIssue21732(t *testing.T) {
38313831
})
38323832
}
38333833
}
3834+
3835+
func TestIssue39999(t *testing.T) {
3836+
store := testkit.CreateMockStore(t)
3837+
3838+
tk := testkit.NewTestKit(t, store)
3839+
3840+
tk.MustExec(`create schema test39999`)
3841+
tk.MustExec(`use test39999`)
3842+
tk.MustExec(`drop table if exists c, t`)
3843+
tk.MustExec("CREATE TABLE `c` (" +
3844+
"`serial_id` varchar(24)," +
3845+
"`occur_trade_date` date," +
3846+
"`txt_account_id` varchar(24)," +
3847+
"`capital_sub_class` varchar(10)," +
3848+
"`occur_amount` decimal(16,2)," +
3849+
"`broker` varchar(10)," +
3850+
"PRIMARY KEY (`txt_account_id`,`occur_trade_date`,`serial_id`) /*T![clustered_index] CLUSTERED */," +
3851+
"KEY `idx_serial_id` (`serial_id`)" +
3852+
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci " +
3853+
"PARTITION BY RANGE COLUMNS(`serial_id`) (" +
3854+
"PARTITION `p202209` VALUES LESS THAN ('20221001')," +
3855+
"PARTITION `p202210` VALUES LESS THAN ('20221101')," +
3856+
"PARTITION `p202211` VALUES LESS THAN ('20221201')" +
3857+
")")
3858+
3859+
tk.MustExec("CREATE TABLE `t` ( " +
3860+
"`txn_account_id` varchar(24), " +
3861+
"`account_id` varchar(32), " +
3862+
"`broker` varchar(10), " +
3863+
"PRIMARY KEY (`txn_account_id`) /*T![clustered_index] CLUSTERED */ " +
3864+
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci")
3865+
3866+
tk.MustExec("INSERT INTO `c` (serial_id, txt_account_id, capital_sub_class, occur_trade_date, occur_amount, broker) VALUES ('2022111700196920','04482786','CUST','2022-11-17',-2.01,'0009')")
3867+
tk.MustExec("INSERT INTO `t` VALUES ('04482786','1142927','0009')")
3868+
3869+
tk.MustExec(`set tidb_partition_prune_mode='dynamic'`)
3870+
tk.MustExec(`analyze table c`)
3871+
tk.MustExec(`analyze table t`)
3872+
query := `select
3873+
/*+ inl_join(c) */
3874+
c.occur_amount
3875+
from
3876+
c
3877+
join t on c.txt_account_id = t.txn_account_id
3878+
and t.broker = '0009'
3879+
and c.occur_trade_date = '2022-11-17'`
3880+
tk.MustQuery("explain " + query).Check(testkit.Rows(""+
3881+
"IndexJoin_22 1.00 root inner join, inner:TableReader_21, outer key:test39999.t.txn_account_id, inner key:test39999.c.txt_account_id, equal cond:eq(test39999.t.txn_account_id, test39999.c.txt_account_id)",
3882+
"├─TableReader_27(Build) 1.00 root data:Selection_26",
3883+
"│ └─Selection_26 1.00 cop[tikv] eq(test39999.t.broker, \"0009\")",
3884+
"│ └─TableFullScan_25 1.00 cop[tikv] table:t keep order:false",
3885+
"└─TableReader_21(Probe) 1.00 root partition:all data:Selection_20",
3886+
" └─Selection_20 1.00 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)",
3887+
" └─TableRangeScan_19 1.00 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false"))
3888+
tk.MustQuery(query).Check(testkit.Rows("-2.01"))
3889+
3890+
// Add the missing partition key part.
3891+
tk.MustExec(`alter table t add column serial_id varchar(24) default '2022111700196920'`)
3892+
query += ` and c.serial_id = t.serial_id`
3893+
tk.MustQuery(query).Check(testkit.Rows("-2.01"))
3894+
tk.MustQuery("explain " + query).Check(testkit.Rows(""+
3895+
`IndexJoin_20 0.80 root inner join, inner:TableReader_19, outer key:test39999.t.txn_account_id, test39999.t.serial_id, inner key:test39999.c.txt_account_id, test39999.c.serial_id, equal cond:eq(test39999.t.serial_id, test39999.c.serial_id), eq(test39999.t.txn_account_id, test39999.c.txt_account_id)`,
3896+
`├─TableReader_25(Build) 0.80 root data:Selection_24`,
3897+
`│ └─Selection_24 0.80 cop[tikv] eq(test39999.t.broker, "0009"), not(isnull(test39999.t.serial_id))`,
3898+
`│ └─TableFullScan_23 1.00 cop[tikv] table:t keep order:false`,
3899+
`└─TableReader_19(Probe) 0.80 root partition:all data:Selection_18`,
3900+
` └─Selection_18 0.80 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)`,
3901+
` └─TableRangeScan_17 0.80 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.serial_id, test39999.t.serial_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false`))
3902+
}

0 commit comments

Comments
 (0)