Skip to content

Commit ffd3afb

Browse files
mjonssti-chi-bot
mjonss
authored and
ti-chi-bot
committed
This is an automated cherry-pick of pingcap#40003
Signed-off-by: ti-chi-bot <[email protected]>
1 parent 10c2cf7 commit ffd3afb

File tree

3 files changed

+143
-47
lines changed

3 files changed

+143
-47
lines changed

‎executor/builder.go‎

Lines changed: 38 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -3381,17 +3381,39 @@ func buildIndexRangeForEachPartition(ctx sessionctx.Context, usedPartitions []ta
33813381
return nextRange, nil
33823382
}
33833383

3384-
func keyColumnsIncludeAllPartitionColumns(keyColumns []int, pe *tables.PartitionExpr) bool {
3385-
tmp := make(map[int]struct{}, len(keyColumns))
3386-
for _, offset := range keyColumns {
3387-
tmp[offset] = struct{}{}
3384+
func getPartitionKeyColOffsets(keyColIDs []int64, pt table.PartitionedTable) []int {
3385+
keyColOffsets := make([]int, len(keyColIDs))
3386+
for i, colID := range keyColIDs {
3387+
offset := -1
3388+
for j, col := range pt.Cols() {
3389+
if colID == col.ID {
3390+
offset = j
3391+
break
3392+
}
3393+
}
3394+
if offset == -1 {
3395+
return nil
3396+
}
3397+
keyColOffsets[i] = offset
3398+
}
3399+
3400+
pe, err := pt.(interface {
3401+
PartitionExpr() (*tables.PartitionExpr, error)
3402+
}).PartitionExpr()
3403+
if err != nil {
3404+
return nil
3405+
}
3406+
3407+
offsetMap := make(map[int]struct{})
3408+
for _, offset := range keyColOffsets {
3409+
offsetMap[offset] = struct{}{}
33883410
}
33893411
for _, offset := range pe.ColumnOffset {
3390-
if _, ok := tmp[offset]; !ok {
3391-
return false
3412+
if _, ok := offsetMap[offset]; !ok {
3413+
return nil
33923414
}
33933415
}
3394-
return true
3416+
return keyColOffsets
33953417
}
33963418

33973419
func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table, schema *expression.Schema, partitionInfo *plannercore.PartitionInfo,
@@ -3406,45 +3428,16 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table
34063428
return nil, false, nil, err
34073429
}
34083430

3409-
// check whether can runtime prune.
3410-
type partitionExpr interface {
3411-
PartitionExpr() (*tables.PartitionExpr, error)
3412-
}
3413-
pe, err := tbl.(partitionExpr).PartitionExpr()
3414-
if err != nil {
3415-
return nil, false, nil, err
3416-
}
3417-
34183431
// recalculate key column offsets
34193432
if len(lookUpContent) == 0 {
34203433
return nil, false, nil, nil
34213434
}
34223435
if lookUpContent[0].keyColIDs == nil {
34233436
return nil, false, nil, plannercore.ErrInternal.GenWithStack("cannot get column IDs when dynamic pruning")
34243437
}
3425-
keyColOffsets := make([]int, len(lookUpContent[0].keyColIDs))
3426-
for i, colID := range lookUpContent[0].keyColIDs {
3427-
offset := -1
3428-
for j, col := range partitionTbl.Cols() {
3429-
if colID == col.ID {
3430-
offset = j
3431-
break
3432-
}
3433-
}
3434-
if offset == -1 {
3435-
return nil, false, nil, plannercore.ErrInternal.GenWithStack("invalid column offset when dynamic pruning")
3436-
}
3437-
keyColOffsets[i] = offset
3438-
}
3439-
3440-
offsetMap := make(map[int]bool)
3441-
for _, offset := range keyColOffsets {
3442-
offsetMap[offset] = true
3443-
}
3444-
for _, offset := range pe.ColumnOffset {
3445-
if _, ok := offsetMap[offset]; !ok {
3446-
return condPruneResult, false, nil, nil
3447-
}
3438+
keyColOffsets := getPartitionKeyColOffsets(lookUpContent[0].keyColIDs, partitionTbl)
3439+
if len(keyColOffsets) == 0 {
3440+
return condPruneResult, false, nil, nil
34483441
}
34493442

34503443
locateKey := make([]types.Datum, len(partitionTbl.Cols()))
@@ -3996,12 +3989,6 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
39963989
}
39973990
tbl, _ := builder.is.TableByID(tbInfo.ID)
39983991
pt := tbl.(table.PartitionedTable)
3999-
pe, err := tbl.(interface {
4000-
PartitionExpr() (*tables.PartitionExpr, error)
4001-
}).PartitionExpr()
4002-
if err != nil {
4003-
return nil, err
4004-
}
40053992
partitionInfo := &v.PartitionInfo
40063993
usedPartitionList, err := builder.partitionPruning(pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames)
40073994
if err != nil {
@@ -4012,8 +3999,12 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
40123999
usedPartitions[p.GetPhysicalID()] = p
40134000
}
40144001
var kvRanges []kv.KeyRange
4002+
var keyColOffsets []int
4003+
if len(lookUpContents) > 0 {
4004+
keyColOffsets = getPartitionKeyColOffsets(lookUpContents[0].keyColIDs, pt)
4005+
}
40154006
if v.IsCommonHandle {
4016-
if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) {
4007+
if len(keyColOffsets) > 0 {
40174008
locateKey := make([]types.Datum, e.Schema().Len())
40184009
kvRanges = make([]kv.KeyRange, 0, len(lookUpContents))
40194010
// lookUpContentsByPID groups lookUpContents by pid(partition) so that kv ranges for same partition can be merged.
@@ -4055,7 +4046,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
40554046

40564047
handles, lookUpContents := dedupHandles(lookUpContents)
40574048

4058-
if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) {
4049+
if len(keyColOffsets) > 0 {
40594050
locateKey := make([]types.Datum, e.Schema().Len())
40604051
kvRanges = make([]kv.KeyRange, 0, len(lookUpContents))
40614052
for _, content := range lookUpContents {

‎executor/index_lookup_join_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,8 +437,11 @@ PARTITIONS 1`)
437437

438438
// Why does the t2.prefiller need be at least 2^32 ? If smaller the bug will not appear!?!
439439
tk.MustExec("insert into t2 values ( pow(2,32), 1, 1), ( pow(2,32)+1, 2, 0)")
440+
tk.MustExec(`analyze table t1`)
441+
tk.MustExec(`analyze table t2`)
440442

441443
// Why must it be = 1 and not 2?
444+
<<<<<<< HEAD
442445
tk.MustQuery("explain select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows("" +
443446
"IndexJoin_15 10.00 root inner join, inner:TableReader_14, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)]\n" +
444447
"[├─HashAgg_25(Build) 8.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller]\n" +
@@ -448,6 +451,17 @@ PARTITIONS 1`)
448451
"[│ └─TableFullScan_23 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo]\n" +
449452
"[└─TableReader_14(Probe) 1.00 root partition:all data:TableRangeScan_13]\n" +
450453
"[ └─TableRangeScan_13 1.00 cop[tikv] table:t1 range: decided by [test.t2.prefiller], keep order:false, stats:pseudo"))
454+
=======
455+
tk.MustQuery("explain format='brief' select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows(""+
456+
`IndexJoin 1.25 root inner join, inner:TableReader, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)`,
457+
`├─HashAgg(Build) 1.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller`,
458+
`│ └─TableReader 1.00 root data:HashAgg`,
459+
`│ └─HashAgg 1.00 cop[tikv] group by:test.t2.prefiller, `,
460+
`│ └─Selection 1.00 cop[tikv] eq(test.t2.postfiller, 1)`,
461+
`│ └─TableFullScan 2.00 cop[tikv] table:t2 keep order:false`,
462+
`└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan`,
463+
` └─TableRangeScan 1.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo`))
464+
>>>>>>> 4a72171ffb (*: Fix issue 39999, used wrong column id list for checking partitions (#40003))
451465
tk.MustQuery("show warnings").Check(testkit.Rows())
452466
// without fix it fails with: "runtime error: index out of range [0] with length 0"
453467
tk.MustQuery("select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows())

‎executor/partition_table_test.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3615,3 +3615,94 @@ func TestIssue35181(t *testing.T) {
36153615
tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'")
36163616
tk.MustExec(`insert into t select * from t where a=3000`)
36173617
}
3618+
<<<<<<< HEAD
3619+
=======
3620+
3621+
func TestIssue21732(t *testing.T) {
3622+
store := testkit.CreateMockStore(t)
3623+
3624+
tk := testkit.NewTestKit(t, store)
3625+
for _, mode := range []variable.PartitionPruneMode{variable.StaticOnly, variable.DynamicOnly} {
3626+
testkit.WithPruneMode(tk, mode, func() {
3627+
tk.MustExec("create database TestIssue21732")
3628+
tk.MustExec("use TestIssue21732")
3629+
tk.MustExec("drop table if exists p")
3630+
tk.MustExec(`create table p (a int, b int GENERATED ALWAYS AS (3*a-2*a) VIRTUAL) partition by hash(b) partitions 2;`)
3631+
tk.MustExec("alter table p add unique index idx (a, b);")
3632+
tk.MustExec("insert into p (a) values (1),(2),(3);")
3633+
tk.MustExec("select * from p ignore index (idx);")
3634+
tk.MustQuery("select * from p use index (idx)").Sort().Check(testkit.Rows("1 1", "2 2", "3 3"))
3635+
tk.MustExec("drop database TestIssue21732")
3636+
})
3637+
}
3638+
}
3639+
3640+
func TestIssue39999(t *testing.T) {
3641+
store := testkit.CreateMockStore(t)
3642+
3643+
tk := testkit.NewTestKit(t, store)
3644+
3645+
tk.MustExec(`create schema test39999`)
3646+
tk.MustExec(`use test39999`)
3647+
tk.MustExec(`drop table if exists c, t`)
3648+
tk.MustExec("CREATE TABLE `c` (" +
3649+
"`serial_id` varchar(24)," +
3650+
"`occur_trade_date` date," +
3651+
"`txt_account_id` varchar(24)," +
3652+
"`capital_sub_class` varchar(10)," +
3653+
"`occur_amount` decimal(16,2)," +
3654+
"`broker` varchar(10)," +
3655+
"PRIMARY KEY (`txt_account_id`,`occur_trade_date`,`serial_id`) /*T![clustered_index] CLUSTERED */," +
3656+
"KEY `idx_serial_id` (`serial_id`)" +
3657+
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci " +
3658+
"PARTITION BY RANGE COLUMNS(`serial_id`) (" +
3659+
"PARTITION `p202209` VALUES LESS THAN ('20221001')," +
3660+
"PARTITION `p202210` VALUES LESS THAN ('20221101')," +
3661+
"PARTITION `p202211` VALUES LESS THAN ('20221201')" +
3662+
")")
3663+
3664+
tk.MustExec("CREATE TABLE `t` ( " +
3665+
"`txn_account_id` varchar(24), " +
3666+
"`account_id` varchar(32), " +
3667+
"`broker` varchar(10), " +
3668+
"PRIMARY KEY (`txn_account_id`) /*T![clustered_index] CLUSTERED */ " +
3669+
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci")
3670+
3671+
tk.MustExec("INSERT INTO `c` (serial_id, txt_account_id, capital_sub_class, occur_trade_date, occur_amount, broker) VALUES ('2022111700196920','04482786','CUST','2022-11-17',-2.01,'0009')")
3672+
tk.MustExec("INSERT INTO `t` VALUES ('04482786','1142927','0009')")
3673+
3674+
tk.MustExec(`set tidb_partition_prune_mode='dynamic'`)
3675+
tk.MustExec(`analyze table c`)
3676+
tk.MustExec(`analyze table t`)
3677+
query := `select
3678+
/*+ inl_join(c) */
3679+
c.occur_amount
3680+
from
3681+
c
3682+
join t on c.txt_account_id = t.txn_account_id
3683+
and t.broker = '0009'
3684+
and c.occur_trade_date = '2022-11-17'`
3685+
tk.MustQuery("explain " + query).Check(testkit.Rows(""+
3686+
"IndexJoin_22 1.00 root inner join, inner:TableReader_21, outer key:test39999.t.txn_account_id, inner key:test39999.c.txt_account_id, equal cond:eq(test39999.t.txn_account_id, test39999.c.txt_account_id)",
3687+
"├─TableReader_27(Build) 1.00 root data:Selection_26",
3688+
"│ └─Selection_26 1.00 cop[tikv] eq(test39999.t.broker, \"0009\")",
3689+
"│ └─TableFullScan_25 1.00 cop[tikv] table:t keep order:false",
3690+
"└─TableReader_21(Probe) 1.00 root partition:all data:Selection_20",
3691+
" └─Selection_20 1.00 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)",
3692+
" └─TableRangeScan_19 1.00 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false"))
3693+
tk.MustQuery(query).Check(testkit.Rows("-2.01"))
3694+
3695+
// Add the missing partition key part.
3696+
tk.MustExec(`alter table t add column serial_id varchar(24) default '2022111700196920'`)
3697+
query += ` and c.serial_id = t.serial_id`
3698+
tk.MustQuery(query).Check(testkit.Rows("-2.01"))
3699+
tk.MustQuery("explain " + query).Check(testkit.Rows(""+
3700+
`IndexJoin_20 0.80 root inner join, inner:TableReader_19, outer key:test39999.t.txn_account_id, test39999.t.serial_id, inner key:test39999.c.txt_account_id, test39999.c.serial_id, equal cond:eq(test39999.t.serial_id, test39999.c.serial_id), eq(test39999.t.txn_account_id, test39999.c.txt_account_id)`,
3701+
`├─TableReader_25(Build) 0.80 root data:Selection_24`,
3702+
`│ └─Selection_24 0.80 cop[tikv] eq(test39999.t.broker, "0009"), not(isnull(test39999.t.serial_id))`,
3703+
`│ └─TableFullScan_23 1.00 cop[tikv] table:t keep order:false`,
3704+
`└─TableReader_19(Probe) 0.80 root partition:all data:Selection_18`,
3705+
` └─Selection_18 0.80 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)`,
3706+
` └─TableRangeScan_17 0.80 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.serial_id, test39999.t.serial_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false`))
3707+
}
3708+
>>>>>>> 4a72171ffb (*: Fix issue 39999, used wrong column id list for checking partitions (#40003))

0 commit comments

Comments
 (0)