Skip to content

Commit 3d42e34

Browse files
lcwangchaolcwangchao
authored
ttl: support to split TTL tasks for a table without binary primary key (#55663)
close #55660
1 parent 44f100f commit 3d42e34

File tree

4 files changed

+252
-65
lines changed

4 files changed

+252
-65
lines changed

‎pkg/ttl/cache/BUILD.bazel

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ go_library(
1717
"//pkg/infoschema",
1818
"//pkg/kv",
1919
"//pkg/parser/ast",
20+
"//pkg/parser/charset",
2021
"//pkg/parser/model",
2122
"//pkg/parser/mysql",
2223
"//pkg/parser/terror",
@@ -50,12 +51,13 @@ go_test(
5051
],
5152
embed = [":cache"],
5253
flaky = True,
53-
shard_count = 17,
54+
shard_count = 18,
5455
deps = [
5556
"//pkg/infoschema",
5657
"//pkg/kv",
5758
"//pkg/parser/ast",
5859
"//pkg/parser/model",
60+
"//pkg/parser/mysql",
5961
"//pkg/server",
6062
"//pkg/session",
6163
"//pkg/store/helper",

‎pkg/ttl/cache/split_test.go

Lines changed: 184 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"github.com/pingcap/tidb/pkg/infoschema"
2727
"github.com/pingcap/tidb/pkg/kv"
2828
"github.com/pingcap/tidb/pkg/parser/model"
29+
"github.com/pingcap/tidb/pkg/parser/mysql"
2930
"github.com/pingcap/tidb/pkg/store/helper"
3031
"github.com/pingcap/tidb/pkg/tablecodec"
3132
"github.com/pingcap/tidb/pkg/testkit"
@@ -250,21 +251,21 @@ func createTTLTableWithSQL(t *testing.T, tk *testkit.TestKit, name string, sql s
250251
return ttlTbl
251252
}
252253

253-
func checkRange(t *testing.T, r cache.ScanRange, start, end types.Datum) {
254+
func checkRange(t *testing.T, r cache.ScanRange, start, end types.Datum, msgAndArgs ...any) {
254255
if start.IsNull() {
255-
require.Nil(t, r.Start)
256+
require.Nil(t, r.Start, msgAndArgs...)
256257
} else {
257-
require.Equal(t, 1, len(r.Start))
258-
require.Equal(t, start.Kind(), r.Start[0].Kind())
259-
require.Equal(t, start.GetValue(), r.Start[0].GetValue())
258+
require.Equal(t, 1, len(r.Start), msgAndArgs...)
259+
require.Equal(t, start.Kind(), r.Start[0].Kind(), msgAndArgs...)
260+
require.Equal(t, start.GetValue(), r.Start[0].GetValue(), msgAndArgs...)
260261
}
261262

262263
if end.IsNull() {
263-
require.Nil(t, r.End)
264+
require.Nil(t, r.End, msgAndArgs...)
264265
} else {
265-
require.Equal(t, 1, len(r.End))
266-
require.Equal(t, end.Kind(), r.End[0].Kind())
267-
require.Equal(t, end.GetValue(), r.End[0].GetValue())
266+
require.Equal(t, 1, len(r.End), msgAndArgs...)
267+
require.Equal(t, end.Kind(), r.End[0].Kind(), msgAndArgs...)
268+
require.Equal(t, end.GetValue(), r.End[0].GetValue(), msgAndArgs...)
268269
}
269270
}
270271

@@ -516,47 +517,134 @@ func TestSplitTTLScanRangesWithBytes(t *testing.T) {
516517
createTTLTable(t, tk, "t3", "varchar(32) CHARACTER SET BINARY"),
517518
createTTLTable(t, tk, "t4", "bit(32)"),
518519
create2PKTTLTable(t, tk, "t5", "binary(32)"),
520+
createTTLTable(t, tk, "t6", "char(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"),
521+
createTTLTable(t, tk, "t7", "char(32) CHARACTER SET utf8 COLLATE utf8_bin"),
522+
create2PKTTLTable(t, tk, "t8", "char(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"),
523+
}
524+
525+
cases := []struct {
526+
name string
527+
regionEdges []kv.Handle
528+
splitCnt int
529+
binaryExpected [][]types.Datum
530+
stringExpected [][]types.Datum
531+
}{
532+
{
533+
name: "2 regions with binary split",
534+
regionEdges: []kv.Handle{
535+
bytesHandle(t, []byte{1, 2, 3}),
536+
},
537+
splitCnt: 4,
538+
binaryExpected: [][]types.Datum{
539+
{types.Datum{}, types.NewBytesDatum([]byte{1, 2, 3})},
540+
{types.NewBytesDatum([]byte{1, 2, 3}), types.Datum{}},
541+
},
542+
stringExpected: [][]types.Datum{
543+
{types.Datum{}, types.Datum{}},
544+
},
545+
},
546+
{
547+
name: "6 regions with binary split",
548+
regionEdges: []kv.Handle{
549+
bytesHandle(t, []byte{1, 2, 3}),
550+
bytesHandle(t, []byte{1, 2, 3, 4}),
551+
bytesHandle(t, []byte{1, 2, 3, 4, 5}),
552+
bytesHandle(t, []byte{1, 2, 4}),
553+
bytesHandle(t, []byte{1, 2, 5}),
554+
},
555+
splitCnt: 4,
556+
binaryExpected: [][]types.Datum{
557+
{types.Datum{}, types.NewBytesDatum([]byte{1, 2, 3, 4})},
558+
{types.NewBytesDatum([]byte{1, 2, 3, 4}), types.NewBytesDatum([]byte{1, 2, 4})},
559+
{types.NewBytesDatum([]byte{1, 2, 4}), types.NewBytesDatum([]byte{1, 2, 5})},
560+
{types.NewBytesDatum([]byte{1, 2, 5}), types.Datum{}},
561+
},
562+
stringExpected: [][]types.Datum{
563+
{types.Datum{}, types.Datum{}},
564+
},
565+
},
566+
{
567+
name: "2 regions with utf8 split",
568+
regionEdges: []kv.Handle{
569+
bytesHandle(t, []byte("中文")),
570+
},
571+
splitCnt: 4,
572+
binaryExpected: [][]types.Datum{
573+
{types.Datum{}, types.NewBytesDatum([]byte("中文"))},
574+
{types.NewBytesDatum([]byte("中文")), types.Datum{}},
575+
},
576+
stringExpected: [][]types.Datum{
577+
{types.Datum{}, types.Datum{}},
578+
},
579+
},
580+
{
581+
name: "several regions with mixed split",
582+
regionEdges: []kv.Handle{
583+
bytesHandle(t, []byte("abc")),
584+
bytesHandle(t, []byte("ab\x7f0")),
585+
bytesHandle(t, []byte("ab\xff0")),
586+
bytesHandle(t, []byte("ac\x001")),
587+
bytesHandle(t, []byte("ad\x0a1")),
588+
bytesHandle(t, []byte("ad23")),
589+
bytesHandle(t, []byte("ad230\xff")),
590+
bytesHandle(t, []byte("befh")),
591+
bytesHandle(t, []byte("中文")),
592+
},
593+
splitCnt: 10,
594+
binaryExpected: [][]types.Datum{
595+
{types.Datum{}, types.NewBytesDatum([]byte("abc"))},
596+
{types.NewBytesDatum([]byte("abc")), types.NewBytesDatum([]byte("ab\x7f0"))},
597+
{types.NewBytesDatum([]byte("ab\x7f0")), types.NewBytesDatum([]byte("ab\xff0"))},
598+
{types.NewBytesDatum([]byte("ab\xff0")), types.NewBytesDatum([]byte("ac\x001"))},
599+
{types.NewBytesDatum([]byte("ac\x001")), types.NewBytesDatum([]byte("ad\x0a1"))},
600+
{types.NewBytesDatum([]byte("ad\x0a1")), types.NewBytesDatum([]byte("ad23"))},
601+
{types.NewBytesDatum([]byte("ad23")), types.NewBytesDatum([]byte("ad230\xff"))},
602+
{types.NewBytesDatum([]byte("ad230\xff")), types.NewBytesDatum([]byte("befh"))},
603+
{types.NewBytesDatum([]byte("befh")), types.NewBytesDatum([]byte("中文"))},
604+
{types.NewBytesDatum([]byte("中文")), types.Datum{}},
605+
},
606+
stringExpected: [][]types.Datum{
607+
{types.Datum{}, types.NewStringDatum("abc")},
608+
{types.NewStringDatum("abc"), types.NewStringDatum("ac")},
609+
{types.NewStringDatum("ac"), types.NewStringDatum("ad\n1")},
610+
{types.NewStringDatum("ad\n1"), types.NewStringDatum("ad23")},
611+
{types.NewStringDatum("ad23"), types.NewStringDatum("ad230")},
612+
{types.NewStringDatum("ad230"), types.NewStringDatum("befh")},
613+
{types.NewStringDatum("befh"), types.Datum{}},
614+
},
615+
},
519616
}
520617

521618
tikvStore := newMockTiKVStore(t)
522619
for _, tbl := range tbls {
523-
// test only one region
524-
tikvStore.clearRegions()
525-
ranges, err := tbl.SplitScanRanges(context.TODO(), tikvStore, 4)
526-
require.NoError(t, err)
527-
require.Equal(t, 1, len(ranges))
528-
checkRange(t, ranges[0], types.Datum{}, types.Datum{})
529-
530-
// test share regions with other table
531-
tikvStore.clearRegions()
532-
tikvStore.addRegion(
533-
tablecodec.GenTablePrefix(tbl.ID-1),
534-
tablecodec.GenTablePrefix(tbl.ID+1),
535-
)
536-
ranges, err = tbl.SplitScanRanges(context.TODO(), tikvStore, 4)
537-
require.NoError(t, err)
538-
require.Equal(t, 1, len(ranges))
539-
checkRange(t, ranges[0], types.Datum{}, types.Datum{})
620+
for _, c := range cases {
621+
tikvStore.clearRegions()
622+
require.Greater(t, len(c.regionEdges), 0)
623+
for i, edge := range c.regionEdges {
624+
if i == 0 {
625+
tikvStore.addRegionBeginWithTablePrefix(tbl.ID, edge)
626+
} else {
627+
tikvStore.addRegionWithTablePrefix(tbl.ID, c.regionEdges[i-1], edge)
628+
}
629+
}
630+
tikvStore.addRegionEndWithTablePrefix(c.regionEdges[len(c.regionEdges)-1], tbl.ID)
631+
ranges, err := tbl.SplitScanRanges(context.TODO(), tikvStore, c.splitCnt)
632+
require.NoError(t, err)
633+
634+
keyTp := tbl.KeyColumnTypes[0]
635+
var expected [][]types.Datum
636+
if keyTp.GetType() == mysql.TypeBit || mysql.HasBinaryFlag(keyTp.GetFlag()) {
637+
expected = c.binaryExpected
638+
} else {
639+
expected = c.stringExpected
640+
}
540641

541-
// test one table has multiple regions
542-
tikvStore.clearRegions()
543-
tikvStore.addRegionBeginWithTablePrefix(tbl.ID, bytesHandle(t, []byte{1, 2, 3}))
544-
tikvStore.addRegionWithTablePrefix(
545-
tbl.ID, bytesHandle(t, []byte{1, 2, 3}), bytesHandle(t, []byte{1, 2, 3, 4}))
546-
tikvStore.addRegionWithTablePrefix(
547-
tbl.ID, bytesHandle(t, []byte{1, 2, 3, 4}), bytesHandle(t, []byte{1, 2, 3, 4, 5}))
548-
tikvStore.addRegionWithTablePrefix(
549-
tbl.ID, bytesHandle(t, []byte{1, 2, 3, 4, 5}), bytesHandle(t, []byte{1, 2, 4}))
550-
tikvStore.addRegionWithTablePrefix(
551-
tbl.ID, bytesHandle(t, []byte{1, 2, 4}), bytesHandle(t, []byte{1, 2, 5}))
552-
tikvStore.addRegionEndWithTablePrefix(bytesHandle(t, []byte{1, 2, 5}), tbl.ID)
553-
ranges, err = tbl.SplitScanRanges(context.TODO(), tikvStore, 4)
554-
require.NoError(t, err)
555-
require.Equal(t, 4, len(ranges))
556-
checkRange(t, ranges[0], types.Datum{}, types.NewBytesDatum([]byte{1, 2, 3, 4}))
557-
checkRange(t, ranges[1], types.NewBytesDatum([]byte{1, 2, 3, 4}), types.NewBytesDatum([]byte{1, 2, 4}))
558-
checkRange(t, ranges[2], types.NewBytesDatum([]byte{1, 2, 4}), types.NewBytesDatum([]byte{1, 2, 5}))
559-
checkRange(t, ranges[3], types.NewBytesDatum([]byte{1, 2, 5}), types.Datum{})
642+
require.Equal(t, len(expected), len(ranges), "tbl: %s, case: %s", tbl.Name, c.name)
643+
for i, r := range ranges {
644+
checkRange(t, r, expected[i][0], expected[i][1],
645+
"tbl: %s, case: %s, i: %d", tbl.Name, c.name, i)
646+
}
647+
}
560648
}
561649
}
562650

@@ -565,10 +653,12 @@ func TestNoTTLSplitSupportTables(t *testing.T) {
565653
tk := testkit.NewTestKit(t, store)
566654

567655
tbls := []*cache.PhysicalTable{
568-
createTTLTable(t, tk, "t1", "char(32) CHARACTER SET UTF8MB4"),
569-
createTTLTable(t, tk, "t2", "varchar(32) CHARACTER SET UTF8MB4"),
570-
createTTLTable(t, tk, "t4", "decimal(32, 2)"),
571-
create2PKTTLTable(t, tk, "t5", "char(32) CHARACTER SET UTF8MB4"),
656+
createTTLTable(t, tk, "t1", "decimal(32, 2)"),
657+
createTTLTable(t, tk, "t2", "date"),
658+
createTTLTable(t, tk, "t3", "datetime"),
659+
createTTLTable(t, tk, "t4", "timestamp"),
660+
createTTLTable(t, tk, "t5", "varchar(32) character set utf8mb4 collate utf8mb4_general_ci"),
661+
createTTLTable(t, tk, "t6", "varchar(32) character set utf8mb4 collate utf8mb4_0900_ai_ci"),
572662
}
573663

574664
tikvStore := newMockTiKVStore(t)
@@ -827,6 +917,51 @@ func TestGetNextBytesHandleDatum(t *testing.T) {
827917
}
828918
}
829919

920+
func TestGetASCIIPrefixDatumFromBytes(t *testing.T) {
921+
cases := []struct {
922+
bytes []byte
923+
expected string
924+
}{
925+
{bytes: nil, expected: ""},
926+
{bytes: []byte{}, expected: ""},
927+
{bytes: []byte{0}, expected: ""},
928+
{bytes: []byte{1}, expected: ""},
929+
{bytes: []byte{8}, expected: ""},
930+
{bytes: []byte{9}, expected: "\t"},
931+
{bytes: []byte{10}, expected: "\n"},
932+
{bytes: []byte{11}, expected: ""},
933+
{bytes: []byte{12}, expected: ""},
934+
{bytes: []byte{13}, expected: "\r"},
935+
{bytes: []byte{14}, expected: ""},
936+
{bytes: []byte{0x19}, expected: ""},
937+
{bytes: []byte{0x20}, expected: " "},
938+
{bytes: []byte{0x21}, expected: "!"},
939+
{bytes: []byte{0x7D}, expected: "}"},
940+
{bytes: []byte{0x7E}, expected: "~"},
941+
{bytes: []byte{0x7F}, expected: ""},
942+
{bytes: []byte{0xFF}, expected: ""},
943+
{bytes: []byte{0x0, 'a', 'b'}, expected: ""},
944+
{bytes: []byte{0xFF, 'a', 'b'}, expected: ""},
945+
{bytes: []byte{'0', '1', 0x0, 'a', 'b'}, expected: "01"},
946+
{bytes: []byte{'0', '1', 0x15, 'a', 'b'}, expected: "01"},
947+
{bytes: []byte{'0', '1', 0xFF, 'a', 'b'}, expected: "01"},
948+
{bytes: []byte{'a', 'b', 0x0}, expected: "ab"},
949+
{bytes: []byte{'a', 'b', 0x15}, expected: "ab"},
950+
{bytes: []byte{'a', 'b', 0xFF}, expected: "ab"},
951+
{bytes: []byte("ab\rcd\tef\nAB!~GH()tt ;;"), expected: "ab\rcd\tef\nAB!~GH()tt ;;"},
952+
{bytes: []byte("中文"), expected: ""},
953+
{bytes: []byte("cn中文"), expected: "cn"},
954+
{bytes: []byte("😀"), expected: ""},
955+
{bytes: []byte("emoji😀"), expected: "emoji"},
956+
}
957+
958+
for i, c := range cases {
959+
d := cache.GetASCIIPrefixDatumFromBytes(c.bytes)
960+
require.Equalf(t, types.KindString, d.Kind(), "i: %d", i)
961+
require.Equalf(t, c.expected, d.GetString(), "i: %d, bs: %v", i, c.bytes)
962+
}
963+
}
964+
830965
func TestGetNextIntHandle(t *testing.T) {
831966
tblID := int64(7)
832967
cases := []struct {

0 commit comments

Comments
 (0)