From 07890ff8a54d49e5c1eec15129cf470b4e0e4f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E5=B2=9A?= <36239017+YuJuncen@users.noreply.github.com> Date: Tue, 24 Dec 2024 13:57:51 +0800 Subject: [PATCH] br/stream: allow pitr to create oversized indices (#58433) close pingcap/tidb#58430 --- br/pkg/task/common_test.go | 2 +- br/pkg/task/restore.go | 6 +++--- br/pkg/task/stream.go | 3 +++ br/tests/br_pitr/config/tidb-max-index-length.toml | 11 +++++++++++ br/tests/br_pitr/incremental_data/ingest_repair.sql | 3 +++ br/tests/br_pitr/prepare_data/ingest_repair.sql | 4 +++- br/tests/br_pitr/run.sh | 13 ++++++++++--- 7 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 br/tests/br_pitr/config/tidb-max-index-length.toml diff --git a/br/pkg/task/common_test.go b/br/pkg/task/common_test.go index 1ad0114764684..fceee4b09101a 100644 --- a/br/pkg/task/common_test.go +++ b/br/pkg/task/common_test.go @@ -87,7 +87,7 @@ func TestUrlNoQuery(t *testing.T) { func TestTiDBConfigUnchanged(t *testing.T) { cfg := config.GetGlobalConfig() - restoreConfig := enableTiDBConfig() + restoreConfig := tweakLocalConfForRestore() require.NotEqual(t, config.GetGlobalConfig(), cfg) restoreConfig() require.Equal(t, config.GetGlobalConfig(), cfg) diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go index 270606ae763e3..d64e057b40bfe 100644 --- a/br/pkg/task/restore.go +++ b/br/pkg/task/restore.go @@ -923,7 +923,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } // pre-set TiDB config for restore - restoreDBConfig := enableTiDBConfig() + restoreDBConfig := tweakLocalConfForRestore() defer restoreDBConfig() if client.GetSupportPolicy() { @@ -1251,9 +1251,9 @@ func restorePostWork( } } -// enableTiDBConfig tweaks some of configs of TiDB to make the restore progress go well. +// tweakLocalConfForRestore tweaks some of configs of TiDB to make the restore progress go well. // return a function that could restore the config to origin. -func enableTiDBConfig() func() { +func tweakLocalConfForRestore() func() { restoreConfig := config.RestoreFunc() config.UpdateGlobal(func(conf *config.Config) { // set max-index-length before execute DDLs and create tables diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go index 8dcc75aff38dc..3b6f12a85a39d 100644 --- a/br/pkg/task/stream.go +++ b/br/pkg/task/stream.go @@ -1263,6 +1263,9 @@ func restoreStream( ctx, cancelFn := context.WithCancel(c) defer cancelFn() + restoreCfg := tweakLocalConfForRestore() + defer restoreCfg() + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan( "restoreStream", diff --git a/br/tests/br_pitr/config/tidb-max-index-length.toml b/br/tests/br_pitr/config/tidb-max-index-length.toml new file mode 100644 index 0000000000000..9ed75ad154693 --- /dev/null +++ b/br/tests/br_pitr/config/tidb-max-index-length.toml @@ -0,0 +1,11 @@ +# config of tidb + +max-index-length = 12288 + +[security] +ssl-ca = "/tmp/backup_restore_test/certs/ca.pem" +ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem" +ssl-key = "/tmp/backup_restore_test/certs/tidb.key" +cluster-ssl-ca = "/tmp/backup_restore_test/certs/ca.pem" +cluster-ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem" +cluster-ssl-key = "/tmp/backup_restore_test/certs/tidb.key" diff --git a/br/tests/br_pitr/incremental_data/ingest_repair.sql b/br/tests/br_pitr/incremental_data/ingest_repair.sql index f82a78e6df3cd..0f0b9b825ed55 100644 --- a/br/tests/br_pitr/incremental_data/ingest_repair.sql +++ b/br/tests/br_pitr/incremental_data/ingest_repair.sql @@ -46,3 +46,6 @@ ALTER TABLE test.pairs9 CHANGE y y2 varchar(20); -- test partition ALTER TABLE test.pairs10 ADD INDEX i1(y); + + +CREATE INDEX huge ON test.huge_idx(blob1, blob2); diff --git a/br/tests/br_pitr/prepare_data/ingest_repair.sql b/br/tests/br_pitr/prepare_data/ingest_repair.sql index fc81fa1ced06e..763e6894b3701 100644 --- a/br/tests/br_pitr/prepare_data/ingest_repair.sql +++ b/br/tests/br_pitr/prepare_data/ingest_repair.sql @@ -43,4 +43,6 @@ INSERT INTO test.pairs10 VALUES (1,1,"1"),(2,2,"2"),(3,3,"3"),(4,4,"4"),(5,5,"5" -- test no need to repair CREATE TABLE test.pairs11(x int auto_increment primary key, y int DEFAULT RAND(), z int DEFAULT RAND()); INSERT INTO test.pairs11 VALUES (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(); -ALTER TABLE test.pairs11 ADD UNIQUE KEY u1(x, y); \ No newline at end of file +ALTER TABLE test.pairs11 ADD UNIQUE KEY u1(x, y); + +CREATE TABLE test.huge_idx(id int AUTO_INCREMENT, blob1 varchar(1000), blob2 varchar(1000)); \ No newline at end of file diff --git a/br/tests/br_pitr/run.sh b/br/tests/br_pitr/run.sh index 7ab0142147749..ee84c04438b96 100644 --- a/br/tests/br_pitr/run.sh +++ b/br/tests/br_pitr/run.sh @@ -22,9 +22,16 @@ CUR=$(cd `dirname $0`; pwd) PREFIX="pitr_backup" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*. res_file="$TEST_DIR/sql_res.$TEST_NAME.txt" +restart_services_allowing_huge_index() { + echo "restarting services with huge indices enabled..." + stop_services + start_services --tidb-cfg "$CUR/config/tidb-max-index-length.toml" + echo "restart services done..." +} + # start a new cluster echo "restart a services" -restart_services +restart_services_allowing_huge_index # prepare the data echo "prepare the data" @@ -94,11 +101,11 @@ done # start a new cluster echo "restart a services" -restart_services +restart_services_allowing_huge_index # PITR restore echo "run pitr" -run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1 +run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1 || ( cat $res_file && exit 1 ) # check something in downstream cluster echo "check br log"