br/stream: allow pitr to create oversized indices (#58433) (#58527)

close #58430

Author: ti-chi-bot

br/pkg/task/common_test.go

@@ -99,7 +99,7 @@ func TestUrlNoQuery(t *testing.T) {
 
 func TestTiDBConfigUnchanged(t *testing.T) {
 	cfg := config.GetGlobalConfig()
-	restoreConfig := enableTiDBConfig()
+	restoreConfig := tweakLocalConfForRestore()
 	require.NotEqual(t, config.GetGlobalConfig(), cfg)
 	restoreConfig()
 	require.Equal(t, config.GetGlobalConfig(), cfg)

br/pkg/task/restore.go

@@ -972,7 +972,7 @@ func runSnapshotRestore(c context.Context, mgr *conn.Mgr, g glue.Glue, cmdName s
 	}
 
 	// pre-set TiDB config for restore
-	restoreDBConfig := enableTiDBConfig()
+	restoreDBConfig := tweakLocalConfForRestore()
 	defer restoreDBConfig()
 
 	if client.GetSupportPolicy() {
@@ -1407,9 +1407,9 @@ func filterRestoreFiles(
 	return
 }
 
-// enableTiDBConfig tweaks some of configs of TiDB to make the restore progress go well.
+// tweakLocalConfForRestore tweaks some of configs of TiDB to make the restore progress go well.
 // return a function that could restore the config to origin.
-func enableTiDBConfig() func() {
+func tweakLocalConfForRestore() func() {
 	restoreConfig := config.RestoreFunc()
 	config.UpdateGlobal(func(conf *config.Config) {
 		// set max-index-length before execute DDLs and create tables

br/pkg/task/stream.go

@@ -1287,6 +1287,9 @@ func restoreStream(
 	ctx, cancelFn := context.WithCancel(c)
 	defer cancelFn()
 
+	restoreCfg := tweakLocalConfForRestore()
+	defer restoreCfg()
+
 	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
 		span1 := span.Tracer().StartSpan(
 			"restoreStream",

br/tests/br_pitr/config/tidb-max-index-length.toml

@@ -0,0 +1,11 @@
+# config of tidb
+
+max-index-length = 12288
+
+[security]
+ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+ssl-key = "/tmp/backup_restore_test/certs/tidb.key"
+cluster-ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+cluster-ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+cluster-ssl-key = "/tmp/backup_restore_test/certs/tidb.key"

br/tests/br_pitr/incremental_data/ingest_repair.sql

@@ -46,3 +46,6 @@ ALTER TABLE test.pairs9 CHANGE y y2 varchar(20);
 
 -- test partition
 ALTER TABLE test.pairs10 ADD INDEX i1(y);
+
+
+CREATE INDEX huge ON test.huge_idx(blob1, blob2);

br/tests/br_pitr/prepare_data/ingest_repair.sql

@@ -43,4 +43,6 @@ INSERT INTO test.pairs10 VALUES (1,1,"1"),(2,2,"2"),(3,3,"3"),(4,4,"4"),(5,5,"5"
 -- test no need to repair
 CREATE TABLE test.pairs11(x int auto_increment primary key, y int DEFAULT RAND(), z int DEFAULT RAND());
 INSERT INTO test.pairs11 VALUES (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
-ALTER TABLE test.pairs11 ADD UNIQUE KEY u1(x, y);
+ALTER TABLE test.pairs11 ADD UNIQUE KEY u1(x, y);
+
+CREATE TABLE test.huge_idx(id int AUTO_INCREMENT, blob1 varchar(1000), blob2 varchar(1000));

br/tests/br_pitr/run.sh

@@ -23,9 +23,15 @@ PREFIX="pitr_backup" # NOTICE: don't start with 'br' because `restart services`
 res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
 TASK_NAME="br_pitr"
 
+restart_services_allowing_huge_index() {
+    echo "restarting services with huge indices enabled..."
+    stop_services
+    start_services --tidb-cfg "$CUR/config/tidb-max-index-length.toml"
+    echo "restart services done..."
+}
+
 # start a new cluster
-echo "restart a services"
-restart_services
+restart_services_allowing_huge_index
 
 # prepare the data
 echo "prepare the data"
@@ -92,8 +98,7 @@ check_result() {
 }
 
 # start a new cluster
-echo "restart services"
-restart_services
+restart_services_allowing_huge_index
 
 # non-compliant operation
 echo "non compliant operation"
@@ -107,13 +112,17 @@ fi
 # PITR restore
 echo "run pitr"
 run_sql "DROP DATABASE __TiDB_BR_Temporary_Log_Restore_Checkpoint;"
+<<<<<<< HEAD
 run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1
+=======
+run_sql "DROP DATABASE __TiDB_BR_Temporary_Custom_SST_Restore_Checkpoint;"
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1 || ( cat $res_file && exit 1 )
+>>>>>>> 384f858a6c8 (br/stream: allow pitr to create oversized indices (#58433))
 
 check_result
 
 # start a new cluster for incremental + log
-echo "restart services"
-restart_services
+restart_services_allowing_huge_index
 
 echo "run snapshot restore#2"
 run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" 
@@ -125,7 +134,7 @@ check_result
 
 # start a new cluster for incremental + log
 echo "restart services"
-restart_services
+restart_services_allowing_huge_index
 
 echo "run snapshot restore#3"
 run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" 
@@ -139,8 +148,7 @@ if [ $restore_fail -ne 1 ]; then
 fi
 
 # start a new cluster for corruption
-echo "restart a services"
-restart_services
+restart_services_allowing_huge_index
 
 file_corruption() {
     echo "corrupt the whole log files"
@@ -166,8 +174,7 @@ if [ $restore_fail -ne 1 ]; then
 fi
 
 # start a new cluster for corruption
-echo "restart a services"
-restart_services
+restart_services_allowing_huge_index
 
 file_lost() {
     echo "lost the whole log files"