This is an automated cherry-pick of pingcap#58433

Signed-off-by: ti-chi-bot <[email protected]>

Author: YuJuncen

br/pkg/task/common_test.go

@@ -43,7 +43,7 @@ func TestUrlNoQuery(t *testing.T) {
 
 func TestTiDBConfigUnchanged(t *testing.T) {
 	cfg := config.GetGlobalConfig()
-	restoreConfig := enableTiDBConfig()
+	restoreConfig := tweakLocalConfForRestore()
 	require.NotEqual(t, config.GetGlobalConfig(), cfg)
 	restoreConfig()
 	require.Equal(t, config.GetGlobalConfig(), cfg)

br/pkg/task/restore.go

@@ -659,7 +659,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 	}
 
 	// pre-set TiDB config for restore
-	restoreDBConfig := enableTiDBConfig()
+	restoreDBConfig := tweakLocalConfForRestore()
 	defer restoreDBConfig()
 
 	if client.GetSupportPolicy() {
@@ -899,6 +899,7 @@ func filterRestoreFiles(
 	return
 }
 
+<<<<<<< HEAD
 // restorePreWork executes some prepare work before restore.
 // TODO make this function returns a restore post work.
 func restorePreWork(ctx context.Context, client *restore.Client, mgr *conn.Mgr, switchToImport bool) (pdutil.UndoFunc, error) {
@@ -935,8 +936,11 @@ func restorePostWork(
 }
 
 // enableTiDBConfig tweaks some of configs of TiDB to make the restore progress go well.
+=======
+// tweakLocalConfForRestore tweaks some of configs of TiDB to make the restore progress go well.
+>>>>>>> 384f858a6c8 (br/stream: allow pitr to create oversized indices (#58433))
 // return a function that could restore the config to origin.
-func enableTiDBConfig() func() {
+func tweakLocalConfForRestore() func() {
 	restoreConfig := config.RestoreFunc()
 	config.UpdateGlobal(func(conf *config.Config) {
 		// set max-index-length before execute DDLs and create tables

br/pkg/task/stream.go

@@ -1194,6 +1194,9 @@ func restoreStream(
 	ctx, cancelFn := context.WithCancel(c)
 	defer cancelFn()
 
+	restoreCfg := tweakLocalConfForRestore()
+	defer restoreCfg()
+
 	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
 		span1 := span.Tracer().StartSpan(
 			"restoreStream",

br/tests/br_pitr/config/tidb-max-index-length.toml

@@ -0,0 +1,11 @@
+# config of tidb
+
+max-index-length = 12288
+
+[security]
+ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+ssl-key = "/tmp/backup_restore_test/certs/tidb.key"
+cluster-ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+cluster-ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+cluster-ssl-key = "/tmp/backup_restore_test/certs/tidb.key"

br/tests/br_pitr/incremental_data/ingest_repair.sql

@@ -0,0 +1,51 @@
+-- basic test: [INDEX/UNIQUE], [COMMENT], [INDEXTYPE], [INVISIBLE], [EXPRESSION]
+ALTER TABLE test.pairs ADD INDEX i1(y, z) USING HASH COMMENT "edelw;fe?fewfe\nefwe" INVISIBLE;
+ALTER TABLE test.pairs ADD UNIQUE KEY u1(x, y) USING RTREE VISIBLE;
+ALTER TABLE test.pairs ADD INDEX i2(y, (z + 1)) USING BTREE COMMENT "123";
+ALTER TABLE test.pairs ADD UNIQUE KEY u2(x, (y+1)) USING HASH COMMENT "243";
+
+-- test: [COLUMN LENGTH], [EXPRESSION], [PRIMARY]
+ALTER TABLE test.pairs2 ADD INDEX i1(y, z(10));
+ALTER TABLE test.pairs2 ADD UNIQUE KEY u1(y, z(10), (y * 2)) USING RTREE VISIBLE;
+ALTER TABLE test.pairs2 ADD PRIMARY KEY (x) USING HASH;
+
+-- test: [MULTIVALUED]
+ALTER TABLE test.pairs3 ADD INDEX zips2((CAST(custinfo->'$.zipcode' AS UNSIGNED ARRAY)));
+
+-- test: DROP operation
+ALTER TABLE test.pairs4 ADD INDEX i1(y, z) USING HASH COMMENT "edelw;fe?fewfe\nefwe" INVISIBLE;
+ALTER TABLE test.pairs4 ADD UNIQUE KEY u1(x, y) USING RTREE VISIBLE;
+ALTER TABLE test.pairs4 ADD INDEX i2(y, (z + 1)) USING BTREE COMMENT "123";
+ALTER TABLE test.pairs4 ADD UNIQUE KEY u2(x, (y+1)) USING HASH COMMENT "243";
+ALTER TABLE test.pairs4 DROP INDEX i1;
+ALTER TABLE test.pairs4 DROP INDEX u1;
+ALTER TABLE test.pairs4 DROP INDEX i2;
+ALTER TABLE test.pairs4 DROP INDEX u2;
+
+-- test: DROP operation
+ALTER TABLE test.pairs5 ADD INDEX i1(y, z(10));
+ALTER TABLE test.pairs5 ADD UNIQUE KEY u1(y, z(10), (y * 2)) USING RTREE VISIBLE;
+ALTER TABLE test.pairs5 ADD PRIMARY KEY (x) USING HASH;
+ALTER TABLE test.pairs5 DROP INDEX i1;
+ALTER TABLE test.pairs5 DROP INDEX u1;
+ALTER TABLE test.pairs5 DROP INDEX `PRIMARY`;
+
+-- test: [strange string in EXPRESSION], [rename operation]
+ALTER TABLE test.pairs6 ADD INDEX zips2((CAST(`cust``;info`->'$.zipcode' AS UNSIGNED ARRAY)));
+ALTER TABLE test.pairs6 ADD INDEX i1(`nam``;e`, (`nam``;e` * 2));
+RENAME TABLE test.pairs6 TO test.pairs7;
+ALTER TABLE test.pairs7 RENAME INDEX i1 to i2;
+
+-- future test: [MODIFY COLUMN operation]
+ALTER TABLE test.pairs8 ADD INDEX i1(y);
+ALTER TABLE test.pairs8 MODIFY y varchar(20);
+
+-- future test: [CHANGE COLUMN operation]
+ALTER TABLE test.pairs9 ADD INDEX i1(y);
+ALTER TABLE test.pairs9 CHANGE y y2 varchar(20);
+
+-- test partition
+ALTER TABLE test.pairs10 ADD INDEX i1(y);
+
+
+CREATE INDEX huge ON test.huge_idx(blob1, blob2);

br/tests/br_pitr/prepare_data/ingest_repair.sql

@@ -22,9 +22,15 @@ CUR=$(cd `dirname $0`; pwd)
 PREFIX="pitr_backup" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*.
 res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
 
+restart_services_allowing_huge_index() {
+    echo "restarting services with huge indices enabled..."
+    stop_services
+    start_services --tidb-cfg "$CUR/config/tidb-max-index-length.toml"
+    echo "restart services done..."
+}
+
 # start a new cluster
-echo "restart a services"
-restart_services
+restart_services_allowing_huge_index
 
 # prepare the data
 echo "prepare the data"
@@ -91,11 +97,16 @@ done
 # ...
 
 # start a new cluster
+<<<<<<< HEAD
 echo "restart a services"
 restart_services
+=======
+restart_services_allowing_huge_index
+>>>>>>> 384f858a6c8 (br/stream: allow pitr to create oversized indices (#58433))
 
 # PITR restore
 echo "run pitr"
+<<<<<<< HEAD
 run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1
 
 # check something in downstream cluster
@@ -109,3 +120,85 @@ run_sql "select * from mysql.gc_delete_range_done"
 run_sql "select count(*) DELETE_RANGE_CNT from (select * from mysql.gc_delete_range union all select * from mysql.gc_delete_range_done) del_range group by ts order by DELETE_RANGE_CNT desc limit 1;"
 expect_delete_range=$(($incremental_delete_range_count-$prepare_delete_range_count))
 check_contains "DELETE_RANGE_CNT: $expect_delete_range"
+=======
+run_sql "DROP DATABASE __TiDB_BR_Temporary_Log_Restore_Checkpoint;"
+run_sql "DROP DATABASE __TiDB_BR_Temporary_Custom_SST_Restore_Checkpoint;"
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1 || ( cat $res_file && exit 1 )
+
+check_result
+
+# start a new cluster for incremental + log
+restart_services_allowing_huge_index
+
+echo "run snapshot restore#2"
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" 
+
+echo "run incremental restore + log restore"
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/inc" > $res_file 2>&1
+
+check_result
+
+# start a new cluster for incremental + log
+echo "restart services"
+restart_services_allowing_huge_index
+
+echo "run snapshot restore#3"
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" 
+
+echo "run incremental restore but failed"
+restore_fail=0
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/inc_fail" || restore_fail=1
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success on incremental restore'
+    exit 1
+fi
+
+# start a new cluster for corruption
+restart_services_allowing_huge_index
+
+file_corruption() {
+    echo "corrupt the whole log files"
+    for filename in $(find $TEST_DIR/$PREFIX/log -regex ".*\.log" | grep -v "schema-meta"); do
+        echo "corrupt the log file $filename"
+        filename_temp=$filename"_temp"
+        echo "corruption" > $filename_temp
+        cat $filename >> $filename_temp
+        mv $filename_temp $filename
+        truncate -s -11 $filename
+    done
+}
+
+# file corruption
+file_corruption
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-remaining-attempts-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success on file corruption'
+    exit 1
+fi
+
+# start a new cluster for corruption
+restart_services_allowing_huge_index
+
+file_lost() {
+    echo "lost the whole log files"
+    for filename in $(find $TEST_DIR/$PREFIX/log -regex ".*\.log" | grep -v "schema-meta"); do
+        echo "lost the log file $filename"
+        filename_temp=$filename"_temp"
+        mv $filename $filename_temp
+    done
+}
+
+# file lost
+file_lost
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-remaining-attempts-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success on file lost'
+    exit 1
+fi
+>>>>>>> 384f858a6c8 (br/stream: allow pitr to create oversized indices (#58433))