!430 7.1.8-5: 更新 v8.5.1 的代码到 7.1.8-5 || tidb_security_advanced_branch_pr=95

* executor: fix a bug that global temporary table send cop request (pingcap#588…
* statistics: fix the panic when to async load stats with dropped index …
* executor: fix prepared protocol charset (pingcap#58872) (pingcap#58931)
* *: Update client-go and verify all read ts (pingcap#58909)
* integration test: fix test case "br_pitr" (pingcap#58876)
* session: add indexes for `mysql.analyze_jobs` (pingcap#58134) (pingcap#58355)
* ddl: fix args count for modify column (pingcap#58855) (pingcap#58858)
* planner: correct plan when scan tidb related cluster table with KeepOr…
* planner: Fix vector not truncated after CBO (pingcap#58809) (pingcap#58844)
* ddl: Fix vector index for high dimensional vectors (pingcap#58717) (pingcap#58835)
* ddl: Fix issue with concurrent update getting reverted by BackfillData…
* statistics: stats cache set default quota as 20%  (pingcap#58013) (pingcap#58817)
* executor: change the evaluation order of columns in `Update` and `Inse…
* statistics: add recover to protect background task (pingcap#58739) (pingcap#58767)
* ttl: fix the infinite waiting for delRateLimiter when `tidb_ttl_delete…
* ttl: reduce some warnings logs when locking TTL tasks (pingcap#58306) (pingcap#58783)
* ttl: retry the rows when del rate limiter returns error in delWorker (…
* ttl: reschedule task to other instances when shrinking worker (pingcap#57703) (pingcap#58778)
* ttl: fix the issue that one task losing heartbeat will block other tas…
* ttl: fix the issue that the task is not cancelled after transfering ow…
* ddl: fix job state overridden when concurrent updates don't overlap in…
* ttl: set the job history status to `cancelled` if it's removed in GC a…
* ttl: fix the timezone issue and panic in the caller of `getSession` (#…
* ddl: fix version syncer doesn't print who hasn't synced on partial syn…
* ttl: fix the issue that `DROP TABLE` / `ALTER TABLE` will keep job run…
* br/stream: allow pitr to create oversized indices (pingcap#58433) (pingcap#58527)
* ttl: set a result for timeout scan task during shrinking scan worker (…
* executor: fix time zone issue when querying slow log (pingcap#58455) (pingcap#58577)
* table: fix the issue that the default value for `BIT` column is wrong …
* statistics: temporarily skip handling errors for DDL events (pingcap#58609) (pingcap#58634)
* sessionctx: fix null max value to leading wrong warning (pingcap#57898) (pingcap#57935)
* planner: convert cartesian semi join with other nulleq condition to cr…
* planner: fix idxMergePartPlans forget to deal with RootTaskConds (pingcap#585…
* domain: change some stats log level as WARN (pingcap#58316) (pingcap#58555)
* planner: quickly get total count from index/column (pingcap#58365) (pingcap#58431)
* planner, expr: eval readonly user var during plan phase (pingcap#54462) (pingcap#58540)
* metrics: add col/idx name(s) for BackfillProgressGauge and BackfillTot…
* br: refactor test to use wait checkpoint method (pingcap#57612) (pingcap#58498)
* executor: reuse chunk in hash join v2 during restoring (pingcap#56936) (pingcap#58018)
* executor: fix goroutine leak when exceed quota in hash agg (pingcap#58078) (pingcap#58462)
* copr: fix the issue that busy threshold may redirect batch copr to fol…
* statistics: skip non-exicted table when to init stats (pingcap#58381) (pingcap#58394)
* planner: fix incorrectly using the schema for plan cache (pingcap#57964) (pingcap#58090)
* *: use DDL subscriber updating stats meta (pingcap#57872) (pingcap#58387)
* planner, runtime_filter: Remove redundant logs whose meaning can be di…
* statistics: remove dead code (pingcap#58412) (pingcap#58442)
* planner: Use/force to apply prefer range scan (pingcap#56928) (pingcap#58444)
* statistics: gc the statistics correctly after drop the database (pingcap#5730…
* ddl: Fixed partition interval from DayMinute to just Minute. (pingcap#57738) (pingcap#58019)
* executor: Enlarge the timeout for fetching TiFlash system tables (pingcap#579…

Author: joccau

DEPS.bzl

@@ -6985,13 +6985,13 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sha256 = "cdcad188042c4d716dd9d4a304a2e36bc9d4edccaf86a19b85b1682f01df193c",
-        strip_prefix = "github.com/tikv/client-go/[email protected].20241121061241-006dfb024c26",
+        sha256 = "f896c60fc81339cc03e7dcb38b70bcfa368bbb2a94c6c7f3f5da6a4795d237f0",
+        strip_prefix = "github.com/tikv/client-go/[email protected].20250113074216-d66e460ff577",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241121061241-006dfb024c26.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241121061241-006dfb024c26.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241121061241-006dfb024c26.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241121061241-006dfb024c26.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250113074216-d66e460ff577.zip",
+            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250113074216-d66e460ff577.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250113074216-d66e460ff577.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250113074216-d66e460ff577.zip",
         ],
     )
     go_repository(

br/pkg/restore/snap_client/systable_restore_test.go

@@ -116,5 +116,5 @@ func TestCheckSysTableCompatibility(t *testing.T) {
 //
 // The above variables are in the file br/pkg/restore/systable_restore.go
 func TestMonitorTheSystemTableIncremental(t *testing.T) {
-	require.Equal(t, int64(218), session.CurrentBootstrapVersion)
+	require.Equal(t, int64(219), session.CurrentBootstrapVersion)
 }

br/pkg/task/common_test.go

@@ -99,7 +99,7 @@ func TestUrlNoQuery(t *testing.T) {
 
 func TestTiDBConfigUnchanged(t *testing.T) {
 	cfg := config.GetGlobalConfig()
-	restoreConfig := enableTiDBConfig()
+	restoreConfig := tweakLocalConfForRestore()
 	require.NotEqual(t, config.GetGlobalConfig(), cfg)
 	restoreConfig()
 	require.Equal(t, config.GetGlobalConfig(), cfg)

br/pkg/task/restore.go

@@ -980,7 +980,7 @@ func runSnapshotRestore(c context.Context, mgr *conn.Mgr, g glue.Glue, cmdName s
 	}
 
 	// pre-set TiDB config for restore
-	restoreDBConfig := enableTiDBConfig()
+	restoreDBConfig := tweakLocalConfForRestore()
 	defer restoreDBConfig()
 
 	if client.GetSupportPolicy() {
@@ -1415,9 +1415,9 @@ func filterRestoreFiles(
 	return
 }
 
-// enableTiDBConfig tweaks some of configs of TiDB to make the restore progress go well.
+// tweakLocalConfForRestore tweaks some of configs of TiDB to make the restore progress go well.
 // return a function that could restore the config to origin.
-func enableTiDBConfig() func() {
+func tweakLocalConfForRestore() func() {
 	restoreConfig := config.RestoreFunc()
 	config.UpdateGlobal(func(conf *config.Config) {
 		// set max-index-length before execute DDLs and create tables

br/pkg/task/stream.go

@@ -1287,6 +1287,9 @@ func restoreStream(
 	ctx, cancelFn := context.WithCancel(c)
 	defer cancelFn()
 
+	restoreCfg := tweakLocalConfForRestore()
+	defer restoreCfg()
+
 	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
 		span1 := span.Tracer().StartSpan(
 			"restoreStream",

br/tests/br_encryption/run.sh

@@ -59,39 +59,6 @@ insert_additional_data() {
     done
 }
 
-wait_log_checkpoint_advance() {
-    echo "wait for log checkpoint to advance"
-    sleep 10
-    local current_ts=$(python3 -c "import time; print(int(time.time() * 1000) << 18)")
-    echo "current ts: $current_ts"
-    i=0
-    while true; do
-        # extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
-        log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name $TASK_NAME --json 2>br.log)
-        echo "log backup status: $log_backup_status"
-        local checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length  == 0 then .[0].checkpoint else empty end')
-        echo "checkpoint ts: $checkpoint_ts"
-
-        # check whether the checkpoint ts is a number
-        if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
-            if [ $checkpoint_ts -gt $current_ts ]; then
-                echo "the checkpoint has advanced"
-                break
-            fi
-            echo "the checkpoint hasn't advanced"
-            i=$((i+1))
-            if [ "$i" -gt 50 ]; then
-                echo 'the checkpoint lag is too large'
-                exit 1
-            fi
-            sleep 10
-        else
-            echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
-            exit 1
-        fi
-    done
-}
-
 calculate_checksum() {
     local db=$1
     local checksum=$(run_sql "USE $db; ADMIN CHECKSUM TABLE $TABLE;" | awk '/CHECKSUM/{print $2}')
@@ -170,7 +137,7 @@ run_backup_restore_test() {
         checksum_ori[${i}]=$(calculate_checksum "$DB${i}") || { echo "Failed to calculate checksum after insertion"; exit 1; }
     done
 
-    wait_log_checkpoint_advance || { echo "Failed to wait for log checkpoint"; exit 1; }
+    . "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME || { echo "Failed to wait for log checkpoint"; exit 1; }
 
     #sanity check pause still works
     run_br log pause --task-name $TASK_NAME --pd $PD_ADDR || { echo "Failed to pause log backup"; exit 1; }
@@ -270,7 +237,7 @@ test_backup_encrypted_restore_unencrypted() {
     # Insert additional test data
     insert_additional_data "insert_after_full_backup" || { echo "Failed to insert additional data"; exit 1; }
 
-    wait_log_checkpoint_advance || { echo "Failed to wait for log checkpoint"; exit 1; }
+    . "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME || { echo "Failed to wait for log checkpoint"; exit 1; }
 
 
     # Stop and clean the cluster

br/tests/br_pitr/config/tidb-max-index-length.toml

@@ -0,0 +1,11 @@
+# config of tidb
+
+max-index-length = 12288
+
+[security]
+ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+ssl-key = "/tmp/backup_restore_test/certs/tidb.key"
+cluster-ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+cluster-ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+cluster-ssl-key = "/tmp/backup_restore_test/certs/tidb.key"

br/tests/br_pitr/incremental_data/ingest_repair.sql

@@ -46,3 +46,6 @@ ALTER TABLE test.pairs9 CHANGE y y2 varchar(20);
 
 -- test partition
 ALTER TABLE test.pairs10 ADD INDEX i1(y);
+
+
+CREATE INDEX huge ON test.huge_idx(blob1, blob2);

br/tests/br_pitr/prepare_data/ingest_repair.sql

@@ -43,4 +43,6 @@ INSERT INTO test.pairs10 VALUES (1,1,"1"),(2,2,"2"),(3,3,"3"),(4,4,"4"),(5,5,"5"
 -- test no need to repair
 CREATE TABLE test.pairs11(x int auto_increment primary key, y int DEFAULT RAND(), z int DEFAULT RAND());
 INSERT INTO test.pairs11 VALUES (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
-ALTER TABLE test.pairs11 ADD UNIQUE KEY u1(x, y);
+ALTER TABLE test.pairs11 ADD UNIQUE KEY u1(x, y);
+
+CREATE TABLE test.huge_idx(id int AUTO_INCREMENT, blob1 varchar(1000), blob2 varchar(1000));

br/tests/br_pitr/run.sh

@@ -21,10 +21,17 @@ CUR=$(cd `dirname $0`; pwd)
 # const value
 PREFIX="pitr_backup" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*.
 res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
+TASK_NAME="br_pitr"
+
+restart_services_allowing_huge_index() {
+    echo "restarting services with huge indices enabled..."
+    stop_services
+    start_services --tidb-cfg "$CUR/config/tidb-max-index-length.toml"
+    echo "restart services done..."
+}
 
 # start a new cluster
-echo "restart a services"
-restart_services
+restart_services_allowing_huge_index
 
 # prepare the data
 echo "prepare the data"
@@ -38,7 +45,7 @@ echo "prepare_delete_range_count: $prepare_delete_range_count"
 
 # start the log backup task
 echo "start log task"
-run_br --pd $PD_ADDR log start --task-name integration_test -s "local://$TEST_DIR/$PREFIX/log"
+run_br --pd $PD_ADDR log start --task-name $TASK_NAME -s "local://$TEST_DIR/$PREFIX/log"
 
 # run snapshot backup
 echo "run snapshot backup"
@@ -70,39 +77,8 @@ incremental_delete_range_count=$(run_sql "select count(*) DELETE_RANGE_CNT from
 echo "incremental_delete_range_count: $incremental_delete_range_count"
 
 # wait checkpoint advance
-echo "wait checkpoint advance"
-sleep 10
 current_ts=$(python3 -c "import time; print(int(time.time() * 1000) << 18)")
-echo "current ts: $current_ts"
-i=0
-while true; do
-    # extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
-    log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name integration_test --json 2>br.log)
-    echo "log backup status: $log_backup_status"
-    checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length  == 0 then .[0].checkpoint else empty end')
-    echo "checkpoint ts: $checkpoint_ts"
-
-    # check whether the checkpoint ts is a number
-    if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
-        # check whether the checkpoint has advanced
-        if [ $checkpoint_ts -gt $current_ts ]; then
-            echo "the checkpoint has advanced"
-            break
-        fi
-        # the checkpoint hasn't advanced
-        echo "the checkpoint hasn't advanced"
-        i=$((i+1))
-        if [ "$i" -gt 50 ]; then
-            echo 'the checkpoint lag is too large'
-            exit 1
-        fi
-        sleep 10
-    else
-        # unknown status, maybe somewhere is wrong
-        echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
-        exit 1
-    fi
-done
+. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME
 
 # dump some info from upstream cluster
 # ...
@@ -122,8 +98,7 @@ check_result() {
 }
 
 # start a new cluster
-echo "restart services"
-restart_services
+restart_services_allowing_huge_index
 
 # non-compliant operation
 echo "non compliant operation"
@@ -142,8 +117,7 @@ run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-bac
 check_result
 
 # start a new cluster for incremental + log
-echo "restart services"
-restart_services
+restart_services_allowing_huge_index
 
 echo "run snapshot restore#2"
 run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" 
@@ -155,7 +129,7 @@ check_result
 
 # start a new cluster for incremental + log
 echo "restart services"
-restart_services
+restart_services_allowing_huge_index
 
 echo "run snapshot restore#3"
 run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" 
@@ -169,8 +143,7 @@ if [ $restore_fail -ne 1 ]; then
 fi
 
 # start a new cluster for corruption
-echo "restart a services"
-restart_services
+restart_services_allowing_huge_index
 
 file_corruption() {
     echo "corrupt the whole log files"
@@ -196,8 +169,7 @@ if [ $restore_fail -ne 1 ]; then
 fi
 
 # start a new cluster for corruption
-echo "restart a services"
-restart_services
+restart_services_allowing_huge_index
 
 file_lost() {
     echo "lost the whole log files"