Skip to content

Commit 18e2ca6

Browse files
ti-chi-botti-chi-bot
authored
br: refactor test to use wait checkpoint method (#57612) (#59284)
close #57609
1 parent d91b100 commit 18e2ca6

File tree

4 files changed

+59
-70
lines changed

4 files changed

+59
-70
lines changed

‎br/pkg/utils/backoff.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,7 @@ func NewRawClientBackoffStrategy() Backoffer {
295295
return &RawClientBackoffStrategy{
296296
Attempts: rawClientMaxAttempts,
297297
BaseBackoff: rawClientDelayTime,
298-
MaxBackoff: rawClientMaxAttempts,
298+
MaxBackoff: rawClientMaxDelayTime,
299299
}
300300
}
301301

‎br/tests/br_pitr/run.sh

Lines changed: 4 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ CUR=$(cd `dirname $0`; pwd)
2121
# const value
2222
PREFIX="pitr_backup" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*.
2323
res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
24+
TASK_NAME="br_pitr"
2425

2526
restart_services_allowing_huge_index() {
2627
echo "restarting services with huge indices enabled..."
@@ -45,7 +46,7 @@ echo "prepare_delete_range_count: $prepare_delete_range_count"
4546

4647
# start the log backup task
4748
echo "start log task"
48-
run_br --pd $PD_ADDR log start --task-name integration_test -s "local://$TEST_DIR/$PREFIX/log"
49+
run_br --pd $PD_ADDR log start --task-name $TASK_NAME -s "local://$TEST_DIR/$PREFIX/log"
4950

5051
# run snapshot backup
5152
echo "run snapshot backup"
@@ -62,39 +63,8 @@ incremental_delete_range_count=$(run_sql "select count(*) DELETE_RANGE_CNT from
6263
echo "incremental_delete_range_count: $incremental_delete_range_count"
6364

6465
# wait checkpoint advance
65-
echo "wait checkpoint advance"
66-
sleep 10
67-
current_ts=$(echo $(($(date +%s%3N) << 18)))
68-
echo "current ts: $current_ts"
69-
i=0
70-
while true; do
71-
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
72-
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --pd $PD_ADDR log status --task-name integration_test --json 2>/dev/null)
73-
echo "log backup status: $log_backup_status"
74-
checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
75-
echo "checkpoint ts: $checkpoint_ts"
76-
77-
# check whether the checkpoint ts is a number
78-
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
79-
# check whether the checkpoint has advanced
80-
if [ $checkpoint_ts -gt $current_ts ]; then
81-
echo "the checkpoint has advanced"
82-
break
83-
fi
84-
# the checkpoint hasn't advanced
85-
echo "the checkpoint hasn't advanced"
86-
i=$((i+1))
87-
if [ "$i" -gt 50 ]; then
88-
echo 'the checkpoint lag is too large'
89-
exit 1
90-
fi
91-
sleep 10
92-
else
93-
# unknown status, maybe somewhere is wrong
94-
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
95-
exit 1
96-
fi
97-
done
66+
current_ts=$(python3 -c "import time; print(int(time.time() * 1000) << 18)")
67+
. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME
9868

9969
# dump some info from upstream cluster
10070
# ...

‎br/tests/br_pitr_failpoint/run.sh

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
set -eu
1818
. run_services
1919
CUR=$(cd `dirname $0`; pwd)
20+
TASK_NAME="br_pitr_failpoint"
2021

2122
# const value
2223
PREFIX="pitr_backup_failpoint" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*.
@@ -42,7 +43,7 @@ sql_pid=$!
4243

4344
# start the log backup task
4445
echo "start log task"
45-
run_br --pd $PD_ADDR log start --task-name integration_test -s "local://$TEST_DIR/$PREFIX/log"
46+
run_br --pd $PD_ADDR log start --task-name $TASK_NAME -s "local://$TEST_DIR/$PREFIX/log"
4647

4748
# wait until the index creation is running
4849
retry_cnt=0
@@ -121,42 +122,9 @@ check_contains "Column_name: y"
121122
check_contains "Column_name: z"
122123

123124
# wait checkpoint advance
124-
echo "wait checkpoint advance"
125-
sleep 10
126-
current_ts=$(echo $(($(date +%s%3N) << 18)))
127-
echo "current ts: $current_ts"
128-
i=0
129-
while true; do
130-
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
131-
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --pd $PD_ADDR log status --task-name integration_test --json 2>/dev/null)
132-
echo "log backup status: $log_backup_status"
133-
checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
134-
echo "checkpoint ts: $checkpoint_ts"
135-
136-
# check whether the checkpoint ts is a number
137-
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
138-
# check whether the checkpoint has advanced
139-
if [ $checkpoint_ts -gt $current_ts ]; then
140-
echo "the checkpoint has advanced"
141-
break
142-
fi
143-
# the checkpoint hasn't advanced
144-
echo "the checkpoint hasn't advanced"
145-
i=$((i+1))
146-
if [ "$i" -gt 50 ]; then
147-
echo 'the checkpoint lag is too large'
148-
exit 1
149-
fi
150-
sleep 10
151-
else
152-
# unknown status, maybe somewhere is wrong
153-
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
154-
exit 1
155-
fi
156-
done
125+
. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME
157126

158127
# start a new cluster
159-
echo "restart a services"
160128
restart_services
161129

162130
# PITR restore - 1

‎br/tests/br_test_utils.sh

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/sh
2+
#
3+
# Copyright 2024 PingCAP, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -eux
18+
19+
wait_log_checkpoint_advance() {
20+
local task_name=${1:-$TASK_NAME}
21+
echo "wait for log checkpoint to advance for task: $task_name"
22+
sleep 10
23+
local current_ts=$(python3 -c "import time; print(int(time.time() * 1000) << 18)")
24+
echo "current ts: $current_ts"
25+
i=0
26+
while true; do
27+
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
28+
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name $task_name --json 2>br.log)
29+
echo "log backup status: $log_backup_status"
30+
local checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
31+
echo "checkpoint ts: $checkpoint_ts"
32+
33+
# check whether the checkpoint ts is a number
34+
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
35+
if [ $checkpoint_ts -gt $current_ts ]; then
36+
echo "the checkpoint has advanced"
37+
break
38+
fi
39+
echo "the checkpoint hasn't advanced"
40+
i=$((i+1))
41+
if [ "$i" -gt 50 ]; then
42+
echo 'the checkpoint lag is too large'
43+
exit 1
44+
fi
45+
sleep 10
46+
else
47+
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
48+
exit 1
49+
fi
50+
done
51+
}

0 commit comments

Comments
 (0)