Skip to content

Commit 9cc4a20

Browse files
Tristan1900Tristan1900
authored
br: refactor test to use wait checkpoint method (#57612)
close #57609
1 parent 0374ae2 commit 9cc4a20

File tree

7 files changed

+69
-207
lines changed

7 files changed

+69
-207
lines changed

‎br/tests/br_encryption/run.sh

Lines changed: 2 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -59,39 +59,6 @@ insert_additional_data() {
5959
done
6060
}
6161

62-
wait_log_checkpoint_advance() {
63-
echo "wait for log checkpoint to advance"
64-
sleep 10
65-
local current_ts=$(python3 -c "import time; print(int(time.time() * 1000) << 18)")
66-
echo "current ts: $current_ts"
67-
i=0
68-
while true; do
69-
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
70-
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name $TASK_NAME --json 2>br.log)
71-
echo "log backup status: $log_backup_status"
72-
local checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
73-
echo "checkpoint ts: $checkpoint_ts"
74-
75-
# check whether the checkpoint ts is a number
76-
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
77-
if [ $checkpoint_ts -gt $current_ts ]; then
78-
echo "the checkpoint has advanced"
79-
break
80-
fi
81-
echo "the checkpoint hasn't advanced"
82-
i=$((i+1))
83-
if [ "$i" -gt 50 ]; then
84-
echo 'the checkpoint lag is too large'
85-
exit 1
86-
fi
87-
sleep 10
88-
else
89-
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
90-
exit 1
91-
fi
92-
done
93-
}
94-
9562
calculate_checksum() {
9663
local db=$1
9764
local checksum=$(run_sql "USE $db; ADMIN CHECKSUM TABLE $TABLE;" | awk '/CHECKSUM/{print $2}')
@@ -170,7 +137,7 @@ run_backup_restore_test() {
170137
checksum_ori[${i}]=$(calculate_checksum "$DB${i}") || { echo "Failed to calculate checksum after insertion"; exit 1; }
171138
done
172139

173-
wait_log_checkpoint_advance || { echo "Failed to wait for log checkpoint"; exit 1; }
140+
. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME || { echo "Failed to wait for log checkpoint"; exit 1; }
174141

175142
#sanity check pause still works
176143
run_br log pause --task-name $TASK_NAME --pd $PD_ADDR || { echo "Failed to pause log backup"; exit 1; }
@@ -270,7 +237,7 @@ test_backup_encrypted_restore_unencrypted() {
270237
# Insert additional test data
271238
insert_additional_data "insert_after_full_backup" || { echo "Failed to insert additional data"; exit 1; }
272239

273-
wait_log_checkpoint_advance || { echo "Failed to wait for log checkpoint"; exit 1; }
240+
. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME || { echo "Failed to wait for log checkpoint"; exit 1; }
274241

275242

276243
# Stop and clean the cluster

‎br/tests/br_pitr/run.sh

Lines changed: 3 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ CUR=$(cd `dirname $0`; pwd)
2121
# const value
2222
PREFIX="pitr_backup" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*.
2323
res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
24+
TASK_NAME="br_pitr"
2425

2526
# start a new cluster
2627
echo "restart a services"
@@ -38,7 +39,7 @@ echo "prepare_delete_range_count: $prepare_delete_range_count"
3839

3940
# start the log backup task
4041
echo "start log task"
41-
run_br --pd $PD_ADDR log start --task-name integration_test -s "local://$TEST_DIR/$PREFIX/log"
42+
run_br --pd $PD_ADDR log start --task-name $TASK_NAME -s "local://$TEST_DIR/$PREFIX/log"
4243

4344
# run snapshot backup
4445
echo "run snapshot backup"
@@ -70,39 +71,8 @@ incremental_delete_range_count=$(run_sql "select count(*) DELETE_RANGE_CNT from
7071
echo "incremental_delete_range_count: $incremental_delete_range_count"
7172

7273
# wait checkpoint advance
73-
echo "wait checkpoint advance"
74-
sleep 10
7574
current_ts=$(python3 -c "import time; print(int(time.time() * 1000) << 18)")
76-
echo "current ts: $current_ts"
77-
i=0
78-
while true; do
79-
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
80-
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name integration_test --json 2>br.log)
81-
echo "log backup status: $log_backup_status"
82-
checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
83-
echo "checkpoint ts: $checkpoint_ts"
84-
85-
# check whether the checkpoint ts is a number
86-
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
87-
# check whether the checkpoint has advanced
88-
if [ $checkpoint_ts -gt $current_ts ]; then
89-
echo "the checkpoint has advanced"
90-
break
91-
fi
92-
# the checkpoint hasn't advanced
93-
echo "the checkpoint hasn't advanced"
94-
i=$((i+1))
95-
if [ "$i" -gt 50 ]; then
96-
echo 'the checkpoint lag is too large'
97-
exit 1
98-
fi
99-
sleep 10
100-
else
101-
# unknown status, maybe somewhere is wrong
102-
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
103-
exit 1
104-
fi
105-
done
75+
. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME
10676

10777
# dump some info from upstream cluster
10878
# ...

‎br/tests/br_pitr_failpoint/run.sh

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
set -eu
1818
. run_services
1919
CUR=$(cd `dirname $0`; pwd)
20+
TASK_NAME="br_pitr_failpoint"
2021

2122
# const value
2223
PREFIX="pitr_backup_failpoint" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*.
@@ -42,7 +43,7 @@ sql_pid=$!
4243

4344
# start the log backup task
4445
echo "start log task"
45-
run_br --pd $PD_ADDR log start --task-name integration_test -s "local://$TEST_DIR/$PREFIX/log"
46+
run_br --pd $PD_ADDR log start --task-name $TASK_NAME -s "local://$TEST_DIR/$PREFIX/log"
4647

4748
# wait until the index creation is running
4849
retry_cnt=0
@@ -121,42 +122,9 @@ check_contains "Column_name: y"
121122
check_contains "Column_name: z"
122123

123124
# wait checkpoint advance
124-
echo "wait checkpoint advance"
125-
sleep 10
126-
current_ts=$(echo $(($(date +%s%3N) << 18)))
127-
echo "current ts: $current_ts"
128-
i=0
129-
while true; do
130-
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
131-
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name integration_test --json 2>/dev/null)
132-
echo "log backup status: $log_backup_status"
133-
checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
134-
echo "checkpoint ts: $checkpoint_ts"
135-
136-
# check whether the checkpoint ts is a number
137-
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
138-
# check whether the checkpoint has advanced
139-
if [ $checkpoint_ts -gt $current_ts ]; then
140-
echo "the checkpoint has advanced"
141-
break
142-
fi
143-
# the checkpoint hasn't advanced
144-
echo "the checkpoint hasn't advanced"
145-
i=$((i+1))
146-
if [ "$i" -gt 50 ]; then
147-
echo 'the checkpoint lag is too large'
148-
exit 1
149-
fi
150-
sleep 10
151-
else
152-
# unknown status, maybe somewhere is wrong
153-
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
154-
exit 1
155-
fi
156-
done
125+
. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME
157126

158127
# start a new cluster
159-
echo "restart a services"
160128
restart_services
161129

162130
# PITR restore - 1

‎br/tests/br_pitr_gc_safepoint/run.sh

Lines changed: 4 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@ CUR=$(cd `dirname $0`; pwd)
2121
# const value
2222
PREFIX="pitr_backup" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*.
2323
res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
24+
TASK_NAME="br_pitr_gc_safepoint"
2425

2526
# start a new cluster
2627
echo "restart a services"
2728
restart_services
2829

2930
# start the log backup task
3031
echo "start log task"
31-
run_br --pd $PD_ADDR log start --task-name integration_test -s "local://$TEST_DIR/$PREFIX/log"
32+
run_br --pd $PD_ADDR log start --task-name $TASK_NAME -s "local://$TEST_DIR/$PREFIX/log"
3233

3334
# prepare the data
3435
echo "prepare the data"
@@ -41,41 +42,9 @@ prepare_delete_range_count=$(run_sql "select count(*) DELETE_RANGE_CNT from (sel
4142
echo "prepare_delete_range_count: $prepare_delete_range_count"
4243

4344
# wait checkpoint advance
44-
echo "wait checkpoint advance"
45-
sleep 10
46-
current_ts=$(echo $(($(date +%s%3N) << 18)))
47-
echo "current ts: $current_ts"
48-
i=0
49-
while true; do
50-
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
51-
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name integration_test --json 2>br.log)
52-
echo "log backup status: $log_backup_status"
53-
checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
54-
echo "checkpoint ts: $checkpoint_ts"
45+
. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance "$TASK_NAME"
5546

56-
# check whether the checkpoint ts is a number
57-
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
58-
# check whether the checkpoint has advanced
59-
if [ $checkpoint_ts -gt $current_ts ]; then
60-
echo "the checkpoint has advanced"
61-
break
62-
fi
63-
# the checkpoint hasn't advanced
64-
echo "the checkpoint hasn't advanced"
65-
i=$((i+1))
66-
if [ "$i" -gt 50 ]; then
67-
echo 'the checkpoint lag is too large'
68-
exit 1
69-
fi
70-
sleep 10
71-
else
72-
# unknown status, maybe somewhere is wrong
73-
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
74-
exit 1
75-
fi
76-
done
77-
78-
run_br --pd $PD_ADDR log pause --task-name integration_test
47+
run_br --pd $PD_ADDR log pause --task-name $TASK_NAME
7948

8049
safe_point=$(run_pd_ctl -u https://$PD_ADDR service-gc-safepoint)
8150

‎br/tests/br_restore_checkpoint/run.sh

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ CUR=$(cd `dirname $0`; pwd)
2222
PREFIX="checkpoint" # NOTICE: don't start with 'br' because `restart services` would remove file/directory br*.
2323
DB=$TEST_NAME
2424
res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
25+
TASK_NAME="br_restore_checkpoint"
2526

2627
# start a new cluster
27-
echo "restart a services"
2828
restart_services
2929

3030
# prepare snapshot data
@@ -37,7 +37,7 @@ run_sql "INSERT INTO $DB.tbl2 values (2, 'b');"
3737

3838
# start the log backup task
3939
echo "start log task"
40-
run_br --pd $PD_ADDR log start --task-name integration_test -s "local://$TEST_DIR/$PREFIX/log"
40+
run_br --pd $PD_ADDR log start --task-name $TASK_NAME -s "local://$TEST_DIR/$PREFIX/log"
4141

4242
# run snapshot backup
4343
echo "run snapshot backup"
@@ -53,41 +53,9 @@ run_sql "INSERT INTO $DB.tbl3 values (33, 'cc');"
5353

5454
# wait checkpoint advance
5555
echo "wait checkpoint advance"
56-
sleep 10
57-
current_ts=$(echo $(($(date +%s%3N) << 18)))
58-
echo "current ts: $current_ts"
59-
i=0
60-
while true; do
61-
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
62-
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name integration_test --json 2>br.log)
63-
echo "log backup status: $log_backup_status"
64-
checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
65-
echo "checkpoint ts: $checkpoint_ts"
66-
67-
# check whether the checkpoint ts is a number
68-
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
69-
# check whether the checkpoint has advanced
70-
if [ $checkpoint_ts -gt $current_ts ]; then
71-
echo "the checkpoint has advanced"
72-
break
73-
fi
74-
# the checkpoint hasn't advanced
75-
echo "the checkpoint hasn't advanced"
76-
i=$((i+1))
77-
if [ "$i" -gt 50 ]; then
78-
echo 'the checkpoint lag is too large'
79-
exit 1
80-
fi
81-
sleep 10
82-
else
83-
# unknown status, maybe somewhere is wrong
84-
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
85-
exit 1
86-
fi
87-
done
56+
. "$CUR/../br_test_utils.sh" && wait_log_checkpoint_advance $TASK_NAME
8857

8958
# start a new cluster
90-
echo "restart a services"
9159
restart_services
9260

9361
# PITR but failed in the snapshot restore stage

‎br/tests/br_test_utils.sh

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/sh
2+
#
3+
# Copyright 2024 PingCAP, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -eux
18+
19+
wait_log_checkpoint_advance() {
20+
local task_name=${1:-$TASK_NAME}
21+
echo "wait for log checkpoint to advance for task: $task_name"
22+
sleep 10
23+
local current_ts=$(python3 -c "import time; print(int(time.time() * 1000) << 18)")
24+
echo "current ts: $current_ts"
25+
i=0
26+
while true; do
27+
# extract the checkpoint ts of the log backup task. If there is some error, the checkpoint ts should be empty
28+
log_backup_status=$(unset BR_LOG_TO_TERM && run_br --skip-goleak --pd $PD_ADDR log status --task-name $task_name --json 2>br.log)
29+
echo "log backup status: $log_backup_status"
30+
local checkpoint_ts=$(echo "$log_backup_status" | head -n 1 | jq 'if .[0].last_errors | length == 0 then .[0].checkpoint else empty end')
31+
echo "checkpoint ts: $checkpoint_ts"
32+
33+
# check whether the checkpoint ts is a number
34+
if [ $checkpoint_ts -gt 0 ] 2>/dev/null; then
35+
if [ $checkpoint_ts -gt $current_ts ]; then
36+
echo "the checkpoint has advanced"
37+
break
38+
fi
39+
echo "the checkpoint hasn't advanced"
40+
i=$((i+1))
41+
if [ "$i" -gt 50 ]; then
42+
echo 'the checkpoint lag is too large'
43+
exit 1
44+
fi
45+
sleep 10
46+
else
47+
echo "TEST: [$TEST_NAME] failed to wait checkpoint advance!"
48+
exit 1
49+
fi
50+
done
51+
}

0 commit comments

Comments
 (0)