Skip to content

Commit eb871f8

Browse files
lance6716lance6716
authored
lightning: add PK to internal tables (#57480)
close #57479
1 parent d42a36d commit eb871f8

File tree

14 files changed

+144
-140
lines changed

14 files changed

+144
-140
lines changed

‎lightning/tests/lightning_config_max_error/run.sh

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ duplicated_row_count=$(( ${total_row_count} - ${uniq_row_count} ))
2828
remaining_row_count=$(( ${uniq_row_count} + ${duplicated_row_count}/2 ))
2929

3030
run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
31-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
31+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v4'
3232
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
3333

3434
stderr_file="/tmp/${TEST_NAME}.stderr"
@@ -47,7 +47,7 @@ EOF
4747
cat "${stderr_file}"
4848
grep -q "${err_msg}" "${stderr_file}"
4949

50-
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v3'
50+
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v4'
5151
# Although conflict error number exceeds the max-error limit,
5252
# all the conflict errors are recorded,
5353
# because recording of conflict errors are executed batch by batch (batch size 1024),
@@ -57,12 +57,12 @@ check_contains "COUNT(*): ${duplicated_row_count}"
5757
# import a second time
5858

5959
run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
60-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
60+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v4'
6161
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
6262

6363
run_lightning --backend local --config "${mydir}/normal_config.toml"
6464

65-
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v3'
65+
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v4'
6666
check_contains "COUNT(*): ${duplicated_row_count}"
6767

6868
# Check remaining records in the target table
@@ -72,40 +72,40 @@ check_contains "COUNT(*): ${remaining_row_count}"
7272
# import a third time
7373

7474
run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
75-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
75+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v4'
7676
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
7777

7878
run_lightning --backend local --config "${mydir}/normal_config_old_style.toml"
7979

80-
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v3'
80+
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v4'
8181
check_contains "COUNT(*): ${duplicated_row_count}"
8282

8383
# Check remaining records in the target table
8484
run_sql 'SELECT COUNT(*) FROM mytest.testtbl'
8585
check_contains "COUNT(*): ${remaining_row_count}"
8686

8787
# import a fourth time
88-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records'
88+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records_v2'
8989
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
9090
! run_lightning --backend local --config "${mydir}/ignore_config.toml"
9191
[ $? -eq 0 ]
9292
tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Config:ErrInvalidConfig]conflict.strategy cannot be set to \\\"ignore\\\" when use tikv-importer.backend = \\\"local\\\""
9393

94-
# Check tidb backend record duplicate entry in conflict_records table
95-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records'
94+
# Check tidb backend record duplicate entry in conflict_records_v2 table
95+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records_v2'
9696
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
9797
run_lightning --backend tidb --config "${mydir}/tidb.toml"
98-
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records'
98+
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records_v2'
9999
check_contains "COUNT(*): 15"
100-
run_sql 'SELECT * FROM lightning_task_info.conflict_records WHERE offset = 149'
100+
run_sql 'SELECT * FROM lightning_task_info.conflict_records_v2 WHERE offset = 149'
101101
check_contains "error: Error 1062 (23000): Duplicate entry '5' for key 'testtbl.PRIMARY'"
102102
check_contains "row_data: ('5','bbb05')"
103103

104-
# Check max-error-record can limit the size of conflict_records table
104+
# Check max-error-record can limit the size of conflict_records_v2 table
105105
run_sql 'DROP DATABASE IF EXISTS lightning_task_info'
106106
run_sql 'DROP DATABASE IF EXISTS mytest'
107107
run_lightning --backend tidb --config "${mydir}/tidb-limit-record.toml" 2>&1 | grep "\`lightning_task_info\`.\`conflict_view\`" | grep -q "5"
108-
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records'
108+
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records_v2'
109109
check_contains "COUNT(*): 5"
110110

111111
# Check conflict.threshold
@@ -121,8 +121,8 @@ run_sql 'DROP DATABASE IF EXISTS mytest'
121121
rm "${TEST_DIR}/lightning.log"
122122
run_lightning --backend tidb --config "${mydir}/tidb-error.toml" 2>&1 | grep -q "Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'"
123123
check_contains "Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'" "${TEST_DIR}/lightning.log"
124-
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records'
124+
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records_v2'
125125
check_contains "COUNT(*): 1"
126-
run_sql 'SELECT * FROM lightning_task_info.conflict_records'
126+
run_sql 'SELECT * FROM lightning_task_info.conflict_records_v2'
127127
check_contains "error: Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'"
128128
check_contains "row_data: ('1','bbb01')"

‎lightning/tests/lightning_duplicate_detection/run.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ verify_detected_rows() {
5353
done
5454
done
5555
mapfile -t expect_rows < <(for row in "${expect_rows[@]}"; do echo "$row"; done | sort | uniq)
56-
mapfile -t actual_rows < <(run_sql "SELECT row_data FROM lightning_task_info.conflict_error_v3 WHERE table_name = \"\`dup_detect\`.\`${table}\`\"" |
56+
mapfile -t actual_rows < <(run_sql "SELECT row_data FROM lightning_task_info.conflict_error_v4 WHERE table_name = \"\`dup_detect\`.\`${table}\`\"" |
5757
grep "row_data:" | sed 's/^.*(//' | sed 's/).*$//' | sed 's/"//g' | sed 's/, */,/g' | sort | uniq)
5858
equal=0
5959
if [ "${#actual_rows[@]}" = "${#expect_rows[@]}" ]; then

‎lightning/tests/lightning_duplicate_detection_new/run.sh

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,14 @@ if [ "$expected_rows" != "$actual_rows" ] || [ "$expected_pks" != "$actual_pks"
4040
echo "local backend replace strategy result is not equal to tidb backend"
4141
exit 1
4242
fi
43-
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records"
43+
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records_v2"
4444
check_contains "count(*): 227"
45-
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records WHERE error = ''"
45+
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records_v2 WHERE error = ''"
4646
check_contains "count(*): 0"
47-
run_sql "SELECT * FROM lightning_task_info.conflict_records WHERE row_id = 12"
47+
run_sql "SELECT * FROM lightning_task_info.conflict_records_v2 WHERE row_id = 12"
4848
check_contains "(171,'yRxZE',9201592769833450947,'xs3d',5,4,283270321)"
4949
check_contains "[kv:1062]Duplicate entry '171' for key 'dup_detect.PRIMARY'"
50-
run_sql "SELECT * FROM lightning_task_info.conflict_records WHERE row_id = 1"
50+
run_sql "SELECT * FROM lightning_task_info.conflict_records_v2 WHERE row_id = 1"
5151
check_contains "(87,'nEoKu',7836621565948506759,'y6',48,0,177543185)"
5252
check_contains "[kv:1062]Duplicate entry '0-177543185' for key 'dup_detect.uniq_col6_col7'"
5353

@@ -61,11 +61,11 @@ expected_pks=$(run_sql "SELECT group_concat(col1 ORDER BY col1) AS pks FROM test
6161
cleanup
6262
run_lightning --backend local --config "$CUR/local-error.toml" --log-file "$LOG_FILE" 2>&1 | grep -q "duplicate key in table \`test\`.\`dup_detect\` caused by index .*, but because checkpoint is off we can't have more details"
6363
grep -q "duplicate key in table \`test\`.\`dup_detect\` caused by index .*, but because checkpoint is off we can't have more details" "$LOG_FILE"
64-
run_sql "SELECT * FROM lightning_task_info.conflict_records"
64+
run_sql "SELECT * FROM lightning_task_info.conflict_records_v2"
6565
check_contains "error: duplicate key in table \`test\`.\`dup_detect\`"
6666
run_lightning --backend local --config "$CUR/local-error.toml" --log-file "$LOG_FILE" --enable-checkpoint=1 2>&1 | grep -q "duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)"
6767
grep -q "duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)" "$LOG_FILE"
68-
run_sql "SELECT * FROM lightning_task_info.conflict_records"
68+
run_sql "SELECT * FROM lightning_task_info.conflict_records_v2"
6969
check_contains "error: duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are"
7070
check_contains "restore table \`test\`.\`dup_detect\` failed: duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)" "$LOG_FILE"
7171
run_lightning_ctl --enable-checkpoint=1 --backend local --config "$CUR/local-error.toml" --checkpoint-error-destroy="\`test\`.\`dup_detect\`"
@@ -89,6 +89,6 @@ run_lightning_ctl --enable-checkpoint=1 --backend local --config "$CUR/local-rep
8989
run_lightning --enable-checkpoint=1 --backend local --config "$CUR/local-replace.toml" --log-file "$LOG_FILE"
9090
run_sql "SELECT count(*) FROM test.dup_detect"
9191
check_contains "count(*): 174"
92-
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records"
92+
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records_v2"
9393
check_contains "count(*): 227"
9494
check_not_contains "duplicate detection start" "$LOG_FILE"

‎lightning/tests/lightning_duplicate_resolution_error/run.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ check_cluster_version 5 2 0 'duplicate detection' || exit 0
2121
mydir=$(dirname "${BASH_SOURCE[0]}")
2222

2323
run_sql 'DROP TABLE IF EXISTS dup_resolve.a'
24-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
24+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v4'
2525
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
2626

2727
! run_lightning --backend local --config "${mydir}/config.toml"

‎lightning/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ check_cluster_version 5 2 0 'duplicate detection' || exit 0
2121
mydir=$(dirname "${BASH_SOURCE[0]}")
2222

2323
run_sql 'DROP TABLE IF EXISTS dup_resolve.a'
24-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
24+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v4'
2525
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
2626

2727
! run_lightning --backend local --config "${mydir}/config.toml"

‎lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ check_cluster_version 5 2 0 'duplicate detection' || exit 0
2121
mydir=$(dirname "${BASH_SOURCE[0]}")
2222

2323
run_sql 'DROP TABLE IF EXISTS dup_resolve.a'
24-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
24+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v4'
2525
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
2626

2727
! run_lightning --backend local --config "${mydir}/config.toml"

‎lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/run.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ check_cluster_version 5 2 0 'duplicate detection' || exit 0
2121
mydir=$(dirname "${BASH_SOURCE[0]}")
2222

2323
run_sql 'DROP TABLE IF EXISTS dup_resolve.a'
24-
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
24+
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v4'
2525
run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
2626

2727
! run_lightning --backend local --config "${mydir}/config.toml"

‎lightning/tests/lightning_duplicate_resolution_merge/run.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,10 @@ run_sql 'admin check table dup_resolve.a'
4242
run_sql 'select count(*) from dup_resolve.a'
4343
check_contains 'count(*): 10'
4444

45-
run_sql 'select count(*) from lightning_task_info.conflict_records'
45+
run_sql 'select count(*) from lightning_task_info.conflict_records_v2'
4646
check_contains 'count(*): 16'
4747

48-
run_sql 'select count(*) from lightning_task_info.conflict_error_v3'
48+
run_sql 'select count(*) from lightning_task_info.conflict_error_v4'
4949
check_contains 'count(*): 4'
5050

5151
run_sql 'select count(*) from lightning_task_info.conflict_view'

‎lightning/tests/lightning_sqlmode/run.sh

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,28 +61,28 @@ run_sql 'SELECT min(id), max(id) FROM sqlmodedb.t'
6161
check_contains 'min(id): 4'
6262
check_contains 'max(id): 4'
6363

64-
run_sql 'SELECT count(*) FROM sqlmodedb_lightning_task_info.type_error_v1'
64+
run_sql 'SELECT count(*) FROM sqlmodedb_lightning_task_info.type_error_v2'
6565
check_contains 'count(*): 4'
6666

67-
run_sql 'SELECT path, `offset`, error, row_data FROM sqlmodedb_lightning_task_info.type_error_v1 WHERE table_name = "`sqlmodedb`.`t`" AND row_data LIKE "(1,%";'
67+
run_sql 'SELECT path, `offset`, error, row_data FROM sqlmodedb_lightning_task_info.type_error_v2 WHERE table_name = "`sqlmodedb`.`t`" AND row_data LIKE "(1,%";'
6868
check_contains 'path: sqlmodedb.t.1.sql'
6969
check_contains 'offset: 53'
7070
check_contains 'cannot convert datum from unsigned bigint to type timestamp.'
7171
check_contains "row_data: (1,9,128,'too long','x,y,z')"
7272

73-
run_sql 'SELECT path, `offset`, error, row_data FROM sqlmodedb_lightning_task_info.type_error_v1 WHERE table_name = "`sqlmodedb`.`t`" AND row_data LIKE "(2,%";'
73+
run_sql 'SELECT path, `offset`, error, row_data FROM sqlmodedb_lightning_task_info.type_error_v2 WHERE table_name = "`sqlmodedb`.`t`" AND row_data LIKE "(2,%";'
7474
check_contains 'path: sqlmodedb.t.1.sql'
7575
check_contains 'offset: 100'
7676
check_contains "Incorrect timestamp value: '2000-00-00 00:00:00'"
7777
check_contains "row_data: (2,'2000-00-00 00:00:00',-99999,'🤩',3)"
7878

79-
run_sql 'SELECT path, `offset`, error, row_data FROM sqlmodedb_lightning_task_info.type_error_v1 WHERE table_name = "`sqlmodedb`.`t`" AND row_data LIKE "(3,%";'
79+
run_sql 'SELECT path, `offset`, error, row_data FROM sqlmodedb_lightning_task_info.type_error_v2 WHERE table_name = "`sqlmodedb`.`t`" AND row_data LIKE "(3,%";'
8080
check_contains 'path: sqlmodedb.t.1.sql'
8181
check_contains 'offset: 149'
8282
check_contains "Incorrect timestamp value: '9999-12-31 23:59:59'"
8383
check_contains "row_data: (3,'9999-12-31 23:59:59','NaN',x'99','x+y')"
8484

85-
run_sql 'SELECT path, `offset`, error, row_data FROM sqlmodedb_lightning_task_info.type_error_v1 WHERE table_name = "`sqlmodedb`.`t`" AND row_data LIKE "(5,%";'
85+
run_sql 'SELECT path, `offset`, error, row_data FROM sqlmodedb_lightning_task_info.type_error_v2 WHERE table_name = "`sqlmodedb`.`t`" AND row_data LIKE "(5,%";'
8686
check_contains 'path: sqlmodedb.t.1.sql'
8787
check_contains 'offset: 237'
8888
check_contains "Column 'a' cannot be null"

‎lightning/tidb-lightning.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ strategy = ""
103103
# precheck-conflict-before-import = false
104104
# Controls the maximum number of conflict errors that can be handled when strategy is "replace" or "ignore". You can set it only when the strategy is "replace" or "ignore". The default value is 10000. If you set a value larger than 10000, the import process might experience performance degradation.
105105
# threshold = 10000
106-
# Controls the maximum number of records in the `conflict_records` table. The default value is 10000.
106+
# Controls the maximum number of records in the `conflict_records_v2` table. The default value is 10000.
107107
# Starting from v8.1.0, there is no need to configure `max-record-rows` manually, because TiDB Lightning automatically assigns the value of `max-record-rows` with the value of `threshold`, regardless of the user input. `max-record-rows` will be deprecated in a future release.
108108
# In the physical import mode, if the strategy is "replace", the conflict records that are overwritten are recorded.
109109
# In the logical import mode, if the strategy is "ignore", the conflict records that are ignored are recorded; if the strategy is "replace", the conflict records are not recorded.

0 commit comments

Comments
 (0)