Skip to content

Commit 54f83eb

Browse files
lichunzhuti-chi-bot
lichunzhu
authored and
ti-chi-bot
committed
This is an automated cherry-pick of pingcap#45374
Signed-off-by: ti-chi-bot <[email protected]>
1 parent 905a155 commit 54f83eb

File tree

6 files changed

+114
-10
lines changed

6 files changed

+114
-10
lines changed

‎dumpling/export/ir_impl.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,7 @@ func newMultiQueriesChunk(queries []string, colLength int) *multiQueriesChunk {
353353
func (td *multiQueriesChunk) Start(tctx *tcontext.Context, conn *sql.Conn) error {
354354
td.tctx = tctx
355355
td.conn = conn
356+
td.SQLRowIter = nil
356357
return nil
357358
}
358359

‎dumpling/export/metadata.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,12 @@ func (m *globalMetadata) writeGlobalMetaData() error {
219219
if err != nil {
220220
return err
221221
}
222-
defer tearDown(m.tctx)
223-
224-
return write(m.tctx, fileWriter, m.String())
222+
err = write(m.tctx, fileWriter, m.String())
223+
tearDownErr := tearDown(m.tctx)
224+
if err == nil {
225+
return tearDownErr
226+
}
227+
return err
225228
}
226229

227230
func getValidStr(str []string, idx int) string {

‎dumpling/export/writer.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,10 +237,13 @@ func (w *Writer) tryToWriteTableData(tctx *tcontext.Context, meta TableMeta, ir
237237
for {
238238
fileWriter, tearDown := buildInterceptFileWriter(tctx, w.extStorage, fileName, conf.CompressType)
239239
n, err := format.WriteInsert(tctx, conf, meta, ir, fileWriter, w.metrics)
240-
tearDown(tctx)
240+
tearDownErr := tearDown(tctx)
241241
if err != nil {
242242
return err
243243
}
244+
if tearDownErr != nil {
245+
return tearDownErr
246+
}
244247

245248
if w, ok := fileWriter.(*InterceptFileWriter); ok && !w.SomethingIsWritten {
246249
break
@@ -275,6 +278,7 @@ func writeMetaToFile(tctx *tcontext.Context, target, metaSQL string, s storage.E
275278
if err != nil {
276279
return errors.Trace(err)
277280
}
281+
<<<<<<< HEAD
278282
defer tearDown(tctx)
279283

280284
return WriteMeta(tctx, &metaData{
@@ -283,7 +287,18 @@ func writeMetaToFile(tctx *tcontext.Context, target, metaSQL string, s storage.E
283287
specCmts: []string{
284288
"/*!40101 SET NAMES binary*/;",
285289
},
290+
=======
291+
err = WriteMeta(tctx, &metaData{
292+
target: target,
293+
metaSQL: metaSQL,
294+
specCmts: getSpecialComments(w.conf.ServerInfo.ServerType),
295+
>>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374))
286296
}, fileWriter)
297+
tearDownErr := tearDown(tctx)
298+
if err == nil {
299+
return tearDownErr
300+
}
301+
return err
287302
}
288303

289304
type outputFileNamer struct {

‎dumpling/export/writer_test.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ import (
1212
"testing"
1313

1414
"github.com/DATA-DOG/go-sqlmock"
15+
<<<<<<< HEAD
16+
=======
17+
"github.com/pingcap/failpoint"
18+
"github.com/pingcap/tidb/br/pkg/version"
19+
>>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374))
1520
tcontext "github.com/pingcap/tidb/dumpling/context"
1621
"github.com/pingcap/tidb/util/promutil"
1722
"github.com/stretchr/testify/require"
@@ -70,7 +75,17 @@ func TestWriteTableMeta(t *testing.T) {
7075
require.NoError(t, err)
7176
bytes, err := ioutil.ReadFile(p)
7277
require.NoError(t, err)
78+
<<<<<<< HEAD
7379
require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes))
80+
=======
81+
require.Equal(t, "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;\n/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes))
82+
83+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile", "return(true)"))
84+
defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile=return(true)")
85+
86+
err = writer.WriteTableMeta("test", "t", "CREATE TABLE t (a INT)")
87+
require.ErrorContains(t, err, "injected error: fail to close meta file")
88+
>>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374))
7489
}
7590

7691
func TestWriteViewMeta(t *testing.T) {
@@ -137,6 +152,13 @@ func TestWriteTableData(t *testing.T) {
137152
"(3,'male','[email protected]','020-1256','healthy'),\n" +
138153
"(4,'female','[email protected]','020-1235','healthy');\n"
139154
require.Equal(t, expected, string(bytes))
155+
156+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile", "return(true)"))
157+
defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile=return(true)")
158+
159+
tableIR = newMockTableIR("test", "employee", data, specCmts, colTypes)
160+
err = writer.WriteTableData(tableIR, tableIR, 0)
161+
require.ErrorContains(t, err, "injected error: fail to close data file")
140162
}
141163

142164
func TestWriteTableDataWithFileSize(t *testing.T) {

‎dumpling/export/writer_util.go

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,7 @@ func writeBytes(tctx *tcontext.Context, writer storage.ExternalFileWriter, p []b
451451
return errors.Trace(err)
452452
}
453453

454-
func buildFileWriter(tctx *tcontext.Context, s storage.ExternalStorage, fileName string, compressType storage.CompressType) (storage.ExternalFileWriter, func(ctx context.Context), error) {
454+
func buildFileWriter(tctx *tcontext.Context, s storage.ExternalStorage, fileName string, compressType storage.CompressType) (storage.ExternalFileWriter, func(ctx context.Context) error, error) {
455455
fileName += compressFileSuffix(compressType)
456456
fullPath := s.URI() + "/" + fileName
457457
writer, err := storage.WithCompression(s, compressType).Create(tctx, fileName)
@@ -462,20 +462,24 @@ func buildFileWriter(tctx *tcontext.Context, s storage.ExternalStorage, fileName
462462
return nil, nil, errors.Trace(err)
463463
}
464464
tctx.L().Debug("opened file", zap.String("path", fullPath))
465-
tearDownRoutine := func(ctx context.Context) {
465+
tearDownRoutine := func(ctx context.Context) error {
466466
err := writer.Close(ctx)
467+
failpoint.Inject("FailToCloseMetaFile", func(_ failpoint.Value) {
468+
err = errors.New("injected error: fail to close meta file")
469+
})
467470
if err == nil {
468-
return
471+
return nil
469472
}
470473
err = errors.Trace(err)
471474
tctx.L().Warn("fail to close file",
472475
zap.String("path", fullPath),
473476
zap.Error(err))
477+
return err
474478
}
475479
return writer, tearDownRoutine, nil
476480
}
477481

478-
func buildInterceptFileWriter(pCtx *tcontext.Context, s storage.ExternalStorage, fileName string, compressType storage.CompressType) (storage.ExternalFileWriter, func(context.Context)) {
482+
func buildInterceptFileWriter(pCtx *tcontext.Context, s storage.ExternalStorage, fileName string, compressType storage.CompressType) (storage.ExternalFileWriter, func(context.Context) error) {
479483
fileName += compressFileSuffix(compressType)
480484
var writer storage.ExternalFileWriter
481485
fullPath := s.URI() + "/" + fileName
@@ -497,17 +501,21 @@ func buildInterceptFileWriter(pCtx *tcontext.Context, s storage.ExternalStorage,
497501
}
498502
fileWriter.initRoutine = initRoutine
499503

500-
tearDownRoutine := func(ctx context.Context) {
504+
tearDownRoutine := func(ctx context.Context) error {
501505
if writer == nil {
502-
return
506+
return nil
503507
}
504508
pCtx.L().Debug("tear down lazy file writer...", zap.String("path", fullPath))
505509
err := writer.Close(ctx)
510+
failpoint.Inject("FailToCloseDataFile", func(_ failpoint.Value) {
511+
err = errors.New("injected error: fail to close data file")
512+
})
506513
if err != nil {
507514
pCtx.L().Warn("fail to close file",
508515
zap.String("path", fullPath),
509516
zap.Error(err))
510517
}
518+
return err
511519
}
512520
return fileWriter, tearDownRoutine
513521
}

‎dumpling/tests/basic/run.sh

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,3 +128,58 @@ run_dumpling --consistency lock -B "$DB_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling
128128
cnt=$(grep -w "$DB_NAME" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}-schema-create.sql|wc -l)
129129
echo "records count is ${cnt}"
130130
[ "$cnt" = 1 ]
131+
<<<<<<< HEAD
132+
=======
133+
134+
echo "Test for recording network usage."
135+
run_sql "drop database if exists test_db;"
136+
run_sql "create database test_db;"
137+
run_sql "create table test_db.test_table (a int primary key);"
138+
run_sql "insert into test_db.test_table values (1),(2),(3),(4),(5),(6),(7),(8);"
139+
140+
export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/SetIOTotalBytes=return(1)"
141+
run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
142+
cnt=$(grep "IOTotalBytes=" ${DUMPLING_OUTPUT_DIR}/dumpling.log | grep -v "IOTotalBytes=0" | wc -l)
143+
[ "$cnt" -ge 1 ]
144+
145+
echo "Test for failing to close meta/data file"
146+
export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile=1*return"
147+
rm ${DUMPLING_OUTPUT_DIR}/dumpling.log
148+
set +e
149+
run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
150+
set -e
151+
cnt=$(grep -w "dump failed error stack info" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
152+
[ "$cnt" -ge 1 ]
153+
154+
# dumpling retry will make it succeed
155+
export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile=1*return"
156+
export DUMPLING_TEST_PORT=4000
157+
run_sql "drop database if exists test_db;"
158+
run_sql "create database test_db;"
159+
run_sql "create table test_db.test_table (a int primary key);"
160+
run_sql "insert into test_db.test_table values (1),(2),(3),(4),(5),(6),(7),(8);"
161+
rm ${DUMPLING_OUTPUT_DIR}/dumpling.log
162+
set +e
163+
run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
164+
set -e
165+
cnt=$(grep -w "dump data successfully" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
166+
[ "$cnt" -ge 1 ]
167+
cnt=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/test_db.test_table.000000000.sql|wc -l)
168+
echo "records count is ${cnt}"
169+
[ "$cnt" -eq 8 ]
170+
171+
export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile=5*return"
172+
rm ${DUMPLING_OUTPUT_DIR}/dumpling.log
173+
set +e
174+
run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
175+
set -e
176+
cnt=$(grep -w "dump failed error stack info" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
177+
[ "$cnt" -ge 1 ]
178+
179+
echo "Test for empty query result, should success."
180+
run_sql "drop database if exists test_db;"
181+
run_sql "create database test_db;"
182+
run_sql "create table test_db.test_table (a int primary key);"
183+
export GO_FAILPOINTS=""
184+
run_dumpling --sql "select * from test_db.test_table" --filetype csv > ${DUMPLING_OUTPUT_DIR}/dumpling.log
185+
>>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374))

0 commit comments

Comments
 (0)