Skip to content

Commit 1eca5b3

Browse files
joechenrhti-chi-bot
authored andcommitted
This is an automated cherry-pick of pingcap#55477
Signed-off-by: ti-chi-bot <[email protected]>
1 parent 9f4dec9 commit 1eca5b3

File tree

23 files changed

+11274
-0
lines changed

23 files changed

+11274
-0
lines changed

br/pkg/lightning/errormanager/errormanager.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,13 @@ const (
121121
`
122122

123123
insertIntoConflictErrorData = `
124+
<<<<<<< HEAD:br/pkg/lightning/errormanager/errormanager.go
124125
INSERT INTO %s.` + ConflictErrorTableName + `
125126
(task_id, table_name, index_name, key_data, row_data, raw_key, raw_value, raw_handle, raw_row)
127+
=======
128+
INSERT IGNORE INTO %s.` + ConflictErrorTableName + `
129+
(task_id, table_name, index_name, key_data, row_data, raw_key, raw_value, raw_handle, raw_row, kv_type)
130+
>>>>>>> 91beef4bb14 (*: disable insert null to not-null column for single-row insertion in non-strict mode (#55477)):pkg/lightning/errormanager/errormanager.go
126131
VALUES
127132
`
128133

br/pkg/lightning/errormanager/errormanager_test.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,7 @@ func TestReplaceConflictKeys(t *testing.T) {
482482
WillReturnResult(sqlmock.NewResult(1, 1))
483483
mockDB.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_task_info`\\.conflict_error_v1.*").
484484
WillReturnResult(sqlmock.NewResult(2, 1))
485+
<<<<<<< HEAD:br/pkg/lightning/errormanager/errormanager_test.go
485486
mockDB.ExpectQuery("\\QSELECT raw_key, index_name, raw_value, raw_handle FROM `lightning_task_info`.conflict_error_v1 WHERE table_name = ? AND index_name <> 'PRIMARY' ORDER BY raw_key\\E").
486487
WillReturnRows(sqlmock.NewRows([]string{"raw_key", "index_name", "raw_value", "raw_handle"}).
487488
AddRow(data1IndexKey, "uni_b", data1IndexValue, data1RowKey).
@@ -492,6 +493,42 @@ func TestReplaceConflictKeys(t *testing.T) {
492493
WillReturnRows(sqlmock.NewRows([]string{"raw_key", "raw_value", "raw_handle"}).
493494
AddRow(data1RowKey, data1RowValue, data1RowKey).
494495
AddRow(data1RowKey, data3RowValue, data1RowKey))
496+
=======
497+
mockDB.ExpectExec("CREATE OR REPLACE VIEW `lightning_task_info`\\.conflict_view.*").
498+
WillReturnResult(sqlmock.NewResult(3, 1))
499+
mockDB.ExpectQuery("\\QSELECT _tidb_rowid, raw_key, index_name, raw_value, raw_handle FROM `lightning_task_info`.conflict_error_v3 WHERE table_name = ? AND kv_type = 0 AND _tidb_rowid >= ? and _tidb_rowid < ? ORDER BY _tidb_rowid LIMIT ?\\E").
500+
WillReturnRows(sqlmock.NewRows([]string{"_tidb_rowid", "raw_key", "index_name", "raw_value", "raw_handle"}).
501+
AddRow(1, data1IndexKey, "uni_b", data1IndexValue, data1RowKey).
502+
AddRow(2, data1IndexKey, "uni_b", data2IndexValue, data2RowKey).
503+
AddRow(3, data3IndexKey, "uni_b", data3IndexValue, data3RowKey).
504+
AddRow(4, data3IndexKey, "uni_b", data4IndexValue, data4RowKey))
505+
mockDB.ExpectBegin()
506+
mockDB.ExpectExec("INSERT IGNORE INTO `lightning_task_info`\\.conflict_error_v3.*").
507+
WithArgs(0, "test", nil, nil, data2RowKey, data2RowValue, 2,
508+
0, "test", nil, nil, data4RowKey, data4RowValue, 2).
509+
WillReturnResult(driver.ResultNoRows)
510+
mockDB.ExpectCommit()
511+
for i := 0; i < 2; i++ {
512+
mockDB.ExpectQuery("\\QSELECT _tidb_rowid, raw_key, index_name, raw_value, raw_handle FROM `lightning_task_info`.conflict_error_v3 WHERE table_name = ? AND kv_type = 0 AND _tidb_rowid >= ? and _tidb_rowid < ? ORDER BY _tidb_rowid LIMIT ?\\E").
513+
WillReturnRows(sqlmock.NewRows([]string{"_tidb_rowid", "raw_key", "index_name", "raw_value", "raw_handle"}))
514+
}
515+
mockDB.ExpectQuery("\\QSELECT _tidb_rowid, raw_key, raw_value FROM `lightning_task_info`.conflict_error_v3 WHERE table_name = ? AND kv_type <> 0 AND _tidb_rowid >= ? and _tidb_rowid < ? ORDER BY _tidb_rowid LIMIT ?\\E").
516+
WillReturnRows(sqlmock.NewRows([]string{"_tidb_rowid", "raw_key", "raw_value"}).
517+
AddRow(1, data1RowKey, data1RowValue).
518+
AddRow(2, data1RowKey, data3RowValue))
519+
for i := 0; i < 2; i++ {
520+
mockDB.ExpectQuery("\\QSELECT _tidb_rowid, raw_key, raw_value FROM `lightning_task_info`.conflict_error_v3 WHERE table_name = ? AND kv_type <> 0 AND _tidb_rowid >= ? and _tidb_rowid < ? ORDER BY _tidb_rowid LIMIT ?\\E").
521+
WillReturnRows(sqlmock.NewRows([]string{"_tidb_rowid", "raw_key", "raw_value"}))
522+
}
523+
mockDB.ExpectBegin()
524+
mockDB.ExpectExec("DELETE FROM `lightning_task_info`\\.conflict_error_v3.*").
525+
WillReturnResult(sqlmock.NewResult(0, 2))
526+
mockDB.ExpectCommit()
527+
mockDB.ExpectBegin()
528+
mockDB.ExpectExec("DELETE FROM `lightning_task_info`\\.conflict_error_v3.*").
529+
WillReturnResult(sqlmock.NewResult(0, 0))
530+
mockDB.ExpectCommit()
531+
>>>>>>> 91beef4bb14 (*: disable insert null to not-null column for single-row insertion in non-strict mode (#55477)):pkg/lightning/errormanager/errormanager_test.go
495532

496533
cfg := config.NewConfig()
497534
cfg.TikvImporter.DuplicateResolution = config.DupeResAlgReplace

pkg/ddl/reorg.go

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,169 @@ func newContext(store kv.Storage) sessionctx.Context {
9595

9696
const defaultWaitReorgTimeout = 10 * time.Second
9797

98+
<<<<<<< HEAD
9899
// ReorgWaitTimeout is the timeout that wait ddl in write reorganization stage.
99100
var ReorgWaitTimeout = 5 * time.Second
101+
=======
102+
ctx := newReorgExprCtx()
103+
evalCtx := ctx.GetStaticEvalCtx().Apply(
104+
exprstatic.WithSQLMode(reorgMeta.SQLMode),
105+
exprstatic.WithLocation(loc),
106+
exprstatic.WithTypeFlags(reorgTypeFlagsWithSQLMode(reorgMeta.SQLMode)),
107+
exprstatic.WithErrLevelMap(reorgErrLevelsWithSQLMode(reorgMeta.SQLMode)),
108+
exprstatic.WithWarnHandler(warnHandler),
109+
)
110+
return ctx.Apply(exprstatic.WithEvalCtx(evalCtx)), nil
111+
}
112+
113+
// reorgTableMutateContext implements table.MutateContext for reorganization.
114+
type reorgTableMutateContext struct {
115+
exprCtx exprctx.ExprContext
116+
encodingConfig tblctx.RowEncodingConfig
117+
mutateBuffers *tblctx.MutateBuffers
118+
shardID *variable.RowIDShardGenerator
119+
reservedRowIDAlloc stmtctx.ReservedRowIDAlloc
120+
}
121+
122+
// AlternativeAllocators implements table.MutateContext.AlternativeAllocators.
123+
func (*reorgTableMutateContext) AlternativeAllocators(*model.TableInfo) (autoid.Allocators, bool) {
124+
// No alternative allocators for all tables because temporary tables
125+
// are not supported (temporary tables do not have any data in TiKV) in reorganization.
126+
return autoid.Allocators{}, false
127+
}
128+
129+
// GetExprCtx implements table.MutateContext.GetExprCtx.
130+
func (ctx *reorgTableMutateContext) GetExprCtx() exprctx.ExprContext {
131+
return ctx.exprCtx
132+
}
133+
134+
// ConnectionID implements table.MutateContext.ConnectionID.
135+
func (*reorgTableMutateContext) ConnectionID() uint64 {
136+
return 0
137+
}
138+
139+
// InRestrictedSQL implements table.MutateContext.InRestrictedSQL.
140+
func (*reorgTableMutateContext) InRestrictedSQL() bool {
141+
return false
142+
}
143+
144+
// TxnAssertionLevel implements table.MutateContext.TxnAssertionLevel.
145+
func (*reorgTableMutateContext) TxnAssertionLevel() variable.AssertionLevel {
146+
// Because only `index.Create` and `index.Delete` are invoked in reorganization which does not use this method,
147+
// we can just return `AssertionLevelOff`.
148+
return variable.AssertionLevelOff
149+
}
150+
151+
// EnableMutationChecker implements table.MutateContext.EnableMutationChecker.
152+
func (*reorgTableMutateContext) EnableMutationChecker() bool {
153+
// Because only `index.Create` and `index.Delete` are invoked in reorganization which does not use this method,
154+
// we can just return false.
155+
return false
156+
}
157+
158+
// GetRowEncodingConfig implements table.MutateContext.GetRowEncodingConfig.
159+
func (ctx *reorgTableMutateContext) GetRowEncodingConfig() tblctx.RowEncodingConfig {
160+
return ctx.encodingConfig
161+
}
162+
163+
// GetMutateBuffers implements table.MutateContext.GetMutateBuffers.
164+
func (ctx *reorgTableMutateContext) GetMutateBuffers() *tblctx.MutateBuffers {
165+
return ctx.mutateBuffers
166+
}
167+
168+
// GetRowIDShardGenerator implements table.MutateContext.GetRowIDShardGenerator.
169+
func (ctx *reorgTableMutateContext) GetRowIDShardGenerator() *variable.RowIDShardGenerator {
170+
return ctx.shardID
171+
}
172+
173+
// GetReservedRowIDAlloc implements table.MutateContext.GetReservedRowIDAlloc.
174+
func (ctx *reorgTableMutateContext) GetReservedRowIDAlloc() (*stmtctx.ReservedRowIDAlloc, bool) {
175+
return &ctx.reservedRowIDAlloc, true
176+
}
177+
178+
// GetStatisticsSupport implements table.MutateContext.GetStatisticsSupport.
179+
func (*reorgTableMutateContext) GetStatisticsSupport() (tblctx.StatisticsSupport, bool) {
180+
// We can just return `(nil, false)` because:
181+
// - Only `index.Create` and `index.Delete` are invoked in reorganization which does not use this method.
182+
// - DDL reorg do need to collect statistics in this way.
183+
return nil, false
184+
}
185+
186+
// GetCachedTableSupport implements table.MutateContext.GetCachedTableSupport.
187+
func (*reorgTableMutateContext) GetCachedTableSupport() (tblctx.CachedTableSupport, bool) {
188+
// We can just return `(nil, false)` because:
189+
// - Only `index.Create` and `index.Delete` are invoked in reorganization which does not use this method.
190+
// - It is not allowed to execute DDL on a cached table.
191+
return nil, false
192+
}
193+
194+
// GetTemporaryTableSupport implements table.MutateContext.GetTemporaryTableSupport.
195+
func (*reorgTableMutateContext) GetTemporaryTableSupport() (tblctx.TemporaryTableSupport, bool) {
196+
// We can just return `(nil, false)` because:
197+
// - Only `index.Create` and `index.Delete` are invoked in reorganization which does not use this method.
198+
// - Temporary tables do not have any data in TiKV.
199+
return nil, false
200+
}
201+
202+
// GetExchangePartitionDMLSupport implements table.MutateContext.GetExchangePartitionDMLSupport.
203+
func (*reorgTableMutateContext) GetExchangePartitionDMLSupport() (tblctx.ExchangePartitionDMLSupport, bool) {
204+
// We can just return `(nil, false)` because:
205+
// - Only `index.Create` and `index.Delete` are invoked in reorganization which does not use this method.
206+
return nil, false
207+
}
208+
209+
// newReorgTableMutateContext creates a new table.MutateContext for reorganization.
210+
func newReorgTableMutateContext(exprCtx exprctx.ExprContext) table.MutateContext {
211+
rowEncoder := &rowcodec.Encoder{
212+
Enable: variable.GetDDLReorgRowFormat() != variable.DefTiDBRowFormatV1,
213+
}
214+
215+
encodingConfig := tblctx.RowEncodingConfig{
216+
IsRowLevelChecksumEnabled: rowEncoder.Enable,
217+
RowEncoder: rowEncoder,
218+
}
219+
220+
return &reorgTableMutateContext{
221+
exprCtx: exprCtx,
222+
encodingConfig: encodingConfig,
223+
mutateBuffers: tblctx.NewMutateBuffers(&variable.WriteStmtBufs{}),
224+
// Though currently, `RowIDShardGenerator` is not required in DDL reorg,
225+
// we still provide a valid one to keep the context complete and to avoid panic if it is used in the future.
226+
shardID: variable.NewRowIDShardGenerator(
227+
rand.New(rand.NewSource(time.Now().UnixNano())), // #nosec G404
228+
variable.DefTiDBShardAllocateStep,
229+
),
230+
}
231+
}
232+
233+
func reorgTypeFlagsWithSQLMode(mode mysql.SQLMode) types.Flags {
234+
return types.StrictFlags.
235+
WithTruncateAsWarning(!mode.HasStrictMode()).
236+
WithIgnoreInvalidDateErr(mode.HasAllowInvalidDatesMode()).
237+
WithIgnoreZeroInDate(!mode.HasStrictMode() || mode.HasAllowInvalidDatesMode()).
238+
WithCastTimeToYearThroughConcat(true)
239+
}
240+
241+
func reorgErrLevelsWithSQLMode(mode mysql.SQLMode) errctx.LevelMap {
242+
return errctx.LevelMap{
243+
errctx.ErrGroupTruncate: errctx.ResolveErrLevel(false, !mode.HasStrictMode()),
244+
errctx.ErrGroupBadNull: errctx.ResolveErrLevel(false, !mode.HasStrictMode()),
245+
errctx.ErrGroupNoDefault: errctx.ResolveErrLevel(false, !mode.HasStrictMode()),
246+
errctx.ErrGroupDividedByZero: errctx.ResolveErrLevel(
247+
!mode.HasErrorForDivisionByZeroMode(),
248+
!mode.HasStrictMode(),
249+
),
250+
}
251+
}
252+
253+
func reorgTimeZoneWithTzLoc(tzLoc *model.TimeZoneLocation) (*time.Location, error) {
254+
if tzLoc == nil {
255+
// It is set to SystemLocation to be compatible with nil LocationInfo.
256+
return timeutil.SystemLocation(), nil
257+
}
258+
return tzLoc.GetLocation()
259+
}
260+
>>>>>>> 91beef4bb14 (*: disable insert null to not-null column for single-row insertion in non-strict mode (#55477))
100261

101262
func (rc *reorgCtx) notifyJobState(state model.JobState) {
102263
atomic.StoreInt32((*int32)(&rc.jobState), int32(state))

0 commit comments

Comments
 (0)