Skip to content

Commit dd437da

Browse files
Lloyd-PottigerLloyd-Pottiger
authored
Merge branch 'master' into return-system-table-by-grpc
2 parents 5caee25 + d77243e commit dd437da

File tree

139 files changed

+2178
-320
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

139 files changed

+2178
-320
lines changed

‎DEPS.bzl

Lines changed: 45 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2705,8 +2705,8 @@ def go_deps():
27052705
name = "com_github_lufia_plan9stats",
27062706
build_file_proto_mode = "disable_global",
27072707
importpath = "github.com/lufia/plan9stats",
2708-
sum = "h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=",
2709-
version = "v0.0.0-20211012122336-39d0f177ccd0",
2708+
sum = "h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik=",
2709+
version = "v0.0.0-20230326075908-cb1d2100619a",
27102710
)
27112711
go_repository(
27122712
name = "com_github_magiconair_properties",
@@ -3373,8 +3373,8 @@ def go_deps():
33733373
name = "com_github_pingcap_kvproto",
33743374
build_file_proto_mode = "disable_global",
33753375
importpath = "github.com/pingcap/kvproto",
3376-
sum = "h1:lOtHtTItLlc9R+Vg/hU2klOOs+pjKLT2Cq+CEJgjvIQ=",
3377-
version = "v0.0.0-20230403051650-e166ae588106",
3376+
sum = "h1:PWkMSJSDaOuLNKCV84K3tQ9stZuZPN8E148jRPD9TcA=",
3377+
version = "v0.0.0-20230407040905-68d0eebd564a",
33783378
)
33793379
go_repository(
33803380
name = "com_github_pingcap_log",
@@ -3387,8 +3387,8 @@ def go_deps():
33873387
name = "com_github_pingcap_sysutil",
33883388
build_file_proto_mode = "disable_global",
33893389
importpath = "github.com/pingcap/sysutil",
3390-
sum = "h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM=",
3391-
version = "v0.0.0-20220114020952-ea68d2dbf5b4",
3390+
sum = "h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I=",
3391+
version = "v1.0.1-0.20230407040306-fb007c5aff21",
33923392
)
33933393
go_repository(
33943394
name = "com_github_pingcap_tidb_dashboard",
@@ -3460,8 +3460,8 @@ def go_deps():
34603460
name = "com_github_power_devops_perfstat",
34613461
build_file_proto_mode = "disable_global",
34623462
importpath = "github.com/power-devops/perfstat",
3463-
sum = "h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=",
3464-
version = "v0.0.0-20210106213030-5aafc221ea8c",
3463+
sum = "h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig=",
3464+
version = "v0.0.0-20221212215047-62379fc7944b",
34653465
)
34663466
go_repository(
34673467
name = "com_github_prometheus_client_golang",
@@ -3753,9 +3753,24 @@ def go_deps():
37533753
name = "com_github_shirou_gopsutil_v3",
37543754
build_file_proto_mode = "disable_global",
37553755
importpath = "github.com/shirou/gopsutil/v3",
3756-
sum = "h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQUExU=",
3757-
version = "v3.23.2",
3756+
sum = "h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE=",
3757+
version = "v3.23.3",
37583758
)
3759+
go_repository(
3760+
name = "com_github_shoenig_go_m1cpu",
3761+
build_file_proto_mode = "disable",
3762+
importpath = "github.com/shoenig/go-m1cpu",
3763+
sum = "h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ=",
3764+
version = "v0.1.5",
3765+
)
3766+
go_repository(
3767+
name = "com_github_shoenig_test",
3768+
build_file_proto_mode = "disable",
3769+
importpath = "github.com/shoenig/test",
3770+
sum = "h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c=",
3771+
version = "v0.6.3",
3772+
)
3773+
37593774
go_repository(
37603775
name = "com_github_shopify_goreferrer",
37613776
build_file_proto_mode = "disable_global",
@@ -4538,6 +4553,14 @@ def go_deps():
45384553
version = "v0.6.0",
45394554
)
45404555

4556+
go_repository(
4557+
name = "com_google_cloud_go_apikeys",
4558+
build_file_proto_mode = "disable",
4559+
importpath = "cloud.google.com/go/apikeys",
4560+
sum = "h1:B9CdHFZTFjVti89tmyXXrO+7vSNo2jvZuHG8zD5trdQ=",
4561+
version = "v0.6.0",
4562+
)
4563+
45414564
go_repository(
45424565
name = "com_google_cloud_go_appengine",
45434566
build_file_proto_mode = "disable",
@@ -5869,8 +5892,8 @@ def go_deps():
58695892
name = "org_golang_google_genproto",
58705893
build_file_proto_mode = "disable_global",
58715894
importpath = "google.golang.org/genproto",
5872-
sum = "h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08=",
5873-
version = "v0.0.0-20230331144136-dcfb400f0633",
5895+
sum = "h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU=",
5896+
version = "v0.0.0-20230403163135-c38d8f061ccd",
58745897
)
58755898
go_repository(
58765899
name = "org_golang_google_grpc",
@@ -5947,8 +5970,8 @@ def go_deps():
59475970
name = "org_golang_x_net",
59485971
build_file_proto_mode = "disable_global",
59495972
importpath = "golang.org/x/net",
5950-
sum = "h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=",
5951-
version = "v0.8.0",
5973+
sum = "h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=",
5974+
version = "v0.9.0",
59525975
)
59535976
go_repository(
59545977
name = "org_golang_x_oauth2",
@@ -5968,22 +5991,22 @@ def go_deps():
59685991
name = "org_golang_x_sys",
59695992
build_file_proto_mode = "disable_global",
59705993
importpath = "golang.org/x/sys",
5971-
sum = "h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=",
5972-
version = "v0.6.0",
5994+
sum = "h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=",
5995+
version = "v0.7.0",
59735996
)
59745997
go_repository(
59755998
name = "org_golang_x_term",
59765999
build_file_proto_mode = "disable_global",
59776000
importpath = "golang.org/x/term",
5978-
sum = "h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=",
5979-
version = "v0.6.0",
6001+
sum = "h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=",
6002+
version = "v0.7.0",
59806003
)
59816004
go_repository(
59826005
name = "org_golang_x_text",
59836006
build_file_proto_mode = "disable_global",
59846007
importpath = "golang.org/x/text",
5985-
sum = "h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=",
5986-
version = "v0.8.0",
6008+
sum = "h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=",
6009+
version = "v0.9.0",
59876010
)
59886011
go_repository(
59896012
name = "org_golang_x_time",
@@ -6137,8 +6160,8 @@ def go_deps():
61376160
name = "org_uber_go_multierr",
61386161
build_file_proto_mode = "disable_global",
61396162
importpath = "go.uber.org/multierr",
6140-
sum = "h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=",
6141-
version = "v1.9.0",
6163+
sum = "h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=",
6164+
version = "v1.11.0",
61426165
)
61436166
go_repository(
61446167
name = "org_uber_go_tools",

‎br/pkg/lightning/mydump/loader.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,14 @@ func (m *MDDatabaseMeta) GetSchema(ctx context.Context, store storage.ExternalSt
6969

7070
// MDTableMeta contains some parsed metadata for a table in the source by MyDumper Loader.
7171
type MDTableMeta struct {
72-
DB string
73-
Name string
74-
SchemaFile FileInfo
75-
DataFiles []FileInfo
76-
charSet string
77-
TotalSize int64
78-
IndexRatio float64
72+
DB string
73+
Name string
74+
SchemaFile FileInfo
75+
DataFiles []FileInfo
76+
charSet string
77+
TotalSize int64
78+
IndexRatio float64
79+
// default to true, and if we do precheck, this var is updated using data sampling result, so it's not accurate.
7980
IsRowOrdered bool
8081
}
8182

‎cmd/ddltest/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ go_test(
1212
],
1313
flaky = True,
1414
race = "on",
15+
shard_count = 6,
1516
deps = [
1617
"//config",
1718
"//domain",

‎config/BUILD.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ go_test(
3737
data = glob(["**"]),
3838
embed = [":config"],
3939
flaky = True,
40-
shard_count = 2,
40+
shard_count = 23,
4141
deps = [
4242
"//testkit/testsetup",
4343
"//util/logutil",

‎ddl/column.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1150,9 +1150,6 @@ func (w *worker) updateCurrentElement(t table.Table, reorgInfo *reorgInfo) error
11501150
reorgInfo.StartKey, reorgInfo.EndKey = originalStartHandle, originalEndHandle
11511151
}
11521152

1153-
// Update the element in the reorgCtx to keep the atomic access for daemon-worker.
1154-
w.getReorgCtx(reorgInfo.Job.ID).setCurrentElement(reorgInfo.elements[i+1])
1155-
11561153
// Update the element in the reorgInfo for updating the reorg meta below.
11571154
reorgInfo.currElement = reorgInfo.elements[i+1]
11581155
// Write the reorg info to store so the whole reorganize process can recover from panic.

‎ddl/ddl.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -522,7 +522,7 @@ func (dc *ddlCtx) getReorgCtx(jobID int64) *reorgCtx {
522522
return dc.reorgCtx.reorgCtxMap[jobID]
523523
}
524524

525-
func (dc *ddlCtx) newReorgCtx(jobID int64, startKey []byte, currElement *meta.Element, rowCount int64) *reorgCtx {
525+
func (dc *ddlCtx) newReorgCtx(jobID int64, rowCount int64) *reorgCtx {
526526
dc.reorgCtx.Lock()
527527
defer dc.reorgCtx.Unlock()
528528
existedRC, ok := dc.reorgCtx.reorgCtxMap[jobID]
@@ -534,7 +534,6 @@ func (dc *ddlCtx) newReorgCtx(jobID int64, startKey []byte, currElement *meta.El
534534
rc.doneCh = make(chan error, 1)
535535
// initial reorgCtx
536536
rc.setRowCount(rowCount)
537-
rc.setCurrentElement(currElement)
538537
rc.mu.warnings = make(map[errors.ErrorID]*terror.Error)
539538
rc.mu.warningsCount = make(map[errors.ErrorID]int64)
540539
rc.references.Add(1)

‎ddl/ddl_api.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6703,6 +6703,7 @@ func (d *ddl) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde
67036703
}
67046704

67056705
tzName, tzOffset := ddlutil.GetTimeZone(ctx)
6706+
charset, collate := ctx.GetSessionVars().GetCharsetInfo()
67066707
job := &model.Job{
67076708
SchemaID: schema.ID,
67086709
TableID: t.Meta().ID,
@@ -6718,6 +6719,8 @@ func (d *ddl) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde
67186719
},
67196720
Args: []interface{}{unique, indexName, indexPartSpecifications, indexOption, hiddenCols, global},
67206721
Priority: ctx.GetSessionVars().DDLReorgPriority,
6722+
Charset: charset,
6723+
Collate: collate,
67216724
}
67226725

67236726
err = d.DoDDLJob(ctx, job)

‎ddl/ddl_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ const testLease = 5 * time.Millisecond
4343
type DDLForTest interface {
4444
// SetInterceptor sets the interceptor.
4545
SetInterceptor(h Interceptor)
46-
NewReorgCtx(jobID int64, startKey []byte, currElement *meta.Element, rowCount int64) *reorgCtx
46+
NewReorgCtx(jobID int64, rowCount int64) *reorgCtx
4747
GetReorgCtx(jobID int64) *reorgCtx
4848
RemoveReorgCtx(id int64)
4949
}
@@ -62,8 +62,8 @@ func (rc *reorgCtx) IsReorgCanceled() bool {
6262
}
6363

6464
// NewReorgCtx exports for testing.
65-
func (d *ddl) NewReorgCtx(jobID int64, startKey []byte, currElement *meta.Element, rowCount int64) *reorgCtx {
66-
return d.newReorgCtx(jobID, startKey, currElement, rowCount)
65+
func (d *ddl) NewReorgCtx(jobID int64, rowCount int64) *reorgCtx {
66+
return d.newReorgCtx(jobID, rowCount)
6767
}
6868

6969
// GetReorgCtx exports for testing.

‎ddl/ddl_worker_test.go

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -317,20 +317,16 @@ func TestUsingReorgCtx(t *testing.T) {
317317
wg := util.WaitGroupWrapper{}
318318
wg.Run(func() {
319319
jobID := int64(1)
320-
startKey := []byte("skey")
321-
ele := &meta.Element{ID: 1, TypeKey: nil}
322320
for i := 0; i < 500; i++ {
323-
d.(ddl.DDLForTest).NewReorgCtx(jobID, startKey, ele, 0)
321+
d.(ddl.DDLForTest).NewReorgCtx(jobID, 0)
324322
d.(ddl.DDLForTest).GetReorgCtx(jobID).IsReorgCanceled()
325323
d.(ddl.DDLForTest).RemoveReorgCtx(jobID)
326324
}
327325
})
328326
wg.Run(func() {
329327
jobID := int64(1)
330-
startKey := []byte("skey")
331-
ele := &meta.Element{ID: 1, TypeKey: nil}
332328
for i := 0; i < 500; i++ {
333-
d.(ddl.DDLForTest).NewReorgCtx(jobID, startKey, ele, 0)
329+
d.(ddl.DDLForTest).NewReorgCtx(jobID, 0)
334330
d.(ddl.DDLForTest).GetReorgCtx(jobID).IsReorgCanceled()
335331
d.(ddl.DDLForTest).RemoveReorgCtx(jobID)
336332
}

‎ddl/index.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -699,6 +699,9 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
699699
if job.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge {
700700
ingest.LitBackCtxMgr.Unregister(job.ID)
701701
}
702+
logutil.BgLogger().Info("[ddl] run add index job done",
703+
zap.String("charset", job.Charset),
704+
zap.String("collation", job.Collate))
702705
default:
703706
err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("index", tblInfo.State)
704707
}

0 commit comments

Comments
 (0)