Merge remote-tracking branch 'upstream/master' into test-fix-ut-create-drop-create

Author: tangenta

.codecov.yml

@@ -72,24 +72,24 @@ flag_management:
 
 ignore:
   - "LICENSES"
-  - "*_test.go"
+  - "**/*_test.go"
   - ".git"
-  - "*.yml"
-  - "*.md"
-  - "cmd/.*"
-  - "docs/.*"
-  - "vendor/.*"
-  - "pkg/ddl/failtest/.*"
-  - "pkg/ddl/testutil/.*"
-  - "pkg/executor/seqtest/.*"
-  - "pkg/metrics/.*"
-  - "pkg/expression/generator/.*"
-  - "br/pkg/mock/.*"
-  - "pkg/testkit/**/*"
-  - "pkg/server/internal/testutil/**/*"
-  - "pkg/statistics/handle/cache/internal/testutil/.*"
+  - "**/*.yml"
+  - "**/*.md"
+  - "cmd"
+  - "docs"
+  - "vendor"
+  - "pkg/ddl/failtest"
+  - "pkg/ddl/testutil"
+  - "pkg/executor/seqtest"
+  - "pkg/metrics"
+  - "pkg/expression/generator"
+  - "br/pkg/mock"
+  - "pkg/testkit"
+  - "pkg/server/internal/testutil"
+  - "pkg/statistics/handle/cache/internal/testutil"
   - "pkg/session/testutil.go"
   - "pkg/store/mockstore/unistore/testutil.go"
-  - "k8s.io/apimachinery/pkg/**/*"
-  - "build/**/*"
+  - "k8s.io/apimachinery/pkg"
+  - "build"
 

.github/dependabot.yml

@@ -10,6 +10,7 @@ updates:
     allow:
       - dependency-name: "golang.org/*"
       - dependency-name: "github.com/golangci/golangci-lint"
+      - dependency-name: "github.com/mgechev/revive"
       - dependency-name: "github.com/prometheus/*"
       - dependency-name: "go.uber.org/*"
     open-pull-requests-limit: 2

.gitignore

@@ -13,6 +13,7 @@ tests/integrationtest/integration-test.out
 tests/integrationtest/integrationtest_tidb-server
 tests/integrationtest/portgenerator
 tests/integrationtest/s/
+tests/integrationtest/replayer/
 *.fail.go
 tools/bin/
 vendor
@@ -39,3 +40,4 @@ bazel-tidb
 *.log.json
 genkeyword
 test_coverage
+coverage.dat

.golangci.yml

@@ -25,6 +25,7 @@ linters:
     - predeclared
     - revive
     - lll
+    - gofmt
 
 linters-settings:
   staticcheck:
@@ -389,6 +390,13 @@ linters-settings:
       - name: waitgroup-by-value
         severity: warning
         disabled: false
+  gofmt:
+    # https://golangci-lint.run/usage/linters/#gofmt
+    # disable for faster check
+    simplify: false
+    rewrite-rules:
+      - pattern: 'interface{}'
+        replacement: 'any'
 issues:
   exclude-rules:
     - path: _test\.go

DEPS.bzl

@@ -44,7 +44,7 @@ check: check-bazel-prepare parser_yacc check-parallel lint tidy testSuite errdoc
 
 fmt:
 	@echo "gofmt (simplify)"
-	@gofmt -s -l -w $(FILES) 2>&1 | $(FAIL_ON_STDOUT)
+	@gofmt -s -l -w -r 'interface{} -> any' $(FILES) 2>&1 | $(FAIL_ON_STDOUT)
 
 check-static: tools/bin/golangci-lint
 	GO111MODULE=on CGO_ENABLED=0 tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES)) --config .golangci.yml
@@ -137,22 +137,19 @@ ut: tools/bin/ut tools/bin/xprog failpoint-enable
 gotest_in_verify_ci: tools/bin/xprog tools/bin/ut failpoint-enable
 	@echo "Running gotest_in_verify_ci"
 	@mkdir -p $(TEST_COVERAGE_DIR)
-	@export TZ='Asia/Shanghai'; \
 	tools/bin/ut --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" --coverprofile "$(TEST_COVERAGE_DIR)/tidb_cov.unit_test.out" --except unstable.txt || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 	@$(CLEAN_UT_BINARY)
 
 gotest_unstable_in_verify_ci: tools/bin/xprog tools/bin/ut failpoint-enable
-	@echo "Running gotest_in_verify_ci"
+	@echo "Running gotest_unstable_in_verify_ci"
 	@mkdir -p $(TEST_COVERAGE_DIR)
-	@export TZ='Asia/Shanghai'; \
 	tools/bin/ut --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" --coverprofile "$(TEST_COVERAGE_DIR)/tidb_cov.unit_test.out" --only unstable.txt || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 	@$(CLEAN_UT_BINARY)
 
 race: failpoint-enable
 	@mkdir -p $(TEST_COVERAGE_DIR)
-	@export TZ='Asia/Shanghai'; \
 	tools/bin/ut --race --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" --coverprofile "$(TEST_COVERAGE_DIR)/tidb_cov.unit_test" --except unstable.txt || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 	@$(CLEAN_UT_BINARY)
@@ -390,10 +387,11 @@ gen_mock: mockgen
 	tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor TaskTable,Pool,TaskExecutor,Extension > pkg/disttask/framework/mock/task_executor_mock.go
 	tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/disttask/framework/scheduler Scheduler,CleanUpRoutine,TaskManager > pkg/disttask/framework/mock/scheduler_mock.go
 	tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/disttask/framework/scheduler Extension > pkg/disttask/framework/scheduler/mock/scheduler_mock.go
-	tools/bin/mockgen -package execute github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/execute SubtaskExecutor > pkg/disttask/framework/mock/execute/execute_mock.go
+	tools/bin/mockgen -package execute github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/execute StepExecutor > pkg/disttask/framework/mock/execute/execute_mock.go
 	tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/disttask/importinto MiniTaskExecutor > pkg/disttask/importinto/mock/import_mock.go
 	tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/disttask/framework/planner LogicalPlan,PipelineSpec > pkg/disttask/framework/mock/plan_mock.go
 	tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/util/sqlexec RestrictedSQLExecutor > pkg/util/sqlexec/mock/restricted_sql_executor_mock.go
+	tools/bin/mockgen -package mockstorage github.com/pingcap/tidb/br/pkg/storage ExternalStorage > br/pkg/mock/storage/storage.go
 
 # There is no FreeBSD environment for GitHub actions. So cross-compile on Linux
 # but that doesn't work with CGO_ENABLED=1, so disable cgo. The reason to have

MODULE.bazel.lock

@@ -104,6 +104,7 @@ approvers:
   - you06
   - youjiali1995
   - YuJuncen
+  - ywqzzy
   - zanmato1984
   - zhaoxinyu
   - zhouqiang-cl
@@ -145,7 +146,7 @@ reviewers:
   - TonsnakeLin
   - tsthght
   - TszKitLo40
-  - ywqzzy
+  - xzhangxian1008
   - zhangjinpeng1987
   - zhongzc
   - zhuo-zhi

Makefile

@@ -16,23 +16,23 @@ versions.check(minimum_bazel_version = "6.0.0")
 
 http_archive(
     name = "io_bazel_rules_go",
-    sha256 = "c8035e8ae248b56040a65ad3f0b7434712e2037e5dfdcebfe97576e620422709",
+    sha256 = "80a98277ad1311dacd837f9b16db62887702e9f1d1c4c9f796d0121a46c8e184",
     urls = [
-        "http://bazel-cache.pingcap.net:8080/bazelbuild/rules_go/releases/download/v0.44.0/rules_go-v0.44.0.zip",
-        "http://ats.apps.svc/bazelbuild/rules_go/releases/download/v0.44.0/rules_go-v0.44.0.zip",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.44.0/rules_go-v0.44.0.zip",
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.44.0/rules_go-v0.44.0.zip",
+        "http://bazel-cache.pingcap.net:8080/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip",
+        "http://ats.apps.svc/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip",
     ],
 )
 
 http_archive(
     name = "bazel_gazelle",
-    sha256 = "b7387f72efb59f876e4daae42f1d3912d0d45563eac7cb23d1de0b094ab588cf",
+    sha256 = "32938bda16e6700063035479063d9d24c60eda8d79fd4739563f50d331cb3209",
     urls = [
-        "http://bazel-cache.pingcap.net:8080/bazelbuild/bazel-gazelle/releases/download/v0.34.0/bazel-gazelle-v0.34.0.tar.gz",
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.34.0/bazel-gazelle-v0.34.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.34.0/bazel-gazelle-v0.34.0.tar.gz",
-        "http://ats.apps.svc/bazelbuild/bazel-gazelle/releases/download/v0.34.0/bazel-gazelle-v0.34.0.tar.gz",
+        "http://bazel-cache.pingcap.net:8080/bazelbuild/bazel-gazelle/releases/download/v0.35.0/bazel-gazelle-v0.35.0.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.35.0/bazel-gazelle-v0.35.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.35.0/bazel-gazelle-v0.35.0.tar.gz",
+        "http://ats.apps.svc/bazelbuild/bazel-gazelle/releases/download/v0.35.0/bazel-gazelle-v0.35.0.tar.gz",
     ],
 )
 
@@ -64,7 +64,7 @@ go_download_sdk(
         "https://mirrors.aliyun.com/golang/{}",
         "https://dl.google.com/go/{}",
     ],
-    version = "1.21.5",
+    version = "1.21.6",
 )
 
 go_register_toolchains(

OWNERS

@@ -45,10 +45,8 @@ func runBackupCommand(command *cobra.Command, cmdName string) error {
 		return nil
 	}
 
-	if cfg.IgnoreStats {
-		// Do not run stat worker in BR.
-		session.DisableStats4Test()
-	}
+	// No need to cache the coproceesor result
+	config.GetGlobalConfig().TiKVClient.CoprCache.CapacityMB = 0
 
 	if err := task.RunBackup(ctx, tidbGlue, cmdName, &cfg); err != nil {
 		log.Error("failed to backup", zap.Error(err))
@@ -110,6 +108,8 @@ func NewBackupCommand() *cobra.Command {
 			build.LogInfo(build.BR)
 			utils.LogEnvVariables()
 			task.LogArguments(c)
+			// Do not run stat worker in BR.
+			session.DisableStats4Test()
 
 			// Do not run ddl worker in BR.
 			config.GetGlobalConfig().Instance.TiDBEnableDDL.Store(false)