Skip to content

Commit 1bc1b64

Browse files
TemaTema
committed
*: integrate circuitbreaker for get region calls to PD
Signed-off-by: artem_danilov <[email protected]>
1 parent 6180fb7 commit 1bc1b64

File tree

8 files changed

+58
-9
lines changed

8 files changed

+58
-9
lines changed

‎DEPS.bzl

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7037,13 +7037,13 @@ def go_deps():
70377037
name = "com_github_tikv_client_go_v2",
70387038
build_file_proto_mode = "disable_global",
70397039
importpath = "github.com/tikv/client-go/v2",
7040-
sha256 = "1715838db1804050b4c45a54e9bb00d9f8e281fb10dd2bc755296e3a7c2fcf63",
7041-
strip_prefix = "github.com/tikv/client-go/[email protected].20250107062519-2eba2f614b38",
7040+
sha256 = "7e28ef7cc71dbb1b7403a2cb3189dafabee368a78133dab0ca93274148c660b5",
7041+
strip_prefix = "github.com/tikv/client-go/[email protected].20250115040613-be4b478c11de",
70427042
urls = [
7043-
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250107062519-2eba2f614b38.zip",
7044-
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250107062519-2eba2f614b38.zip",
7045-
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250107062519-2eba2f614b38.zip",
7046-
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250107062519-2eba2f614b38.zip",
7043+
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250115040613-be4b478c11de.zip",
7044+
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250115040613-be4b478c11de.zip",
7045+
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250115040613-be4b478c11de.zip",
7046+
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250115040613-be4b478c11de.zip",
70477047
],
70487048
)
70497049
go_repository(

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ require (
109109
github.com/stretchr/testify v1.10.0
110110
github.com/tdakkota/asciicheck v0.3.0
111111
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
112-
github.com/tikv/client-go/v2 v2.0.8-0.20250107062519-2eba2f614b38
112+
github.com/tikv/client-go/v2 v2.0.8-0.20250115040613-be4b478c11de
113113
github.com/tikv/pd/client v0.0.0-20250107032658-5c4ab57d68de
114114
github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3
115115
github.com/twmb/murmur3 v1.1.6

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -830,8 +830,8 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf
830830
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
831831
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
832832
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
833-
github.com/tikv/client-go/v2 v2.0.8-0.20250107062519-2eba2f614b38 h1:pHuqJqpQaxYZwB7vr64hrWU2ayMADXnleLEd2dBuih0=
834-
github.com/tikv/client-go/v2 v2.0.8-0.20250107062519-2eba2f614b38/go.mod h1:YMP2WbENjh0V0PaNY9LPRh7bLto2MmmuRE0zJ7IAZIA=
833+
github.com/tikv/client-go/v2 v2.0.8-0.20250115040613-be4b478c11de h1:evmmaCKxKOPsL5HycdJp+g2Y+LghmH/Bmw2ZwBFKwgs=
834+
github.com/tikv/client-go/v2 v2.0.8-0.20250115040613-be4b478c11de/go.mod h1:YMP2WbENjh0V0PaNY9LPRh7bLto2MmmuRE0zJ7IAZIA=
835835
github.com/tikv/pd/client v0.0.0-20250107032658-5c4ab57d68de h1:fefo6U56UMca1CfwY/FVhfVcinyDgXgffQQp9lfMeLg=
836836
github.com/tikv/pd/client v0.0.0-20250107032658-5c4ab57d68de/go.mod h1:6xcCIcECSowarZ9+AqdURngHzS+t5w2x3qhwg1moj4o=
837837
github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3 h1:y4mJRFlM6fUyPhoXuFg/Yu02fg/nIPFMOY8tOqppoFg=

‎pkg/domain/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ go_library(
110110
"@com_github_tikv_pd_client//:client",
111111
"@com_github_tikv_pd_client//http",
112112
"@com_github_tikv_pd_client//opt",
113+
"@com_github_tikv_pd_client//pkg/circuitbreaker",
113114
"@com_github_tikv_pd_client//resource_group/controller",
114115
"@io_etcd_go_etcd_client_v3//:client",
115116
"@io_etcd_go_etcd_client_v3//concurrency",

‎pkg/domain/domain_sysvars.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,10 @@ import (
2222
"github.com/pingcap/tidb/pkg/kv"
2323
"github.com/pingcap/tidb/pkg/meta"
2424
"github.com/pingcap/tidb/pkg/sessionctx/variable"
25+
"github.com/tikv/client-go/v2/tikv"
2526
pd "github.com/tikv/pd/client"
2627
"github.com/tikv/pd/client/opt"
28+
"github.com/tikv/pd/client/pkg/circuitbreaker"
2729
)
2830

2931
// initDomainSysVars() is called when a domain is initialized.
@@ -45,6 +47,8 @@ func (do *Domain) initDomainSysVars() {
4547
variable.SetLowResolutionTSOUpdateInterval = do.setLowResolutionTSOUpdateInterval
4648

4749
variable.ChangeSchemaCacheSize = do.changeSchemaCacheSize
50+
51+
variable.ChangePDMetadataCircuitBreakerErrorRateThresholdPct = do.changePDMetadataCircuitBreakerErrorRateThresholdPct
4852
}
4953

5054
// setStatsCacheCapacity sets statsCache cap
@@ -150,3 +154,9 @@ func (do *Domain) changeSchemaCacheSize(ctx context.Context, size uint64) error
150154
do.infoCache.Data.SetCacheCapacity(size)
151155
return nil
152156
}
157+
158+
func (do *Domain) changePDMetadataCircuitBreakerErrorRateThresholdPct(errorRatePct uint32) {
159+
tikv.ChangePDRegionMetaCircuitBreakerSettings(func(config *circuitbreaker.Settings) {
160+
config.ErrorRateThresholdPct = errorRatePct
161+
})
162+
}

‎pkg/sessionctx/variable/sysvar.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3447,6 +3447,14 @@ var defaultSysVars = []*SysVar{
34473447
return (*SetPDClientDynamicOption.Load())(TiDBTSOClientRPCMode, val)
34483448
},
34493449
},
3450+
{Scope: ScopeGlobal, Name: TiDBCircuitBreakerPDMetadataErrorRateThresholdPct, Value: strconv.Itoa(DefTiDBCircuitBreakerPDMetaErrorRatePct), Type: TypeUnsigned, MinValue: 0, MaxValue: 100,
3451+
SetGlobal: func(_ context.Context, s *SessionVars, val string) error {
3452+
if ChangePDMetadataCircuitBreakerErrorRateThresholdPct != nil {
3453+
ChangePDMetadataCircuitBreakerErrorRateThresholdPct(uint32(tidbOptPositiveInt32(val, DefTiDBCircuitBreakerPDMetaErrorRatePct)))
3454+
}
3455+
return nil
3456+
},
3457+
},
34503458
}
34513459

34523460
// GlobalSystemVariableInitialValue gets the default value for a system variable including ones that are dynamically set (e.g. based on the store)

‎pkg/sessionctx/variable/sysvar_test.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1742,6 +1742,30 @@ func TestTiDBSchemaCacheSize(t *testing.T) {
17421742
require.Error(t, err)
17431743
}
17441744

1745+
func TestTiDBCircuitBreakerPDMetadataErrorRateThresholdPct(t *testing.T) {
1746+
sv := GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdPct)
1747+
vars := NewSessionVars(nil)
1748+
1749+
// Too low, will get raised to the min value
1750+
val, err := sv.Validate(vars, "-1", ScopeGlobal)
1751+
require.NoError(t, err)
1752+
require.Equal(t, strconv.FormatInt(GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdPct).MinValue, 10), val)
1753+
warn := vars.StmtCtx.GetWarnings()[0].Err
1754+
require.Equal(t, "[variable:1292]Truncated incorrect tidb_cb_pd_metadata_error_rate_threshold_pct value: '-1'", warn.Error())
1755+
1756+
// Too high, will get lowered to the max value
1757+
val, err = sv.Validate(vars, "101", ScopeGlobal)
1758+
require.NoError(t, err)
1759+
require.Equal(t, strconv.FormatUint(GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdPct).MaxValue, 10), val)
1760+
warn = vars.StmtCtx.GetWarnings()[1].Err
1761+
require.Equal(t, "[variable:1292]Truncated incorrect tidb_cb_pd_metadata_error_rate_threshold_pct value: '101'", warn.Error())
1762+
1763+
// valid
1764+
val, err = sv.Validate(vars, "10", ScopeGlobal)
1765+
require.NoError(t, err)
1766+
require.Equal(t, "10", val)
1767+
}
1768+
17451769
func TestEnableWindowFunction(t *testing.T) {
17461770
vars := NewSessionVars(nil)
17471771
require.Equal(t, vars.EnableWindowFunction, DefEnableWindowFunction)

‎pkg/sessionctx/variable/tidb_vars.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1218,6 +1218,9 @@ const (
12181218
// TiDBTSOClientRPCMode controls how the TSO client performs the TSO RPC requests. It internally controls the
12191219
// concurrency of the RPC. This variable provides an approach to tune the latency of getting timestamps from PD.
12201220
TiDBTSOClientRPCMode = "tidb_tso_client_rpc_mode"
1221+
// This variable is used to set percent of errors to trip the circuit breaker for get region calls to PD
1222+
// https://github.com/tikv/rfcs/blob/master/text/0115-circuit-breaker.md
1223+
TiDBCircuitBreakerPDMetadataErrorRateThresholdPct = "tidb_cb_pd_metadata_error_rate_threshold_pct"
12211224
)
12221225

12231226
// TiDB intentional limits
@@ -1571,6 +1574,7 @@ const (
15711574
DefOptEnableProjectionPushDown = true
15721575
DefTiDBEnableSharedLockPromotion = false
15731576
DefTiDBTSOClientRPCMode = TSOClientRPCModeDefault
1577+
DefTiDBCircuitBreakerPDMetaErrorRatePct = 0
15741578
)
15751579

15761580
// Process global variables.
@@ -1730,6 +1734,8 @@ var (
17301734
EnableStatsOwner func() error = nil
17311735
// DisableStatsOwner is the func registered by stats to disable running stats in this instance.
17321736
DisableStatsOwner func() error = nil
1737+
// ChangePDMetadataCircuitBreakerErrorRateThresholdPct changes the error rate threshold of the PD metadata circuit breaker.
1738+
ChangePDMetadataCircuitBreakerErrorRateThresholdPct func(uint32) = nil
17331739
)
17341740

17351741
// Hooks functions for Cluster Resource Control.

0 commit comments

Comments
 (0)