Skip to content

Commit c952360

Browse files
JmPotatoti-chi-bot
JmPotato
authored and
ti-chi-bot
committed
This is an automated cherry-pick of pingcap#61795
Signed-off-by: ti-chi-bot <[email protected]>
1 parent 65c78b2 commit c952360

File tree

6 files changed

+789
-5
lines changed

6 files changed

+789
-5
lines changed

‎pkg/ddl/tests/resourcegroup/resource_group_test.go

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,100 @@ func TestResourceGroupRunaway(t *testing.T) {
352352
tk.MustGetErrCode("select /*+ resource_group(rg4) */ * from t", mysql.ErrResourceGroupQueryRunawayQuarantine)
353353
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, watch_text from mysql.tidb_runaway_watch", nil,
354354
testkit.Rows("rg3 select /*+ resource_group(rg3) */ * from t", "rg4 select /*+ resource_group(rg4) */ * from t"), maxWaitDuration, tryInterval)
355+
<<<<<<< HEAD:pkg/ddl/tests/resourcegroup/resource_group_test.go
355356
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/store/copr/sleepCoprAfterReq"))
357+
=======
358+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/checkThresholds"))
359+
360+
tk.MustExec("create resource group rg5 BURSTABLE=UNLIMITED RU_PER_SEC=2000 QUERY_LIMIT=(PROCESSED_KEYS=10 action KILL WATCH EXACT)")
361+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/checkThresholds", "return(true)"))
362+
err = tk.QueryToErr("select /*+ resource_group(rg5) */ * from t")
363+
require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query")
364+
tk.MustGetErrCode("select /*+ resource_group(rg5) */ * from t", mysql.ErrResourceGroupQueryRunawayQuarantine)
365+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/checkThresholds"))
366+
}
367+
368+
func TestResourceGroupRunawayExceedTiDBSide(t *testing.T) {
369+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC", `return(true)`))
370+
defer func() {
371+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC"))
372+
}()
373+
store, dom := testkit.CreateMockStoreAndDomain(t)
374+
sv := server.CreateMockServer(t, store)
375+
sv.SetDomain(dom)
376+
defer sv.Close()
377+
378+
conn1 := server.CreateMockConn(t, sv)
379+
tk := testkit.NewTestKitWithSession(t, store, conn1.Context().Session)
380+
381+
go dom.ExpensiveQueryHandle().SetSessionManager(sv).Run()
382+
tk.MustExec("set global tidb_enable_resource_control='on'")
383+
require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost"}, nil, nil, nil))
384+
385+
tk.MustExec("use test")
386+
tk.MustExec("create table t(a int)")
387+
tk.MustExec("insert into t values(1)")
388+
tk.MustExec("create resource group rg1 RU_PER_SEC=1000 QUERY_LIMIT=(EXEC_ELAPSED='50ms' ACTION=KILL)")
389+
390+
err := tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.5) from t")
391+
require.ErrorContains(t, err, "[executor:8253]Query execution was interrupted, identified as runaway query")
392+
393+
tryInterval := time.Millisecond * 100
394+
maxWaitDuration := time.Second * 5
395+
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, match_type from mysql.tidb_runaway_queries", nil,
396+
testkit.Rows("rg1 select /*+ resource_group(rg1) */ sleep(0.5) from t identify"), maxWaitDuration, tryInterval)
397+
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, start_time from mysql.tidb_runaway_queries", nil,
398+
nil, maxWaitDuration, tryInterval)
399+
}
400+
401+
func TestResourceGroupRunawayFlood(t *testing.T) {
402+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC", `return(true)`))
403+
defer func() {
404+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC"))
405+
}()
406+
store := testkit.CreateMockStore(t)
407+
tk := testkit.NewTestKit(t, store)
408+
require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost"}, nil, nil, nil))
409+
410+
tk.MustExec("use test")
411+
tk.MustExec("create table t(a int)")
412+
tk.MustExec("insert into t values(1)")
413+
tk.MustExec("set global tidb_enable_resource_control='on'")
414+
tk.MustExec("create resource group rg1 RU_PER_SEC=1000 QUERY_LIMIT=(EXEC_ELAPSED='50ms' ACTION=KILL)")
415+
tk.MustQuery("select /*+ resource_group(rg1) */ * from t").Check(testkit.Rows("1"))
416+
417+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/store/copr/sleepCoprRequest", fmt.Sprintf("return(%d)", 60)))
418+
defer func() {
419+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/store/copr/sleepCoprRequest"))
420+
}()
421+
err := tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.1) from t")
422+
require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query")
423+
424+
tryInterval := time.Millisecond * 100
425+
maxWaitDuration := time.Second * 5
426+
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, repeats, match_type from mysql.tidb_runaway_queries", nil,
427+
testkit.Rows("rg1 select /*+ resource_group(rg1) */ sleep(0.1) from t 1 identify"), maxWaitDuration, tryInterval)
428+
// wait for the runaway watch to be cleaned up
429+
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, repeats from mysql.tidb_runaway_queries", nil,
430+
nil, maxWaitDuration, tryInterval)
431+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC"))
432+
433+
// check thrice to make sure the runaway query be regarded as a repeated query.
434+
err = tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.2) from t")
435+
require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query")
436+
err = tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.3) from t")
437+
require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query")
438+
// using FastRunawayGC to trigger flush
439+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC", `return(true)`))
440+
err = tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.4) from t")
441+
require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query")
442+
// only have one runaway query
443+
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, repeats, match_type from mysql.tidb_runaway_queries", nil,
444+
testkit.Rows("rg1 select /*+ resource_group(rg1) */ sleep(0.2) from t 3 identify"), maxWaitDuration, tryInterval)
445+
// wait for the runaway watch to be cleaned up
446+
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, repeats from mysql.tidb_runaway_queries", nil,
447+
nil, maxWaitDuration, tryInterval)
448+
>>>>>>> 2d0da8ee789 (fix(runaway): resolve the dead channel in UpdateNewAndDoneWatch (#61795)):pkg/resourcegroup/tests/resource_group_test.go
356449
}
357450

358451
func TestAlreadyExistsDefaultResourceGroup(t *testing.T) {

‎pkg/domain/domain.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1319,7 +1319,8 @@ func (do *Domain) Init(
13191319
do.wg.Run(do.topNSlowQueryLoop, "topNSlowQueryLoop")
13201320
do.wg.Run(do.infoSyncerKeeper, "infoSyncerKeeper")
13211321
do.wg.Run(do.globalConfigSyncerKeeper, "globalConfigSyncerKeeper")
1322-
do.wg.Run(do.runawayStartLoop, "runawayStartLoop")
1322+
do.wg.Run(do.runawayManager.RunawayRecordFlushLoop, "runawayRecordFlushLoop")
1323+
do.wg.Run(do.runawayManager.RunawayWatchSyncLoop, "runawayWatchSyncLoop")
13231324
do.wg.Run(do.requestUnitsWriterLoop, "requestUnitsWriterLoop")
13241325
if !skipRegisterToDashboard {
13251326
do.wg.Run(do.topologySyncerKeeper, "topologySyncerKeeper")

‎pkg/domain/runaway.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,18 @@ import (
1818
"context"
1919
"net"
2020
"strconv"
21+
<<<<<<< HEAD
2122
"strings"
2223
"sync"
2324
"time"
25+
=======
26+
>>>>>>> 2d0da8ee789 (fix(runaway): resolve the dead channel in UpdateNewAndDoneWatch (#61795))
2427

2528
"github.com/pingcap/errors"
2629
"github.com/pingcap/failpoint"
2730
rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
2831
"github.com/pingcap/tidb/pkg/domain/infosync"
32+
<<<<<<< HEAD
2933
"github.com/pingcap/tidb/pkg/domain/resourcegroup"
3034
"github.com/pingcap/tidb/pkg/kv"
3135
"github.com/pingcap/tidb/pkg/metrics"
@@ -37,11 +41,15 @@ import (
3741
"github.com/pingcap/tidb/pkg/types"
3842
"github.com/pingcap/tidb/pkg/util"
3943
"github.com/pingcap/tidb/pkg/util/chunk"
44+
=======
45+
"github.com/pingcap/tidb/pkg/resourcegroup/runaway"
46+
>>>>>>> 2d0da8ee789 (fix(runaway): resolve the dead channel in UpdateNewAndDoneWatch (#61795))
4047
"github.com/pingcap/tidb/pkg/util/logutil"
4148
"github.com/pingcap/tidb/pkg/util/sqlexec"
4249
"github.com/tikv/client-go/v2/tikv"
4350
pd "github.com/tikv/pd/client"
4451
rmclient "github.com/tikv/pd/client/resource_group/controller"
52+
<<<<<<< HEAD
4553
"go.uber.org/zap"
4654
)
4755

@@ -56,6 +64,8 @@ const (
5664

5765
maxIDRetries = 3
5866
runawayLoopLogErrorIntervalCount = 1800
67+
=======
68+
>>>>>>> 2d0da8ee789 (fix(runaway): resolve the dead channel in UpdateNewAndDoneWatch (#61795))
5969
)
6070

6171
var systemSchemaCIStr = model.NewCIStr("mysql")
@@ -140,6 +150,7 @@ func (do *Domain) deleteExpiredRows(tableName, colName string, expiredDuration t
140150
}
141151
}
142152
}
153+
<<<<<<< HEAD
143154

144155
func (do *Domain) runawayStartLoop() {
145156
defer util.Recover(metrics.LabelDomain, "runawayStartLoop", nil, false)
@@ -657,3 +668,5 @@ func (*SystemTableReader) Read(exec sqlexec.RestrictedSQLExecutor, genFn func()
657668
)
658669
return rows, err
659670
}
671+
=======
672+
>>>>>>> 2d0da8ee789 (fix(runaway): resolve the dead channel in UpdateNewAndDoneWatch (#61795))

‎pkg/executor/internal/querywatch/query_watch_test.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,13 @@ import (
2727
)
2828

2929
func TestQueryWatch(t *testing.T) {
30+
<<<<<<< HEAD
31+
=======
32+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC", `return(true)`))
33+
defer func() {
34+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC"))
35+
}()
36+
>>>>>>> 2d0da8ee789 (fix(runaway): resolve the dead channel in UpdateNewAndDoneWatch (#61795))
3037
store := testkit.CreateMockStore(t)
3138
tk := testkit.NewTestKit(t, store)
3239
tk.MustExec("use test")
@@ -36,6 +43,10 @@ func TestQueryWatch(t *testing.T) {
3643
tk.MustExec("insert into t2 values(1)")
3744
tk.MustExec("create table t3(a int)")
3845
tk.MustExec("insert into t3 values(1)")
46+
<<<<<<< HEAD
47+
=======
48+
49+
>>>>>>> 2d0da8ee789 (fix(runaway): resolve the dead channel in UpdateNewAndDoneWatch (#61795))
3950
err := tk.QueryToErr("query watch add sql text exact to 'select * from test.t1'")
4051
require.ErrorContains(t, err, "must set runaway config for resource group `default`")
4152
err = tk.QueryToErr("query watch add resource group rg2 action DRYRUN sql text exact to 'select * from test.t1'")
@@ -139,12 +150,9 @@ func TestQueryWatchIssue56897(t *testing.T) {
139150
defer func() {
140151
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC"))
141152
}()
142-
store, dom := testkit.CreateMockStoreAndDomain(t)
153+
store := testkit.CreateMockStore(t)
143154
tk := testkit.NewTestKit(t, store)
144155
tk.MustExec("use test")
145-
require.Eventually(t, func() bool {
146-
return dom.RunawayManager().IsSyncerInitialized()
147-
}, 20*time.Second, 300*time.Millisecond)
148156
tk.MustQuery("QUERY WATCH ADD ACTION KILL SQL TEXT SIMILAR TO 'use test';").Check((testkit.Rows("1")))
149157
time.Sleep(1 * time.Second)
150158
_, err := tk.Exec("use test")

0 commit comments

Comments
 (0)