Skip to content

Commit 98fe03d

Browse files
ti-chi-botti-chi-bot
authored
pkg/util: refine tidb_server_memory_limit to make the cpu usage more stable (#48927) (#49061)
close #48741
1 parent f80b618 commit 98fe03d

File tree

2 files changed

+124
-12
lines changed

2 files changed

+124
-12
lines changed

‎util/gctuner/memory_limit_tuner.go

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,11 @@ var GlobalMemoryLimitTuner = &memoryLimitTuner{}
3333
// So we can change memory limit dynamically to avoid frequent GC when memory usage is greater than the limit.
3434
type memoryLimitTuner struct {
3535
finalizer *finalizer
36-
isTuning atomicutil.Bool
36+
isValidValueSet atomicutil.Bool
3737
percentage atomicutil.Float64
38-
waitingReset atomicutil.Bool
38+
adjustPercentageInProgress atomicutil.Bool
39+
serverMemLimitBeforeAdjust atomicutil.Uint64
40+
percentageBeforeAdjust atomicutil.Float64
3941
nextGCTriggeredByMemoryLimit atomicutil.Bool
4042
}
4143

@@ -56,7 +58,7 @@ func WaitMemoryLimitTunerExitInTest() {
5658
// tuning check the memory nextGC and judge whether this GC is trigger by memory limit.
5759
// Go runtime ensure that it will be called serially.
5860
func (t *memoryLimitTuner) tuning() {
59-
if !t.isTuning.Load() {
61+
if !t.isValidValueSet.Load() {
6062
return
6163
}
6264
r := memory.ForceReadMemStats()
@@ -72,7 +74,11 @@ func (t *memoryLimitTuner) tuning() {
7274
// - Only if NextGC >= MemoryLimit , the **next** GC will be triggered by MemoryLimit. Thus, we need to reset
7375
// MemoryLimit after the **next** GC happens if needed.
7476
if float64(r.HeapInuse)*ratio > float64(debug.SetMemoryLimit(-1)) {
75-
if t.nextGCTriggeredByMemoryLimit.Load() && t.waitingReset.CompareAndSwap(false, true) {
77+
if t.nextGCTriggeredByMemoryLimit.Load() && t.adjustPercentageInProgress.CompareAndSwap(false, true) {
78+
// It's ok to update `adjustPercentageInProgress`, `serverMemLimitBeforeAdjust` and `percentageBeforeAdjust` not in a transaction.
79+
// The update of memory limit is eventually consistent.
80+
t.serverMemLimitBeforeAdjust.Store(memory.ServerMemoryLimit.Load())
81+
t.percentageBeforeAdjust.Store(t.GetPercentage())
7682
go func() {
7783
if intest.InTest {
7884
memoryGoroutineCntInTest.Inc()
@@ -85,14 +91,21 @@ func (t *memoryLimitTuner) tuning() {
8591
if intest.InTest {
8692
resetInterval = 3 * time.Second
8793
}
94+
failpoint.Inject("mockUpdateGlobalVarDuringAdjustPercentage", func(val failpoint.Value) {
95+
if val, ok := val.(bool); val && ok {
96+
resetInterval = 5 * time.Second
97+
time.Sleep(300 * time.Millisecond)
98+
t.UpdateMemoryLimit()
99+
}
100+
})
88101
failpoint.Inject("testMemoryLimitTuner", func(val failpoint.Value) {
89102
if val, ok := val.(bool); val && ok {
90103
resetInterval = 1 * time.Second
91104
}
92105
})
93106
time.Sleep(resetInterval)
94107
debug.SetMemoryLimit(t.calcMemoryLimit(t.GetPercentage()))
95-
for !t.waitingReset.CompareAndSwap(true, false) {
108+
for !t.adjustPercentageInProgress.CompareAndSwap(true, false) {
96109
continue
97110
}
98111
}()
@@ -128,12 +141,17 @@ func (t *memoryLimitTuner) GetPercentage() float64 {
128141
// UpdateMemoryLimit updates the memory limit.
129142
// This function should be called when `tidb_server_memory_limit` or `tidb_server_memory_limit_gc_trigger` is modified.
130143
func (t *memoryLimitTuner) UpdateMemoryLimit() {
144+
if t.adjustPercentageInProgress.Load() {
145+
if t.serverMemLimitBeforeAdjust.Load() == memory.ServerMemoryLimit.Load() && t.percentageBeforeAdjust.Load() == t.GetPercentage() {
146+
return
147+
}
148+
}
131149
var memoryLimit = t.calcMemoryLimit(t.GetPercentage())
132150
if memoryLimit == math.MaxInt64 {
133-
t.isTuning.Store(false)
151+
t.isValidValueSet.Store(false)
134152
memoryLimit = initGOMemoryLimitValue
135153
} else {
136-
t.isTuning.Store(true)
154+
t.isValidValueSet.Store(true)
137155
}
138156
debug.SetMemoryLimit(memoryLimit)
139157
}

‎util/gctuner/memory_limit_tuner_test.go

Lines changed: 99 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,16 @@ func TestGlobalMemoryTuner(t *testing.T) {
5757
memory.ServerMemoryLimit.Store(1 << 30) // 1GB
5858
GlobalMemoryLimitTuner.SetPercentage(0.8) // 1GB * 80% = 800MB
5959
GlobalMemoryLimitTuner.UpdateMemoryLimit()
60-
require.True(t, GlobalMemoryLimitTuner.isTuning.Load())
60+
require.True(t, GlobalMemoryLimitTuner.isValidValueSet.Load())
6161
defer func() {
6262
// If test.count > 1, wait tuning finished.
6363
require.Eventually(t, func() bool {
6464
//nolint: all_revive
65-
return GlobalMemoryLimitTuner.isTuning.Load()
65+
return GlobalMemoryLimitTuner.isValidValueSet.Load()
6666
}, 5*time.Second, 100*time.Millisecond)
6767
require.Eventually(t, func() bool {
6868
//nolint: all_revive
69-
return !GlobalMemoryLimitTuner.waitingReset.Load()
69+
return !GlobalMemoryLimitTuner.adjustPercentageInProgress.Load()
7070
}, 5*time.Second, 100*time.Millisecond)
7171
require.Eventually(t, func() bool {
7272
//nolint: all_revive
@@ -85,7 +85,7 @@ func TestGlobalMemoryTuner(t *testing.T) {
8585
runtime.ReadMemStats(r)
8686
nextGC := r.NextGC
8787
memoryLimit := GlobalMemoryLimitTuner.calcMemoryLimit(GlobalMemoryLimitTuner.GetPercentage())
88-
// In golang source, nextGC = memoryLimit - three parts memory.
88+
// Refer to golang source code, nextGC = memoryLimit - nonHeapMemory - overageMemory - headroom
8989
require.True(t, nextGC < uint64(memoryLimit))
9090
}
9191

@@ -94,7 +94,7 @@ func TestGlobalMemoryTuner(t *testing.T) {
9494

9595
memory210mb := allocator.alloc(210 << 20)
9696
require.Eventually(t, func() bool {
97-
return GlobalMemoryLimitTuner.waitingReset.Load() && gcNum < getNowGCNum()
97+
return GlobalMemoryLimitTuner.adjustPercentageInProgress.Load() && gcNum < getNowGCNum()
9898
}, 5*time.Second, 100*time.Millisecond)
9999
// Test waiting for reset
100100
require.Eventually(t, func() bool {
@@ -123,3 +123,97 @@ func TestGlobalMemoryTuner(t *testing.T) {
123123
allocator.free(memory210mb)
124124
allocator.free(memory600mb)
125125
}
126+
127+
func TestIssue48741(t *testing.T) {
128+
// Close GOGCTuner
129+
gogcTuner := EnableGOGCTuner.Load()
130+
EnableGOGCTuner.Store(false)
131+
defer EnableGOGCTuner.Store(gogcTuner)
132+
133+
r := &runtime.MemStats{}
134+
getNowGCNum := func() uint32 {
135+
runtime.ReadMemStats(r)
136+
return r.NumGC
137+
}
138+
allocator := &mockAllocator{}
139+
defer allocator.freeAll()
140+
141+
checkIfMemoryLimitIsModified := func() {
142+
memory.ServerMemoryLimit.Store(1500 << 20) // 1.5 GB
143+
144+
// Try to trigger GC by 1GB * 80% = 800MB (tidb_server_memory_limit * tidb_server_memory_limit_gc_trigger)
145+
gcNum := getNowGCNum()
146+
memory810mb := allocator.alloc(810 << 20)
147+
require.Eventually(t,
148+
// Wait for the GC triggered by memory810mb
149+
func() bool {
150+
return GlobalMemoryLimitTuner.adjustPercentageInProgress.Load() && gcNum < getNowGCNum()
151+
},
152+
500*time.Millisecond, 100*time.Millisecond)
153+
154+
gcNumAfterMemory810mb := getNowGCNum()
155+
// After the GC triggered by memory810mb.
156+
time.Sleep(4500 * time.Millisecond)
157+
require.Equal(t, debug.SetMemoryLimit(-1), int64(1500<<20*80/100))
158+
159+
memory700mb := allocator.alloc(200 << 20)
160+
time.Sleep(5 * time.Second)
161+
// The heapInUse is less than 1.5GB * 80% = 1.2GB, so the gc will not be triggered.
162+
require.Equal(t, gcNumAfterMemory810mb, getNowGCNum())
163+
164+
memory150mb := allocator.alloc(300 << 20)
165+
require.Eventually(t,
166+
// Wait for the GC triggered by memory810mb
167+
func() bool {
168+
return GlobalMemoryLimitTuner.adjustPercentageInProgress.Load() && gcNumAfterMemory810mb < getNowGCNum()
169+
},
170+
5*time.Second, 100*time.Millisecond)
171+
172+
time.Sleep(4500 * time.Millisecond)
173+
require.Equal(t, debug.SetMemoryLimit(-1), int64(1500<<20*110/100))
174+
175+
allocator.free(memory810mb)
176+
allocator.free(memory700mb)
177+
allocator.free(memory150mb)
178+
}
179+
180+
checkIfMemoryLimitNotModified := func() {
181+
// Try to trigger GC by 1GB * 80% = 800MB (tidb_server_memory_limit * tidb_server_memory_limit_gc_trigger)
182+
gcNum := getNowGCNum()
183+
memory810mb := allocator.alloc(810 << 20)
184+
require.Eventually(t,
185+
// Wait for the GC triggered by memory810mb
186+
func() bool {
187+
return GlobalMemoryLimitTuner.adjustPercentageInProgress.Load() && gcNum < getNowGCNum()
188+
},
189+
500*time.Millisecond, 100*time.Millisecond)
190+
191+
gcNumAfterMemory810mb := getNowGCNum()
192+
// After the GC triggered by memory810mb.
193+
time.Sleep(4500 * time.Millisecond)
194+
// During the process of adjusting the percentage, the memory limit will be set to 1GB * 110% = 1.1GB.
195+
require.Equal(t, debug.SetMemoryLimit(-1), int64(1<<30*110/100))
196+
197+
require.Eventually(t,
198+
// The GC will be trigged immediately after memoryLimit is set back to 1GB * 80% = 800MB.
199+
func() bool {
200+
return GlobalMemoryLimitTuner.adjustPercentageInProgress.Load() && gcNumAfterMemory810mb < getNowGCNum()
201+
},
202+
2*time.Second, 100*time.Millisecond)
203+
204+
allocator.free(memory810mb)
205+
}
206+
207+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/gctuner/mockUpdateGlobalVarDuringAdjustPercentage", "return(true)"))
208+
defer func() {
209+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/gctuner/mockUpdateGlobalVarDuringAdjustPercentage"))
210+
}()
211+
212+
memory.ServerMemoryLimit.Store(1 << 30) // 1GB
213+
GlobalMemoryLimitTuner.SetPercentage(0.8) // 1GB * 80% = 800MB
214+
GlobalMemoryLimitTuner.UpdateMemoryLimit()
215+
require.Equal(t, debug.SetMemoryLimit(-1), int64(1<<30*80/100))
216+
217+
checkIfMemoryLimitNotModified()
218+
checkIfMemoryLimitIsModified()
219+
}

0 commit comments

Comments
 (0)