cgmon: fix the issue that CPU quota is 0 if it's not limited by cgroup (#61322)

close #61223

Author: YangKeao

pkg/util/cgmon/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "cgmon",
@@ -14,3 +14,15 @@ go_library(
         "@org_uber_go_zap//:zap",
     ],
 )
+
+go_test(
+    name = "cgmon_test",
+    timeout = "short",
+    srcs = ["cgmon_test.go"],
+    embed = [":cgmon"],
+    flaky = True,
+    deps = [
+        "@com_github_shirou_gopsutil_v3//mem",
+        "@com_github_stretchr_testify//require",
+    ],
+)

pkg/util/cgmon/cgmon.go

@@ -38,9 +38,11 @@ var (
 	ctx             context.Context
 	cancel          context.CancelFunc
 	wg              sync.WaitGroup
-	cfgMaxProcs     int
-	lastMaxProcs    int
+	lastCPU         int
 	lastMemoryLimit uint64
+
+	getCgroupCPUPeriodAndQuota = cgroup.GetCPUPeriodAndQuota
+	getCgroupMemoryLimit       = cgroup.GetMemoryLimit
 )
 
 // StartCgroupMonitor uses to start the cgroup monitoring.
@@ -115,45 +117,44 @@ func refreshCgroupCPU() error {
 	quota := runtime.NumCPU()
 
 	// Get CPU quota from cgroup.
-	cpuPeriod, cpuQuota, err := cgroup.GetCPUPeriodAndQuota()
-	if err != nil {
-		return err
-	}
-	if cpuPeriod > 0 && cpuQuota > 0 {
+	cpuPeriod, cpuQuota, err := getCgroupCPUPeriodAndQuota()
+
+	// Only use cgroup CPU quota if it is available. It's possible that the cgroup doesn't have CPU quota set.
+	// Then in some environments, systemd will not enable the cpu controller if cpu quota is not set.
+	if err == nil && cpuPeriod > 0 && cpuQuota > 0 {
 		ratio := float64(cpuQuota) / float64(cpuPeriod)
 		if ratio < float64(quota) {
 			quota = int(math.Ceil(ratio))
 		}
 	}
 
-	if quota != lastMaxProcs {
+	if quota != lastCPU {
 		log.Info("set the maxprocs", zap.Int("quota", quota))
 		metrics.MaxProcs.Set(float64(quota))
-		lastMaxProcs = quota
-	} else if lastMaxProcs == 0 {
-		log.Info("set the maxprocs", zap.Int("cfgMaxProcs", cfgMaxProcs))
-		metrics.MaxProcs.Set(float64(cfgMaxProcs))
-		lastMaxProcs = cfgMaxProcs
+		lastCPU = quota
 	}
-	return nil
+
+	return err
 }
 
 func refreshCgroupMemory() error {
-	memLimit, err := cgroup.GetMemoryLimit()
-	if err != nil {
-		return err
-	}
+	// Get the total memory limit from `procfs`
 	vmem, err := mem.VirtualMemory()
 	if err != nil {
 		return err
 	}
-	if memLimit > vmem.Total {
-		memLimit = vmem.Total
+	memLimit := vmem.Total
+
+	// Only use cgroup memory limit if it is available.
+	cgroupMemLimit, err := getCgroupMemoryLimit()
+	if err == nil && cgroupMemLimit < memLimit {
+		memLimit = cgroupMemLimit
 	}
+
 	if memLimit != lastMemoryLimit {
 		log.Info("set the memory limit", zap.Uint64("memLimit", memLimit))
 		metrics.MemoryLimit.Set(float64(memLimit))
 		lastMemoryLimit = memLimit
 	}
-	return nil
+	return err
 }

pkg/util/cgmon/cgmon_test.go

@@ -0,0 +1,41 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cgmon
+
+import (
+	"errors"
+	"runtime"
+	"testing"
+
+	"github.com/shirou/gopsutil/v3/mem"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUploadDefaultValueWithoutCgroup(t *testing.T) {
+	getCgroupCPUPeriodAndQuota = func() (period int64, quota int64, err error) {
+		return 0, 0, errors.New("mock error")
+	}
+	getCgroupMemoryLimit = func() (uint64, error) {
+		return 0, errors.New("mock error")
+	}
+
+	require.Error(t, refreshCgroupCPU())
+	require.Error(t, refreshCgroupMemory())
+
+	require.Equal(t, runtime.NumCPU(), lastCPU)
+	vmem, err := mem.VirtualMemory()
+	require.NoError(t, err)
+	require.Equal(t, vmem.Total, lastMemoryLimit)
+}