Merge pull request kubernetes#3929 from jayantjain93/memory-estimation

Fixing Kernel Memory Usage estimation for GCE cloud provider
towca · Mar 9, 2021 · f25d81f · f25d81f
2 parents 31e2b43 + 7de393d
commit f25d81f
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 7 deletions.
diff --git a/cluster-autoscaler/cloudprovider/gce/reserved.go b/cluster-autoscaler/cloudprovider/gce/reserved.go
@@ -57,6 +57,15 @@ const (
 	// Reserved memory for software IO TLB
 	swiotlbReservedMemory  = 64 * MiB
 	swiotlbThresholdMemory = 3 * GiB
+
+	// Memory Estimation Correction
+	// correctionConstant is a linear constant for additional reserved memory
+	correctionConstant = 0.00175
+	// maximumCorrectionValue is the max-cap for additional reserved memory
+	maximumCorrectionValue = 248 * MiB
+	// ubuntuSpecificOffset is a constant value that is additionally added to Ubuntu
+	// based distributions as reserved memory
+	ubuntuSpecificOffset = 4 * MiB
 )
 
 // EvictionHard is the struct used to keep parsed values for eviction
@@ -67,7 +76,7 @@ type EvictionHard struct {
 
 // CalculateKernelReserved computes how much memory Linux kernel will reserve.
 // TODO(jkaniuk): account for crashkernel reservation on RHEL / CentOS
-func CalculateKernelReserved(physicalMemory int64, os OperatingSystem) int64 {
+func CalculateKernelReserved(physicalMemory int64, os OperatingSystem, osDistribution OperatingSystemDistribution) int64 {
 	switch os {
 	case OperatingSystemLinux:
 		// Account for memory reserved by kernel
@@ -77,6 +86,18 @@ func CalculateKernelReserved(physicalMemory int64, os OperatingSystem) int64 {
 		if physicalMemory > swiotlbThresholdMemory {
 			reserved += swiotlbReservedMemory
 		}
+
+		// Additional reserved memory to correct estimation
+		// The reason for this value is we detected additional reservation, but we were
+		// unable to find the root cause. Hence, we added a best estimated formula that was
+		// statistically developed.
+		if osDistribution == OperatingSystemDistributionCOS || osDistribution == OperatingSystemDistributionCOSContainerd {
+			reserved += int64(math.Min(correctionConstant*float64(physicalMemory), maximumCorrectionValue))
+		} else if osDistribution == OperatingSystemDistributionUbuntu || osDistribution == OperatingSystemDistributionUbuntuContainerd {
+			reserved += int64(math.Min(correctionConstant*float64(physicalMemory), maximumCorrectionValue))
+			reserved += ubuntuSpecificOffset
+		}
+
 		return reserved
 	case OperatingSystemWindows:
 		return 0

diff --git a/cluster-autoscaler/cloudprovider/gce/reserved_test.go b/cluster-autoscaler/cloudprovider/gce/reserved_test.go
@@ -18,6 +18,7 @@ package gce
 
 import (
 	"fmt"
+	"math"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -27,37 +28,78 @@ func TestCalculateKernelReservedLinux(t *testing.T) {
 	type testCase struct {
 		physicalMemory int64
 		reservedMemory int64
+		osDistribution OperatingSystemDistribution
 	}
 	testCases := []testCase{
 		{
 			physicalMemory: 256 * MiB,
 			reservedMemory: 4*MiB + kernelReservedMemory,
+			osDistribution: OperatingSystemDistributionCOS,
 		},
 		{
 			physicalMemory: 2 * GiB,
 			reservedMemory: 32*MiB + kernelReservedMemory,
+			osDistribution: OperatingSystemDistributionCOS,
 		},
 		{
 			physicalMemory: 3 * GiB,
 			reservedMemory: 48*MiB + kernelReservedMemory,
+			osDistribution: OperatingSystemDistributionCOS,
 		},
 		{
 			physicalMemory: 3.25 * GiB,
 			reservedMemory: 52*MiB + kernelReservedMemory + swiotlbReservedMemory,
+			osDistribution: OperatingSystemDistributionCOS,
 		},
 		{
 			physicalMemory: 4 * GiB,
 			reservedMemory: 64*MiB + kernelReservedMemory + swiotlbReservedMemory,
+			osDistribution: OperatingSystemDistributionCOS,
 		},
 		{
 			physicalMemory: 128 * GiB,
 			reservedMemory: 2*GiB + kernelReservedMemory + swiotlbReservedMemory,
+			osDistribution: OperatingSystemDistributionUbuntu,
+		},
+		{
+			physicalMemory: 256 * MiB,
+			reservedMemory: 4*MiB + kernelReservedMemory,
+			osDistribution: OperatingSystemDistributionUbuntu,
+		},
+		{
+			physicalMemory: 2 * GiB,
+			reservedMemory: 32*MiB + kernelReservedMemory,
+			osDistribution: OperatingSystemDistributionUbuntu,
+		},
+		{
+			physicalMemory: 3 * GiB,
+			reservedMemory: 48*MiB + kernelReservedMemory,
+			osDistribution: OperatingSystemDistributionUbuntu,
+		},
+		{
+			physicalMemory: 3.25 * GiB,
+			reservedMemory: 52*MiB + kernelReservedMemory + swiotlbReservedMemory,
+			osDistribution: OperatingSystemDistributionUbuntu,
+		},
+		{
+			physicalMemory: 4 * GiB,
+			reservedMemory: 64*MiB + kernelReservedMemory + swiotlbReservedMemory,
+			osDistribution: OperatingSystemDistributionUbuntu,
+		},
+		{
+			physicalMemory: 128 * GiB,
+			reservedMemory: 2*GiB + kernelReservedMemory + swiotlbReservedMemory,
+			osDistribution: OperatingSystemDistributionUbuntu,
 		},
 	}
 	for idx, tc := range testCases {
 		t.Run(fmt.Sprintf("%v", idx), func(t *testing.T) {
-			reserved := CalculateKernelReserved(tc.physicalMemory, OperatingSystemLinux)
-			assert.Equal(t, tc.reservedMemory, reserved)
+			reserved := CalculateKernelReserved(tc.physicalMemory, OperatingSystemLinux, tc.osDistribution)
+			if tc.osDistribution == OperatingSystemDistributionUbuntu {
+				assert.Equal(t, tc.reservedMemory+int64(math.Min(correctionConstant*float64(tc.physicalMemory), maximumCorrectionValue)+ubuntuSpecificOffset), reserved)
+			} else if tc.osDistribution == OperatingSystemDistributionCOS {
+				assert.Equal(t, tc.reservedMemory+int64(math.Min(correctionConstant*float64(tc.physicalMemory), maximumCorrectionValue)), reserved)
+			}
 		})
 	}
 }
diff --git a/cluster-autoscaler/cloudprovider/gce/templates.go b/cluster-autoscaler/cloudprovider/gce/templates.go
@@ -63,7 +63,7 @@ func (t *GceTemplateBuilder) BuildCapacity(cpu int64, mem int64, accelerators []
 	}
 
 	capacity[apiv1.ResourceCPU] = *resource.NewQuantity(cpu, resource.DecimalSI)
-	memTotal := mem - CalculateKernelReserved(mem, os)
+	memTotal := mem - CalculateKernelReserved(mem, os, osDistribution)
 	capacity[apiv1.ResourceMemory] = *resource.NewQuantity(memTotal, resource.DecimalSI)
 
 	if accelerators != nil && len(accelerators) > 0 {

diff --git a/cluster-autoscaler/cloudprovider/gce/templates_test.go b/cluster-autoscaler/cloudprovider/gce/templates_test.go
@@ -18,6 +18,7 @@ package gce
 
 import (
 	"fmt"
+	"math"
 	"strings"
 	"testing"
 
@@ -454,19 +455,19 @@ func TestBuildCapacityMemory(t *testing.T) {
 			physicalCpu:            1,
 			physicalMemory:         2 * units.GiB,
 			os:                     OperatingSystemLinux,
-			expectedCapacityMemory: 2*units.GiB - 32*units.MiB - kernelReservedMemory,
+			expectedCapacityMemory: 2*units.GiB - 32*units.MiB - kernelReservedMemory - int64(math.Min(correctionConstant*float64(2*units.GiB), maximumCorrectionValue)),
 		},
 		{
 			physicalCpu:            2,
 			physicalMemory:         4 * units.GiB,
 			os:                     OperatingSystemLinux,
-			expectedCapacityMemory: 4*units.GiB - 64*units.MiB - kernelReservedMemory - swiotlbReservedMemory,
+			expectedCapacityMemory: 4*units.GiB - 64*units.MiB - kernelReservedMemory - swiotlbReservedMemory - int64(math.Min(correctionConstant*float64(4*units.GiB), maximumCorrectionValue)),
 		},
 		{
 			physicalCpu:            32,
 			physicalMemory:         128 * units.GiB,
 			os:                     OperatingSystemLinux,
-			expectedCapacityMemory: 128*units.GiB - 2*units.GiB - kernelReservedMemory - swiotlbReservedMemory,
+			expectedCapacityMemory: 128*units.GiB - 2*units.GiB - kernelReservedMemory - swiotlbReservedMemory - int64(math.Min(correctionConstant*float64(128*units.GiB), maximumCorrectionValue)),
 		},
 		{
 			physicalCpu:            2,