Merge remote-tracking branch 'origin' into tmo

config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml

@@ -260,18 +260,29 @@ spec:
                             description: GracePeriod is the grace period of pod deletion
                             format: int64
                             type: integer
-                          minimumFreeThreshold:
-                            description: 'MinimumFreeThreshold is a threshold for
-                              a node. Once the rootfs free space of current node is
-                              lower than this threshold, the eviction manager will
-                              try to evict some pods. For example: "200Gi", "10%".'
+                          minimumImageFsDiskCapacityThreshold:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            description: 'MinimumImageFsDiskCapacityThreshold is a
+                              threshold for all nodes. The eviction manager will ignore
+                              those nodes whose image fs disk capacity is less than
+                              this threshold. Fox example: "100Gi".'
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                          minimumImageFsFreeThreshold:
+                            description: 'MinimumImageFsFreeThreshold is a threshold
+                              for a node. Once the image rootfs free space of current
+                              node is lower than this threshold, the eviction manager
+                              will try to evict some pods. For example: "200Gi", "10%".'
                             pattern: ^(0|[1-9][0-9]*)(\.[0-9]+)?%?$|^(0|[1-9][0-9]*)([kKmMGTPeE]i?)$
                             type: string
-                          minimumInodesFreeThreshold:
-                            description: 'MinimumInodesFreeThreshold is a threshold
-                              for a node. Once the rootfs free inodes of current node
-                              is lower than this threshold, the eviction manager will
-                              try to evict some pods. For example: "100000", "10%".'
+                          minimumImageFsInodesFreeThreshold:
+                            description: 'MinimumImageFsInodesFreeThreshold is a threshold
+                              for a node. Once the image rootfs free inodes of current
+                              node is lower than this threshold, the eviction manager
+                              will try to evict some pods. For example: "100000",
+                              "10%".'
                             pattern: ^(0|[1-9]\d*)(\.\d+)?%?$|^\d+$
                             type: string
                           podMinimumInodesUsedThreshold:

config/crd/bases/workload.katalyst.kubewharf.io_serviceprofiledescriptors.yaml

@@ -101,6 +101,16 @@ spec:
                     business and system scopes. Examples include policies for custom
                     memory reclamation and I/O limitations specific to a service.
                   properties:
+                    baselinePercent:
+                      description: BaselinePercent marks off a bunch of instances,
+                        and skip applying extended indicator for them; those instances
+                        are defined as baselines, and can be compared with other (experimental/production)
+                        instances to demonstrate the benefits. If BaselinePercent
+                        is not set, we should take all instances as production instances.
+                      format: int32
+                      maximum: 100
+                      minimum: 0
+                      type: integer
                     indicators:
                       description: Indicators defines extend workload characteristics,
                         Indicators can have arbitrary structure. Each kind of Indicator

pkg/apis/config/v1alpha1/adminqos.go

@@ -384,47 +384,52 @@ type RootfsPressureEvictionConfig struct {
 	// +optional
 	EnableRootfsPressureEviction *bool `json:"enableRootfsPressureEviction,omitempty"`
 
-	// MinimumFreeThreshold is a threshold for a node.
-	// Once the rootfs free space of current node is lower than this threshold, the eviction manager will try to evict some pods.
+	// MinimumImageFsFreeThreshold is a threshold for a node.
+	// Once the image rootfs free space of current node is lower than this threshold, the eviction manager will try to evict some pods.
 	// For example: "200Gi", "10%".
 	// +optional
 	// +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]*)(\.[0-9]+)?%?$|^(0|[1-9][0-9]*)([kKmMGTPeE]i?)$`
-	MinimumFreeThreshold string `json:"minimumFreeThreshold,omitempty"`
+	MinimumImageFsFreeThreshold *string `json:"minimumImageFsFreeThreshold,omitempty"`
 
-	// MinimumInodesFreeThreshold is a threshold for a node.
-	// Once the rootfs free inodes of current node is lower than this threshold, the eviction manager will try to evict some pods.
+	// MinimumImageFsInodesFreeThreshold is a threshold for a node.
+	// Once the image rootfs free inodes of current node is lower than this threshold, the eviction manager will try to evict some pods.
 	// For example: "100000", "10%".
 	// +optional
 	// +kubebuilder:validation:Pattern=`^(0|[1-9]\d*)(\.\d+)?%?$|^\d+$`
-	MinimumInodesFreeThreshold string `json:"minimumInodesFreeThreshold,omitempty"`
+	MinimumImageFsInodesFreeThreshold *string `json:"minimumImageFsInodesFreeThreshold,omitempty"`
 
 	// PodMinimumUsedThreshold is a threshold for all pods.
 	// The eviction manager will ignore this pod if its rootfs used in bytes is lower than this threshold.
 	// For example: "200Gi", "1%".
 	// +optional
 	// +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]*)(\.[0-9]+)?%?$|^(0|[1-9][0-9]*)([kKmMGTPeE]i?)$`
-	PodMinimumUsedThreshold string `json:"podMinimumUsedThreshold,omitempty"`
+	PodMinimumUsedThreshold *string `json:"podMinimumUsedThreshold,omitempty"`
 
 	// PodMinimumInodesUsedThreshold is a threshold for all pods.
 	// The eviction manager will ignore this pod if its rootfs inodes used is lower than this threshold.
 	// For example: "1000", "1%".
 	// +optional
 	// +kubebuilder:validation:Pattern=`^(0|[1-9]\d*)(\.\d+)?%?$|^\d+$`
-	PodMinimumInodesUsedThreshold string `json:"podMinimumInodesUsedThreshold,omitempty"`
+	PodMinimumInodesUsedThreshold *string `json:"podMinimumInodesUsedThreshold,omitempty"`
 
 	// ReclaimedQoSPodUsedPriorityThreshold is a threshold for all offline pods.
 	// The eviction manager will prioritize the eviction of offline pods that reach this threshold.
 	// For example: "100Gi", "1%".
 	// +optional
 	// +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]*)(\.[0-9]+)?%?$|^(0|[1-9][0-9]*)([kKmMGTPeE]i?)$`
-	ReclaimedQoSPodUsedPriorityThreshold string `json:"reclaimedQoSPodUsedPriorityThreshold,omitempty"`
+	ReclaimedQoSPodUsedPriorityThreshold *string `json:"reclaimedQoSPodUsedPriorityThreshold,omitempty"`
 
 	// ReclaimedQoSPodInodesUsedPriorityThreshold is a threshold for all offline pods.
 	// The eviction manager will prioritize the eviction of reclaimed pods that reach this threshold.
 	// For example: "500", "1%".
 	// +optional
 	// +kubebuilder:validation:Pattern=`^(0|[1-9]\d*)(\.\d+)?%?$|^\d+$`
-	ReclaimedQoSPodInodesUsedPriorityThreshold string `json:"reclaimedQoSPodInodesUsedPriorityThreshold,omitempty"`
+	ReclaimedQoSPodInodesUsedPriorityThreshold *string `json:"reclaimedQoSPodInodesUsedPriorityThreshold,omitempty"`
+
+	// MinimumImageFsDiskCapacityThreshold is a threshold for all nodes.
+	// The eviction manager will ignore those nodes whose image fs disk capacity is less than this threshold.
+	// Fox example: "100Gi".
+	MinimumImageFsDiskCapacityThreshold *resource.Quantity `json:"minimumImageFsDiskCapacityThreshold,omitempty"`
 
 	// GracePeriod is the grace period of pod deletion
 	// +optional

pkg/apis/workload/v1alpha1/register.go

@@ -52,6 +52,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&ServiceProfileDescriptor{},
 		&ServiceProfileDescriptorList{},
+
+		&TestExtendedIndicators{},
 	)
 
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)

pkg/apis/workload/v1alpha1/types.go

@@ -50,22 +50,13 @@ type ServiceProfileDescriptor struct {
 
 // DecodeNestedObjects decodes extended indicator for known types.
 func (c *ServiceProfileDescriptor) DecodeNestedObjects(d runtime.Decoder) error {
-	var strictDecodingErrs []error
 	for i := range c.Spec.ExtendedIndicator {
 		indicator := &c.Spec.ExtendedIndicator[i]
 		err := indicator.decodeNestedObjects(d)
 		if err != nil {
-			decodingErr := fmt.Errorf("decoding .spec.extendedIndicator[%d]: %w", i, err)
-			if runtime.IsStrictDecodingError(err) {
-				strictDecodingErrs = append(strictDecodingErrs, decodingErr)
-			} else {
-				return decodingErr
-			}
+			return fmt.Errorf("decoding .spec.extendedIndicator[%d]: %w", i, err)
 		}
 	}
-	if len(strictDecodingErrs) > 0 {
-		return runtime.NewStrictDecodingError(strictDecodingErrs)
-	}
 	return nil
 }
 
@@ -126,35 +117,42 @@ type ServiceExtendedIndicatorSpec struct {
 	// Name defines the name of extended module
 	Name string `json:"name"`
 
+	// BaselinePercent marks off a bunch of instances, and skip applying extended indicator
+	// for them; those instances are defined as baselines, and can be compared
+	// with other (experimental/production) instances to demonstrate the benefits.
+	// If BaselinePercent is not set, we should take all instances as production instances.
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:validation:Maximum=100
+	// +optional
+	BaselinePercent *int32 `json:"baselinePercent,omitempty"`
+
 	// Indicators defines extend workload characteristics, Indicators can have arbitrary structure.
 	// Each kind of Indicator must be named using Name, followed by the suffix 'Indicators'.
 	// +optional
 	// +kubebuilder:pruning:PreserveUnknownFields
 	Indicators runtime.RawExtension `json:"indicators,omitempty"`
 }
 
+const (
+	ExtendedIndicatorSuffix = "Indicators"
+)
+
 func (c *ServiceExtendedIndicatorSpec) decodeNestedObjects(d runtime.Decoder) error {
-	gvk := SchemeGroupVersion.WithKind(c.Name + "Indicators")
+	gvk := SchemeGroupVersion.WithKind(c.Name + ExtendedIndicatorSuffix)
 	// dry-run to detect and skip out-of-tree extended indicators.
 	if _, _, err := d.Decode(nil, &gvk, nil); runtime.IsNotRegisteredError(err) {
 		return nil
 	}
 
-	var strictDecodingErr error
 	obj, parsedGvk, err := d.Decode(c.Indicators.Raw, &gvk, nil)
 	if err != nil {
-		decodingArgsErr := fmt.Errorf("decoding extended indicators %s: %w", c.Name, err)
-		if obj != nil && runtime.IsStrictDecodingError(err) {
-			strictDecodingErr = runtime.NewStrictDecodingError([]error{decodingArgsErr})
-		} else {
-			return decodingArgsErr
-		}
-	}
-	if parsedGvk.GroupKind() != gvk.GroupKind() {
+		return fmt.Errorf("decoding extended indicators %s: %w", c.Name, err)
+	} else if parsedGvk.GroupKind() != gvk.GroupKind() {
 		return fmt.Errorf("indicators for %s were not of type %s, got %s", c.Name, gvk.GroupKind(), parsedGvk.GroupKind())
 	}
+
 	c.Indicators.Object = obj
-	return strictDecodingErr
+	return nil
 }
 
 func (c *ServiceExtendedIndicatorSpec) encodeNestedObjects(e runtime.Encoder) error {
@@ -304,3 +302,16 @@ type ServiceProfileDescriptorList struct {
 	// items is the list of SPDs
 	Items []ServiceProfileDescriptor `json:"items"`
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// TestExtendedIndicators just for extended indicator test
+type TestExtendedIndicators struct {
+	metav1.TypeMeta `json:",inline"`
+
+	Indicators *TestIndicators `json:"indicators"`
+}
+
+type TestIndicators struct {
+	TestIndicator string `json:"testIndicator"`
+}

pkg/consts/resource.go

@@ -24,7 +24,8 @@ const (
 
 // const variables for resource names of guaranteed resource
 const (
-	ResourceNetBandwidth v1.ResourceName = "resource.katalyst.kubewharf.io/net_bandwidth"
+	ResourceNetBandwidth    v1.ResourceName = "resource.katalyst.kubewharf.io/net_bandwidth"
+	ResourceMemoryBandwidth v1.ResourceName = "resource.katalyst.kubewharf.io/memory_bandwidth"
 )
 
 // ResourceAnnotationKeyResourceIdentifier nominated the key to override the default name

pkg/consts/spd.go

@@ -32,11 +32,12 @@ const (
 
 // const variables for spd.
 const (
-	// SPDAnnotationBaselineSentinelKey is updated by the SPD controller. It represents
-	// the sentinel pod among all pods managed by this SPD. Agents or controllers
-	// can use this key to determine if a pod falls within the baseline by comparing it
-	// with the pod's createTime and podName.
-	SPDAnnotationBaselineSentinelKey = "spd.katalyst.kubewharf.io/baselineSentinel"
+	// SPDAnnotationBaselineSentinelKey and SPDAnnotationExtendedBaselineSentinelKey is
+	// updated by the SPD controller. It represents the sentinel pod among all pods managed
+	// by this SPD. Agents or controllers can use this key to determine if a pod falls within
+	// the baseline by comparing it with the pod's createTime and podName.
+	SPDAnnotationBaselineSentinelKey         = "spd.katalyst.kubewharf.io/baselineSentinel"
+	SPDAnnotationExtendedBaselineSentinelKey = "spd.katalyst.kubewharf.io/extendedBaselineSentinel"
 
 	SPDBaselinePercentMax = 100
 	SPDBaselinePercentMin = 0