diff --git a/apis/apps/v1alpha1/well_known_labels.go b/apis/apps/v1alpha1/well_known_labels.go index 85e37780ef..5d5cb917f2 100644 --- a/apis/apps/v1alpha1/well_known_labels.go +++ b/apis/apps/v1alpha1/well_known_labels.go @@ -10,6 +10,9 @@ const ( // SpecifiedDeleteKey indicates this object should be deleted, and the value could be the deletion option. SpecifiedDeleteKey = "apps.kruise.io/specified-delete" + // KeepPVCForDeletionKey indicates should keep PVC for reuse when specified delete the pod. + KeepPVCForDeletionKey = "apps.kruise.io/keep-pvc-for-deletion" + // ImagePreDownloadCreatedKey indicates the images of this revision have been pre-downloaded ImagePreDownloadCreatedKey = "apps.kruise.io/pre-predownload-created" diff --git a/pkg/controller/cloneset/sync/cloneset_scale.go b/pkg/controller/cloneset/sync/cloneset_scale.go index fb56c9a0fb..b592b217c9 100644 --- a/pkg/controller/cloneset/sync/cloneset_scale.go +++ b/pkg/controller/cloneset/sync/cloneset_scale.go @@ -36,6 +36,7 @@ import ( "github.com/openkruise/kruise/pkg/util/expectations" "github.com/openkruise/kruise/pkg/util/lifecycle" "github.com/openkruise/kruise/pkg/util/revision" + "github.com/openkruise/kruise/pkg/util/specifieddelete" ) const ( @@ -289,6 +290,9 @@ func (r *realControl) deletePods(cs *appsv1alpha1.CloneSet, podsToDelete []*v1.P modified = true r.recorder.Event(cs, v1.EventTypeNormal, "SuccessfulDelete", fmt.Sprintf("succeed to delete pod %s", pod.Name)) + if specifieddelete.ShouldKeepPVC(pod) { + continue + } // delete pvcs which have the same instance-id for _, pvc := range pvcs { if pvc.Labels[appsv1alpha1.CloneSetInstanceID] != pod.Labels[appsv1alpha1.CloneSetInstanceID] { diff --git a/pkg/controller/cloneset/sync/cloneset_update.go b/pkg/controller/cloneset/sync/cloneset_update.go index 031b55f6dd..1e05d56e7d 100644 --- a/pkg/controller/cloneset/sync/cloneset_update.go +++ b/pkg/controller/cloneset/sync/cloneset_update.go @@ -308,7 +308,8 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc klog.V(2).InfoS("CloneSet started to patch Pod specified-delete for update", "cloneSet", klog.KObj(cs), "pod", klog.KObj(pod), "updateRevision", klog.KObj(updateRevision)) - if patched, err := specifieddelete.PatchPodSpecifiedDelete(c.Client, pod, "true"); err != nil { + keepPVC := !cs.Spec.ScaleStrategy.DisablePVCReuse && utilfeature.DefaultFeatureGate.Enabled(features.CloneSetPVCReuseDuringUpdate) + if patched, err := specifieddelete.PatchPodSpecifiedDelete(c.Client, pod, keepPVC); err != nil { c.recorder.Eventf(cs, v1.EventTypeWarning, "FailedUpdatePodReCreate", "failed to patch pod specified-delete %s for update(revision %s): %v", pod.Name, updateRevision.Name, err) return 0, err diff --git a/pkg/features/kruise_features.go b/pkg/features/kruise_features.go index bfa086fb05..30a649902e 100644 --- a/pkg/features/kruise_features.go +++ b/pkg/features/kruise_features.go @@ -138,6 +138,10 @@ const ( // InPlaceWorkloadVerticalScaling enable CloneSet/Advanced StatefulSet controller to support vertical scaling // of managed Pods. InPlaceWorkloadVerticalScaling featuregate.Feature = "InPlaceWorkloadVerticalScaling" + + // CloneSetPVCReuseDuringUpdate enables CloneSet to reuse PVC during update, and it also depends on + // the spec.scaleStrategy.disablePVCReuse not to be true. + CloneSetPVCReuseDuringUpdate featuregate.Feature = "CloneSetPVCReuseDuringUpdate" ) var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ @@ -175,6 +179,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ StatefulSetAutoResizePVCGate: {Default: false, PreRelease: featuregate.Alpha}, ForceDeleteTimeoutExpectationFeatureGate: {Default: false, PreRelease: featuregate.Alpha}, InPlaceWorkloadVerticalScaling: {Default: false, PreRelease: featuregate.Alpha}, + CloneSetPVCReuseDuringUpdate: {Default: false, PreRelease: featuregate.Alpha}, } func init() { diff --git a/pkg/util/specifieddelete/specified_delete.go b/pkg/util/specifieddelete/specified_delete.go index 8624cd9ace..685737306e 100644 --- a/pkg/util/specifieddelete/specified_delete.go +++ b/pkg/util/specifieddelete/specified_delete.go @@ -18,13 +18,14 @@ package specifieddelete import ( "context" - "fmt" - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/util" ) func IsSpecifiedDelete(obj metav1.Object) bool { @@ -32,15 +33,28 @@ func IsSpecifiedDelete(obj metav1.Object) bool { return ok } -func PatchPodSpecifiedDelete(c client.Client, pod *v1.Pod, value string) (bool, error) { +func ShouldKeepPVC(obj metav1.Object) bool { + return obj.GetLabels()[appsv1alpha1.KeepPVCForDeletionKey] == "true" +} + +func PatchPodSpecifiedDelete(c client.Client, pod *v1.Pod, keepPVC bool) (bool, error) { if _, ok := pod.Labels[appsv1alpha1.SpecifiedDeleteKey]; ok { return false, nil } - body := fmt.Sprintf( - `{"metadata":{"labels":{"%s":"%s"}}}`, - appsv1alpha1.SpecifiedDeleteKey, - value, - ) - return true, c.Patch(context.TODO(), pod, client.RawPatch(types.StrategicMergePatchType, []byte(body))) + body := patchBody{Metadata: patchMeta{Labels: map[string]string{ + appsv1alpha1.SpecifiedDeleteKey: "true", + }}} + if keepPVC { + body.Metadata.Labels[appsv1alpha1.KeepPVCForDeletionKey] = "true" + } + return true, c.Patch(context.TODO(), pod, client.RawPatch(types.StrategicMergePatchType, []byte(util.DumpJSON(body)))) +} + +type patchBody struct { + Metadata patchMeta `json:"metadata"` +} + +type patchMeta struct { + Labels map[string]string `json:"labels"` } diff --git a/pkg/util/specifieddelete/specified_delete_test.go b/pkg/util/specifieddelete/specified_delete_test.go new file mode 100644 index 0000000000..aecf03e36d --- /dev/null +++ b/pkg/util/specifieddelete/specified_delete_test.go @@ -0,0 +1,73 @@ +package specifieddelete + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" +) + +func TestPatchPodSpecifiedDelete(t *testing.T) { + tests := []struct { + pod *corev1.Pod + keepPVC bool + expected *corev1.Pod + }{ + { + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + Spec: corev1.PodSpec{}, + }, + keepPVC: false, + expected: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Labels: map[string]string{ + appsv1alpha1.SpecifiedDeleteKey: "true", + }, + }, + Spec: corev1.PodSpec{}, + }, + }, + { + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + Spec: corev1.PodSpec{}, + }, + keepPVC: true, + expected: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Labels: map[string]string{ + appsv1alpha1.SpecifiedDeleteKey: "true", + appsv1alpha1.KeepPVCForDeletionKey: "true", + }, + }, + Spec: corev1.PodSpec{}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + cli := fake.NewClientBuilder().WithObjects(test.pod).Build() + _, err := PatchPodSpecifiedDelete(cli, test.pod, test.keepPVC) + assert.NoError(t, err) + + // patch will write result object into the given test.pod + if apiequality.Semantic.DeepEqual(test.expected, test.pod) { + t.Fatalf("expected %v but got %v", test.expected, test.pod) + } + }) + } +} diff --git a/test/e2e/apps/cloneset.go b/test/e2e/apps/cloneset.go index ad06800b87..97a3621e39 100644 --- a/test/e2e/apps/cloneset.go +++ b/test/e2e/apps/cloneset.go @@ -1268,6 +1268,9 @@ func testUpdateVolumeClaimTemplates(tester *framework.CloneSetTester, randStr st updateStrategy := appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.RecreateCloneSetUpdateStrategyType} var replicas int = 4 cs := tester.NewCloneSet("clone-"+randStr, int32(replicas), updateStrategy) + // If enable DisablePVCReuse and the CloneSetPVCReuseDuringUpdate feature-gate, CloneSet will reuse PVC during upgrade. + // So if user wants to recreate PVC, then he shouldn't set pvc to be reused. + cs.Spec.ScaleStrategy.DisablePVCReuse = true imageConfig := imageutils.GetConfig(imageutils.Nginx) imageConfig.SetRegistry("docker.io/library") imageConfig.SetVersion("alpine")