From b1731d76dcfdb6aacbf22da4b7d10e2b137bda16 Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 20 Sep 2018 17:13:46 +0800 Subject: [PATCH] Failover fix: no longer modify spec.Replicas (#95) * pd failover: no longer modify TidbCluster.Spec.PD.Replicas * add RequeueError hint * use RequeueError * tikv and tidb failover fix * make govet happy --- pkg/apis/pingcap.com/v1alpha1/tidbcluster.go | 82 +++++- pkg/controller/tidb_control.go | 4 +- .../tidbcluster/tidb_cluster_control.go | 11 +- .../tidbcluster/tidb_cluster_controller.go | 2 +- pkg/manager/member/pd_failover.go | 172 ++++++------- pkg/manager/member/pd_failover_test.go | 235 ++++++++++-------- pkg/manager/member/pd_member_manager.go | 16 +- pkg/manager/member/tidb_failover.go | 54 +--- pkg/manager/member/tidb_failover_test.go | 8 +- pkg/manager/member/tidb_member_manager.go | 9 +- pkg/manager/member/tikv_failover.go | 17 +- pkg/manager/member/tikv_failover_test.go | 35 +-- pkg/manager/member/tikv_member_manager.go | 10 +- 13 files changed, 324 insertions(+), 331 deletions(-) diff --git a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go index 120a3b29419..9f0840ef090 100644 --- a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go +++ b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go @@ -17,10 +17,88 @@ func (mt MemberType) String() string { return string(mt) } -func (tc TidbCluster) PDUpgrading() bool { +func (tc *TidbCluster) PDUpgrading() bool { return tc.Status.PD.Phase == UpgradePhase } -func (tc TidbCluster) TiKVUpgrading() bool { +func (tc *TidbCluster) TiKVUpgrading() bool { return tc.Status.TiKV.Phase == UpgradePhase } + +func (tc *TidbCluster) PDAllPodsStarted() bool { + return tc.PDRealReplicas() == tc.Status.PD.StatefulSet.Replicas +} + +func (tc *TidbCluster) PDAllMembersReady() bool { + if int(tc.PDRealReplicas()) != len(tc.Status.PD.Members) { + return false + } + + for _, member := range tc.Status.PD.Members { + if !member.Health { + return false + } + } + return true +} + +func (tc *TidbCluster) PDAutoFailovering() bool { + if len(tc.Status.PD.FailureMembers) == 0 { + return false + } + + for _, failureMember := range tc.Status.PD.FailureMembers { + if !failureMember.MemberDeleted { + return true + } + } + return false +} + +func (tc *TidbCluster) PDRealReplicas() int32 { + return tc.Spec.PD.Replicas + int32(len(tc.Status.PD.FailureMembers)) +} + +func (tc *TidbCluster) TiKVAllPodsStarted() bool { + return tc.TiKVRealReplicas() == tc.Status.TiKV.StatefulSet.Replicas +} + +func (tc *TidbCluster) TiKVAllStoresReady() bool { + if int(tc.TiKVRealReplicas()) != len(tc.Status.TiKV.Stores) { + return false + } + + for _, store := range tc.Status.TiKV.Stores { + if store.State != TiKVStateUp { + return false + } + } + + return true +} + +func (tc *TidbCluster) TiKVRealReplicas() int32 { + return tc.Spec.TiKV.Replicas + int32(len(tc.Status.TiKV.FailureStores)) +} + +func (tc *TidbCluster) TiDBAllPodsStarted() bool { + return tc.TiDBRealReplicas() == tc.Status.TiDB.StatefulSet.Replicas +} + +func (tc *TidbCluster) TiDBAllMembersReady() bool { + if int(tc.TiDBRealReplicas()) != len(tc.Status.TiDB.Members) { + return false + } + + for _, member := range tc.Status.TiDB.Members { + if !member.Health { + return false + } + } + + return true +} + +func (tc *TidbCluster) TiDBRealReplicas() int32 { + return tc.Spec.TiDB.Replicas + int32(len(tc.Status.TiDB.FailureMembers)) +} diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index a7f1bbe52a0..61046f645c3 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -43,7 +43,7 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo ns := tc.GetNamespace() result := map[string]bool{} - for i := 0; i < int(tc.Spec.TiDB.Replicas); i++ { + for i := 0; i < int(tc.TiDBRealReplicas()); i++ { hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), i) url := fmt.Sprintf("http://%s.%s-tidb-peer.%s:10080/status", hostName, tcName, ns) _, err := tdc.getBodyOK(url) @@ -89,6 +89,6 @@ func (ftd *FakeTiDBControl) SetHealth(healthInfo map[string]bool) { ftd.healthInfo = healthInfo } -func (ftd *FakeTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bool { +func (ftd *FakeTiDBControl) GetHealth(_ *v1alpha1.TidbCluster) map[string]bool { return ftd.healthInfo } diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go index be55f52a2ad..9763d0b3892 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control.go @@ -71,22 +71,15 @@ type defaultTidbClusterControl struct { // UpdateStatefulSet executes the core logic loop for a tidbcluster. func (tcc *defaultTidbClusterControl) UpdateTidbCluster(tc *v1alpha1.TidbCluster) error { // perform the main update function and get the status - oldStatus := tc.Status.DeepCopy() - oldPDReplicas := tc.Spec.PD.Replicas - oldTiKVReplicas := tc.Spec.TiKV.Replicas - oldTiDBReplicas := tc.Spec.TiDB.Replicas - var errs []error + err := tcc.updateTidbCluster(tc) if err != nil { errs = append(errs, err) } - replicasChanged := tc.Spec.PD.Replicas != oldPDReplicas || - tc.Spec.TiKV.Replicas != oldTiKVReplicas || - tc.Spec.TiDB.Replicas != oldTiDBReplicas - if !apiequality.Semantic.DeepEqual(&tc.Status, oldStatus) || replicasChanged { + if !apiequality.Semantic.DeepEqual(&tc.Status, oldStatus) { _, err := tcc.tcControl.UpdateTidbCluster(tc.DeepCopy()) if err != nil { errs = append(errs, err) diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 19415195bdc..66f2ed5d338 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -99,7 +99,7 @@ func NewController( podControl := controller.NewRealPodControl(kubeCli, pdControl, podInformer.Lister(), recorder) pdScaler := mm.NewPDScaler(pdControl, pvcInformer.Lister(), pvcControl) tikvScaler := mm.NewTiKVScaler(pdControl, pvcInformer.Lister(), pvcControl) - pdFailover := mm.NewPDFailover(cli, tcControl, pdControl, pdFailoverPeriod, podInformer.Lister(), podControl, pvcInformer.Lister(), pvcControl, pvInformer.Lister()) + pdFailover := mm.NewPDFailover(cli, pdControl, pdFailoverPeriod, podInformer.Lister(), podControl, pvcInformer.Lister(), pvcControl, pvInformer.Lister()) pdUpgrader := mm.NewPDUpgrader() tikvFailover := mm.NewTiKVFailover(pdControl) tikvUpgrader := mm.NewTiKVUpgrader() diff --git a/pkg/manager/member/pd_failover.go b/pkg/manager/member/pd_failover.go index 34bb7fb5313..10177547309 100644 --- a/pkg/manager/member/pd_failover.go +++ b/pkg/manager/member/pd_failover.go @@ -29,7 +29,6 @@ import ( // TODO add maxFailoverCount type pdFailover struct { cli versioned.Interface - tcControl controller.TidbClusterControlInterface pdControl controller.PDControlInterface pdFailoverPeriod time.Duration podLister corelisters.PodLister @@ -41,7 +40,6 @@ type pdFailover struct { // NewPDFailover returns a pd Failover func NewPDFailover(cli versioned.Interface, - tcControl controller.TidbClusterControlInterface, pdControl controller.PDControlInterface, pdFailoverPeriod time.Duration, podLister corelisters.PodLister, @@ -51,7 +49,6 @@ func NewPDFailover(cli versioned.Interface, pvLister corelisters.PersistentVolumeLister) Failover { return &pdFailover{ cli, - tcControl, pdControl, pdFailoverPeriod, podLister, @@ -64,23 +61,25 @@ func NewPDFailover(cli versioned.Interface, func (pf *pdFailover) Failover(tc *v1alpha1.TidbCluster) error { ns := tc.GetNamespace() tcName := tc.GetName() + healthCount := 0 for _, pdMember := range tc.Status.PD.Members { if pdMember.Health { healthCount++ } } - inQuorum := healthCount > int(tc.Spec.PD.Replicas/2) + inQuorum := healthCount > len(tc.Status.PD.Members)/2 if !inQuorum { - return fmt.Errorf("TidbCluster: %s/%s's pd cluster is not health: %d/%d, can't failover", - ns, tcName, healthCount, tc.Spec.PD.Replicas) + return fmt.Errorf("TidbCluster: %s/%s's pd cluster is not health: %d/%d, replicas: %d, failureCount: %d, can't failover", + ns, tcName, healthCount, tc.PDRealReplicas(), tc.Spec.PD.Replicas, len(tc.Status.PD.FailureMembers)) } + if tc.Status.PD.FailureMembers == nil { tc.Status.PD.FailureMembers = map[string]v1alpha1.PDFailureMember{} } notDeletedCount := 0 - for _, failureMember := range tc.Status.PD.FailureMembers { - if !failureMember.MemberDeleted { + for _, pdMember := range tc.Status.PD.FailureMembers { + if !pdMember.MemberDeleted { notDeletedCount++ } } @@ -98,101 +97,96 @@ func (pf *pdFailover) Failover(tc *v1alpha1.TidbCluster) error { if err != nil { return err } - break + + return controller.RequeueErrorf("marking Pod: %s/%s pd member: %s as failure", ns, podName, pdMember.Name) } } } - // invoke deleteMember api to delete a member from the pd cluster - for podName, failureMember := range tc.Status.PD.FailureMembers { - if !failureMember.MemberDeleted { - err := pf.pdControl.GetPDClient(tc).DeleteMember(failureMember.PodName) - if err != nil { - return err - } - failureMember.MemberDeleted = true - tc.Status.PD.FailureMembers[podName] = failureMember + + var failureMember *v1alpha1.PDFailureMember + var failurePodName string + for podName, pdMember := range tc.Status.PD.FailureMembers { + if !pdMember.MemberDeleted { + failureMember = &pdMember + failurePodName = podName break } } + if failureMember == nil { + return nil + } + + // invoke deleteMember api to delete a member from the pd cluster + err := pf.pdControl.GetPDClient(tc).DeleteMember(failureMember.PodName) + if err != nil { + return err + } + // The order of old PVC deleting and the new Pod creating is not guaranteed by Kubernetes. // If new Pod is created before old PVC deleted, new Pod will reuse old PVC. // So we must try to delete the PVC and Pod of this PD peer over and over, // and let StatefulSet create the new PD peer with the same ordinal, but don't use the tombstone PV - for podName, failureMember := range tc.Status.PD.FailureMembers { - if !failureMember.MemberDeleted { - continue - } - // increase the replicas to add a new PD peer - if failureMember.Replicas+1 > tc.Spec.PD.Replicas { - tc.Spec.PD.Replicas = failureMember.Replicas + 1 - } - pod, err := pf.podLister.Pods(ns).Get(podName) - if err != nil && !errors.IsNotFound(err) { - return err - } - ordinal, err := getOrdinalFromPodName(podName) - if err != nil { - return err - } - pvcName := ordinalPVCName(v1alpha1.PDMemberType, controller.PDMemberName(tcName), ordinal) - pvc, err := pf.pvcLister.PersistentVolumeClaims(ns).Get(pvcName) - if errors.IsNotFound(err) { - if pod != nil && pod.DeletionTimestamp == nil { - err := pf.podControl.DeletePod(tc, pod) - if err != nil { - return err - } + pod, err := pf.podLister.Pods(ns).Get(failurePodName) + if err != nil && !errors.IsNotFound(err) { + return err + } + + ordinal, err := getOrdinalFromPodName(failurePodName) + if err != nil { + return err + } + pvcName := ordinalPVCName(v1alpha1.PDMemberType, controller.PDMemberName(tcName), ordinal) + pvc, err := pf.pvcLister.PersistentVolumeClaims(ns).Get(pvcName) + if errors.IsNotFound(err) { + if pod != nil && pod.DeletionTimestamp == nil { + err := pf.podControl.DeletePod(tc, pod) + if err != nil { + return err } - continue } + setMemberDeleted(tc, failurePodName) + return nil + } + if err != nil { + return err + } + pv, err := pf.pvLister.Get(pvc.Spec.VolumeName) + if errors.IsNotFound(err) { + setMemberDeleted(tc, failurePodName) + return nil + } + if err != nil { + return err + } + if string(pv.UID) != string(failureMember.PVUID) { + setMemberDeleted(tc, failurePodName) + return nil + } + if pod != nil && pod.DeletionTimestamp == nil { + err := pf.podControl.DeletePod(tc, pod) if err != nil { return err } - pv, err := pf.pvLister.Get(pvc.Spec.VolumeName) - if errors.IsNotFound(err) { - continue - } + } + if pvc.DeletionTimestamp == nil { + err = pf.pvcControl.DeletePVC(tc, pvc) if err != nil { return err } - if string(pv.UID) != string(failureMember.PVUID) { - continue - } - if pod != nil && pod.DeletionTimestamp == nil { - err := pf.podControl.DeletePod(tc, pod) - if err != nil { - return err - } - } - if pvc.DeletionTimestamp == nil { - err = pf.pvcControl.DeletePVC(tc, pvc) - if err != nil { - return err - } - } } + + setMemberDeleted(tc, failurePodName) return nil } func (pf *pdFailover) Recover(tc *v1alpha1.TidbCluster) { - defer func() { - tc.Status.PD.FailureMembers = nil - }() - maxReplicas := int32(0) - minReplicas := int32(0) - for _, failureMember := range tc.Status.PD.FailureMembers { - if minReplicas == int32(0) { - minReplicas = failureMember.Replicas - } - if failureMember.Replicas > maxReplicas { - maxReplicas = failureMember.Replicas - } else if failureMember.Replicas < minReplicas { - minReplicas = failureMember.Replicas - } - } - if maxReplicas+1 == tc.Spec.PD.Replicas { - tc.Spec.PD.Replicas = minReplicas - } + tc.Status.PD.FailureMembers = nil +} + +func setMemberDeleted(tc *v1alpha1.TidbCluster, podName string) { + failureMember := tc.Status.PD.FailureMembers[podName] + failureMember.MemberDeleted = true + tc.Status.PD.FailureMembers[podName] = failureMember } func (pf *pdFailover) markThisMemberAsFailure(tc *v1alpha1.TidbCluster, pdMember v1alpha1.PDMember) error { @@ -212,6 +206,7 @@ func (pf *pdFailover) markThisMemberAsFailure(tc *v1alpha1.TidbCluster, pdMember if err != nil { return err } + if tc.Status.PD.FailureMembers == nil { tc.Status.PD.FailureMembers = map[string]v1alpha1.PDFailureMember{} } @@ -222,9 +217,8 @@ func (pf *pdFailover) markThisMemberAsFailure(tc *v1alpha1.TidbCluster, pdMember Replicas: tc.Spec.PD.Replicas, MemberDeleted: false, } - // we must update TidbCluster immediately before delete member, or data may not be consistent - tc, err = pf.tcControl.UpdateTidbCluster(tc) - return err + + return nil } func getOrdinalFromPodName(podName string) (int32, error) { @@ -236,18 +230,6 @@ func getOrdinalFromPodName(podName string) (int32, error) { return int32(ordinalInt), nil } -func allPDMembersAreReady(tc *v1alpha1.TidbCluster) bool { - if int(tc.Spec.PD.Replicas) != len(tc.Status.PD.Members) { - return false - } - for _, member := range tc.Status.PD.Members { - if !member.Health { - return false - } - } - return true -} - type fakePDFailover struct{} // NewFakePDFailover returns a fake Failover diff --git a/pkg/manager/member/pd_failover_test.go b/pkg/manager/member/pd_failover_test.go index d6a957104b1..c98872ef84c 100644 --- a/pkg/manager/member/pd_failover_test.go +++ b/pkg/manager/member/pd_failover_test.go @@ -22,7 +22,6 @@ import ( . "github.com/onsi/gomega" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned/fake" - informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -45,7 +44,6 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed bool delPodFailed bool delPVCFailed bool - tcUpdateFailed bool errExpectFn func(*GomegaWithT, error) expectFn func(*v1alpha1.TidbCluster) } @@ -54,7 +52,7 @@ func TestPDFailoverFailover(t *testing.T) { tc := newTidbClusterForPD() test.update(tc) - pdFailover, pvcIndexer, pvIndexer, podIndexer, fakePDControl, fakePodControl, fakePVCControl, fakeTCControl := newFakePDFailover() + pdFailover, pvcIndexer, pvIndexer, podIndexer, fakePDControl, fakePodControl, fakePVCControl := newFakePDFailover() pdClient := controller.NewFakePDClient() fakePDControl.SetPDClient(tc, pdClient) @@ -86,9 +84,6 @@ func TestPDFailoverFailover(t *testing.T) { if test.delPVCFailed { fakePVCControl.SetDeletePVCError(errors.NewInternalError(fmt.Errorf("API server failed")), 0) } - if test.tcUpdateFailed { - fakeTCControl.SetUpdateTidbClusterError(errors.NewInternalError(fmt.Errorf("API server failed")), 0) - } err := pdFailover.Failover(tc) test.errExpectFn(g, err) @@ -105,14 +100,14 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNil, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) }, }, { - name: "two members are not ready", + name: "two members are not ready, not in quorum", update: twoMembersNotReady, hasPVC: true, hasPV: true, @@ -121,13 +116,34 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: func(g *GomegaWithT, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "pd cluster is not health")).To(Equal(true)) }, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) + }, + }, + { + name: "two members are ready and a failure member", + update: oneFailureMember, + hasPVC: true, + hasPV: true, + hasPod: true, + podWithDeletionTimestamp: false, + delMemberFailed: false, + delPodFailed: false, + delPVCFailed: false, + errExpectFn: errExpectNil, + expectFn: func(tc *v1alpha1.TidbCluster) { + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(1)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + pd1, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(pd1.MemberDeleted).To(Equal(true)) + g.Expect(int(pd1.Replicas)).To(Equal(3)) }, }, { @@ -146,14 +162,14 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNil, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) }, }, { - name: "has one not ready member, lastTransitionTime is zero", + name: "has one not ready member, and exceed deadline, lastTransitionTime is zero", update: func(tc *v1alpha1.TidbCluster) { oneNotReadyMember(tc) pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) @@ -168,51 +184,58 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNil, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) }, }, { - name: "has one not ready member, and exceed deadline, update TidbCluster success", + name: "has one not ready member, and exceed deadline, don't have PVC, have PV", update: oneNotReadyMember, hasPVC: true, - hasPV: true, + hasPV: false, hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, - errExpectFn: errExpectNil, + errExpectFn: errExpectNotNil, expectFn: func(tc *v1alpha1.TidbCluster) { - pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) - pd1, ok := tc.Status.PD.FailureMembers[pd1Name] - g.Expect(ok).To(Equal(true)) - g.Expect(pd1.MemberDeleted).To(Equal(true)) - g.Expect(int(pd1.Replicas)).To(Equal(3)) + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) }, }, { - name: "has one not ready member, and exceed deadline, update TidbCluster failed", + name: "has one not ready member, and exceed deadline, have PVC, don't have PV", update: oneNotReadyMember, hasPVC: true, - hasPV: true, + hasPV: false, hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: true, - errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(strings.Contains(err.Error(), "failover ongoing")).NotTo(Equal(true)) + errExpectFn: errExpectNotNil, + expectFn: func(tc *v1alpha1.TidbCluster) { + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) }, + }, + { + name: "has one not ready member, and exceed deadline, return requeue error", + update: oneNotReadyMember, + hasPVC: true, + hasPV: true, + hasPod: true, + podWithDeletionTimestamp: false, + delMemberFailed: false, + delPodFailed: false, + delPVCFailed: false, + errExpectFn: errExpectRequeue, expectFn: func(tc *v1alpha1.TidbCluster) { - pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) pd1, ok := tc.Status.PD.FailureMembers[pd1Name] g.Expect(ok).To(Equal(true)) g.Expect(pd1.MemberDeleted).To(Equal(false)) @@ -220,8 +243,8 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "has one not ready member, and exceed deadline, don't have PVC", - update: oneNotReadyMember, + name: "has one not ready member, and exceed deadline, don't have PVC, has Pod, delete pod success", + update: oneNotReadyMemberAndAFailureMember, hasPVC: false, hasPV: true, hasPod: true, @@ -229,26 +252,34 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, - errExpectFn: errExpectNotNil, + errExpectFn: errExpectNil, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + pd1, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(pd1.MemberDeleted).To(Equal(true)) + g.Expect(int(pd1.Replicas)).To(Equal(3)) }, }, { - name: "has one not ready member, and exceed deadline, don't have PV", - update: oneNotReadyMember, - hasPVC: true, - hasPV: false, + name: "has one not ready member, and exceed deadline, don't have PVC, has Pod, delete pod failed", + update: oneNotReadyMemberAndAFailureMember, + hasPVC: false, + hasPV: true, hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, - delPodFailed: false, + delPodFailed: true, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNotNil, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + pd1, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(pd1.MemberDeleted).To(Equal(false)) + g.Expect(int(pd1.Replicas)).To(Equal(3)) }, }, { @@ -261,35 +292,18 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: true, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNotNil, expectFn: func(tc *v1alpha1.TidbCluster) { - pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) failMember, ok := tc.Status.PD.FailureMembers[pd1Name] g.Expect(ok).To(Equal(true)) g.Expect(failMember.MemberDeleted).To(Equal(false)) }, }, { - name: "one failure member, don't have pvc, don't have pod, increase the replicas", - update: oneFailureMember, - hasPVC: false, - hasPV: true, - hasPod: false, - podWithDeletionTimestamp: false, - delMemberFailed: false, - delPodFailed: false, - delPVCFailed: false, - tcUpdateFailed: false, - errExpectFn: errExpectNil, - expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) - }, - }, - { - name: "one failure member, don't have pvc, have pod with deletetimestamp, increase the replicas", - update: oneFailureMember, + name: "one failure member, don't have pvc, have pod with deletetimestamp", + update: oneNotReadyMemberAndAFailureMember, hasPVC: false, hasPV: true, hasPod: true, @@ -297,15 +311,18 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNil, expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + failMember, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(failMember.MemberDeleted).To(Equal(true)) }, }, { - name: "one failure member, don't have pvc, have pod without deletetimestamp, delete pod success, increase the replicas", - update: oneFailureMember, + name: "one failure member, don't have pvc, have pod without deletetimestamp, delete pod success", + update: oneNotReadyMemberAndAFailureMember, hasPVC: false, hasPV: true, hasPod: true, @@ -313,15 +330,18 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNil, expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + failMember, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(failMember.MemberDeleted).To(Equal(true)) }, }, { name: "one failure member, don't have pvc, have pod without deletetimestamp, delete pod failed", - update: oneFailureMember, + update: oneNotReadyMemberAndAFailureMember, hasPVC: false, hasPV: true, hasPod: true, @@ -329,15 +349,18 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: true, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNotNil, expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + failMember, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(failMember.MemberDeleted).To(Equal(false)) }, }, { - name: "one failure member, don't have pv, increase the replicas", - update: oneFailureMember, + name: "one failure member, don't have pv", + update: oneNotReadyMemberAndAFailureMember, hasPVC: true, hasPV: false, hasPod: true, @@ -345,16 +368,19 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNil, expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + failMember, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(failMember.MemberDeleted).To(Equal(true)) }, }, { - name: "one failure members, pv uid changed, increase the replicas", + name: "one failure members, pv uid changed", update: func(tc *v1alpha1.TidbCluster) { - oneFailureMember(tc) + oneNotReadyMemberAndAFailureMember(tc) pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) pd1 := tc.Status.PD.FailureMembers[pd1Name] pd1.PVUID = "xxx" @@ -367,31 +393,18 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: false, - tcUpdateFailed: false, errExpectFn: errExpectNil, expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) - }, - }, - { - name: "one failure members, has pod but delete fail", - update: oneFailureMember, - hasPVC: true, - hasPV: true, - hasPod: true, - podWithDeletionTimestamp: false, - delMemberFailed: false, - delPodFailed: true, - delPVCFailed: false, - tcUpdateFailed: false, - errExpectFn: errExpectNotNil, - expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + failMember, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(failMember.MemberDeleted).To(Equal(true)) }, }, { name: "one failure members, pvc delete fail", - update: oneFailureMember, + update: oneNotReadyMemberAndAFailureMember, hasPVC: true, hasPV: true, hasPod: true, @@ -399,10 +412,13 @@ func TestPDFailoverFailover(t *testing.T) { delMemberFailed: false, delPodFailed: false, delPVCFailed: true, - tcUpdateFailed: false, errExpectFn: errExpectNotNil, expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(4)) + g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) + pd1Name := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + failMember, ok := tc.Status.PD.FailureMembers[pd1Name] + g.Expect(ok).To(Equal(true)) + g.Expect(failMember.MemberDeleted).To(Equal(false)) }, }, } @@ -425,7 +441,7 @@ func TestPDFailoverRecovery(t *testing.T) { tc := newTidbClusterForPD() test.update(tc) - pdFailover, _, _, _, _, _, _, _ := newFakePDFailover() + pdFailover, _, _, _, _, _, _ := newFakePDFailover() pdFailover.Recover(tc) test.expectFn(tc) } @@ -434,7 +450,7 @@ func TestPDFailoverRecovery(t *testing.T) { name: "two failure member, user don't modify the replicas", update: func(tc *v1alpha1.TidbCluster) { twoFailureMembers(tc) - tc.Spec.PD.Replicas = 5 + tc.Spec.PD.Replicas = 3 }, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) @@ -478,7 +494,7 @@ func TestPDFailoverRecovery(t *testing.T) { name: "one failure member, user don't modify the replicas", update: func(tc *v1alpha1.TidbCluster) { oneFailureMember(tc) - tc.Spec.PD.Replicas = 4 + tc.Spec.PD.Replicas = 3 }, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.PD.Replicas)).To(Equal(3)) @@ -514,22 +530,19 @@ func TestPDFailoverRecovery(t *testing.T) { } } -func newFakePDFailover() (*pdFailover, cache.Indexer, cache.Indexer, cache.Indexer, *controller.FakePDControl, *controller.FakePodControl, *controller.FakePVCControl, *controller.FakeTidbClusterControl) { +func newFakePDFailover() (*pdFailover, cache.Indexer, cache.Indexer, cache.Indexer, *controller.FakePDControl, *controller.FakePodControl, *controller.FakePVCControl) { cli := fake.NewSimpleClientset() kubeCli := kubefake.NewSimpleClientset() pdControl := controller.NewFakePDControl() kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeCli, 0) - tcInformer := informers.NewSharedInformerFactory(cli, 0).Pingcap().V1alpha1().TidbClusters() podInformer := kubeInformerFactory.Core().V1().Pods() pvcInformer := kubeInformerFactory.Core().V1().PersistentVolumeClaims() pvInformer := kubeInformerFactory.Core().V1().PersistentVolumes() podControl := controller.NewFakePodControl(podInformer) pvcControl := controller.NewFakePVCControl(pvcInformer) - tcControl := controller.NewFakeTidbClusterControl(tcInformer) return &pdFailover{ cli, - tcControl, pdControl, 5 * time.Minute, podInformer.Lister(), @@ -540,7 +553,7 @@ func newFakePDFailover() (*pdFailover, cache.Indexer, cache.Indexer, cache.Index pvcInformer.Informer().GetIndexer(), pvInformer.Informer().GetIndexer(), podInformer.Informer().GetIndexer(), - pdControl, podControl, pvcControl, tcControl + pdControl, podControl, pvcControl } func oneFailureMember(tc *v1alpha1.TidbCluster) { @@ -552,7 +565,7 @@ func oneFailureMember(tc *v1alpha1.TidbCluster) { pd2: {Name: pd2, ID: "2", Health: true}, } tc.Status.PD.FailureMembers = map[string]v1alpha1.PDFailureMember{ - pd1: {Replicas: 3, PVUID: "uid-1"}, + pd1: {Replicas: 3}, } } @@ -580,6 +593,20 @@ func oneNotReadyMember(tc *v1alpha1.TidbCluster) { } } +func oneNotReadyMemberAndAFailureMember(tc *v1alpha1.TidbCluster) { + pd0 := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 0) + pd1 := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) + pd2 := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 2) + tc.Status.PD.Members = map[string]v1alpha1.PDMember{ + pd0: {Name: pd0, ID: "0", Health: true}, + pd1: {Name: pd1, ID: "1", Health: false, LastTransitionTime: metav1.Time{Time: time.Now().Add(-10 * time.Minute)}}, + pd2: {Name: pd2, ID: "2", Health: true}, + } + tc.Status.PD.FailureMembers = map[string]v1alpha1.PDFailureMember{ + pd1: {Replicas: 3, PVUID: "uid-1"}, + } +} + func allMembersReady(tc *v1alpha1.TidbCluster) { pd0 := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 0) pd1 := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 1) diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index 31932f595ab..e2e7d05853e 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -256,11 +256,9 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu } if pmm.autoFailover { - if allPDMembersAreReady(tc) { - if tc.Status.PD.FailureMembers != nil { - pmm.pdFailover.Recover(tc) - } - } else if tc.Spec.PD.Replicas == tc.Status.PD.StatefulSet.Replicas { + if tc.PDAllPodsStarted() && tc.PDAllMembersReady() && tc.Status.PD.FailureMembers != nil { + pmm.pdFailover.Recover(tc) + } else if tc.PDAllPodsStarted() && !tc.PDAllMembersReady() || tc.PDAutoFailovering() { if err := pmm.pdFailover.Failover(tc); err != nil { return err } @@ -445,6 +443,12 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) if storageClassName == "" { storageClassName = controller.DefaultStorageClassName } + failureReplicas := 0 + for _, failureMember := range tc.Status.PD.FailureMembers { + if failureMember.MemberDeleted { + failureReplicas++ + } + } pdSet := &apps.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -454,7 +458,7 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, }, Spec: apps.StatefulSetSpec{ - Replicas: func() *int32 { r := tc.Spec.PD.Replicas; return &r }(), + Replicas: func() *int32 { r := tc.Spec.PD.Replicas + int32(failureReplicas); return &r }(), Selector: pdLabel.LabelSelector(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/manager/member/tidb_failover.go b/pkg/manager/member/tidb_failover.go index 84628e1e3e3..8d8a589b592 100644 --- a/pkg/manager/member/tidb_failover.go +++ b/pkg/manager/member/tidb_failover.go @@ -44,63 +44,13 @@ func (tf *tidbFailover) Failover(tc *v1alpha1.TidbCluster) error { } } - // increase the replicas to add a new TiDB peer - for _, failureMember := range tc.Status.TiDB.FailureMembers { - if failureMember.Replicas+1 > tc.Spec.TiDB.Replicas { - tc.Spec.TiDB.Replicas = failureMember.Replicas + 1 - } - } - return nil } func (tf *tidbFailover) Recover(tc *v1alpha1.TidbCluster) { - maxReplicas := int32(0) - minReplicas := int32(0) - for _, failureMember := range tc.Status.TiDB.FailureMembers { - if minReplicas == int32(0) { - minReplicas = failureMember.Replicas - } - if failureMember.Replicas > maxReplicas { - maxReplicas = failureMember.Replicas - } else if failureMember.Replicas < minReplicas { - minReplicas = failureMember.Replicas - } - } - if maxReplicas+1 == tc.Spec.TiDB.Replicas { - tc.Spec.TiDB.Replicas = minReplicas - } - tc.Status.TiDB.FailureMembers = nil } -func needRecover(tc *v1alpha1.TidbCluster) bool { - if int(tc.Spec.TiDB.Replicas) != len(tc.Status.TiDB.Members) { - return false - } - for _, tidbMember := range tc.Status.TiDB.Members { - if !tidbMember.Health { - return false - } - } - if tc.Status.TiDB.FailureMembers == nil || len(tc.Status.TiDB.FailureMembers) == 0 { - return false - } - return true -} - -func needFailover(tc *v1alpha1.TidbCluster) bool { - if tc.Spec.TiDB.Replicas != tc.Status.TiDB.StatefulSet.Replicas { - return false - } - for _, tidbMember := range tc.Status.TiDB.Members { - if !tidbMember.Health { - return true - } - } - return false -} - type fakeTiDBFailover struct{} // NewFakeTiDBFailover returns a fake Failover @@ -108,10 +58,10 @@ func NewFakeTiDBFailover() Failover { return &fakeTiDBFailover{} } -func (ftf *fakeTiDBFailover) Failover(tc *v1alpha1.TidbCluster) error { +func (ftf *fakeTiDBFailover) Failover(_ *v1alpha1.TidbCluster) error { return nil } -func (ftf *fakeTiDBFailover) Recover(tc *v1alpha1.TidbCluster) { +func (ftf *fakeTiDBFailover) Recover(_ *v1alpha1.TidbCluster) { return } diff --git a/pkg/manager/member/tidb_failover_test.go b/pkg/manager/member/tidb_failover_test.go index 71cc87f66c5..3301384d8fb 100644 --- a/pkg/manager/member/tidb_failover_test.go +++ b/pkg/manager/member/tidb_failover_test.go @@ -87,7 +87,7 @@ func TestFakeTiDBFailoverFailover(t *testing.T) { }, expectFn: func(t *GomegaWithT, tc *v1alpha1.TidbCluster) { t.Expect(len(tc.Status.TiDB.FailureMembers)).To(Equal(1)) - t.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(3)) + t.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(2)) }, }, { @@ -109,7 +109,7 @@ func TestFakeTiDBFailoverFailover(t *testing.T) { }, expectFn: func(t *GomegaWithT, tc *v1alpha1.TidbCluster) { t.Expect(len(tc.Status.TiDB.FailureMembers)).To(Equal(1)) - t.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(3)) + t.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(2)) }, }, } @@ -183,7 +183,7 @@ func TestFakeTiDBFailoverRecover(t *testing.T) { } }, expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(2)) + g.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(3)) g.Expect(len(tc.Status.TiDB.FailureMembers)).To(Equal(0)) }, }, @@ -221,7 +221,7 @@ func TestFakeTiDBFailoverRecover(t *testing.T) { } }, expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(2)) + g.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(4)) g.Expect(len(tc.Status.TiDB.FailureMembers)).To(Equal(0)) }, }, diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 170cab28ac9..997cfee9e92 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -140,11 +140,9 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid } if tmm.autoFailover { - if needRecover(tc) { + if tc.TiDBAllPodsStarted() && tc.TiDBAllMembersReady() && tc.Status.TiDB.FailureMembers != nil { tmm.tidbFailover.Recover(tc) - } - - if needFailover(tc) { + } else if tc.TiDBAllPodsStarted() && !tc.TiDBAllMembersReady() { if err := tmm.tidbFailover.Failover(tc); err != nil { return err } @@ -258,7 +256,6 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust } tidbLabel := label.New().Cluster(tcName).TiDB() - tidbSet := &apps.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: controller.TiDBMemberName(tcName), @@ -267,7 +264,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, }, Spec: apps.StatefulSetSpec{ - Replicas: func() *int32 { r := tc.Spec.TiDB.Replicas; return &r }(), + Replicas: func() *int32 { r := tc.TiDBRealReplicas(); return &r }(), Selector: tidbLabel.LabelSelector(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/manager/member/tikv_failover.go b/pkg/manager/member/tikv_failover.go index 3a79c3972b2..f6612d905d1 100644 --- a/pkg/manager/member/tikv_failover.go +++ b/pkg/manager/member/tikv_failover.go @@ -57,27 +57,14 @@ func (tf *tikvFailover) Failover(tc *v1alpha1.TidbCluster) error { StoreID: store.ID, Replicas: tc.Spec.TiKV.Replicas, } - tc.Spec.TiKV.Replicas++ } } return nil } -func (tf *tikvFailover) Recover(tc *v1alpha1.TidbCluster) { - tc.Status.TiKV.FailureStores = nil -} - -func allTiKVStoresAreReady(tc *v1alpha1.TidbCluster) bool { - if int(tc.Spec.TiKV.Replicas) != len(tc.Status.TiKV.Stores) { - return false - } - for _, store := range tc.Status.TiKV.Stores { - if store.State != v1alpha1.TiKVStateUp { - return false - } - } - return true +func (tf *tikvFailover) Recover(_ *v1alpha1.TidbCluster) { + // Do nothing now } type fakeTiKVFailover struct{} diff --git a/pkg/manager/member/tikv_failover_test.go b/pkg/manager/member/tikv_failover_test.go index efb97dd5506..b5be0ca0a80 100644 --- a/pkg/manager/member/tikv_failover_test.go +++ b/pkg/manager/member/tikv_failover_test.go @@ -82,7 +82,8 @@ func TestTiKVFailoverFailover(t *testing.T) { getCfgErr: false, err: false, expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(int(tc.Spec.TiKV.Replicas)).To(Equal(5)) + g.Expect(int(tc.Spec.TiKV.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(2)) }, }, { @@ -92,6 +93,7 @@ func TestTiKVFailoverFailover(t *testing.T) { err: true, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.TiKV.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(0)) }, }, { @@ -105,6 +107,7 @@ func TestTiKVFailoverFailover(t *testing.T) { err: false, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.TiKV.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(0)) }, }, { @@ -122,6 +125,7 @@ func TestTiKVFailoverFailover(t *testing.T) { err: false, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.TiKV.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(0)) }, }, { @@ -138,6 +142,7 @@ func TestTiKVFailoverFailover(t *testing.T) { err: false, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.TiKV.Replicas)).To(Equal(3)) + g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(0)) }, }, { @@ -161,33 +166,7 @@ func TestTiKVFailoverFailover(t *testing.T) { err: false, expectFn: func(tc *v1alpha1.TidbCluster) { g.Expect(int(tc.Spec.TiKV.Replicas)).To(Equal(3)) - }, - }, - } - for i := range tests { - testFn(&tests[i], t) - } -} - -func TestTiKVFailoverRecover(t *testing.T) { - g := NewGomegaWithT(t) - - type testcase struct { - name string - expectFn func(*v1alpha1.TidbCluster) - } - testFn := func(test *testcase, t *testing.T) { - t.Log(test.name) - tc := newTidbClusterForPD() - tikvFailover, _ := newFakeTiKVFailover() - tikvFailover.Recover(tc) - test.expectFn(tc) - } - tests := []testcase{ - { - name: "normal", - expectFn: func(tc *v1alpha1.TidbCluster) { - g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(0)) + g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(1)) }, }, } diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index c1962856965..8acf3839b87 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -193,11 +193,7 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl } if tkmm.autoFailover { - if allTiKVStoresAreReady(tc) { - if tc.Status.TiKV.FailureStores != nil { - tkmm.tikvFailover.Recover(tc) - } - } else if tc.Spec.TiKV.Replicas == tc.Status.TiKV.StatefulSet.Replicas { + if tc.TiKVAllPodsStarted() && !tc.TiKVAllStoresReady() { if err := tkmm.tikvFailover.Failover(tc); err != nil { return err } @@ -312,7 +308,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, }, Spec: apps.StatefulSetSpec{ - Replicas: func() *int32 { r := tc.Spec.TiKV.Replicas; return &r }(), + Replicas: func() *int32 { r := tc.TiKVRealReplicas(); return &r }(), Selector: tikvLabel.LabelSelector(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -468,7 +464,7 @@ func (tkmm *tikvMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, s // avoid LastHeartbeatTime be overwrite by zero time when pd lost LastHeartbeatTime if status.LastHeartbeatTime.IsZero() { if oldStatus, ok := previousStores[status.ID]; ok { - glog.Warningf("the pod:%s's store LastHeartbeatTime is zero,so will keep in %v", status.PodName, oldStatus.LastHeartbeatTime) + glog.V(4).Infof("the pod:%s's store LastHeartbeatTime is zero,so will keep in %v", status.PodName, oldStatus.LastHeartbeatTime) status.LastHeartbeatTime = oldStatus.LastHeartbeatTime } }