Failover fix: no longer modify spec.Replicas (#95)

* pd failover: no longer modify TidbCluster.Spec.PD.Replicas * add RequeueError hint * use RequeueError * tikv and tidb failover fix * make govet happy
pingcap · Sep 20, 2018 · b1731d7 · b1731d7
1 parent 0002396
commit b1731d7
Show file tree

Hide file tree

Showing 13 changed files with 324 additions and 331 deletions.
diff --git a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go
@@ -17,10 +17,88 @@ func (mt MemberType) String() string {
 	return string(mt)
 }
 
-func (tc TidbCluster) PDUpgrading() bool {
+func (tc *TidbCluster) PDUpgrading() bool {
 	return tc.Status.PD.Phase == UpgradePhase
 }
 
-func (tc TidbCluster) TiKVUpgrading() bool {
+func (tc *TidbCluster) TiKVUpgrading() bool {
 	return tc.Status.TiKV.Phase == UpgradePhase
 }
+
+func (tc *TidbCluster) PDAllPodsStarted() bool {
+	return tc.PDRealReplicas() == tc.Status.PD.StatefulSet.Replicas
+}
+
+func (tc *TidbCluster) PDAllMembersReady() bool {
+	if int(tc.PDRealReplicas()) != len(tc.Status.PD.Members) {
+		return false
+	}
+
+	for _, member := range tc.Status.PD.Members {
+		if !member.Health {
+			return false
+		}
+	}
+	return true
+}
+
+func (tc *TidbCluster) PDAutoFailovering() bool {
+	if len(tc.Status.PD.FailureMembers) == 0 {
+		return false
+	}
+
+	for _, failureMember := range tc.Status.PD.FailureMembers {
+		if !failureMember.MemberDeleted {
+			return true
+		}
+	}
+	return false
+}
+
+func (tc *TidbCluster) PDRealReplicas() int32 {
+	return tc.Spec.PD.Replicas + int32(len(tc.Status.PD.FailureMembers))
+}
+
+func (tc *TidbCluster) TiKVAllPodsStarted() bool {
+	return tc.TiKVRealReplicas() == tc.Status.TiKV.StatefulSet.Replicas
+}
+
+func (tc *TidbCluster) TiKVAllStoresReady() bool {
+	if int(tc.TiKVRealReplicas()) != len(tc.Status.TiKV.Stores) {
+		return false
+	}
+
+	for _, store := range tc.Status.TiKV.Stores {
+		if store.State != TiKVStateUp {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (tc *TidbCluster) TiKVRealReplicas() int32 {
+	return tc.Spec.TiKV.Replicas + int32(len(tc.Status.TiKV.FailureStores))
+}
+
+func (tc *TidbCluster) TiDBAllPodsStarted() bool {
+	return tc.TiDBRealReplicas() == tc.Status.TiDB.StatefulSet.Replicas
+}
+
+func (tc *TidbCluster) TiDBAllMembersReady() bool {
+	if int(tc.TiDBRealReplicas()) != len(tc.Status.TiDB.Members) {
+		return false
+	}
+
+	for _, member := range tc.Status.TiDB.Members {
+		if !member.Health {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (tc *TidbCluster) TiDBRealReplicas() int32 {
+	return tc.Spec.TiDB.Replicas + int32(len(tc.Status.TiDB.FailureMembers))
+}
diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go
@@ -43,7 +43,7 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo
 	ns := tc.GetNamespace()
 
 	result := map[string]bool{}
-	for i := 0; i < int(tc.Spec.TiDB.Replicas); i++ {
+	for i := 0; i < int(tc.TiDBRealReplicas()); i++ {
 		hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), i)
 		url := fmt.Sprintf("http://%s.%s-tidb-peer.%s:10080/status", hostName, tcName, ns)
 		_, err := tdc.getBodyOK(url)
@@ -89,6 +89,6 @@ func (ftd *FakeTiDBControl) SetHealth(healthInfo map[string]bool) {
 	ftd.healthInfo = healthInfo
 }
 
-func (ftd *FakeTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bool {
+func (ftd *FakeTiDBControl) GetHealth(_ *v1alpha1.TidbCluster) map[string]bool {
 	return ftd.healthInfo
 }
diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go
@@ -71,22 +71,15 @@ type defaultTidbClusterControl struct {
 // UpdateStatefulSet executes the core logic loop for a tidbcluster.
 func (tcc *defaultTidbClusterControl) UpdateTidbCluster(tc *v1alpha1.TidbCluster) error {
 	// perform the main update function and get the status
-
 	oldStatus := tc.Status.DeepCopy()
-	oldPDReplicas := tc.Spec.PD.Replicas
-	oldTiKVReplicas := tc.Spec.TiKV.Replicas
-	oldTiDBReplicas := tc.Spec.TiDB.Replicas
-
 	var errs []error
+
 	err := tcc.updateTidbCluster(tc)
 	if err != nil {
 		errs = append(errs, err)
 	}
 
-	replicasChanged := tc.Spec.PD.Replicas != oldPDReplicas ||
-		tc.Spec.TiKV.Replicas != oldTiKVReplicas ||
-		tc.Spec.TiDB.Replicas != oldTiDBReplicas
-	if !apiequality.Semantic.DeepEqual(&tc.Status, oldStatus) || replicasChanged {
+	if !apiequality.Semantic.DeepEqual(&tc.Status, oldStatus) {
 		_, err := tcc.tcControl.UpdateTidbCluster(tc.DeepCopy())
 		if err != nil {
 			errs = append(errs, err)

diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go
@@ -99,7 +99,7 @@ func NewController(
 	podControl := controller.NewRealPodControl(kubeCli, pdControl, podInformer.Lister(), recorder)
 	pdScaler := mm.NewPDScaler(pdControl, pvcInformer.Lister(), pvcControl)
 	tikvScaler := mm.NewTiKVScaler(pdControl, pvcInformer.Lister(), pvcControl)
-	pdFailover := mm.NewPDFailover(cli, tcControl, pdControl, pdFailoverPeriod, podInformer.Lister(), podControl, pvcInformer.Lister(), pvcControl, pvInformer.Lister())
+	pdFailover := mm.NewPDFailover(cli, pdControl, pdFailoverPeriod, podInformer.Lister(), podControl, pvcInformer.Lister(), pvcControl, pvInformer.Lister())
 	pdUpgrader := mm.NewPDUpgrader()
 	tikvFailover := mm.NewTiKVFailover(pdControl)
 	tikvUpgrader := mm.NewTiKVUpgrader()