Feature: telemetry module & important events recording

Dockerfile

@@ -15,6 +15,6 @@ FROM openanolis/anolisos:8.4-x86_64
 WORKDIR /
 COPY --from=builder /workspace/manager .
 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-USER 65532:65532
+USER 65534:65534
 
 ENTRYPOINT ["/manager"]

api/v1alpha1/obcluster_webhook.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// log is for logging in this package.
+var obclusterlog = logf.Log.WithName("obcluster-resource")
+
+func (r *OBCluster) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(r).
+		Complete()
+}
+
+// TODO(user): EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+
+//+kubebuilder:webhook:path=/mutate-oceanbase-oceanbase-com-v1alpha1-obcluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=oceanbase.oceanbase.com,resources=obclusters,verbs=create;update,versions=v1alpha1,name=mobcluster.kb.io,admissionReviewVersions=v1
+
+var _ webhook.Defaulter = &OBCluster{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func (r *OBCluster) Default() {
+	// TODO(user): fill in your defaulting logic.
+}
+
+// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
+//+kubebuilder:webhook:path=/validate-oceanbase-oceanbase-com-v1alpha1-obcluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=oceanbase.oceanbase.com,resources=obclusters,verbs=create;update,versions=v1alpha1,name=vobcluster.kb.io,admissionReviewVersions=v1
+
+var _ webhook.Validator = &OBCluster{}
+
+// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
+func (r *OBCluster) ValidateCreate() (admission.Warnings, error) {
+	obclusterlog.Info("validate create", "name", r.Name)
+
+	return nil, r.validateMutation()
+}
+
+// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
+func (r *OBCluster) ValidateUpdate(old runtime.Object) (admission.Warnings, error) {
+	_ = old
+	obclusterlog.Info("validate update", "name", r.Name)
+
+	return nil, r.validateMutation()
+}
+
+// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
+func (r *OBCluster) ValidateDelete() (admission.Warnings, error) {
+	return nil, nil
+}
+
+func (r *OBCluster) validateMutation() error {
+	// Ignore deleting objects
+	if r.GetDeletionTimestamp() != nil {
+		return nil
+	}
+
+	var allErrs field.ErrorList
+
+	// 0. Validate userSecrets
+	if err := r.checkSecretExistence(r.Namespace, r.Spec.UserSecrets.Root, "root"); err != nil {
+		allErrs = append(allErrs, err)
+	}
+	if err := r.checkSecretExistence(r.Namespace, r.Spec.UserSecrets.ProxyRO, "proxyro"); err != nil {
+		allErrs = append(allErrs, err)
+	}
+	if err := r.checkSecretExistence(r.Namespace, r.Spec.UserSecrets.Operator, "operator"); err != nil {
+		allErrs = append(allErrs, err)
+	}
+	if err := r.checkSecretExistence(r.Namespace, r.Spec.UserSecrets.Monitor, "monitor"); err != nil {
+		allErrs = append(allErrs, err)
+	}
+
+	if len(allErrs) == 0 {
+		return nil
+	}
+	return apierrors.NewInvalid(GroupVersion.WithKind("OBCluster").GroupKind(), r.Name, allErrs)
+}
+
+func (r *OBCluster) checkSecretExistence(ns, secretName, fieldName string) *field.Error {
+	if secretName == "" {
+		return field.Invalid(field.NewPath("spec").Child("userSecrets").Child(fieldName), secretName, fmt.Sprintf("Empty credential %s is not permitted", fieldName))
+	}
+	secret := &v1.Secret{}
+	err := tenantClt.Get(context.Background(), types.NamespacedName{
+		Namespace: ns,
+		Name:      secretName,
+	}, secret)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return field.Invalid(field.NewPath("spec").Child("userSecrets").Child(fieldName), secretName, fmt.Sprintf("Given %s credential %s not found", fieldName, secretName))
+		}
+		return field.InternalError(field.NewPath("spec").Child("userSecrets").Child(fieldName), err)
+	}
+	return nil
+}

api/v1alpha1/obtenant_webhook.go

@@ -97,6 +97,51 @@ func (r *OBTenant) validateMutation() error {
 	}
 	var allErrs field.ErrorList
 
+	// 0. OBCluster must exist
+	cluster := &OBCluster{}
+	err := tenantClt.Get(context.Background(), types.NamespacedName{
+		Namespace: r.GetNamespace(),
+		Name:      r.Spec.ClusterName,
+	}, cluster)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("clusterName"), r.Spec.ClusterName, "Given cluster not found"))
+		} else {
+			allErrs = append(allErrs, field.InternalError(field.NewPath("spec").Child("clusterName"), err))
+		}
+	}
+
+	// 0. Given credentials must exist
+	if r.Spec.Credentials.Root != "" {
+		secret := &v1.Secret{}
+		err = tenantClt.Get(context.Background(), types.NamespacedName{
+			Namespace: r.GetNamespace(),
+			Name:      r.Spec.Credentials.Root,
+		}, secret)
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("credentials").Child("root"), r.Spec.Credentials.Root, "Given root credential not found"))
+			} else {
+				allErrs = append(allErrs, field.InternalError(field.NewPath("spec").Child("credentials").Child("root"), err))
+			}
+		}
+	}
+
+	if r.Spec.Credentials.StandbyRO != "" {
+		secret := &v1.Secret{}
+		err = tenantClt.Get(context.Background(), types.NamespacedName{
+			Namespace: r.GetNamespace(),
+			Name:      r.Spec.Credentials.StandbyRO,
+		}, secret)
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("credentials").Child("standbyRo"), r.Spec.Credentials.StandbyRO, "Given standbyRo credential not found"))
+			} else {
+				allErrs = append(allErrs, field.InternalError(field.NewPath("spec").Child("credentials").Child("standbyRo"), err))
+			}
+		}
+	}
+
 	// 1. Standby tenant must have a source
 	if r.Spec.TenantRole == constants.TenantRoleStandby {
 		if r.Spec.Source == nil {

api/v1alpha1/obtenantbackuppolicy_webhook.go

@@ -185,6 +185,19 @@ func (r *OBTenantBackupPolicy) validateBackupPolicy() error {
 	if r.Spec.ObClusterName == "" {
 		return errors.New("obClusterName is required")
 	}
+
+	cluster := &OBCluster{}
+	err := tenantClt.Get(context.Background(), types.NamespacedName{
+		Namespace: r.GetNamespace(),
+		Name:      r.Spec.ObClusterName,
+	}, cluster)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return field.Invalid(field.NewPath("spec").Child("clusterName"), r.Spec.ObClusterName, "Given cluster not found")
+		}
+		return field.InternalError(field.NewPath("spec").Child("clusterName"), err)
+	}
+
 	if r.Spec.TenantName == "" && r.Spec.TenantCRName == "" {
 		return field.Invalid(field.NewPath("spec").Child("[tenantName | tenantCRName]"), r.Spec.TenantName, "tenantName and tenantCRName are both empty")
 	}
@@ -281,7 +294,7 @@ func (r *OBTenantBackupPolicy) validateBackupPolicy() error {
 		}
 	}
 
-	err := r.validateBackupCrontab()
+	err = r.validateBackupCrontab()
 	if err != nil {
 		return err
 	}

api/v1alpha1/webhook_suite_test.go

@@ -108,6 +108,9 @@ var _ = BeforeSuite(func() {
 	err = (&OBTenantOperation{}).SetupWebhookWithManager(mgr)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = (&OBCluster{}).SetupWebhookWithManager(mgr)
+	Expect(err).NotTo(HaveOccurred())
+
 	//+kubebuilder:scaffold:webhook
 
 	go func() {

charts/oceanbase-cluster/templates/NOTES.txt

@@ -0,0 +1,11 @@
+Welcome to OceanBase Cluster! 
+
+After installing OBCluster chart, you need to wait for the cluster bootstrapped. Bootstrap progress will cost approximately 2~3 minutes which may varies depends on the machine.
+
+You can use the following command to wait for the OBCluster to be ready.
+
+> kubectl wait -n {{ .Release.Namespace }} obcluster {{ .Release.Name }} --for=jsonpath='{.status.status}'=running --timeout=10m
+
+After that, the cluster is ready to handle connections stably. Example command is following:
+
+> mysql -A -h$(kubectl get pods -l ref-obcluster={{ .Release.Name }} -o jsonpath='{.items[0].status.podIP}') -P2881 -uroot -p$(kubectl get secret -n {{ .Release.Namespace }} {{ .Values.userSecrets.root }} -o jsonpath='{.data.password}' | base64 -d)

charts/oceanbase-cluster/templates/obcluster.yaml

@@ -37,10 +37,9 @@ spec:
     - name: {{ $param.name }}
       value: {{ $param.value | quote }}
   {{- end }}
-  {{- if .Values.nfsBackupEnabled }}
+  {{- if .Values.backupVolumeEnabled }}
   backupVolume:
     volume: 
       name: backup
-      nfs:
-      {{- toYaml .Values.nfsBackup | nindent 8 }}
+      {{- toYaml .Values.backupVolume | nindent 6 }}
   {{- end }}

charts/oceanbase-cluster/values.yaml

@@ -20,12 +20,15 @@ monitorResource:
 parameters:
   - name: system_memory
     value: 2G
+  - name: "__min_full_resource_pool_memory"
+    value: "2147483648" # 2G
 
-nfsBackupEnabled: false # set true and config volume if you want to enable backup with NFS
-nfsBackup:
-  server: 1.1.1.1
-  path: /opt/nfs
-  readOnly: false
+backupVolumeEnabled: false # set true and config volume if you want to enable backup volume
+backupVolume:
+  nfs:
+    server: 1.1.1.1
+    path: /opt/nfs
+    readOnly: false
 
 generateUserSecrets: true # if set true, all system user secrets will be generated automatically
 userSecrets:

cmd/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"os"
 
@@ -34,6 +35,7 @@ import (
 	v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1"
 	"github.com/oceanbase/ob-operator/pkg/controller"
 	"github.com/oceanbase/ob-operator/pkg/controller/config"
+	"github.com/oceanbase/ob-operator/pkg/telemetry"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -147,7 +149,7 @@ func main() {
 	if err = (&controller.OBTenantBackupReconciler{
 		Client:   mgr.GetClient(),
 		Scheme:   mgr.GetScheme(),
-		Recorder: mgr.GetEventRecorderFor(config.OBTenantBackupControllerName),
+		Recorder: telemetry.NewRecorder(context.Background(), mgr.GetEventRecorderFor(config.OBTenantBackupControllerName)),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "OBTenantBackup")
 		os.Exit(1)
@@ -160,18 +162,6 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "OBTenantRestore")
 		os.Exit(1)
 	}
-	// if err = (&v1alpha1.OBCluster{}).SetupWebhookWithManager(mgr); err != nil {
-	// 	setupLog.Error(err, "unable to create webhook", "webhook", "OBCluster")
-	// 	os.Exit(1)
-	// }
-	// if err = (&v1alpha1.OBZone{}).SetupWebhookWithManager(mgr); err != nil {
-	// 	setupLog.Error(err, "unable to create webhook", "webhook", "OBZone")
-	// 	os.Exit(1)
-	// }
-	// if err = (&v1alpha1.OBServer{}).SetupWebhookWithManager(mgr); err != nil {
-	// 	setupLog.Error(err, "unable to create webhook", "webhook", "OBServer")
-	// 	os.Exit(1)
-	// }
 	if err = (&controller.OBTenantBackupPolicyReconciler{
 		Client:   mgr.GetClient(),
 		Scheme:   mgr.GetScheme(),
@@ -200,6 +190,10 @@ func main() {
 		setupLog.Error(err, "unable to create webhook", "webhook", "OBTenantOperation")
 		os.Exit(1)
 	}
+	if err = (&v1alpha1.OBCluster{}).SetupWebhookWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "OBCluster")
+		os.Exit(1)
+	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
@@ -211,6 +205,9 @@ func main() {
 		os.Exit(1)
 	}
 
+	rcd := telemetry.NewRecorder(context.Background(), mgr.GetEventRecorderFor("ob-operator"))
+	rcd.GenerateTelemetryRecord(nil, telemetry.ObjectTypeOperator, "Start", "", "start ob-operator", nil)
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

config/crd/kustomization.yaml

@@ -19,7 +19,7 @@ resources:
 patchesStrategicMerge:
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
 # patches here are for enabling the conversion webhook for each CRD
-# - patches/webhook_in_obclusters.yaml
+- patches/webhook_in_obclusters.yaml
 # - patches/webhook_in_obzones.yaml
 # - patches/webhook_in_observers.yaml
 # - patches/webhook_in_obparameters.yaml
@@ -35,7 +35,7 @@ patchesStrategicMerge:
 
 # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
 # patches here are for enabling the CA injection for each CRD
-# - patches/cainjection_in_obclusters.yaml
+- patches/cainjection_in_obclusters.yaml
 # - patches/cainjection_in_obzones.yaml
 # - patches/cainjection_in_observers.yaml
 # - patches/cainjection_in_obparameters.yaml

config/default/kustomization.yaml

@@ -31,7 +31,7 @@ patchesStrategicMerge:
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
 - manager_auth_proxy_patch.yaml
-
+- manager_config_patch.yaml
 
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in

config/default/manager_auth_proxy_patch.yaml

@@ -8,22 +8,6 @@ metadata:
 spec:
   template:
     spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                - key: kubernetes.io/arch
-                  operator: In
-                  values:
-                    - amd64
-                    - arm64
-                    - ppc64le
-                    - s390x
-                - key: kubernetes.io/os
-                  operator: In
-                  values:
-                    - linux
       containers:
       - name: kube-rbac-proxy
         securityContext:

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: oceanbasedev/ob-operator
-  newTag: 2.0.1
+  newTag: 2.1.0-alpha.1