From 8427669923052be57d9cdfc4391ef3bb9b4123d8 Mon Sep 17 00:00:00 2001 From: yuyi Date: Wed, 1 Nov 2023 20:18:20 +0800 Subject: [PATCH 01/18] feat: telemetry data extractor, need more testings --- cmd/main.go | 12 --- deploy/operator.yaml | 2 +- go.mod | 16 ++-- go.sum | 16 ++++ make/debug.mk | 2 +- pkg/telemetry/client.go | 85 ++++++++++++++++++ pkg/telemetry/consts.go | 48 ++++++++++ pkg/telemetry/metrics.go | 123 ++++++++++++++++++++++++++ pkg/telemetry/metrics_test.go | 54 +++++++++++ pkg/telemetry/record.go | 87 ++++++++++++++++++ pkg/telemetry/record_test.go | 102 +++++++++++++++++++++ pkg/telemetry/telemetry_suite_test.go | 13 +++ pkg/telemetry/throttler.go | 100 +++++++++++++++++++++ 13 files changed, 638 insertions(+), 22 deletions(-) create mode 100644 pkg/telemetry/client.go create mode 100644 pkg/telemetry/consts.go create mode 100644 pkg/telemetry/metrics.go create mode 100644 pkg/telemetry/metrics_test.go create mode 100644 pkg/telemetry/record.go create mode 100644 pkg/telemetry/record_test.go create mode 100644 pkg/telemetry/telemetry_suite_test.go create mode 100644 pkg/telemetry/throttler.go diff --git a/cmd/main.go b/cmd/main.go index 9d8e14bfa..55ce8e3c5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -160,18 +160,6 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "OBTenantRestore") os.Exit(1) } - // if err = (&v1alpha1.OBCluster{}).SetupWebhookWithManager(mgr); err != nil { - // setupLog.Error(err, "unable to create webhook", "webhook", "OBCluster") - // os.Exit(1) - // } - // if err = (&v1alpha1.OBZone{}).SetupWebhookWithManager(mgr); err != nil { - // setupLog.Error(err, "unable to create webhook", "webhook", "OBZone") - // os.Exit(1) - // } - // if err = (&v1alpha1.OBServer{}).SetupWebhookWithManager(mgr); err != nil { - // setupLog.Error(err, "unable to create webhook", "webhook", "OBServer") - // os.Exit(1) - // } if err = (&controller.OBTenantBackupPolicyReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 8758574fd..2f538cf44 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -6689,7 +6689,7 @@ spec: type: string create_time: type: string - gmt_create_time: + gmt_create: type: string in_recyclebin: type: string diff --git a/go.mod b/go.mod index 9a0a6abd3..7938f9a92 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/google/uuid v1.3.0 github.com/jmoiron/sqlx v1.3.5 - github.com/onsi/ginkgo/v2 v2.11.0 - github.com/onsi/gomega v1.27.10 + github.com/onsi/ginkgo/v2 v2.13.0 + github.com/onsi/gomega v1.29.0 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.8.1 @@ -35,7 +35,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/imdario/mergo v0.3.6 // indirect @@ -55,13 +55,13 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.3 // indirect + golang.org/x/tools v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect diff --git a/go.sum b/go.sum index db5198d41..8af4c2d23 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -113,8 +115,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -181,6 +187,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= @@ -202,14 +210,20 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -223,6 +237,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/make/debug.mk b/make/debug.mk index 70b904c44..e08238a1b 100644 --- a/make/debug.mk +++ b/make/debug.mk @@ -4,7 +4,7 @@ .PHONY: connect gettenants getpolicy getbackupjobs getcluster getobserver getrestorejobs getpods connect: - $(eval nodeHost = $(shell kubectl get pods -o jsonpath='{.items[1].status.podIP}')) + $(eval nodeHost = $(shell kubectl get pods -o jsonpath='{.items[0].status.podIP}')) ifdef TENANT $(eval secretName = $(shell kubectl get obtenant ${TENANT} -o jsonpath='{.status.credentials.root}')) $(eval tenantName = $(shell kubectl get obtenant ${TENANT} -o jsonpath='{.spec.tenantName}')) diff --git a/pkg/telemetry/client.go b/pkg/telemetry/client.go new file mode 100644 index 000000000..607408335 --- /dev/null +++ b/pkg/telemetry/client.go @@ -0,0 +1,85 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "context" + "fmt" + "os" + + "k8s.io/apimachinery/pkg/runtime" + record "k8s.io/client-go/tools/record" +) + +type Telemetry interface { + record.EventRecorder + GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...ExtraField) +} + +type telemetry struct { + *throttler + record.EventRecorder + + telemetryDisabled bool +} + +// Implement record.EventRecorder interface +func (t *telemetry) Event(object runtime.Object, eventType, reason, message string) { + t.EventRecorder.Event(object, eventType, reason, message) + t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, message, nil) +} + +// Implement record.EventRecorder interface +func (t *telemetry) Eventf(object runtime.Object, eventType, reason, messageFmt string, args ...interface{}) { + t.EventRecorder.Eventf(object, eventType, reason, messageFmt, args...) + t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, fmt.Sprintf(messageFmt, args...), nil) +} + +// Implement record.EventRecorder interface +func (t *telemetry) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...interface{}) { + t.EventRecorder.AnnotatedEventf(object, annotations, eventType, reason, messageFmt, args...) + t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) +} + +func (t *telemetry) GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...ExtraField) { + if t.telemetryDisabled { + return + } + go func(ctx context.Context, ch chan<- *TelemetryRecord) { + // TODO: guard here to mask IP address + select { + case <-ctx.Done(): + return + case ch <- newTelemetryRecord(object, objectType, eventType, reason, message, annotations, extra...): + default: + } + }(t.ctx, t.chanIn()) +} + +func NewTelemetry(recorder record.EventRecorder) Telemetry { + clt := &telemetry{ + EventRecorder: recorder, + } + + if disabled, exist := os.LookupEnv(DisableTelemetryEnvName); exist && disabled == "true" { + clt.telemetryDisabled = true + return clt + } + + clt.throttler = getThrottler() + return clt +} + +func (t *telemetry) Done() { + t.cancel() +} diff --git a/pkg/telemetry/consts.go b/pkg/telemetry/consts.go new file mode 100644 index 000000000..3da8edbf2 --- /dev/null +++ b/pkg/telemetry/consts.go @@ -0,0 +1,48 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +const ( + DefaultThrottlerBufferSize = 100 + DefaultWorkerCount = 10 +) + +const ( + TelemetryReportDevHost = "http://openwebapi.dev.alipay.net" + TelemetryReportTestHost = "http://openwebapi.test.alipay.net" + TelemetryReportProdHost = "https://openwebapi.ocenbase.com" + TelemetryReportHost = TelemetryReportProdHost + TelemetryReportPath = "/api/web/oceanbase/report" +) + +const ( + TelemetryComponent = "ob-operator" +) + +const ( + // devel use + TelemetryRequestSignature = "dbe97393a695335d67de91dd4049ba" +) + +const ( + ResourceBehaviorCreate = "Create" + ResourceBehaviorUpdate = "Update" + ResourceBehaviorDelete = "Delete" + + ResourceBehaviorError = "Error" + ResourceBehaviorNormal = "Normal" +) + +const ( + DisableTelemetryEnvName = "DISABLE_TELEMETRY" +) diff --git a/pkg/telemetry/metrics.go b/pkg/telemetry/metrics.go new file mode 100644 index 000000000..0bf15e4aa --- /dev/null +++ b/pkg/telemetry/metrics.go @@ -0,0 +1,123 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "context" + "crypto/md5" + "encoding/hex" + "net" + "os" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" +) + +type TelemetryEnvMetrics struct { + IPHashes []string `json:"ipHashes"` + K8sNodes []K8sNode `json:"k8sNodes"` +} + +func LocalIP() ([]net.IP, error) { + ifaces, err := net.Interfaces() + if err != nil { + return nil, err + } + var ips []net.IP + for _, i := range ifaces { + addrs, err := i.Addrs() + if err != nil { + return nil, err + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip != nil && !ip.IsLoopback() { + ips = append(ips, ip) + } + } + } + return ips, nil +} + +func K8sNodes() ([]corev1.Node, error) { + var err error + var config *rest.Config + + if _, exist := os.LookupEnv("KUBERNETES_SERVICE_HOST"); exist { + config, err = rest.InClusterConfig() + if err != nil { + return nil, err + } + } else { + home := homedir.HomeDir() + configPath := filepath.Join(home, ".kube", "config") + config, err = clientcmd.BuildConfigFromFlags("", configPath) + if err != nil { + return nil, err + } + } + + k8sClient, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + nodes, err := k8sClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + return nodes.Items, nil +} + +func GetHostMetrics() (*TelemetryEnvMetrics, error) { + metrics := &TelemetryEnvMetrics{ + IPHashes: []string{}, + K8sNodes: []K8sNode{}, + } + ips, err := LocalIP() + if err != nil { + return nil, err + } + for _, ip := range ips { + md5Hash := md5.Sum([]byte(ip.String())) + metrics.IPHashes = append(metrics.IPHashes, hex.EncodeToString(md5Hash[:])) + } + k8sNodes, err := K8sNodes() + if err != nil { + return nil, err + } + for _, node := range k8sNodes { + metrics.K8sNodes = append(metrics.K8sNodes, K8sNode{ + KernelVersion: node.Status.NodeInfo.KernelVersion, + OsImage: node.Status.NodeInfo.OSImage, + ContainerRuntimeVersion: node.Status.NodeInfo.ContainerRuntimeVersion, + KubeletVersion: node.Status.NodeInfo.KubeletVersion, + KubeProxyVersion: node.Status.NodeInfo.KubeProxyVersion, + OperatingSystem: node.Status.NodeInfo.OperatingSystem, + Architecture: node.Status.NodeInfo.Architecture, + }) + } + return metrics, nil +} diff --git a/pkg/telemetry/metrics_test.go b/pkg/telemetry/metrics_test.go new file mode 100644 index 000000000..3fc3c6a74 --- /dev/null +++ b/pkg/telemetry/metrics_test.go @@ -0,0 +1,54 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Telemetry", func() { + Context("Test Telemetry", Label("metrics"), func() { + It("Test LocalIP", func() { + ips, err := LocalIP() + Expect(err).ShouldNot(HaveOccurred()) + Expect(ips).ShouldNot(BeNil()) + }) + + It("Test K8sNodes", func() { + nodes, err := K8sNodes() + Expect(err).ShouldNot(HaveOccurred()) + Expect(nodes).ShouldNot(BeNil()) + }) + + It("Test TelemetryEnvMetrics", func() { + metrics, err := GetHostMetrics() + Expect(err).ShouldNot(HaveOccurred()) + Expect(metrics).ShouldNot(BeNil()) + fmt.Printf("%+v\n", metrics) + }) + + It("Cancel context multiple times", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cancel() + select { + case <-ctx.Done(): + Expect(ctx.Err()).ShouldNot(BeNil()) + } + }) + }) +}) diff --git a/pkg/telemetry/record.go b/pkg/telemetry/record.go new file mode 100644 index 000000000..15e589eed --- /dev/null +++ b/pkg/telemetry/record.go @@ -0,0 +1,87 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "bytes" + "encoding/json" + "io" + "time" +) + +type TelemetryRecord struct { + IpHashes []string `json:"ipHashes"` + Timestamp int64 `json:"timestamp"` + Message string `json:"message"` + ResourceType string `json:"resourceType"` + EventType string `json:"eventType"` + Resource any `json:"resource,omitempty"` + Extra any `json:"extra,omitempty"` + + K8sNodes []K8sNode `json:"k8sNodes,omitempty"` +} + +type ExtraField struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type K8sNode struct { + KernelVersion string `json:"kernelVersion,omitempty"` + OsImage string `json:"osImage,omitempty"` + ContainerRuntimeVersion string `json:"containerRuntimeVersion,omitempty"` + KubeletVersion string `json:"kubeletVersion,omitempty"` + KubeProxyVersion string `json:"kubeProxyVersion,omitempty"` + OperatingSystem string `json:"operatingSystem,omitempty"` + Architecture string `json:"architecture,omitempty"` +} + +type TelemetryUploadBody struct { + Content TelemetryRecord `json:"content"` + Time string `json:"time"` + Component string `json:"component"` +} + +func newTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...ExtraField) *TelemetryRecord { + anno := annotations + if len(extra) > 0 { + if anno == nil { + anno = make(map[string]string) + } + for _, field := range extra { + anno[field.Key] = field.Value + } + } + return &TelemetryRecord{ + Timestamp: time.Now().Unix(), + Message: message, + ResourceType: objectType, + EventType: eventType, + Resource: object, + Extra: anno, + } +} + +// Encode a TelemetryRecord into a io.ReadCloser +func encodeTelemetryRecord(record *TelemetryRecord) (io.ReadCloser, error) { + body := TelemetryUploadBody{ + Content: *record, + Time: time.Unix(record.Timestamp, 0).Format(time.DateTime), + Component: TelemetryComponent, + } + bts, err := json.Marshal(body) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(bts)), nil +} diff --git a/pkg/telemetry/record_test.go b/pkg/telemetry/record_test.go new file mode 100644 index 000000000..794811c10 --- /dev/null +++ b/pkg/telemetry/record_test.go @@ -0,0 +1,102 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/oceanbase/ob-operator/api/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Telemetry record", func() { + It("Marshal TelemetryBody", func() { + record := &TelemetryRecord{ + IpHashes: []string{}, + Timestamp: time.Now().Unix(), + Message: "", + ResourceType: "", + EventType: "", + Resource: nil, + Extra: nil, + K8sNodes: []K8sNode{}, + } + body := TelemetryUploadBody{ + Content: *record, + Time: time.Unix(record.Timestamp, 0).Format(time.DateTime), + Component: TelemetryComponent, + } + _, err := json.Marshal(body) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("Marshal TelemetryRecord", func() { + var err error + var body []byte + record := &TelemetryRecord{ + IpHashes: []string{}, + Timestamp: time.Now().Unix(), + Message: "", + ResourceType: "", + EventType: "", + Resource: nil, + Extra: nil, + K8sNodes: []K8sNode{}, + } + _, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + + record.IpHashes = nil + _, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + + record.IpHashes = []string{"test", "test2", "test3"} + _, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + + record.Extra = "test" + _, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + + record.Extra = []string{"test", "test2", "test3"} + _, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + + record.Extra = []ExtraField{ + { + Key: "test", + Value: "value", + }, + } + _, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + + m := make(map[string]string) + m["test"] = "value" + record.Extra = m + _, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + + record.Resource = v1alpha1.OBCluster{} + body, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + fmt.Printf("%s\n", string(body)) + + record.K8sNodes = []K8sNode{{}, {}} + _, err = json.Marshal(record) + Expect(err).ShouldNot(HaveOccurred()) + }) +}) diff --git a/pkg/telemetry/telemetry_suite_test.go b/pkg/telemetry/telemetry_suite_test.go new file mode 100644 index 000000000..04c897a7c --- /dev/null +++ b/pkg/telemetry/telemetry_suite_test.go @@ -0,0 +1,13 @@ +package telemetry_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestTelemetry(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Telemetry Suite") +} diff --git a/pkg/telemetry/throttler.go b/pkg/telemetry/throttler.go new file mode 100644 index 000000000..0b31f8365 --- /dev/null +++ b/pkg/telemetry/throttler.go @@ -0,0 +1,100 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "context" + "net/http" + "net/url" + "sync" +) + +type throttler struct { + http.Client + + metrics *TelemetryEnvMetrics + ctx context.Context + cancel context.CancelFunc + recordChan chan *TelemetryRecord +} + +var throttlerSingleton *throttler +var throttlerOnce sync.Once + +func getThrottler() *throttler { + throttlerOnce.Do(func() { + throttlerSingleton = &throttler{ + recordChan: make(chan *TelemetryRecord, DefaultThrottlerBufferSize), + } + if metrics, err := GetHostMetrics(); err == nil { + throttlerSingleton.metrics = metrics + } + ctx, cancel := context.WithCancel(context.Background()) + throttlerSingleton.ctx = ctx + throttlerSingleton.cancel = cancel + throttlerSingleton.Client = *http.DefaultClient + + throttlerSingleton.startWorkers() + }) + return throttlerSingleton +} + +func (t *throttler) chanOut() <-chan *TelemetryRecord { + return t.recordChan +} + +func (t *throttler) chanIn() chan<- *TelemetryRecord { + return t.recordChan +} + +func (t *throttler) close() { + if _, ok := <-t.recordChan; ok { + close(t.recordChan) + } +} + +func (t *throttler) sendTelemetryRecord(record *TelemetryRecord) error { + record.IpHashes = t.metrics.IPHashes + body, err := encodeTelemetryRecord(record) + req := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Host: TelemetryReportDevHost, Path: TelemetryReportPath}, + Header: http.Header{ + "content-type": []string{"application/json"}, + "sig": []string{TelemetryRequestSignature}, + }, + Body: body, + } + _, err = t.Client.Do(req) + return err +} + +func (t *throttler) startWorkers() { + for i := 0; i < DefaultWorkerCount; i++ { + go func(ctx context.Context, ch <-chan *TelemetryRecord) error { + for { + select { + case record, ok := <-ch: + if !ok { + // channel closed + return nil + } + _ = t.sendTelemetryRecord(record) + case <-ctx.Done(): + return ctx.Err() + default: + } + } + }(t.ctx, t.chanOut()) + } +} From 507dc862440a351dc9ac804203d882fbc3162f09 Mon Sep 17 00:00:00 2001 From: yuyi Date: Thu, 2 Nov 2023 21:14:50 +0800 Subject: [PATCH 02/18] feat: telemetry module with sig, debug and diabled as env vars --- config/rbac/role.yaml | 8 ++ pkg/controller/obcluster_controller.go | 3 +- pkg/telemetry/configs.go | 27 ++++++ pkg/telemetry/consts.go | 25 +++--- pkg/telemetry/metrics.go | 69 +++++++------- pkg/telemetry/metrics_test.go | 13 ++- pkg/telemetry/models/k8s.go | 23 +++++ pkg/telemetry/models/record.go | 35 ++++++++ pkg/telemetry/record.go | 44 ++------- pkg/telemetry/record_test.go | 15 ++-- pkg/telemetry/sentry.go | 90 +++++++++++++++++++ pkg/telemetry/sentry_test.go | 105 ++++++++++++++++++++++ pkg/telemetry/{client.go => telemetry.go} | 68 +++++++++----- pkg/telemetry/telemetry_test.go | 98 ++++++++++++++++++++ pkg/telemetry/throttler.go | 64 ++++++++----- pkg/telemetry/throttler_test.go | 72 +++++++++++++++ pkg/telemetry/utils.go | 23 +++++ 17 files changed, 638 insertions(+), 144 deletions(-) create mode 100644 pkg/telemetry/configs.go create mode 100644 pkg/telemetry/models/k8s.go create mode 100644 pkg/telemetry/models/record.go create mode 100644 pkg/telemetry/sentry.go create mode 100644 pkg/telemetry/sentry_test.go rename pkg/telemetry/{client.go => telemetry.go} (55%) create mode 100644 pkg/telemetry/telemetry_test.go create mode 100644 pkg/telemetry/throttler_test.go create mode 100644 pkg/telemetry/utils.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6c1454a2b..022f0ef5e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -11,6 +11,14 @@ rules: verbs: - create - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: diff --git a/pkg/controller/obcluster_controller.go b/pkg/controller/obcluster_controller.go index 198ad21d0..2c6f64305 100644 --- a/pkg/controller/obcluster_controller.go +++ b/pkg/controller/obcluster_controller.go @@ -48,6 +48,7 @@ type OBClusterReconciler struct { // +kubebuilder:rbac:groups="",resources=services/status,verbs=get;update;patch // +kubebuilder:rbac:groups="",resources=services/finalizers,verbs=update // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=jobs/status,verbs=get;update;patch // +kubebuilder:rbac:groups=batch,resources=jobs/finalizers,verbs=update @@ -66,11 +67,11 @@ func (r *OBClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( obcluster := &v1alpha1.OBCluster{} err := r.Client.Get(ctx, req.NamespacedName, obcluster) if err != nil { - logger.Error(err, "get obcluster error") if kubeerrors.IsNotFound(err) { // obcluster not found, just return return ctrl.Result{}, nil } + logger.Error(err, "get obcluster error") return ctrl.Result{}, err } diff --git a/pkg/telemetry/configs.go b/pkg/telemetry/configs.go new file mode 100644 index 000000000..781900162 --- /dev/null +++ b/pkg/telemetry/configs.go @@ -0,0 +1,27 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import "os" + +var ( + TelemetryRequestSignature string = os.Getenv(TelemetrySignatureEnvName) +) + +const ( + DefaultThrottlerBufferSize = 30 + DefaultThrottlerWorkerCount = 30 + DefaultWaitThrottlerSeconds = 60 +) + +const TelemetryReportHost = TelemetryReportProdHost diff --git a/pkg/telemetry/consts.go b/pkg/telemetry/consts.go index 3da8edbf2..1544efa9b 100644 --- a/pkg/telemetry/consts.go +++ b/pkg/telemetry/consts.go @@ -13,25 +13,22 @@ See the Mulan PSL v2 for more details. package telemetry const ( - DefaultThrottlerBufferSize = 100 - DefaultWorkerCount = 10 -) + TelemetryReportDevHost = "openwebapi.dev.alipay.net" // http + TelemetryReportTestHost = "openwebapi.test.alipay.net" // http + TelemetryReportProdHost = "openwebapi.ocenbase.com" // https -const ( - TelemetryReportDevHost = "http://openwebapi.dev.alipay.net" - TelemetryReportTestHost = "http://openwebapi.test.alipay.net" - TelemetryReportProdHost = "https://openwebapi.ocenbase.com" - TelemetryReportHost = TelemetryReportProdHost - TelemetryReportPath = "/api/web/oceanbase/report" + TelemetryReportPath = "/api/web/oceanbase/report" ) const ( - TelemetryComponent = "ob-operator" + SchemeHttp = "http" + SchemeHttps = "https" ) +const ContentTypeJson = "application/json" + const ( - // devel use - TelemetryRequestSignature = "dbe97393a695335d67de91dd4049ba" + TelemetryComponent = "ob-operator" ) const ( @@ -44,5 +41,7 @@ const ( ) const ( - DisableTelemetryEnvName = "DISABLE_TELEMETRY" + DisableTelemetryEnvName = "DISABLE_TELEMETRY" + TelemetrySignatureEnvName = "TELEMETRY_SIGNATURE" + TelemetryDebugEnvName = "TELEMETRY_DEBUG" ) diff --git a/pkg/telemetry/metrics.go b/pkg/telemetry/metrics.go index 0bf15e4aa..2cc8eb6eb 100644 --- a/pkg/telemetry/metrics.go +++ b/pkg/telemetry/metrics.go @@ -19,7 +19,9 @@ import ( "net" "os" "path/filepath" + "sync" + "github.com/oceanbase/ob-operator/pkg/telemetry/models" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -28,12 +30,15 @@ import ( "k8s.io/client-go/util/homedir" ) -type TelemetryEnvMetrics struct { - IPHashes []string `json:"ipHashes"` - K8sNodes []K8sNode `json:"k8sNodes"` +type hostMetrics struct { + IPHashes []string `json:"ipHashes"` + K8sNodes []models.K8sNode `json:"k8sNodes"` } -func LocalIP() ([]net.IP, error) { +var telemetryEnvMetrics *hostMetrics +var telemetryEnvMetricsOnce sync.Once + +func getLocalIPs() ([]net.IP, error) { ifaces, err := net.Interfaces() if err != nil { return nil, err @@ -62,7 +67,7 @@ func LocalIP() ([]net.IP, error) { return ips, nil } -func K8sNodes() ([]corev1.Node, error) { +func getK8sNodes() ([]corev1.Node, error) { var err error var config *rest.Config @@ -91,33 +96,33 @@ func K8sNodes() ([]corev1.Node, error) { return nodes.Items, nil } -func GetHostMetrics() (*TelemetryEnvMetrics, error) { - metrics := &TelemetryEnvMetrics{ +func getHostMetrics() *hostMetrics { + telemetryEnvMetrics = &hostMetrics{ IPHashes: []string{}, - K8sNodes: []K8sNode{}, - } - ips, err := LocalIP() - if err != nil { - return nil, err + K8sNodes: []models.K8sNode{}, } - for _, ip := range ips { - md5Hash := md5.Sum([]byte(ip.String())) - metrics.IPHashes = append(metrics.IPHashes, hex.EncodeToString(md5Hash[:])) - } - k8sNodes, err := K8sNodes() - if err != nil { - return nil, err - } - for _, node := range k8sNodes { - metrics.K8sNodes = append(metrics.K8sNodes, K8sNode{ - KernelVersion: node.Status.NodeInfo.KernelVersion, - OsImage: node.Status.NodeInfo.OSImage, - ContainerRuntimeVersion: node.Status.NodeInfo.ContainerRuntimeVersion, - KubeletVersion: node.Status.NodeInfo.KubeletVersion, - KubeProxyVersion: node.Status.NodeInfo.KubeProxyVersion, - OperatingSystem: node.Status.NodeInfo.OperatingSystem, - Architecture: node.Status.NodeInfo.Architecture, - }) - } - return metrics, nil + telemetryEnvMetricsOnce.Do(func() { + ips, err := getLocalIPs() + if err == nil { + for _, ip := range ips { + md5Hash := md5.Sum([]byte(ip.String())) + telemetryEnvMetrics.IPHashes = append(telemetryEnvMetrics.IPHashes, hex.EncodeToString(md5Hash[:])) + } + } + k8sNodes, err := getK8sNodes() + if err == nil { + for _, node := range k8sNodes { + telemetryEnvMetrics.K8sNodes = append(telemetryEnvMetrics.K8sNodes, models.K8sNode{ + KernelVersion: node.Status.NodeInfo.KernelVersion, + OsImage: node.Status.NodeInfo.OSImage, + ContainerRuntimeVersion: node.Status.NodeInfo.ContainerRuntimeVersion, + KubeletVersion: node.Status.NodeInfo.KubeletVersion, + KubeProxyVersion: node.Status.NodeInfo.KubeProxyVersion, + OperatingSystem: node.Status.NodeInfo.OperatingSystem, + Architecture: node.Status.NodeInfo.Architecture, + }) + } + } + }) + return telemetryEnvMetrics } diff --git a/pkg/telemetry/metrics_test.go b/pkg/telemetry/metrics_test.go index 3fc3c6a74..e7f5f5a4e 100644 --- a/pkg/telemetry/metrics_test.go +++ b/pkg/telemetry/metrics_test.go @@ -22,21 +22,20 @@ import ( var _ = Describe("Telemetry", func() { Context("Test Telemetry", Label("metrics"), func() { - It("Test LocalIP", func() { - ips, err := LocalIP() + It("Test getLocalIP", func() { + ips, err := getLocalIPs() Expect(err).ShouldNot(HaveOccurred()) Expect(ips).ShouldNot(BeNil()) }) - It("Test K8sNodes", func() { - nodes, err := K8sNodes() + It("Test getK8sNodes", func() { + nodes, err := getK8sNodes() Expect(err).ShouldNot(HaveOccurred()) Expect(nodes).ShouldNot(BeNil()) }) - It("Test TelemetryEnvMetrics", func() { - metrics, err := GetHostMetrics() - Expect(err).ShouldNot(HaveOccurred()) + It("Test getHostMetrics", func() { + metrics := getHostMetrics() Expect(metrics).ShouldNot(BeNil()) fmt.Printf("%+v\n", metrics) }) diff --git a/pkg/telemetry/models/k8s.go b/pkg/telemetry/models/k8s.go new file mode 100644 index 000000000..0bf7a4acd --- /dev/null +++ b/pkg/telemetry/models/k8s.go @@ -0,0 +1,23 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package models + +type K8sNode struct { + KernelVersion string `json:"kernelVersion,omitempty"` + OsImage string `json:"osImage,omitempty"` + ContainerRuntimeVersion string `json:"containerRuntimeVersion,omitempty"` + KubeletVersion string `json:"kubeletVersion,omitempty"` + KubeProxyVersion string `json:"kubeProxyVersion,omitempty"` + OperatingSystem string `json:"operatingSystem,omitempty"` + Architecture string `json:"architecture,omitempty"` +} diff --git a/pkg/telemetry/models/record.go b/pkg/telemetry/models/record.go new file mode 100644 index 000000000..e02c62458 --- /dev/null +++ b/pkg/telemetry/models/record.go @@ -0,0 +1,35 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package models + +type TelemetryRecord struct { + IpHashes []string `json:"ipHashes"` + Timestamp int64 `json:"timestamp"` + Message string `json:"message"` + Reason string `json:"reason"` + ResourceType string `json:"resourceType"` + EventType string `json:"eventType"` + Resource any `json:"resource,omitempty"` + Extra any `json:"extra,omitempty"` +} + +type ExtraField struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type TelemetryUploadBody struct { + Content TelemetryRecord `json:"content"` + Time string `json:"time"` + Component string `json:"component"` +} diff --git a/pkg/telemetry/record.go b/pkg/telemetry/record.go index 15e589eed..9f45716d8 100644 --- a/pkg/telemetry/record.go +++ b/pkg/telemetry/record.go @@ -17,42 +17,11 @@ import ( "encoding/json" "io" "time" -) - -type TelemetryRecord struct { - IpHashes []string `json:"ipHashes"` - Timestamp int64 `json:"timestamp"` - Message string `json:"message"` - ResourceType string `json:"resourceType"` - EventType string `json:"eventType"` - Resource any `json:"resource,omitempty"` - Extra any `json:"extra,omitempty"` - - K8sNodes []K8sNode `json:"k8sNodes,omitempty"` -} - -type ExtraField struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type K8sNode struct { - KernelVersion string `json:"kernelVersion,omitempty"` - OsImage string `json:"osImage,omitempty"` - ContainerRuntimeVersion string `json:"containerRuntimeVersion,omitempty"` - KubeletVersion string `json:"kubeletVersion,omitempty"` - KubeProxyVersion string `json:"kubeProxyVersion,omitempty"` - OperatingSystem string `json:"operatingSystem,omitempty"` - Architecture string `json:"architecture,omitempty"` -} -type TelemetryUploadBody struct { - Content TelemetryRecord `json:"content"` - Time string `json:"time"` - Component string `json:"component"` -} + "github.com/oceanbase/ob-operator/pkg/telemetry/models" +) -func newTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...ExtraField) *TelemetryRecord { +func newRecordFromEvent(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...models.ExtraField) *models.TelemetryRecord { anno := annotations if len(extra) > 0 { if anno == nil { @@ -62,9 +31,10 @@ func newTelemetryRecord(object any, objectType, eventType, reason, message strin anno[field.Key] = field.Value } } - return &TelemetryRecord{ + return &models.TelemetryRecord{ Timestamp: time.Now().Unix(), Message: message, + Reason: reason, ResourceType: objectType, EventType: eventType, Resource: object, @@ -73,8 +43,8 @@ func newTelemetryRecord(object any, objectType, eventType, reason, message strin } // Encode a TelemetryRecord into a io.ReadCloser -func encodeTelemetryRecord(record *TelemetryRecord) (io.ReadCloser, error) { - body := TelemetryUploadBody{ +func encodeRecord(record *models.TelemetryRecord) (io.ReadCloser, error) { + body := models.TelemetryUploadBody{ Content: *record, Time: time.Unix(record.Timestamp, 0).Format(time.DateTime), Component: TelemetryComponent, diff --git a/pkg/telemetry/record_test.go b/pkg/telemetry/record_test.go index 794811c10..1ab51891a 100644 --- a/pkg/telemetry/record_test.go +++ b/pkg/telemetry/record_test.go @@ -18,13 +18,14 @@ import ( "time" "github.com/oceanbase/ob-operator/api/v1alpha1" + "github.com/oceanbase/ob-operator/pkg/telemetry/models" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("Telemetry record", func() { It("Marshal TelemetryBody", func() { - record := &TelemetryRecord{ + record := &models.TelemetryRecord{ IpHashes: []string{}, Timestamp: time.Now().Unix(), Message: "", @@ -32,9 +33,8 @@ var _ = Describe("Telemetry record", func() { EventType: "", Resource: nil, Extra: nil, - K8sNodes: []K8sNode{}, } - body := TelemetryUploadBody{ + body := models.TelemetryUploadBody{ Content: *record, Time: time.Unix(record.Timestamp, 0).Format(time.DateTime), Component: TelemetryComponent, @@ -46,7 +46,7 @@ var _ = Describe("Telemetry record", func() { It("Marshal TelemetryRecord", func() { var err error var body []byte - record := &TelemetryRecord{ + record := &models.TelemetryRecord{ IpHashes: []string{}, Timestamp: time.Now().Unix(), Message: "", @@ -54,7 +54,6 @@ var _ = Describe("Telemetry record", func() { EventType: "", Resource: nil, Extra: nil, - K8sNodes: []K8sNode{}, } _, err = json.Marshal(record) Expect(err).ShouldNot(HaveOccurred()) @@ -75,7 +74,7 @@ var _ = Describe("Telemetry record", func() { _, err = json.Marshal(record) Expect(err).ShouldNot(HaveOccurred()) - record.Extra = []ExtraField{ + record.Extra = []models.ExtraField{ { Key: "test", Value: "value", @@ -94,9 +93,5 @@ var _ = Describe("Telemetry record", func() { body, err = json.Marshal(record) Expect(err).ShouldNot(HaveOccurred()) fmt.Printf("%s\n", string(body)) - - record.K8sNodes = []K8sNode{{}, {}} - _, err = json.Marshal(record) - Expect(err).ShouldNot(HaveOccurred()) }) }) diff --git a/pkg/telemetry/sentry.go b/pkg/telemetry/sentry.go new file mode 100644 index 000000000..59f307c40 --- /dev/null +++ b/pkg/telemetry/sentry.go @@ -0,0 +1,90 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "fmt" + + "github.com/oceanbase/ob-operator/api/v1alpha1" +) + +func objectSentry(object any) { + if object == nil { + return + } + switch object.(type) { + case *v1alpha1.OBCluster: + cluster, ok := object.(*v1alpha1.OBCluster) + if !ok { + return + } + processOBCluster(cluster) + case *v1alpha1.OBTenant: + tenant, ok := object.(*v1alpha1.OBTenant) + if !ok { + return + } + processOBTenant(tenant) + case *v1alpha1.OBServer: + server, ok := object.(*v1alpha1.OBServer) + if !ok { + return + } + processOBServer(server) + case *v1alpha1.OBZone: + zone, ok := object.(*v1alpha1.OBZone) + if !ok { + return + } + processOBZone(zone) + } +} + +func processOBCluster(cluster *v1alpha1.OBCluster) { + fmt.Printf("[OBCluster Before] %+v\n", cluster) + if cluster.Spec.BackupVolume != nil && cluster.Spec.BackupVolume.Volume != nil && cluster.Spec.BackupVolume.Volume.NFS != nil { + cluster.Spec.BackupVolume.Volume.NFS.Server = md5Hash(cluster.Spec.BackupVolume.Volume.NFS.Server) + } + fmt.Printf("[OBCluster After] %+v\n", cluster) +} + +func processOBServer(server *v1alpha1.OBServer) { + fmt.Printf("[OBServer Before] %+v\n", server) + server.Status.PodIp = md5Hash(server.Status.PodIp) + server.Status.NodeIp = md5Hash(server.Status.NodeIp) + fmt.Printf("[OBServer After] %+v\n", server) +} + +func processOBTenant(tenant *v1alpha1.OBTenant) { + fmt.Printf("[OBTenant After] %+v\n", tenant) + for i := range tenant.Status.Pools { + for j := range tenant.Status.Pools[i].Units { + tenant.Status.Pools[i].Units[j].ServerIP = md5Hash(tenant.Status.Pools[i].Units[j].ServerIP) + if tenant.Status.Pools[i].Units[j].Migrate.ServerIP != "" { + tenant.Status.Pools[i].Units[j].Migrate.ServerIP = md5Hash(tenant.Status.Pools[i].Units[j].Migrate.ServerIP) + } + } + } + fmt.Printf("[OBTenant After] %+v\n", tenant) +} + +func processOBZone(zone *v1alpha1.OBZone) { + fmt.Printf("[OBZone Before] %+v\n", zone) + for i := range zone.Status.OBServerStatus { + zone.Status.OBServerStatus[i].Server = md5Hash(zone.Status.OBServerStatus[i].Server) + } + if zone.Spec.BackupVolume != nil && zone.Spec.BackupVolume.Volume != nil && zone.Spec.BackupVolume.Volume.NFS != nil { + zone.Spec.BackupVolume.Volume.NFS.Server = md5Hash(zone.Spec.BackupVolume.Volume.NFS.Server) + } + fmt.Printf("[OBZone After] %+v\n", zone) +} diff --git a/pkg/telemetry/sentry_test.go b/pkg/telemetry/sentry_test.go new file mode 100644 index 000000000..aec340374 --- /dev/null +++ b/pkg/telemetry/sentry_test.go @@ -0,0 +1,105 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "github.com/oceanbase/ob-operator/api/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +var _ = Describe("Telemetry sentry", Label("sentry"), func() { + It("cluster sentry", func() { + cluster := &v1alpha1.OBCluster{} + objectSentry(cluster) + + beforeServer := "1.2.3.4" + cluster.Spec.BackupVolume = &v1alpha1.BackupVolumeSpec{ + Volume: &corev1.Volume{ + Name: "backup", + VolumeSource: corev1.VolumeSource{ + NFS: &corev1.NFSVolumeSource{ + Server: beforeServer, + Path: "/backup/opt", + }, + }, + }, + } + objectSentry(cluster) + Expect(cluster.Spec.BackupVolume.Volume.NFS.Server).ShouldNot(Equal(beforeServer)) + }) + + It("server sentry", func() { + podIP := "127.0.1.2" + nodeIP := "128.0.1.2" + server := &v1alpha1.OBServer{ + Status: v1alpha1.OBServerStatus{ + PodIp: podIP, + NodeIp: nodeIP, + }, + } + objectSentry(server) + Expect(server.Status.PodIp).ShouldNot(Equal(podIP)) + Expect(server.Status.NodeIp).ShouldNot(Equal(nodeIP)) + }) + + It("tenant sentry", func() { + beforeIp := "1.2.3.4" + tenant := &v1alpha1.OBTenant{ + Status: v1alpha1.OBTenantStatus{ + Pools: []v1alpha1.ResourcePoolStatus{{ + Units: []v1alpha1.UnitStatus{{ + UnitId: 0, + ServerIP: beforeIp, + ServerPort: 0, + Status: "", + Migrate: v1alpha1.MigrateServerStatus{ + ServerIP: beforeIp, + }, + }}, + }}, + }, + } + objectSentry(tenant) + Expect(tenant.Status.Pools[0].Units[0].ServerIP).ShouldNot(Equal(beforeIp)) + Expect(tenant.Status.Pools[0].Units[0].Migrate.ServerIP).ShouldNot(Equal(beforeIp)) + }) + + It("zone sentry", func() { + beforeIp := "1.2.3.4" + zone := &v1alpha1.OBZone{ + Status: v1alpha1.OBZoneStatus{ + OBServerStatus: []v1alpha1.OBServerReplicaStatus{{ + Server: beforeIp, + }}, + }, + Spec: v1alpha1.OBZoneSpec{ + BackupVolume: &v1alpha1.BackupVolumeSpec{ + Volume: &corev1.Volume{ + Name: "backup", + VolumeSource: corev1.VolumeSource{ + NFS: &corev1.NFSVolumeSource{ + Server: beforeIp, + Path: "/backup/opt", + }, + }, + }, + }, + }, + } + objectSentry(zone) + Expect(zone.Status.OBServerStatus[0].Server).ShouldNot(Equal(beforeIp)) + Expect(zone.Spec.BackupVolume.Volume.NFS.Server).ShouldNot(Equal(beforeIp)) + }) +}) diff --git a/pkg/telemetry/client.go b/pkg/telemetry/telemetry.go similarity index 55% rename from pkg/telemetry/client.go rename to pkg/telemetry/telemetry.go index 607408335..bd7be5799 100644 --- a/pkg/telemetry/client.go +++ b/pkg/telemetry/telemetry.go @@ -16,70 +16,98 @@ import ( "context" "fmt" "os" + "time" + "github.com/oceanbase/ob-operator/pkg/telemetry/models" "k8s.io/apimachinery/pkg/runtime" record "k8s.io/client-go/tools/record" ) type Telemetry interface { record.EventRecorder - GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...ExtraField) + GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...models.ExtraField) + GetHostMetrics() *hostMetrics + Done() } type telemetry struct { *throttler + *hostMetrics record.EventRecorder telemetryDisabled bool } +func NewTelemetry(recorder record.EventRecorder) Telemetry { + clt := &telemetry{ + EventRecorder: recorder, + } + + // if telemetry is disabled, return a dummy telemetry as original event recorder + if os.Getenv(DisableTelemetryEnvName) == "true" { + clt.telemetryDisabled = true + return clt + } + // no signature means telemetry is disabled + if TelemetryRequestSignature == "" { + clt.telemetryDisabled = true + return clt + } + clt.hostMetrics = getHostMetrics() + clt.throttler = getThrottler() + return clt +} + // Implement record.EventRecorder interface func (t *telemetry) Event(object runtime.Object, eventType, reason, message string) { t.EventRecorder.Event(object, eventType, reason, message) - t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, message, nil) + t.generateFromEvent(object, nil, eventType, reason, message) } // Implement record.EventRecorder interface func (t *telemetry) Eventf(object runtime.Object, eventType, reason, messageFmt string, args ...interface{}) { t.EventRecorder.Eventf(object, eventType, reason, messageFmt, args...) - t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, fmt.Sprintf(messageFmt, args...), nil) + t.generateFromEvent(object, nil, eventType, reason, messageFmt, args...) } // Implement record.EventRecorder interface func (t *telemetry) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...interface{}) { t.EventRecorder.AnnotatedEventf(object, annotations, eventType, reason, messageFmt, args...) - t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) + t.generateFromEvent(object, annotations, eventType, reason, messageFmt, args...) } -func (t *telemetry) GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...ExtraField) { +// Use Event, Eventf, AnnotatedEventf first +func (t *telemetry) GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...models.ExtraField) { if t.telemetryDisabled { return } - go func(ctx context.Context, ch chan<- *TelemetryRecord) { + go func(ctx context.Context, ch chan<- *models.TelemetryRecord) { // TODO: guard here to mask IP address + objectSentry(object) + record := newRecordFromEvent(object, objectType, eventType, reason, message, annotations, extra...) + record.IpHashes = t.hostMetrics.IPHashes select { case <-ctx.Done(): return - case ch <- newTelemetryRecord(object, objectType, eventType, reason, message, annotations, extra...): + case ch <- record: + case <-time.After(DefaultWaitThrottlerSeconds * time.Second): default: } }(t.ctx, t.chanIn()) } -func NewTelemetry(recorder record.EventRecorder) Telemetry { - clt := &telemetry{ - EventRecorder: recorder, - } - - if disabled, exist := os.LookupEnv(DisableTelemetryEnvName); exist && disabled == "true" { - clt.telemetryDisabled = true - return clt - } +func (t *telemetry) Done() { + t.throttler.close() +} - clt.throttler = getThrottler() - return clt +func (t *telemetry) GetHostMetrics() *hostMetrics { + return t.hostMetrics } -func (t *telemetry) Done() { - t.cancel() +func (t *telemetry) generateFromEvent(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...interface{}) { + if object == nil { + t.GenerateTelemetryRecord(nil, "Unknown", eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) + } else { + t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) + } } diff --git a/pkg/telemetry/telemetry_test.go b/pkg/telemetry/telemetry_test.go new file mode 100644 index 000000000..1a026ddab --- /dev/null +++ b/pkg/telemetry/telemetry_test.go @@ -0,0 +1,98 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "time" + + "github.com/oceanbase/ob-operator/api/v1alpha1" + . "github.com/onsi/ginkgo/v2" + + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type fakeEventRecorder struct{} + +func (f *fakeEventRecorder) Event(object runtime.Object, eventtype, reason, message string) { +} +func (f *fakeEventRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...any) { +} +func (f *fakeEventRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...any) { +} + +var _ = Describe("Telemetry", Label("telemetry"), Ordered, func() { + var telemetry Telemetry + tenant := &v1alpha1.OBTenant{ + TypeMeta: metav1.TypeMeta{ + Kind: "OBTenant", + APIVersion: "oceanbase.oceanbase.com/v1alpha1", + }, + } + + BeforeAll(func() { + telemetry = NewTelemetry(&fakeEventRecorder{}) + Expect(telemetry).ShouldNot(BeNil()) + }) + + AfterAll(func() { + By("Wait for telemetry to finish, watch the output") + time.Sleep(2 * time.Second) + telemetry.Done() + }) + + It("Empty pointer", func() { + Expect(tenant).NotTo(BeNil()) + Expect(tenant.GetObjectKind().GroupVersionKind().Kind).Should(Equal("OBTenant")) + }) + + It("GetHostMetrics", func() { + metrics := telemetry.GetHostMetrics() + Expect(metrics).ShouldNot(BeNil()) + }) + + It("Event", func() { + telemetry.Event(tenant, "event", "some reasons", "test") + }) + + It("Eventf", func() { + telemetry.Eventf(tenant, "eventf", "some reasons", "hello %s", "world") + }) + + It("AnnotatedEventf", func() { + annos := map[string]string{ + "hello": "world", + } + telemetry.AnnotatedEventf(tenant, annos, "annotatedEventf", "some reasons", "hello %s", "world") + }) + + It("Generate recorder directly", func() { + tenant := v1alpha1.OBTenant{ + Spec: v1alpha1.OBTenantSpec{ + ClusterName: "test", + TenantName: "t1", + UnitNumber: 1, + Charset: "test", + Collate: "test", + ConnectWhiteList: "test", + TenantRole: "STANDBY", + Credentials: v1alpha1.TenantCredentials{ + Root: "root", + StandbyRO: "standby-ro", + }, + }} + telemetry.GenerateTelemetryRecord(tenant, "arbitrary", "generateTelemetryRecord", "some reason", "test", nil) + }) +}) diff --git a/pkg/telemetry/throttler.go b/pkg/telemetry/throttler.go index 0b31f8365..e14f3dd9c 100644 --- a/pkg/telemetry/throttler.go +++ b/pkg/telemetry/throttler.go @@ -14,18 +14,22 @@ package telemetry import ( "context" + "fmt" + "io" "net/http" "net/url" + "os" "sync" + + "github.com/oceanbase/ob-operator/pkg/telemetry/models" ) type throttler struct { - http.Client - - metrics *TelemetryEnvMetrics + debug bool + client http.Client ctx context.Context cancel context.CancelFunc - recordChan chan *TelemetryRecord + recordChan chan *models.TelemetryRecord } var throttlerSingleton *throttler @@ -34,54 +38,56 @@ var throttlerOnce sync.Once func getThrottler() *throttler { throttlerOnce.Do(func() { throttlerSingleton = &throttler{ - recordChan: make(chan *TelemetryRecord, DefaultThrottlerBufferSize), - } - if metrics, err := GetHostMetrics(); err == nil { - throttlerSingleton.metrics = metrics + recordChan: make(chan *models.TelemetryRecord, DefaultThrottlerBufferSize), } + + throttlerSingleton.debug = os.Getenv(TelemetryDebugEnvName) == "true" ctx, cancel := context.WithCancel(context.Background()) throttlerSingleton.ctx = ctx throttlerSingleton.cancel = cancel - throttlerSingleton.Client = *http.DefaultClient + throttlerSingleton.client = *http.DefaultClient throttlerSingleton.startWorkers() }) return throttlerSingleton } -func (t *throttler) chanOut() <-chan *TelemetryRecord { +func (t *throttler) chanOut() <-chan *models.TelemetryRecord { return t.recordChan } -func (t *throttler) chanIn() chan<- *TelemetryRecord { +func (t *throttler) chanIn() chan<- *models.TelemetryRecord { return t.recordChan } func (t *throttler) close() { - if _, ok := <-t.recordChan; ok { - close(t.recordChan) - } + t.cancel() } -func (t *throttler) sendTelemetryRecord(record *TelemetryRecord) error { - record.IpHashes = t.metrics.IPHashes - body, err := encodeTelemetryRecord(record) +func (t *throttler) sendTelemetryRecord(record *models.TelemetryRecord) (*http.Response, error) { + body, err := encodeRecord(record) + if err != nil { + return nil, err + } req := &http.Request{ Method: http.MethodPost, - URL: &url.URL{Host: TelemetryReportDevHost, Path: TelemetryReportPath}, + URL: &url.URL{ + Scheme: SchemeHttp, + Host: TelemetryReportTestHost, + Path: TelemetryReportPath, + }, Header: http.Header{ - "content-type": []string{"application/json"}, + "content-type": []string{ContentTypeJson}, "sig": []string{TelemetryRequestSignature}, }, Body: body, } - _, err = t.Client.Do(req) - return err + return t.client.Do(req) } func (t *throttler) startWorkers() { - for i := 0; i < DefaultWorkerCount; i++ { - go func(ctx context.Context, ch <-chan *TelemetryRecord) error { + for i := 0; i < DefaultThrottlerWorkerCount; i++ { + go func(ctx context.Context, ch <-chan *models.TelemetryRecord) error { for { select { case record, ok := <-ch: @@ -89,7 +95,17 @@ func (t *throttler) startWorkers() { // channel closed return nil } - _ = t.sendTelemetryRecord(record) + res, err := t.sendTelemetryRecord(record) + if t.debug { + if err != nil { + fmt.Printf("send telemetry record error: %v\n", err) + } + bts, err := io.ReadAll(res.Body) + if err != nil { + fmt.Printf("read response body error: %v\n", err) + } + fmt.Printf("[Event %s.%s] %s\n", record.ResourceType, record.EventType, string(bts)) + } case <-ctx.Done(): return ctx.Err() default: diff --git a/pkg/telemetry/throttler_test.go b/pkg/telemetry/throttler_test.go new file mode 100644 index 000000000..73f9ae11b --- /dev/null +++ b/pkg/telemetry/throttler_test.go @@ -0,0 +1,72 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "fmt" + "io" + "time" + + "github.com/oceanbase/ob-operator/pkg/telemetry/models" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Telemetry throttler", Ordered, Label("throttler"), func() { + var throttler *throttler + + BeforeAll(func() { + throttler = getThrottler() + Expect(throttler).ShouldNot(BeNil()) + }) + + AfterAll(func() { + throttler.close() + }) + + It("Send telemetry record", func() { + res, err := throttler.sendTelemetryRecord(&models.TelemetryRecord{ + IpHashes: []string{}, + Timestamp: time.Now().Unix(), + Message: "dev", + ResourceType: "dev", + EventType: "test", + Resource: nil, + Extra: nil, + }) + Expect(err).ShouldNot(HaveOccurred()) + bts, err := io.ReadAll(res.Body) + Expect(err).ShouldNot(HaveOccurred()) + fmt.Printf("%s\n", string(bts)) + }) + + It("Send telemetry record", func() { + res, err := throttler.sendTelemetryRecord(&models.TelemetryRecord{ + IpHashes: []string{}, + Timestamp: time.Now().Unix(), + Message: "dev", + ResourceType: "dev", + EventType: "test", + Resource: map[string]interface{}{ + "test": "test", + "ips": []string{"ip1", "ip2"}, + "k8sNodes": []models.K8sNode{{}, {}}, + }, + Extra: nil, + }) + Expect(err).ShouldNot(HaveOccurred()) + bts, err := io.ReadAll(res.Body) + Expect(err).ShouldNot(HaveOccurred()) + fmt.Printf("%s\n", string(bts)) + }) +}) diff --git a/pkg/telemetry/utils.go b/pkg/telemetry/utils.go new file mode 100644 index 000000000..d4416a79b --- /dev/null +++ b/pkg/telemetry/utils.go @@ -0,0 +1,23 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "crypto/md5" + "encoding/hex" +) + +func md5Hash(s string) string { + bts := md5.Sum([]byte(s)) + return hex.EncodeToString(bts[:]) +} From edf19345099b75e983e99b87e59d2bf87b397318 Mon Sep 17 00:00:00 2001 From: yuyi Date: Thu, 2 Nov 2023 21:22:50 +0800 Subject: [PATCH 03/18] fix: add license to telemetry_suite_test.go --- pkg/telemetry/telemetry_suite_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/telemetry/telemetry_suite_test.go b/pkg/telemetry/telemetry_suite_test.go index 04c897a7c..0209b6d3e 100644 --- a/pkg/telemetry/telemetry_suite_test.go +++ b/pkg/telemetry/telemetry_suite_test.go @@ -1,3 +1,15 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + package telemetry_test import ( From 86fee9b9de4f28f1995d0866c05e5b021a10aa90 Mon Sep 17 00:00:00 2001 From: yuyi Date: Thu, 2 Nov 2023 21:34:40 +0800 Subject: [PATCH 04/18] fix: lint --- pkg/telemetry/configs.go | 4 +--- pkg/telemetry/metrics.go | 3 ++- pkg/telemetry/record_test.go | 5 ++-- pkg/telemetry/sentry.go | 41 ++++++++++----------------------- pkg/telemetry/sentry_test.go | 3 ++- pkg/telemetry/telemetry.go | 9 ++++---- pkg/telemetry/telemetry_test.go | 5 ++-- pkg/telemetry/throttler.go | 14 +++++------ pkg/telemetry/throttler_test.go | 3 ++- 9 files changed, 36 insertions(+), 51 deletions(-) diff --git a/pkg/telemetry/configs.go b/pkg/telemetry/configs.go index 781900162..b721503fe 100644 --- a/pkg/telemetry/configs.go +++ b/pkg/telemetry/configs.go @@ -14,9 +14,7 @@ package telemetry import "os" -var ( - TelemetryRequestSignature string = os.Getenv(TelemetrySignatureEnvName) -) +var TelemetryRequestSignature = os.Getenv(TelemetrySignatureEnvName) const ( DefaultThrottlerBufferSize = 30 diff --git a/pkg/telemetry/metrics.go b/pkg/telemetry/metrics.go index 2cc8eb6eb..ff528712f 100644 --- a/pkg/telemetry/metrics.go +++ b/pkg/telemetry/metrics.go @@ -21,13 +21,14 @@ import ( "path/filepath" "sync" - "github.com/oceanbase/ob-operator/pkg/telemetry/models" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" + + "github.com/oceanbase/ob-operator/pkg/telemetry/models" ) type hostMetrics struct { diff --git a/pkg/telemetry/record_test.go b/pkg/telemetry/record_test.go index 1ab51891a..30d1b6cc9 100644 --- a/pkg/telemetry/record_test.go +++ b/pkg/telemetry/record_test.go @@ -17,10 +17,11 @@ import ( "fmt" "time" - "github.com/oceanbase/ob-operator/api/v1alpha1" - "github.com/oceanbase/ob-operator/pkg/telemetry/models" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/oceanbase/ob-operator/api/v1alpha1" + "github.com/oceanbase/ob-operator/pkg/telemetry/models" ) var _ = Describe("Telemetry record", func() { diff --git a/pkg/telemetry/sentry.go b/pkg/telemetry/sentry.go index 59f307c40..3bf09d940 100644 --- a/pkg/telemetry/sentry.go +++ b/pkg/telemetry/sentry.go @@ -22,51 +22,34 @@ func objectSentry(object any) { if object == nil { return } - switch object.(type) { - case *v1alpha1.OBCluster: - cluster, ok := object.(*v1alpha1.OBCluster) - if !ok { - return - } + if cluster, ok := object.(*v1alpha1.OBCluster); ok { processOBCluster(cluster) - case *v1alpha1.OBTenant: - tenant, ok := object.(*v1alpha1.OBTenant) - if !ok { - return - } + } else if tenant, ok := object.(*v1alpha1.OBTenant); ok { processOBTenant(tenant) - case *v1alpha1.OBServer: - server, ok := object.(*v1alpha1.OBServer) - if !ok { - return - } + } else if server, ok := object.(*v1alpha1.OBServer); ok { processOBServer(server) - case *v1alpha1.OBZone: - zone, ok := object.(*v1alpha1.OBZone) - if !ok { - return - } + } else if zone, ok := object.(*v1alpha1.OBZone); ok { processOBZone(zone) } } func processOBCluster(cluster *v1alpha1.OBCluster) { - fmt.Printf("[OBCluster Before] %+v\n", cluster) + _, _ = fmt.Printf("[OBCluster Before] %+v\n", cluster) if cluster.Spec.BackupVolume != nil && cluster.Spec.BackupVolume.Volume != nil && cluster.Spec.BackupVolume.Volume.NFS != nil { cluster.Spec.BackupVolume.Volume.NFS.Server = md5Hash(cluster.Spec.BackupVolume.Volume.NFS.Server) } - fmt.Printf("[OBCluster After] %+v\n", cluster) + _, _ = fmt.Printf("[OBCluster After] %+v\n", cluster) } func processOBServer(server *v1alpha1.OBServer) { - fmt.Printf("[OBServer Before] %+v\n", server) + _, _ = fmt.Printf("[OBServer Before] %+v\n", server) server.Status.PodIp = md5Hash(server.Status.PodIp) server.Status.NodeIp = md5Hash(server.Status.NodeIp) - fmt.Printf("[OBServer After] %+v\n", server) + _, _ = fmt.Printf("[OBServer After] %+v\n", server) } func processOBTenant(tenant *v1alpha1.OBTenant) { - fmt.Printf("[OBTenant After] %+v\n", tenant) + _, _ = fmt.Printf("[OBTenant After] %+v\n", tenant) for i := range tenant.Status.Pools { for j := range tenant.Status.Pools[i].Units { tenant.Status.Pools[i].Units[j].ServerIP = md5Hash(tenant.Status.Pools[i].Units[j].ServerIP) @@ -75,16 +58,16 @@ func processOBTenant(tenant *v1alpha1.OBTenant) { } } } - fmt.Printf("[OBTenant After] %+v\n", tenant) + _, _ = fmt.Printf("[OBTenant After] %+v\n", tenant) } func processOBZone(zone *v1alpha1.OBZone) { - fmt.Printf("[OBZone Before] %+v\n", zone) + _, _ = fmt.Printf("[OBZone Before] %+v\n", zone) for i := range zone.Status.OBServerStatus { zone.Status.OBServerStatus[i].Server = md5Hash(zone.Status.OBServerStatus[i].Server) } if zone.Spec.BackupVolume != nil && zone.Spec.BackupVolume.Volume != nil && zone.Spec.BackupVolume.Volume.NFS != nil { zone.Spec.BackupVolume.Volume.NFS.Server = md5Hash(zone.Spec.BackupVolume.Volume.NFS.Server) } - fmt.Printf("[OBZone After] %+v\n", zone) + _, _ = fmt.Printf("[OBZone After] %+v\n", zone) } diff --git a/pkg/telemetry/sentry_test.go b/pkg/telemetry/sentry_test.go index aec340374..d301f9c61 100644 --- a/pkg/telemetry/sentry_test.go +++ b/pkg/telemetry/sentry_test.go @@ -13,10 +13,11 @@ See the Mulan PSL v2 for more details. package telemetry import ( - "github.com/oceanbase/ob-operator/api/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + + "github.com/oceanbase/ob-operator/api/v1alpha1" ) var _ = Describe("Telemetry sentry", Label("sentry"), func() { diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index bd7be5799..322deb4ae 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -18,9 +18,10 @@ import ( "os" "time" - "github.com/oceanbase/ob-operator/pkg/telemetry/models" "k8s.io/apimachinery/pkg/runtime" record "k8s.io/client-go/tools/record" + + "github.com/oceanbase/ob-operator/pkg/telemetry/models" ) type Telemetry interface { @@ -65,13 +66,13 @@ func (t *telemetry) Event(object runtime.Object, eventType, reason, message stri } // Implement record.EventRecorder interface -func (t *telemetry) Eventf(object runtime.Object, eventType, reason, messageFmt string, args ...interface{}) { +func (t *telemetry) Eventf(object runtime.Object, eventType, reason, messageFmt string, args ...any) { t.EventRecorder.Eventf(object, eventType, reason, messageFmt, args...) t.generateFromEvent(object, nil, eventType, reason, messageFmt, args...) } // Implement record.EventRecorder interface -func (t *telemetry) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...interface{}) { +func (t *telemetry) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...any) { t.EventRecorder.AnnotatedEventf(object, annotations, eventType, reason, messageFmt, args...) t.generateFromEvent(object, annotations, eventType, reason, messageFmt, args...) } @@ -104,7 +105,7 @@ func (t *telemetry) GetHostMetrics() *hostMetrics { return t.hostMetrics } -func (t *telemetry) generateFromEvent(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...interface{}) { +func (t *telemetry) generateFromEvent(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...any) { if object == nil { t.GenerateTelemetryRecord(nil, "Unknown", eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) } else { diff --git a/pkg/telemetry/telemetry_test.go b/pkg/telemetry/telemetry_test.go index 1a026ddab..ba0b27790 100644 --- a/pkg/telemetry/telemetry_test.go +++ b/pkg/telemetry/telemetry_test.go @@ -15,13 +15,12 @@ package telemetry import ( "time" - "github.com/oceanbase/ob-operator/api/v1alpha1" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + + "github.com/oceanbase/ob-operator/api/v1alpha1" ) type fakeEventRecorder struct{} diff --git a/pkg/telemetry/throttler.go b/pkg/telemetry/throttler.go index e14f3dd9c..58d1cd10f 100644 --- a/pkg/telemetry/throttler.go +++ b/pkg/telemetry/throttler.go @@ -87,28 +87,28 @@ func (t *throttler) sendTelemetryRecord(record *models.TelemetryRecord) (*http.R func (t *throttler) startWorkers() { for i := 0; i < DefaultThrottlerWorkerCount; i++ { - go func(ctx context.Context, ch <-chan *models.TelemetryRecord) error { + go func(ctx context.Context, ch <-chan *models.TelemetryRecord) { for { select { case record, ok := <-ch: if !ok { // channel closed - return nil + return } res, err := t.sendTelemetryRecord(record) if t.debug { if err != nil { - fmt.Printf("send telemetry record error: %v\n", err) + _, _ = fmt.Printf("send telemetry record error: %v\n", err) } bts, err := io.ReadAll(res.Body) if err != nil { - fmt.Printf("read response body error: %v\n", err) + _, _ = fmt.Printf("read response body error: %v\n", err) } - fmt.Printf("[Event %s.%s] %s\n", record.ResourceType, record.EventType, string(bts)) + _, _ = fmt.Printf("[Event %s.%s] %s\n", record.ResourceType, record.EventType, string(bts)) } case <-ctx.Done(): - return ctx.Err() - default: + _, _ = fmt.Println(ctx.Err()) + return } } }(t.ctx, t.chanOut()) diff --git a/pkg/telemetry/throttler_test.go b/pkg/telemetry/throttler_test.go index 73f9ae11b..3194fc78f 100644 --- a/pkg/telemetry/throttler_test.go +++ b/pkg/telemetry/throttler_test.go @@ -17,9 +17,10 @@ import ( "io" "time" - "github.com/oceanbase/ob-operator/pkg/telemetry/models" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/oceanbase/ob-operator/pkg/telemetry/models" ) var _ = Describe("Telemetry throttler", Ordered, Label("throttler"), func() { From b0e09f7e535568f4a25e00063dd32ebcb9dbadd0 Mon Sep 17 00:00:00 2001 From: yuyi Date: Thu, 2 Nov 2023 21:39:39 +0800 Subject: [PATCH 05/18] chore: restore go.mod --- go.mod | 16 ++++++++-------- go.sum | 16 ---------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 7938f9a92..9a0a6abd3 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/google/uuid v1.3.0 github.com/jmoiron/sqlx v1.3.5 - github.com/onsi/ginkgo/v2 v2.13.0 - github.com/onsi/gomega v1.29.0 + github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/gomega v1.27.10 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.8.1 @@ -35,7 +35,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/imdario/mergo v0.3.6 // indirect @@ -55,13 +55,13 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/term v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect diff --git a/go.sum b/go.sum index 8af4c2d23..db5198d41 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -115,12 +113,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -187,8 +181,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= @@ -210,20 +202,14 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -237,8 +223,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From b26401cbdc1e50a1ba2ed9d376b7152d3424a8b8 Mon Sep 17 00:00:00 2001 From: yuyi Date: Fri, 3 Nov 2023 11:56:01 +0800 Subject: [PATCH 06/18] chore: set image of operator manager to oceanbasedev/... for test convinience --- config/manager/kustomization.yaml | 2 +- deploy/operator.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index e5d954caf..bbe1337ed 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: oceanbasedev/ob-operator - newTag: 2.0.1 + newTag: 2.0.1-alpha.1 diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 2f538cf44..8a2fd498d 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -12622,7 +12622,7 @@ spec: - --manager-namespace=oceanbase-system command: - /manager - image: oceanbase/ob-operator:2.0.0 + image: oceanbasedev/ob-operator:2.0.1-alpha.1 livenessProbe: httpGet: path: /healthz From 9163999843e44bd2c2bfa2c022cca8d484aa57e3 Mon Sep 17 00:00:00 2001 From: yuyi Date: Fri, 3 Nov 2023 16:49:39 +0800 Subject: [PATCH 07/18] fix: telemetry host metrics logical mistake --- pkg/telemetry/metrics.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/telemetry/metrics.go b/pkg/telemetry/metrics.go index ff528712f..95d7b5321 100644 --- a/pkg/telemetry/metrics.go +++ b/pkg/telemetry/metrics.go @@ -98,11 +98,11 @@ func getK8sNodes() ([]corev1.Node, error) { } func getHostMetrics() *hostMetrics { - telemetryEnvMetrics = &hostMetrics{ - IPHashes: []string{}, - K8sNodes: []models.K8sNode{}, - } telemetryEnvMetricsOnce.Do(func() { + telemetryEnvMetrics = &hostMetrics{ + IPHashes: []string{}, + K8sNodes: []models.K8sNode{}, + } ips, err := getLocalIPs() if err == nil { for _, ip := range ips { From 5dba21299ec2c78a1d25a73fcff338fec2544460 Mon Sep 17 00:00:00 2001 From: yuyi Date: Mon, 6 Nov 2023 17:03:54 +0800 Subject: [PATCH 08/18] feat: introduce telemetry module --- config/default/kustomization.yaml | 2 +- config/default/manager_auth_proxy_patch.yaml | 16 ------ config/manager/kustomization.yaml | 2 +- config/manager/manager.yaml | 45 +++++++++------ deploy/operator.yaml | 21 ++++++- deploy/tenant.yaml | 2 +- deploy/tenant_restore.yaml | 2 - pkg/controller/obcluster_controller.go | 2 + pkg/controller/obparameter_controller.go | 2 + pkg/controller/observer_controller.go | 12 ++-- pkg/controller/obtenant_controller.go | 12 ++-- pkg/controller/obtenantbackup_controller.go | 56 ++++++++++++++++--- .../obtenantbackuppolicy_controller.go | 2 + .../obtenantoperation_controller.go | 12 ++-- pkg/controller/obtenantrestore_controller.go | 23 +++----- pkg/controller/obzone_controller.go | 12 ++-- pkg/resource/obcluster_manager.go | 5 +- pkg/resource/obcluster_task.go | 14 ++++- pkg/resource/obparameter_manager.go | 2 + pkg/resource/observer_manager.go | 12 ++-- pkg/resource/obtenant_manager.go | 17 ++++-- pkg/resource/obtenant_task.go | 5 +- pkg/resource/obtenantbackuppolicy_manager.go | 3 + pkg/resource/obtenantoperation_manager.go | 12 ++-- pkg/resource/obtenantrestore_manager.go | 12 ++-- pkg/resource/obtenantrestore_task.go | 2 + pkg/resource/obzone_manager.go | 12 ++-- pkg/resource/template_manager.go | 12 ++-- pkg/telemetry/logging.go | 44 +++++++++++++++ pkg/telemetry/sentry.go | 24 ++++---- pkg/telemetry/telemetry.go | 20 +++++-- pkg/telemetry/telemetry_test.go | 3 +- pkg/telemetry/throttler.go | 15 ++--- 33 files changed, 289 insertions(+), 148 deletions(-) create mode 100644 pkg/telemetry/logging.go diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index bc13e3dc0..749e98ef9 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -31,7 +31,7 @@ patchesStrategicMerge: # If you want your controller-manager to expose the /metrics # endpoint w/o any authn/z, please comment the following line. - manager_auth_proxy_patch.yaml - +- manager_config_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index f06445fee..14f6112f3 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -8,22 +8,6 @@ metadata: spec: template: spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux containers: - name: kube-rbac-proxy securityContext: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index bbe1337ed..7f0a32564 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: oceanbasedev/ob-operator - newTag: 2.0.1-alpha.1 + newTag: 2.0.1-alpha.2 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index f6fc5cb01..c16d052b2 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -40,22 +40,28 @@ spec: # according to the platforms which are supported by your solution. # It is considered best practice to support multiple architectures. You can # build your manager image using the makefile target docker-buildx. - # affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: kubernetes.io/arch - # operator: In - # values: - # - amd64 - # - arm64 - # - ppc64le - # - s390x - # - key: kubernetes.io/os - # operator: In - # values: - # - linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/hostname + operator: In + values: + - sqaappnoxdnv62s2011161204053.sa128 + - key: node-role.kubernetes.io/master + operator: Exists securityContext: runAsNonRoot: true # TODO(user): For common cases that do not require escalating privileges @@ -72,6 +78,11 @@ spec: - --leader-elect image: controller:latest name: manager + env: + # - name: TELEMETRY_DEBUG + # value: "true" + - name: TELEMETRY_SIGNATURE + value: "dbe97393a695335d67de91dd4049ba" securityContext: allowPrivilegeEscalation: false capabilities: @@ -93,7 +104,7 @@ spec: # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: - cpu: 1 + cpu: "1" memory: 1Gi requests: cpu: 10m diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 8a2fd498d..1339e1a90 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -11963,6 +11963,14 @@ rules: verbs: - create - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -12614,6 +12622,12 @@ spec: operator: In values: - linux + - key: kubernetes.io/hostname + operator: In + values: + - sqaappnoxdnv62s2011161204053.sa128 + - key: node-role.kubernetes.io/master + operator: Exists containers: - args: - --health-probe-bind-address=:8081 @@ -12622,7 +12636,10 @@ spec: - --manager-namespace=oceanbase-system command: - /manager - image: oceanbasedev/ob-operator:2.0.1-alpha.1 + env: + - name: TELEMETRY_SIGNATURE + value: dbe97393a695335d67de91dd4049ba + image: oceanbasedev/ob-operator:2.0.1-alpha.2 livenessProbe: httpGet: path: /healthz @@ -12642,7 +12659,7 @@ spec: periodSeconds: 10 resources: limits: - cpu: 1 + cpu: "1" memory: 1Gi requests: cpu: 10m diff --git a/deploy/tenant.yaml b/deploy/tenant.yaml index d7f89fac8..f966da02e 100644 --- a/deploy/tenant.yaml +++ b/deploy/tenant.yaml @@ -2,7 +2,7 @@ apiVersion: oceanbase.oceanbase.com/v1alpha1 kind: OBTenant metadata: name: t1 - namespace: oceanbase + # namespace: oceanbase spec: obcluster: test tenantName: t1 diff --git a/deploy/tenant_restore.yaml b/deploy/tenant_restore.yaml index 536e3b51b..450eec094 100644 --- a/deploy/tenant_restore.yaml +++ b/deploy/tenant_restore.yaml @@ -16,8 +16,6 @@ spec: standbyRo: t1s-ro source: restore: - # sourceUri: "file:///ob-backup/t1/data_backup_custom1,file:///ob-backup/t1/log_archive_custom1" - # path in xxxSource should archiveSource: type: NFS path: "t1/log_archive_custom" diff --git a/pkg/controller/obcluster_controller.go b/pkg/controller/obcluster_controller.go index 2c6f64305..f16385306 100644 --- a/pkg/controller/obcluster_controller.go +++ b/pkg/controller/obcluster_controller.go @@ -29,6 +29,7 @@ import ( v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) // OBClusterReconciler reconciles a OBCluster object @@ -84,6 +85,7 @@ func (r *OBClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( Client: r.Client, Recorder: r.Recorder, Logger: &logger, + Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), } coordinator := resource.NewCoordinator(obclusterManager, &logger) return coordinator.Coordinate() diff --git a/pkg/controller/obparameter_controller.go b/pkg/controller/obparameter_controller.go index d77e2af8f..667042ab0 100644 --- a/pkg/controller/obparameter_controller.go +++ b/pkg/controller/obparameter_controller.go @@ -28,6 +28,7 @@ import ( v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) // OBParameterReconciler reconciles a OBParameter object @@ -72,6 +73,7 @@ func (r *OBParameterReconciler) Reconcile(ctx context.Context, req ctrl.Request) Client: r.Client, Recorder: r.Recorder, Logger: &logger, + Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), } coordinator := resource.NewCoordinator(obparameterManager, &logger) return coordinator.Coordinate() diff --git a/pkg/controller/observer_controller.go b/pkg/controller/observer_controller.go index 731c2c67a..943265858 100644 --- a/pkg/controller/observer_controller.go +++ b/pkg/controller/observer_controller.go @@ -32,6 +32,7 @@ import ( v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) // OBServerReconciler reconciles a OBServer object @@ -77,11 +78,12 @@ func (r *OBServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // create observer manager observerManager := &resource.OBServerManager{ - Ctx: ctx, - OBServer: observer, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, + Ctx: ctx, + OBServer: observer, + Client: r.Client, + Recorder: r.Recorder, + Logger: &logger, + Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), } // execute finalizers diff --git a/pkg/controller/obtenant_controller.go b/pkg/controller/obtenant_controller.go index d8b02d4df..8a64c8b2c 100644 --- a/pkg/controller/obtenant_controller.go +++ b/pkg/controller/obtenant_controller.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" "github.com/oceanbase/ob-operator/pkg/util/codec" v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" @@ -84,11 +85,12 @@ func (r *OBTenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // create observer manager obtenantManager := &resource.OBTenantManager{ - Ctx: ctx, - OBTenant: obtenant, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, + Ctx: ctx, + OBTenant: obtenant, + Client: r.Client, + Recorder: r.Recorder, + Logger: &logger, + Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), } coordinator := resource.NewCoordinator(obtenantManager, &logger) diff --git a/pkg/controller/obtenantbackup_controller.go b/pkg/controller/obtenantbackup_controller.go index 2162746f7..54f4772de 100644 --- a/pkg/controller/obtenantbackup_controller.go +++ b/pkg/controller/obtenantbackup_controller.go @@ -19,16 +19,19 @@ package controller import ( "context" "fmt" + "sync" "time" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" "github.com/pkg/errors" @@ -41,10 +44,12 @@ import ( // OBTenantBackupReconciler reconciles a OBTenantBackup object type OBTenantBackupReconciler struct { client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder + Scheme *runtime.Scheme + Recorder record.EventRecorder + Telemetry telemetry.Telemetry - con *operation.OceanbaseOperationManager + telemetryOnce sync.Once + con *operation.OceanbaseOperationManager } //+kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obtenantbackups,verbs=get;list;watch;create;update;patch;delete @@ -124,23 +129,31 @@ func (r *OBTenantBackupReconciler) createBackupJobInOB(ctx context.Context, job password, err := resource.ReadPassword(r.Client, job.Namespace, job.Spec.EncryptionSecret) if err != nil { logger.Error(err, "failed to read backup encryption secret") - r.Recorder.Event(job, "Warning", "ReadBackupEncryptionSecretFailed", err.Error()) + r.getTelemetry(ctx).Event(job, "Warning", "ReadBackupEncryptionSecretFailed", err.Error()) } else if password != "" { err = con.SetBackupPassword(password) if err != nil { logger.Error(err, "failed to set backup password") - r.Recorder.Event(job, "Warning", "SetBackupPasswordFailed", err.Error()) + r.getTelemetry(ctx).Event(job, "Warning", "SetBackupPasswordFailed", err.Error()) } } } latest, err := con.CreateAndReturnBackupJob(job.Spec.Type) if err != nil { logger.Error(err, "failed to create and return backup job") + r.getTelemetry(ctx).Event(job, "Warning", "CreateAndReturnBackupJobFailed", err.Error()) return err } job.Status.BackupJob = latest - return r.Status().Update(ctx, job) + err = r.retryUpdateStatus(ctx, job) + if err != nil { + logger.Error(err, "failed to update status") + r.getTelemetry(ctx).Event(job, "Warning", "UpdateStatusFailed", err.Error()) + return err + } + r.getTelemetry(ctx).Event(job, "Create", "", "create backup job successfully") + return nil } // TODO: Calculate the progress of running jobs @@ -187,7 +200,7 @@ func (r *OBTenantBackupReconciler) maintainRunningBackupJob(ctx context.Context, case "CANCELED": job.Status.Status = constants.BackupJobStatusCanceled } - return r.Client.Status().Update(ctx, job) + return r.retryUpdateStatus(ctx, job) } func (r *OBTenantBackupReconciler) maintainRunningBackupCleanJob(ctx context.Context, job *v1alpha1.OBTenantBackup) error { @@ -219,7 +232,7 @@ func (r *OBTenantBackupReconciler) maintainRunningBackupCleanJob(ctx context.Con case "DOING": job.Status.Status = constants.BackupJobStatusRunning } - return r.Client.Status().Update(ctx, job) + return r.retryUpdateStatus(ctx, job) } return nil @@ -252,7 +265,7 @@ func (r *OBTenantBackupReconciler) maintainRunningArchiveLogJob(ctx context.Cont case "SUSPEND": job.Status.Status = constants.BackupJobStatusSuspend } - return r.Client.Status().Update(ctx, job) + return r.retryUpdateStatus(ctx, job) } return nil @@ -279,3 +292,28 @@ func (r *OBTenantBackupReconciler) getObOperationClient(ctx context.Context, job r.con = con return con, nil } + +func (r *OBTenantBackupReconciler) retryUpdateStatus(ctx context.Context, job *v1alpha1.OBTenantBackup) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + newestJob := &v1alpha1.OBTenantBackup{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: job.GetNamespace(), + Name: job.GetName(), + }, newestJob) + if err != nil { + return client.IgnoreNotFound(err) + } + newestJob.Status = job.Status + return r.Status().Update(ctx, newestJob) + }) +} + +func (r *OBTenantBackupReconciler) getTelemetry(ctx context.Context) telemetry.Telemetry { + if r.Telemetry != nil { + return r.Telemetry + } + r.telemetryOnce.Do(func() { + r.Telemetry = telemetry.NewTelemetry(ctx, r.Recorder) + }) + return r.Telemetry +} diff --git a/pkg/controller/obtenantbackuppolicy_controller.go b/pkg/controller/obtenantbackuppolicy_controller.go index a0a80cdd8..73cf81d0e 100644 --- a/pkg/controller/obtenantbackuppolicy_controller.go +++ b/pkg/controller/obtenantbackuppolicy_controller.go @@ -27,6 +27,7 @@ import ( "github.com/oceanbase/ob-operator/api/v1alpha1" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) // OBTenantBackupPolicyReconciler reconciles a OBTenantBackupPolicy object @@ -74,6 +75,7 @@ func (r *OBTenantBackupPolicyReconciler) Reconcile(ctx context.Context, req ctrl Client: r.Client, Recorder: r.Recorder, Logger: &logger, + Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), } coordinator := resource.NewCoordinator(mgr, &logger) diff --git a/pkg/controller/obtenantoperation_controller.go b/pkg/controller/obtenantoperation_controller.go index a15805790..475f2374b 100644 --- a/pkg/controller/obtenantoperation_controller.go +++ b/pkg/controller/obtenantoperation_controller.go @@ -27,6 +27,7 @@ import ( v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) // OBTenantOperationReconciler reconciles a OBTenantOperation object @@ -57,11 +58,12 @@ func (r *OBTenantOperationReconciler) Reconcile(ctx context.Context, req ctrl.Re } mgr := &resource.ObTenantOperationManager{ - Ctx: ctx, - Resource: operation, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, + Ctx: ctx, + Resource: operation, + Client: r.Client, + Recorder: r.Recorder, + Logger: &logger, + Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), } coordinator := resource.NewCoordinator(mgr, &logger) diff --git a/pkg/controller/obtenantrestore_controller.go b/pkg/controller/obtenantrestore_controller.go index 5acd83ded..aa90967f0 100644 --- a/pkg/controller/obtenantrestore_controller.go +++ b/pkg/controller/obtenantrestore_controller.go @@ -28,6 +28,7 @@ import ( v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) // OBTenantRestoreReconciler reconciles a OBTenantRestore object @@ -58,23 +59,13 @@ func (r *OBTenantRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, client.IgnoreNotFound(err) } - // finalizerName := "obtenantrestore.finalizers.oceanbase.com" - // // examine DeletionTimestamp to determine if the policy is under deletion - // if restore.ObjectMeta.DeletionTimestamp.IsZero() { - // if !controllerutil.ContainsFinalizer(restore, finalizerName) { - // controllerutil.AddFinalizer(restore, finalizerName) - // if err := r.Update(ctx, restore); err != nil { - // return ctrl.Result{}, err - // } - // } - // } - mgr := &resource.ObTenantRestoreManager{ - Ctx: ctx, - Resource: restore, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, + Ctx: ctx, + Resource: restore, + Client: r.Client, + Recorder: r.Recorder, + Logger: &logger, + Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), } coordinator := resource.NewCoordinator(mgr, &logger) diff --git a/pkg/controller/obzone_controller.go b/pkg/controller/obzone_controller.go index 23a8d243b..818947be1 100644 --- a/pkg/controller/obzone_controller.go +++ b/pkg/controller/obzone_controller.go @@ -28,6 +28,7 @@ import ( v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" "github.com/oceanbase/ob-operator/pkg/resource" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) // OBZoneReconciler reconciles a OBZone object @@ -71,11 +72,12 @@ func (r *OBZoneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // create zone manager obzoneManager := &resource.OBZoneManager{ - Ctx: ctx, - OBZone: obzone, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, + Ctx: ctx, + OBZone: obzone, + Client: r.Client, + Recorder: r.Recorder, + Logger: &logger, + Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), } coordinator := resource.NewCoordinator(obzoneManager, &logger) return coordinator.Coordinate() diff --git a/pkg/resource/obcluster_manager.go b/pkg/resource/obcluster_manager.go index 863d556fc..7892d4f89 100644 --- a/pkg/resource/obcluster_manager.go +++ b/pkg/resource/obcluster_manager.go @@ -31,6 +31,7 @@ import ( taskname "github.com/oceanbase/ob-operator/pkg/task/const/task/name" taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/task/status" "github.com/oceanbase/ob-operator/pkg/task/strategy" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) type OBClusterManager struct { @@ -39,6 +40,7 @@ type OBClusterManager struct { OBCluster *v1alpha1.OBCluster Client client.Client Recorder record.EventRecorder + Telemetry telemetry.Telemetry Logger *logr.Logger } @@ -48,6 +50,7 @@ func (m *OBClusterManager) IsNewResource() bool { func (m *OBClusterManager) InitStatus() { m.Logger.Info("newly created cluster, init status") + m.Telemetry.Event(m.OBCluster, "Init", "", "newly created cluster, init status") status := v1alpha1.OBClusterStatus{ Image: m.OBCluster.Spec.OBServerTemplate.Image, Status: clusterstatus.New, @@ -309,7 +312,7 @@ func (m *OBClusterManager) GetTaskFunc(name string) (func() error, error) { } func (m *OBClusterManager) PrintErrEvent(err error) { - m.Recorder.Event(m.OBCluster, corev1.EventTypeWarning, "task exec failed", err.Error()) + m.Telemetry.Event(m.OBCluster, corev1.EventTypeWarning, "task exec failed", err.Error()) } func (m *OBClusterManager) listOBZones() (*v1alpha1.OBZoneList, error) { diff --git a/pkg/resource/obcluster_task.go b/pkg/resource/obcluster_task.go index ae33f06f1..22d374de8 100644 --- a/pkg/resource/obcluster_task.go +++ b/pkg/resource/obcluster_task.go @@ -292,6 +292,8 @@ func (m *OBClusterManager) Bootstrap() error { err = manager.Bootstrap(bootstrapServers) if err != nil { m.Logger.Error(err, "bootstrap failed") + } else { + m.Telemetry.Event(m.OBCluster, "Bootstrap", "", "Bootstrap successfully") } return err } @@ -780,7 +782,12 @@ func (m *OBClusterManager) CreateServiceForMonitor() error { Type: corev1.ServiceTypeClusterIP, }, } - return m.Client.Create(m.Ctx, &monitorService) + err := m.Client.Create(m.Ctx, &monitorService) + if err != nil { + return errors.Wrap(err, "Create monitor service") + } + m.Telemetry.Event(m.OBCluster, "MaintainedAfterBootstrap", "", "Create monitor service successfully") + return nil } func (m *OBClusterManager) RestoreEssentialParameters() error { @@ -798,9 +805,9 @@ func (m *OBClusterManager) RestoreEssentialParameters() error { }, contextSecret) if err != nil { m.Logger.Error(err, "Failed to get context secret") - return nil // parameter can be set manually, just return here and emit an event - // TODO: emit an event + m.Telemetry.Event(m.OBCluster, "Warning", "Restore essential parameters failed", err.Error()) + return nil } encodedParameters := string(contextSecret.Data[oceanbaseconst.EssentialParametersKey]) @@ -820,5 +827,6 @@ func (m *OBClusterManager) RestoreEssentialParameters() error { } } _ = m.Client.Delete(m.Ctx, contextSecret) + m.Telemetry.Event(m.OBCluster, "Upgrade", "", "Restore essential parameters successfully") return nil } diff --git a/pkg/resource/obparameter_manager.go b/pkg/resource/obparameter_manager.go index 16c3dc771..103f76539 100644 --- a/pkg/resource/obparameter_manager.go +++ b/pkg/resource/obparameter_manager.go @@ -27,6 +27,7 @@ import ( "github.com/oceanbase/ob-operator/pkg/oceanbase/operation" taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/task/status" "github.com/oceanbase/ob-operator/pkg/task/strategy" + "github.com/oceanbase/ob-operator/pkg/telemetry" v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" oceanbaseconst "github.com/oceanbase/ob-operator/pkg/const/oceanbase" @@ -43,6 +44,7 @@ type OBParameterManager struct { OBParameter *v1alpha1.OBParameter Client client.Client Recorder record.EventRecorder + Telemetry telemetry.Telemetry Logger *logr.Logger } diff --git a/pkg/resource/observer_manager.go b/pkg/resource/observer_manager.go index 00feb563f..c181db930 100644 --- a/pkg/resource/observer_manager.go +++ b/pkg/resource/observer_manager.go @@ -18,6 +18,7 @@ import ( "github.com/oceanbase/ob-operator/pkg/oceanbase/model" taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/task/status" "github.com/oceanbase/ob-operator/pkg/task/strategy" + "github.com/oceanbase/ob-operator/pkg/telemetry" corev1 "k8s.io/api/core/v1" kubeerrors "k8s.io/apimachinery/pkg/api/errors" @@ -42,11 +43,12 @@ import ( type OBServerManager struct { ResourceManager - Ctx context.Context - OBServer *v1alpha1.OBServer - Client client.Client - Recorder record.EventRecorder - Logger *logr.Logger + Ctx context.Context + OBServer *v1alpha1.OBServer + Client client.Client + Recorder record.EventRecorder + Telemetry telemetry.Telemetry + Logger *logr.Logger } func (m *OBServerManager) GetTaskFunc(name string) (func() error, error) { diff --git a/pkg/resource/obtenant_manager.go b/pkg/resource/obtenant_manager.go index 1575cc6da..6ed21f4c0 100644 --- a/pkg/resource/obtenant_manager.go +++ b/pkg/resource/obtenant_manager.go @@ -39,15 +39,17 @@ import ( taskname "github.com/oceanbase/ob-operator/pkg/task/const/task/name" taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/task/status" "github.com/oceanbase/ob-operator/pkg/task/strategy" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) type OBTenantManager struct { ResourceManager - OBTenant *v1alpha1.OBTenant - Ctx context.Context - Client client.Client - Recorder record.EventRecorder - Logger *logr.Logger + OBTenant *v1alpha1.OBTenant + Ctx context.Context + Client client.Client + Recorder record.EventRecorder + Telemetry telemetry.Telemetry + Logger *logr.Logger } // TODO add lock to be thread safe, and read/write whitelist from/to DB @@ -88,9 +90,12 @@ func (m *OBTenantManager) InitStatus() { if m.OBTenant.Spec.Source != nil && m.OBTenant.Spec.Source.Restore != nil { m.OBTenant.Status.Status = tenantstatus.Restoring + m.Telemetry.Event(m.OBTenant, "InitRestore", "", "start restoring") } else if m.OBTenant.Spec.Source != nil && m.OBTenant.Spec.Source.Tenant != nil { + m.Telemetry.Event(m.OBTenant, "InitEmptyStandby", "", "start creating empty standby") m.OBTenant.Status.Status = tenantstatus.CreatingEmptyStandby } else { + m.Telemetry.Event(m.OBTenant, "Init", "", "start creating") m.OBTenant.Status.Status = tenantstatus.CreatingTenant } } @@ -328,7 +333,7 @@ func (m *OBTenantManager) GetTaskFlow() (*task.TaskFlow, error) { } func (m *OBTenantManager) PrintErrEvent(err error) { - m.Recorder.Event(m.OBTenant, corev1.EventTypeWarning, "task exec failed", err.Error()) + m.Telemetry.Event(m.OBTenant, corev1.EventTypeWarning, "task exec failed", err.Error()) } // ---------- K8S API Helper ---------- diff --git a/pkg/resource/obtenant_task.go b/pkg/resource/obtenant_task.go index a286a96ee..827432899 100644 --- a/pkg/resource/obtenant_task.go +++ b/pkg/resource/obtenant_task.go @@ -386,10 +386,12 @@ func (m *OBTenantManager) createTenant() error { err = oceanbaseOperationManager.AddTenant(tenantSQLParam) if err != nil { + m.Telemetry.Event(m.OBTenant, corev1.EventTypeWarning, "failed to create OBTenant", err.Error()) return err } GlobalWhiteListMap[tenantName] = m.OBTenant.Spec.ConnectWhiteList // Create user or change password of root, do not return error + m.Telemetry.Event(m.OBTenant, "Create", "", "create OBTenant successfully") return nil } @@ -1086,7 +1088,7 @@ func (m *OBTenantManager) CreateUserWithCredentialSecrets() error { } err := m.createUserWithCredentials() if err != nil { - m.Recorder.Event(m.OBTenant, corev1.EventTypeWarning, "Failed to create user or change password", err.Error()) + m.Telemetry.Event(m.OBTenant, corev1.EventTypeWarning, "Failed to create user or change password", err.Error()) m.Logger.Error(err, "Failed to create user or change password, please check the credential secrets") } @@ -1179,6 +1181,7 @@ func (m *OBTenantManager) CreateEmptyStandbyTenant() error { if err != nil { return err } + m.Telemetry.Event(m.OBTenant, "CreateEmptyStandby", "", "Succeed to create empty standby tenant") return nil } diff --git a/pkg/resource/obtenantbackuppolicy_manager.go b/pkg/resource/obtenantbackuppolicy_manager.go index 9ef4e7b22..151647506 100644 --- a/pkg/resource/obtenantbackuppolicy_manager.go +++ b/pkg/resource/obtenantbackuppolicy_manager.go @@ -37,6 +37,7 @@ import ( taskname "github.com/oceanbase/ob-operator/pkg/task/const/task/name" taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/task/status" "github.com/oceanbase/ob-operator/pkg/task/strategy" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) type ObTenantBackupPolicyManager struct { @@ -45,6 +46,7 @@ type ObTenantBackupPolicyManager struct { BackupPolicy *v1alpha1.OBTenantBackupPolicy Client client.Client Recorder record.EventRecorder + Telemetry telemetry.Telemetry Logger *logr.Logger con *operation.OceanbaseOperationManager @@ -115,6 +117,7 @@ func (m *ObTenantBackupPolicyManager) InitStatus() { m.BackupPolicy.Status = v1alpha1.OBTenantBackupPolicyStatus{ Status: constants.BackupPolicyStatusPreparing, } + m.Telemetry.Event(m.BackupPolicy, "Init", "", "init status") err = m.syncTenantInformation() if err != nil { m.PrintErrEvent(err) diff --git a/pkg/resource/obtenantoperation_manager.go b/pkg/resource/obtenantoperation_manager.go index a30501617..f1e02a2be 100644 --- a/pkg/resource/obtenantoperation_manager.go +++ b/pkg/resource/obtenantoperation_manager.go @@ -33,16 +33,18 @@ import ( taskname "github.com/oceanbase/ob-operator/pkg/task/const/task/name" taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/task/status" "github.com/oceanbase/ob-operator/pkg/task/strategy" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) type ObTenantOperationManager struct { ResourceManager - Ctx context.Context - Resource *v1alpha1.OBTenantOperation - Client client.Client - Recorder record.EventRecorder - Logger *logr.Logger + Ctx context.Context + Resource *v1alpha1.OBTenantOperation + Client client.Client + Recorder record.EventRecorder + Telemetry telemetry.Telemetry + Logger *logr.Logger con *operation.OceanbaseOperationManager } diff --git a/pkg/resource/obtenantrestore_manager.go b/pkg/resource/obtenantrestore_manager.go index d294042a4..e4d3c92e5 100644 --- a/pkg/resource/obtenantrestore_manager.go +++ b/pkg/resource/obtenantrestore_manager.go @@ -33,16 +33,18 @@ import ( taskname "github.com/oceanbase/ob-operator/pkg/task/const/task/name" taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/task/status" "github.com/oceanbase/ob-operator/pkg/task/strategy" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) type ObTenantRestoreManager struct { ResourceManager - Ctx context.Context - Resource *v1alpha1.OBTenantRestore - Client client.Client - Recorder record.EventRecorder - Logger *logr.Logger + Ctx context.Context + Resource *v1alpha1.OBTenantRestore + Client client.Client + Recorder record.EventRecorder + Telemetry telemetry.Telemetry + Logger *logr.Logger con *operation.OceanbaseOperationManager } diff --git a/pkg/resource/obtenantrestore_task.go b/pkg/resource/obtenantrestore_task.go index 2f6a8f1b6..e729d2069 100644 --- a/pkg/resource/obtenantrestore_task.go +++ b/pkg/resource/obtenantrestore_task.go @@ -112,11 +112,13 @@ func (m *OBTenantManager) WatchRestoreJobToFinish() error { if runningRestore.Status.Status == constants.RestoreJobSuccessful { break } else if runningRestore.Status.Status == constants.RestoreJobFailed { + m.Telemetry.Event(m.OBTenant, "RestoreJobFailed", "", "restore job failed") return errors.New("Restore job failed") } time.Sleep(5 * time.Second) } GlobalWhiteListMap[m.OBTenant.Spec.TenantName] = m.OBTenant.Spec.ConnectWhiteList + m.Telemetry.Event(m.OBTenant, "RestoreJobFinished", "", "restore job finished successfully") return nil } diff --git a/pkg/resource/obzone_manager.go b/pkg/resource/obzone_manager.go index 84a94a9fd..b4a5a4de3 100644 --- a/pkg/resource/obzone_manager.go +++ b/pkg/resource/obzone_manager.go @@ -35,15 +35,17 @@ import ( taskname "github.com/oceanbase/ob-operator/pkg/task/const/task/name" taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/task/status" "github.com/oceanbase/ob-operator/pkg/task/strategy" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) type OBZoneManager struct { ResourceManager - Ctx context.Context - OBZone *v1alpha1.OBZone - Client client.Client - Recorder record.EventRecorder - Logger *logr.Logger + Ctx context.Context + OBZone *v1alpha1.OBZone + Client client.Client + Recorder record.EventRecorder + Telemetry telemetry.Telemetry + Logger *logr.Logger } func (m *OBZoneManager) IsNewResource() bool { diff --git a/pkg/resource/template_manager.go b/pkg/resource/template_manager.go index 3f6bdf6ab..eb0453f41 100644 --- a/pkg/resource/template_manager.go +++ b/pkg/resource/template_manager.go @@ -23,16 +23,18 @@ import ( v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" "github.com/oceanbase/ob-operator/pkg/oceanbase/operation" "github.com/oceanbase/ob-operator/pkg/task" + "github.com/oceanbase/ob-operator/pkg/telemetry" ) type ObResourceManager[T client.Object] struct { ResourceManager - Ctx context.Context - Resource T - Client client.Client - Recorder record.EventRecorder - Logger *logr.Logger + Ctx context.Context + Resource T + Client client.Client + Recorder record.EventRecorder + Logger *logr.Logger + Telemetry telemetry.Telemetry con *operation.OceanbaseOperationManager } diff --git a/pkg/telemetry/logging.go b/pkg/telemetry/logging.go new file mode 100644 index 000000000..85ec7eead --- /dev/null +++ b/pkg/telemetry/logging.go @@ -0,0 +1,44 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "io" + "log" + "os" + "sync" +) + +var lg *log.Logger +var loggerOnce sync.Once +var debugMode = os.Getenv(TelemetryDebugEnvName) == "true" + +func configLogger() { + if debugMode { + file, err := os.OpenFile("/tmp/telemetry.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + // log.Println("Failed to open log file:", err) + lg = log.New(io.Discard, "[Telemetry] ", log.LstdFlags|log.Lshortfile) + } else { + lg = log.New(file, "[Telemetry] ", log.LstdFlags|log.Lshortfile) + } + } else { + // if not in debug mode, discard all logs + lg = log.New(io.Discard, "[Telemetry] ", log.LstdFlags|log.Lshortfile) + } +} + +func getLogger() *log.Logger { + loggerOnce.Do(configLogger) + return lg +} diff --git a/pkg/telemetry/sentry.go b/pkg/telemetry/sentry.go index 3bf09d940..42dee82f4 100644 --- a/pkg/telemetry/sentry.go +++ b/pkg/telemetry/sentry.go @@ -13,7 +13,7 @@ See the Mulan PSL v2 for more details. package telemetry import ( - "fmt" + "k8s.io/apimachinery/pkg/runtime" "github.com/oceanbase/ob-operator/api/v1alpha1" ) @@ -23,33 +23,28 @@ func objectSentry(object any) { return } if cluster, ok := object.(*v1alpha1.OBCluster); ok { - processOBCluster(cluster) + debugWrapper(processOBCluster, cluster, "OBCluster") } else if tenant, ok := object.(*v1alpha1.OBTenant); ok { - processOBTenant(tenant) + debugWrapper(processOBTenant, tenant, "OBTenant") } else if server, ok := object.(*v1alpha1.OBServer); ok { - processOBServer(server) + debugWrapper(processOBServer, server, "OBServer") } else if zone, ok := object.(*v1alpha1.OBZone); ok { - processOBZone(zone) + debugWrapper(processOBZone, zone, "OBZone") } } func processOBCluster(cluster *v1alpha1.OBCluster) { - _, _ = fmt.Printf("[OBCluster Before] %+v\n", cluster) if cluster.Spec.BackupVolume != nil && cluster.Spec.BackupVolume.Volume != nil && cluster.Spec.BackupVolume.Volume.NFS != nil { cluster.Spec.BackupVolume.Volume.NFS.Server = md5Hash(cluster.Spec.BackupVolume.Volume.NFS.Server) } - _, _ = fmt.Printf("[OBCluster After] %+v\n", cluster) } func processOBServer(server *v1alpha1.OBServer) { - _, _ = fmt.Printf("[OBServer Before] %+v\n", server) server.Status.PodIp = md5Hash(server.Status.PodIp) server.Status.NodeIp = md5Hash(server.Status.NodeIp) - _, _ = fmt.Printf("[OBServer After] %+v\n", server) } func processOBTenant(tenant *v1alpha1.OBTenant) { - _, _ = fmt.Printf("[OBTenant After] %+v\n", tenant) for i := range tenant.Status.Pools { for j := range tenant.Status.Pools[i].Units { tenant.Status.Pools[i].Units[j].ServerIP = md5Hash(tenant.Status.Pools[i].Units[j].ServerIP) @@ -58,16 +53,19 @@ func processOBTenant(tenant *v1alpha1.OBTenant) { } } } - _, _ = fmt.Printf("[OBTenant After] %+v\n", tenant) } func processOBZone(zone *v1alpha1.OBZone) { - _, _ = fmt.Printf("[OBZone Before] %+v\n", zone) for i := range zone.Status.OBServerStatus { zone.Status.OBServerStatus[i].Server = md5Hash(zone.Status.OBServerStatus[i].Server) } if zone.Spec.BackupVolume != nil && zone.Spec.BackupVolume.Volume != nil && zone.Spec.BackupVolume.Volume.NFS != nil { zone.Spec.BackupVolume.Volume.NFS.Server = md5Hash(zone.Spec.BackupVolume.Volume.NFS.Server) } - _, _ = fmt.Printf("[OBZone After] %+v\n", zone) +} + +func debugWrapper[T runtime.Object](processor func(T), object T, objectType string) { + getLogger().Printf("[%s Before] %+v\n", objectType, object) + processor(object) + getLogger().Printf("[%s After] %+v\n", objectType, object) } diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index 322deb4ae..acd832ba8 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -18,6 +18,7 @@ import ( "os" "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" record "k8s.io/client-go/tools/record" @@ -36,11 +37,13 @@ type telemetry struct { *hostMetrics record.EventRecorder + ctx context.Context telemetryDisabled bool } -func NewTelemetry(recorder record.EventRecorder) Telemetry { +func NewTelemetry(ctx context.Context, recorder record.EventRecorder) Telemetry { clt := &telemetry{ + ctx: ctx, EventRecorder: recorder, } @@ -61,19 +64,19 @@ func NewTelemetry(recorder record.EventRecorder) Telemetry { // Implement record.EventRecorder interface func (t *telemetry) Event(object runtime.Object, eventType, reason, message string) { - t.EventRecorder.Event(object, eventType, reason, message) + t.EventRecorder.Event(object, t.transformEventType(eventType), reason, message) t.generateFromEvent(object, nil, eventType, reason, message) } // Implement record.EventRecorder interface func (t *telemetry) Eventf(object runtime.Object, eventType, reason, messageFmt string, args ...any) { - t.EventRecorder.Eventf(object, eventType, reason, messageFmt, args...) + t.EventRecorder.Eventf(object, t.transformEventType(eventType), reason, messageFmt, args...) t.generateFromEvent(object, nil, eventType, reason, messageFmt, args...) } // Implement record.EventRecorder interface func (t *telemetry) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...any) { - t.EventRecorder.AnnotatedEventf(object, annotations, eventType, reason, messageFmt, args...) + t.EventRecorder.AnnotatedEventf(object, annotations, t.transformEventType(eventType), reason, messageFmt, args...) t.generateFromEvent(object, annotations, eventType, reason, messageFmt, args...) } @@ -112,3 +115,12 @@ func (t *telemetry) generateFromEvent(object runtime.Object, annotations map[str t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) } } + +func (t *telemetry) transformEventType(eventType string) string { + switch eventType { + case "Error", "Warning", "error": + return corev1.EventTypeWarning + default: + return corev1.EventTypeNormal + } +} diff --git a/pkg/telemetry/telemetry_test.go b/pkg/telemetry/telemetry_test.go index ba0b27790..338ca9ccc 100644 --- a/pkg/telemetry/telemetry_test.go +++ b/pkg/telemetry/telemetry_test.go @@ -13,6 +13,7 @@ See the Mulan PSL v2 for more details. package telemetry import ( + "context" "time" . "github.com/onsi/ginkgo/v2" @@ -42,7 +43,7 @@ var _ = Describe("Telemetry", Label("telemetry"), Ordered, func() { } BeforeAll(func() { - telemetry = NewTelemetry(&fakeEventRecorder{}) + telemetry = NewTelemetry(context.TODO(), &fakeEventRecorder{}) Expect(telemetry).ShouldNot(BeNil()) }) diff --git a/pkg/telemetry/throttler.go b/pkg/telemetry/throttler.go index 58d1cd10f..a4d1d627a 100644 --- a/pkg/telemetry/throttler.go +++ b/pkg/telemetry/throttler.go @@ -14,18 +14,15 @@ package telemetry import ( "context" - "fmt" "io" "net/http" "net/url" - "os" "sync" "github.com/oceanbase/ob-operator/pkg/telemetry/models" ) type throttler struct { - debug bool client http.Client ctx context.Context cancel context.CancelFunc @@ -41,13 +38,13 @@ func getThrottler() *throttler { recordChan: make(chan *models.TelemetryRecord, DefaultThrottlerBufferSize), } - throttlerSingleton.debug = os.Getenv(TelemetryDebugEnvName) == "true" ctx, cancel := context.WithCancel(context.Background()) throttlerSingleton.ctx = ctx throttlerSingleton.cancel = cancel throttlerSingleton.client = *http.DefaultClient throttlerSingleton.startWorkers() + getLogger().Println("telemetry throttler started", "#worker:", DefaultThrottlerWorkerCount) }) return throttlerSingleton } @@ -96,18 +93,18 @@ func (t *throttler) startWorkers() { return } res, err := t.sendTelemetryRecord(record) - if t.debug { + if debugMode { if err != nil { - _, _ = fmt.Printf("send telemetry record error: %v\n", err) + getLogger().Printf("send telemetry record error: %v\n", err) } bts, err := io.ReadAll(res.Body) if err != nil { - _, _ = fmt.Printf("read response body error: %v\n", err) + getLogger().Printf("read response body error: %v\n", err) } - _, _ = fmt.Printf("[Event %s.%s] %s\n", record.ResourceType, record.EventType, string(bts)) + getLogger().Printf("[Event %s.%s] %s\n", record.ResourceType, record.EventType, string(bts)) } case <-ctx.Done(): - _, _ = fmt.Println(ctx.Err()) + getLogger().Println(ctx.Err()) return } } From 1916121faab950d8e85cc86d6cbebd563d043086 Mon Sep 17 00:00:00 2001 From: yuyi Date: Tue, 7 Nov 2023 12:00:51 +0800 Subject: [PATCH 09/18] fix: obcluster chart --- charts/oceanbase-cluster/templates/NOTES.txt | 11 +++++++++++ charts/oceanbase-cluster/templates/obcluster.yaml | 5 ++--- charts/oceanbase-cluster/values.yaml | 11 ++++++----- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/charts/oceanbase-cluster/templates/NOTES.txt b/charts/oceanbase-cluster/templates/NOTES.txt index e69de29bb..f204a0f5e 100644 --- a/charts/oceanbase-cluster/templates/NOTES.txt +++ b/charts/oceanbase-cluster/templates/NOTES.txt @@ -0,0 +1,11 @@ +Welcome to OceanBase Cluster! + +After installing OBCluster chart, you need to wait for the cluster bootstrapped. Bootstrap progress will cost approximately 2~3 minutes which may varies depends on the machine. + +You can use the following command to wait for the OBCluster to be ready. + +> kubectl wait -n {{ .Release.Namespace }} obcluster {{ .Release.Name }} --for=jsonpath='{.status.status}'=running --timeout=10m + +After that, the cluster is ready to handle connections stably. Example command is following: + +> mysql -A -h$(kubectl get pods -l ref-obcluster={{ .Release.Name }} -o jsonpath='{.items[0].status.podIP}') -P2881 -uroot -p$(kubectl get secret -n {{ .Release.Namespace }} {{ .Values.userSecrets.root }} -o jsonpath='{.data.password}' | base64 -d) \ No newline at end of file diff --git a/charts/oceanbase-cluster/templates/obcluster.yaml b/charts/oceanbase-cluster/templates/obcluster.yaml index ed5695d1d..5aa68a409 100644 --- a/charts/oceanbase-cluster/templates/obcluster.yaml +++ b/charts/oceanbase-cluster/templates/obcluster.yaml @@ -37,10 +37,9 @@ spec: - name: {{ $param.name }} value: {{ $param.value | quote }} {{- end }} - {{- if .Values.nfsBackupEnabled }} + {{- if .Values.backupVolumeEnabled }} backupVolume: volume: name: backup - nfs: - {{- toYaml .Values.nfsBackup | nindent 8 }} + {{- toYaml .Values.backupVolume | nindent 6 }} {{- end }} \ No newline at end of file diff --git a/charts/oceanbase-cluster/values.yaml b/charts/oceanbase-cluster/values.yaml index 4f4d0d209..6f006242f 100644 --- a/charts/oceanbase-cluster/values.yaml +++ b/charts/oceanbase-cluster/values.yaml @@ -23,11 +23,12 @@ parameters: - name: "__min_full_resource_pool_memory" value: "2147483648" # 2G -nfsBackupEnabled: false # set true and config volume if you want to enable backup with NFS -nfsBackup: - server: 1.1.1.1 - path: /opt/nfs - readOnly: false +backupVolumeEnabled: false # set true and config volume if you want to enable backup volume +backupVolume: + nfs: + server: 1.1.1.1 + path: /opt/nfs + readOnly: false generateUserSecrets: true # if set true, all system user secrets will be generated automatically userSecrets: From ec6455321220825549b5ff639cd9161341f45367 Mon Sep 17 00:00:00 2001 From: yuyi Date: Tue, 7 Nov 2023 12:06:46 +0800 Subject: [PATCH 10/18] fix: tenant webhook suplement --- api/v1alpha1/obtenant_webhook.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/api/v1alpha1/obtenant_webhook.go b/api/v1alpha1/obtenant_webhook.go index adc194b39..aac557236 100644 --- a/api/v1alpha1/obtenant_webhook.go +++ b/api/v1alpha1/obtenant_webhook.go @@ -111,6 +111,37 @@ func (r *OBTenant) validateMutation() error { } } + // 0. Given credentials must exist + if r.Spec.Credentials.Root != "" { + secret := &v1.Secret{} + err = tenantClt.Get(context.Background(), types.NamespacedName{ + Namespace: r.GetNamespace(), + Name: r.Spec.Credentials.Root, + }, secret) + if err != nil { + if apierrors.IsNotFound(err) { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("credentials").Child("root"), r.Spec.Credentials.Root, "Given root credential not found")) + } else { + allErrs = append(allErrs, field.InternalError(field.NewPath("spec").Child("credentials").Child("root"), err)) + } + } + } + + if r.Spec.Credentials.StandbyRO != "" { + secret := &v1.Secret{} + err = tenantClt.Get(context.Background(), types.NamespacedName{ + Namespace: r.GetNamespace(), + Name: r.Spec.Credentials.StandbyRO, + }, secret) + if err != nil { + if apierrors.IsNotFound(err) { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("credentials").Child("standbyRo"), r.Spec.Credentials.StandbyRO, "Given standbyRo credential not found")) + } else { + allErrs = append(allErrs, field.InternalError(field.NewPath("spec").Child("credentials").Child("standbyRo"), err)) + } + } + } + // 1. Standby tenant must have a source if r.Spec.TenantRole == constants.TenantRoleStandby { if r.Spec.Source == nil { From 47b6e507152a27b5672f63de504f3c35e6a35ab0 Mon Sep 17 00:00:00 2001 From: yuyi Date: Tue, 7 Nov 2023 14:18:58 +0800 Subject: [PATCH 11/18] chore: add adminssion webhook for obcluster resource --- api/v1alpha1/obcluster_webhook.go | 123 +++++++++++++++++++++++++++++ api/v1alpha1/webhook_suite_test.go | 3 + cmd/main.go | 4 + config/webhook/manifests.yaml | 40 ++++++++++ 4 files changed, 170 insertions(+) create mode 100644 api/v1alpha1/obcluster_webhook.go diff --git a/api/v1alpha1/obcluster_webhook.go b/api/v1alpha1/obcluster_webhook.go new file mode 100644 index 000000000..c4469de43 --- /dev/null +++ b/api/v1alpha1/obcluster_webhook.go @@ -0,0 +1,123 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// log is for logging in this package. +var obclusterlog = logf.Log.WithName("obcluster-resource") + +func (r *OBCluster) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +//+kubebuilder:webhook:path=/mutate-oceanbase-oceanbase-com-v1alpha1-obcluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=oceanbase.oceanbase.com,resources=obclusters,verbs=create;update,versions=v1alpha1,name=mobcluster.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &OBCluster{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *OBCluster) Default() { + // TODO(user): fill in your defaulting logic. +} + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +//+kubebuilder:webhook:path=/validate-oceanbase-oceanbase-com-v1alpha1-obcluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=oceanbase.oceanbase.com,resources=obclusters,verbs=create;update,versions=v1alpha1,name=vobcluster.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &OBCluster{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *OBCluster) ValidateCreate() (admission.Warnings, error) { + obclusterlog.Info("validate create", "name", r.Name) + + return nil, r.validateMutation() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *OBCluster) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + _ = old + obclusterlog.Info("validate update", "name", r.Name) + + return nil, r.validateMutation() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *OBCluster) ValidateDelete() (admission.Warnings, error) { + return nil, nil +} + +func (r *OBCluster) validateMutation() error { + // Ignore deleting objects + if r.GetDeletionTimestamp() != nil { + return nil + } + + var allErrs field.ErrorList + + // 0. Validate userSecrets + if err := r.checkSecretExistence(r.Namespace, r.Spec.UserSecrets.Root, "root"); err != nil { + allErrs = append(allErrs, err) + } + if err := r.checkSecretExistence(r.Namespace, r.Spec.UserSecrets.ProxyRO, "proxyro"); err != nil { + allErrs = append(allErrs, err) + } + if err := r.checkSecretExistence(r.Namespace, r.Spec.UserSecrets.Operator, "operator"); err != nil { + allErrs = append(allErrs, err) + } + if err := r.checkSecretExistence(r.Namespace, r.Spec.UserSecrets.Monitor, "monitor"); err != nil { + allErrs = append(allErrs, err) + } + + if len(allErrs) == 0 { + return nil + } + return apierrors.NewInvalid(GroupVersion.WithKind("OBCluster").GroupKind(), r.Name, allErrs) +} + +func (r *OBCluster) checkSecretExistence(ns, secretName, fieldName string) *field.Error { + if secretName == "" { + return field.Invalid(field.NewPath("spec").Child("userSecrets").Child(fieldName), secretName, fmt.Sprintf("Empty credential %s is not permitted", fieldName)) + } + secret := &v1.Secret{} + err := tenantClt.Get(context.Background(), types.NamespacedName{ + Namespace: ns, + Name: secretName, + }, secret) + if err != nil { + if apierrors.IsNotFound(err) { + return field.Invalid(field.NewPath("spec").Child("userSecrets").Child(fieldName), secretName, fmt.Sprintf("Given %s credential %s not found", fieldName, secretName)) + } + return field.InternalError(field.NewPath("spec").Child("userSecrets").Child(fieldName), err) + } + return nil +} diff --git a/api/v1alpha1/webhook_suite_test.go b/api/v1alpha1/webhook_suite_test.go index fb5dc15e8..38316a708 100644 --- a/api/v1alpha1/webhook_suite_test.go +++ b/api/v1alpha1/webhook_suite_test.go @@ -108,6 +108,9 @@ var _ = BeforeSuite(func() { err = (&OBTenantOperation{}).SetupWebhookWithManager(mgr) Expect(err).NotTo(HaveOccurred()) + err = (&OBCluster{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:webhook go func() { diff --git a/cmd/main.go b/cmd/main.go index 175edbd4f..962b2c84f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -190,6 +190,10 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "OBTenantOperation") os.Exit(1) } + if err = (&v1alpha1.OBCluster{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "OBCluster") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index ca5be9e3e..95223b999 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -4,6 +4,26 @@ kind: MutatingWebhookConfiguration metadata: name: mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-oceanbase-oceanbase-com-v1alpha1-obcluster + failurePolicy: Fail + name: mobcluster.kb.io + rules: + - apiGroups: + - oceanbase.oceanbase.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - obclusters + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -70,6 +90,26 @@ kind: ValidatingWebhookConfiguration metadata: name: validating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-oceanbase-oceanbase-com-v1alpha1-obcluster + failurePolicy: Fail + name: vobcluster.kb.io + rules: + - apiGroups: + - oceanbase.oceanbase.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - obclusters + sideEffects: None - admissionReviewVersions: - v1 clientConfig: From c48c401c1aaacb74708561de77e93e7ee07e8828 Mon Sep 17 00:00:00 2001 From: yuyi Date: Tue, 7 Nov 2023 15:30:27 +0800 Subject: [PATCH 12/18] refactor: rename the main interface to Recorder --- cmd/main.go | 4 +- pkg/controller/obcluster_controller.go | 3 +- pkg/controller/obparameter_controller.go | 3 +- pkg/controller/observer_controller.go | 11 ++- pkg/controller/obtenant_controller.go | 11 ++- pkg/controller/obtenantbackup_controller.go | 6 +- .../obtenantbackuppolicy_controller.go | 3 +- .../obtenantoperation_controller.go | 11 ++- pkg/controller/obtenantrestore_controller.go | 11 ++- pkg/controller/obzone_controller.go | 11 ++- pkg/resource/obcluster_manager.go | 8 +- pkg/resource/obcluster_task.go | 8 +- pkg/resource/obparameter_manager.go | 4 +- pkg/resource/observer_manager.go | 12 ++- pkg/resource/obtenant_manager.go | 20 +++-- pkg/resource/obtenant_task.go | 8 +- pkg/resource/obtenantbackuppolicy_manager.go | 6 +- pkg/resource/obtenantoperation_manager.go | 12 ++- pkg/resource/obtenantrestore_manager.go | 12 ++- pkg/resource/obtenantrestore_task.go | 4 +- pkg/resource/obzone_manager.go | 12 ++- pkg/resource/template_manager.go | 12 ++- pkg/telemetry/configs.go | 8 +- pkg/telemetry/consts.go | 15 ++-- pkg/telemetry/init.go | 40 ++++++++++ pkg/telemetry/logging.go | 1 - pkg/telemetry/misc_test.go | 75 +++++++++++++++++++ pkg/telemetry/{telemetry.go => recorder.go} | 41 +++++----- .../{telemetry_test.go => recorder_test.go} | 4 +- pkg/telemetry/throttler.go | 5 +- 30 files changed, 236 insertions(+), 145 deletions(-) create mode 100644 pkg/telemetry/init.go create mode 100644 pkg/telemetry/misc_test.go rename pkg/telemetry/{telemetry.go => recorder.go} (67%) rename pkg/telemetry/{telemetry_test.go => recorder_test.go} (96%) diff --git a/cmd/main.go b/cmd/main.go index 962b2c84f..5ec4147a0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -205,8 +205,8 @@ func main() { os.Exit(1) } - telem := telemetry.NewTelemetry(context.Background(), mgr.GetEventRecorderFor("ob-operator")) - telem.GenerateTelemetryRecord(nil, "Operator", "Start", "", "start ob-operator", nil) + rcd := telemetry.NewRecorder(context.Background(), mgr.GetEventRecorderFor("ob-operator")) + rcd.GenerateTelemetryRecord(nil, telemetry.ObjectTypeOperator, "Start", "", "start ob-operator", nil) setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { diff --git a/pkg/controller/obcluster_controller.go b/pkg/controller/obcluster_controller.go index f16385306..0260fc8ee 100644 --- a/pkg/controller/obcluster_controller.go +++ b/pkg/controller/obcluster_controller.go @@ -83,9 +83,8 @@ func (r *OBClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( Ctx: ctx, OBCluster: obcluster, Client: r.Client, - Recorder: r.Recorder, Logger: &logger, - Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), + Recorder: telemetry.NewRecorder(ctx, r.Recorder), } coordinator := resource.NewCoordinator(obclusterManager, &logger) return coordinator.Coordinate() diff --git a/pkg/controller/obparameter_controller.go b/pkg/controller/obparameter_controller.go index 667042ab0..158c8953a 100644 --- a/pkg/controller/obparameter_controller.go +++ b/pkg/controller/obparameter_controller.go @@ -71,9 +71,8 @@ func (r *OBParameterReconciler) Reconcile(ctx context.Context, req ctrl.Request) Ctx: ctx, OBParameter: obparameter, Client: r.Client, - Recorder: r.Recorder, Logger: &logger, - Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), + Recorder: telemetry.NewRecorder(ctx, r.Recorder), } coordinator := resource.NewCoordinator(obparameterManager, &logger) return coordinator.Coordinate() diff --git a/pkg/controller/observer_controller.go b/pkg/controller/observer_controller.go index 943265858..dd4a5f38a 100644 --- a/pkg/controller/observer_controller.go +++ b/pkg/controller/observer_controller.go @@ -78,12 +78,11 @@ func (r *OBServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // create observer manager observerManager := &resource.OBServerManager{ - Ctx: ctx, - OBServer: observer, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, - Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), + Ctx: ctx, + OBServer: observer, + Client: r.Client, + Logger: &logger, + Recorder: telemetry.NewRecorder(ctx, r.Recorder), } // execute finalizers diff --git a/pkg/controller/obtenant_controller.go b/pkg/controller/obtenant_controller.go index 8a64c8b2c..37b0d504d 100644 --- a/pkg/controller/obtenant_controller.go +++ b/pkg/controller/obtenant_controller.go @@ -85,12 +85,11 @@ func (r *OBTenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // create observer manager obtenantManager := &resource.OBTenantManager{ - Ctx: ctx, - OBTenant: obtenant, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, - Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), + Ctx: ctx, + OBTenant: obtenant, + Client: r.Client, + Logger: &logger, + Recorder: telemetry.NewRecorder(ctx, r.Recorder), } coordinator := resource.NewCoordinator(obtenantManager, &logger) diff --git a/pkg/controller/obtenantbackup_controller.go b/pkg/controller/obtenantbackup_controller.go index 42d9d0802..adeb13c24 100644 --- a/pkg/controller/obtenantbackup_controller.go +++ b/pkg/controller/obtenantbackup_controller.go @@ -46,7 +46,7 @@ type OBTenantBackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Telemetry telemetry.Telemetry + Telemetry telemetry.Recorder telemetryOnce sync.Once } @@ -303,12 +303,12 @@ func (r *OBTenantBackupReconciler) retryUpdateStatus(ctx context.Context, job *v }) } -func (r *OBTenantBackupReconciler) getTelemetry(ctx context.Context) telemetry.Telemetry { +func (r *OBTenantBackupReconciler) getTelemetry(ctx context.Context) telemetry.Recorder { if r.Telemetry != nil { return r.Telemetry } r.telemetryOnce.Do(func() { - r.Telemetry = telemetry.NewTelemetry(ctx, r.Recorder) + r.Telemetry = telemetry.NewRecorder(ctx, r.Recorder) }) return r.Telemetry } diff --git a/pkg/controller/obtenantbackuppolicy_controller.go b/pkg/controller/obtenantbackuppolicy_controller.go index 73cf81d0e..d30006261 100644 --- a/pkg/controller/obtenantbackuppolicy_controller.go +++ b/pkg/controller/obtenantbackuppolicy_controller.go @@ -73,9 +73,8 @@ func (r *OBTenantBackupPolicyReconciler) Reconcile(ctx context.Context, req ctrl Ctx: ctx, BackupPolicy: policy, Client: r.Client, - Recorder: r.Recorder, Logger: &logger, - Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), + Recorder: telemetry.NewRecorder(ctx, r.Recorder), } coordinator := resource.NewCoordinator(mgr, &logger) diff --git a/pkg/controller/obtenantoperation_controller.go b/pkg/controller/obtenantoperation_controller.go index 475f2374b..85a6b7239 100644 --- a/pkg/controller/obtenantoperation_controller.go +++ b/pkg/controller/obtenantoperation_controller.go @@ -58,12 +58,11 @@ func (r *OBTenantOperationReconciler) Reconcile(ctx context.Context, req ctrl.Re } mgr := &resource.ObTenantOperationManager{ - Ctx: ctx, - Resource: operation, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, - Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), + Ctx: ctx, + Resource: operation, + Client: r.Client, + Logger: &logger, + Recorder: telemetry.NewRecorder(ctx, r.Recorder), } coordinator := resource.NewCoordinator(mgr, &logger) diff --git a/pkg/controller/obtenantrestore_controller.go b/pkg/controller/obtenantrestore_controller.go index aa90967f0..7d2ae7d69 100644 --- a/pkg/controller/obtenantrestore_controller.go +++ b/pkg/controller/obtenantrestore_controller.go @@ -60,12 +60,11 @@ func (r *OBTenantRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Requ } mgr := &resource.ObTenantRestoreManager{ - Ctx: ctx, - Resource: restore, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, - Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), + Ctx: ctx, + Resource: restore, + Client: r.Client, + Logger: &logger, + Recorder: telemetry.NewRecorder(ctx, r.Recorder), } coordinator := resource.NewCoordinator(mgr, &logger) diff --git a/pkg/controller/obzone_controller.go b/pkg/controller/obzone_controller.go index 818947be1..0f6633ba1 100644 --- a/pkg/controller/obzone_controller.go +++ b/pkg/controller/obzone_controller.go @@ -72,12 +72,11 @@ func (r *OBZoneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // create zone manager obzoneManager := &resource.OBZoneManager{ - Ctx: ctx, - OBZone: obzone, - Client: r.Client, - Recorder: r.Recorder, - Logger: &logger, - Telemetry: telemetry.NewTelemetry(ctx, r.Recorder), + Ctx: ctx, + OBZone: obzone, + Client: r.Client, + Logger: &logger, + Recorder: telemetry.NewRecorder(ctx, r.Recorder), } coordinator := resource.NewCoordinator(obzoneManager, &logger) return coordinator.Coordinate() diff --git a/pkg/resource/obcluster_manager.go b/pkg/resource/obcluster_manager.go index 7892d4f89..01f7e3cc7 100644 --- a/pkg/resource/obcluster_manager.go +++ b/pkg/resource/obcluster_manager.go @@ -18,7 +18,6 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,8 +38,7 @@ type OBClusterManager struct { Ctx context.Context OBCluster *v1alpha1.OBCluster Client client.Client - Recorder record.EventRecorder - Telemetry telemetry.Telemetry + Recorder telemetry.Recorder Logger *logr.Logger } @@ -50,7 +48,7 @@ func (m *OBClusterManager) IsNewResource() bool { func (m *OBClusterManager) InitStatus() { m.Logger.Info("newly created cluster, init status") - m.Telemetry.Event(m.OBCluster, "Init", "", "newly created cluster, init status") + m.Recorder.Event(m.OBCluster, "Init", "", "newly created cluster, init status") status := v1alpha1.OBClusterStatus{ Image: m.OBCluster.Spec.OBServerTemplate.Image, Status: clusterstatus.New, @@ -312,7 +310,7 @@ func (m *OBClusterManager) GetTaskFunc(name string) (func() error, error) { } func (m *OBClusterManager) PrintErrEvent(err error) { - m.Telemetry.Event(m.OBCluster, corev1.EventTypeWarning, "task exec failed", err.Error()) + m.Recorder.Event(m.OBCluster, corev1.EventTypeWarning, "task exec failed", err.Error()) } func (m *OBClusterManager) listOBZones() (*v1alpha1.OBZoneList, error) { diff --git a/pkg/resource/obcluster_task.go b/pkg/resource/obcluster_task.go index 22d374de8..00943e2ea 100644 --- a/pkg/resource/obcluster_task.go +++ b/pkg/resource/obcluster_task.go @@ -293,7 +293,7 @@ func (m *OBClusterManager) Bootstrap() error { if err != nil { m.Logger.Error(err, "bootstrap failed") } else { - m.Telemetry.Event(m.OBCluster, "Bootstrap", "", "Bootstrap successfully") + m.Recorder.Event(m.OBCluster, "Bootstrap", "", "Bootstrap successfully") } return err } @@ -786,7 +786,7 @@ func (m *OBClusterManager) CreateServiceForMonitor() error { if err != nil { return errors.Wrap(err, "Create monitor service") } - m.Telemetry.Event(m.OBCluster, "MaintainedAfterBootstrap", "", "Create monitor service successfully") + m.Recorder.Event(m.OBCluster, "MaintainedAfterBootstrap", "", "Create monitor service successfully") return nil } @@ -806,7 +806,7 @@ func (m *OBClusterManager) RestoreEssentialParameters() error { if err != nil { m.Logger.Error(err, "Failed to get context secret") // parameter can be set manually, just return here and emit an event - m.Telemetry.Event(m.OBCluster, "Warning", "Restore essential parameters failed", err.Error()) + m.Recorder.Event(m.OBCluster, "Warning", "Restore essential parameters failed", err.Error()) return nil } @@ -827,6 +827,6 @@ func (m *OBClusterManager) RestoreEssentialParameters() error { } } _ = m.Client.Delete(m.Ctx, contextSecret) - m.Telemetry.Event(m.OBCluster, "Upgrade", "", "Restore essential parameters successfully") + m.Recorder.Event(m.OBCluster, "Upgrade", "", "Restore essential parameters successfully") return nil } diff --git a/pkg/resource/obparameter_manager.go b/pkg/resource/obparameter_manager.go index 103f76539..1d0d77bb5 100644 --- a/pkg/resource/obparameter_manager.go +++ b/pkg/resource/obparameter_manager.go @@ -20,7 +20,6 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -43,8 +42,7 @@ type OBParameterManager struct { Ctx context.Context OBParameter *v1alpha1.OBParameter Client client.Client - Recorder record.EventRecorder - Telemetry telemetry.Telemetry + Recorder telemetry.Recorder Logger *logr.Logger } diff --git a/pkg/resource/observer_manager.go b/pkg/resource/observer_manager.go index c181db930..44ee7e6d9 100644 --- a/pkg/resource/observer_manager.go +++ b/pkg/resource/observer_manager.go @@ -23,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" kubeerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" apipod "k8s.io/kubernetes/pkg/api/v1/pod" "sigs.k8s.io/controller-runtime/pkg/client" @@ -43,12 +42,11 @@ import ( type OBServerManager struct { ResourceManager - Ctx context.Context - OBServer *v1alpha1.OBServer - Client client.Client - Recorder record.EventRecorder - Telemetry telemetry.Telemetry - Logger *logr.Logger + Ctx context.Context + OBServer *v1alpha1.OBServer + Client client.Client + Recorder telemetry.Recorder + Logger *logr.Logger } func (m *OBServerManager) GetTaskFunc(name string) (func() error, error) { diff --git a/pkg/resource/obtenant_manager.go b/pkg/resource/obtenant_manager.go index 6ed21f4c0..0e24e1388 100644 --- a/pkg/resource/obtenant_manager.go +++ b/pkg/resource/obtenant_manager.go @@ -25,7 +25,6 @@ import ( kubeerrors "k8s.io/apimachinery/pkg/api/errors" kuberesource "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -44,12 +43,11 @@ import ( type OBTenantManager struct { ResourceManager - OBTenant *v1alpha1.OBTenant - Ctx context.Context - Client client.Client - Recorder record.EventRecorder - Telemetry telemetry.Telemetry - Logger *logr.Logger + OBTenant *v1alpha1.OBTenant + Ctx context.Context + Client client.Client + Recorder telemetry.Recorder + Logger *logr.Logger } // TODO add lock to be thread safe, and read/write whitelist from/to DB @@ -90,12 +88,12 @@ func (m *OBTenantManager) InitStatus() { if m.OBTenant.Spec.Source != nil && m.OBTenant.Spec.Source.Restore != nil { m.OBTenant.Status.Status = tenantstatus.Restoring - m.Telemetry.Event(m.OBTenant, "InitRestore", "", "start restoring") + m.Recorder.Event(m.OBTenant, "InitRestore", "", "start restoring") } else if m.OBTenant.Spec.Source != nil && m.OBTenant.Spec.Source.Tenant != nil { - m.Telemetry.Event(m.OBTenant, "InitEmptyStandby", "", "start creating empty standby") + m.Recorder.Event(m.OBTenant, "InitEmptyStandby", "", "start creating empty standby") m.OBTenant.Status.Status = tenantstatus.CreatingEmptyStandby } else { - m.Telemetry.Event(m.OBTenant, "Init", "", "start creating") + m.Recorder.Event(m.OBTenant, "Init", "", "start creating") m.OBTenant.Status.Status = tenantstatus.CreatingTenant } } @@ -333,7 +331,7 @@ func (m *OBTenantManager) GetTaskFlow() (*task.TaskFlow, error) { } func (m *OBTenantManager) PrintErrEvent(err error) { - m.Telemetry.Event(m.OBTenant, corev1.EventTypeWarning, "task exec failed", err.Error()) + m.Recorder.Event(m.OBTenant, corev1.EventTypeWarning, "task exec failed", err.Error()) } // ---------- K8S API Helper ---------- diff --git a/pkg/resource/obtenant_task.go b/pkg/resource/obtenant_task.go index 827432899..df3f64f8f 100644 --- a/pkg/resource/obtenant_task.go +++ b/pkg/resource/obtenant_task.go @@ -386,12 +386,12 @@ func (m *OBTenantManager) createTenant() error { err = oceanbaseOperationManager.AddTenant(tenantSQLParam) if err != nil { - m.Telemetry.Event(m.OBTenant, corev1.EventTypeWarning, "failed to create OBTenant", err.Error()) + m.Recorder.Event(m.OBTenant, corev1.EventTypeWarning, "failed to create OBTenant", err.Error()) return err } GlobalWhiteListMap[tenantName] = m.OBTenant.Spec.ConnectWhiteList // Create user or change password of root, do not return error - m.Telemetry.Event(m.OBTenant, "Create", "", "create OBTenant successfully") + m.Recorder.Event(m.OBTenant, "Create", "", "create OBTenant successfully") return nil } @@ -1088,7 +1088,7 @@ func (m *OBTenantManager) CreateUserWithCredentialSecrets() error { } err := m.createUserWithCredentials() if err != nil { - m.Telemetry.Event(m.OBTenant, corev1.EventTypeWarning, "Failed to create user or change password", err.Error()) + m.Recorder.Event(m.OBTenant, corev1.EventTypeWarning, "Failed to create user or change password", err.Error()) m.Logger.Error(err, "Failed to create user or change password, please check the credential secrets") } @@ -1181,7 +1181,7 @@ func (m *OBTenantManager) CreateEmptyStandbyTenant() error { if err != nil { return err } - m.Telemetry.Event(m.OBTenant, "CreateEmptyStandby", "", "Succeed to create empty standby tenant") + m.Recorder.Event(m.OBTenant, "CreateEmptyStandby", "", "Succeed to create empty standby tenant") return nil } diff --git a/pkg/resource/obtenantbackuppolicy_manager.go b/pkg/resource/obtenantbackuppolicy_manager.go index 151647506..4594b85cf 100644 --- a/pkg/resource/obtenantbackuppolicy_manager.go +++ b/pkg/resource/obtenantbackuppolicy_manager.go @@ -23,7 +23,6 @@ import ( kubeerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -45,8 +44,7 @@ type ObTenantBackupPolicyManager struct { Ctx context.Context BackupPolicy *v1alpha1.OBTenantBackupPolicy Client client.Client - Recorder record.EventRecorder - Telemetry telemetry.Telemetry + Recorder telemetry.Recorder Logger *logr.Logger con *operation.OceanbaseOperationManager @@ -117,7 +115,7 @@ func (m *ObTenantBackupPolicyManager) InitStatus() { m.BackupPolicy.Status = v1alpha1.OBTenantBackupPolicyStatus{ Status: constants.BackupPolicyStatusPreparing, } - m.Telemetry.Event(m.BackupPolicy, "Init", "", "init status") + m.Recorder.Event(m.BackupPolicy, "Init", "", "init status") err = m.syncTenantInformation() if err != nil { m.PrintErrEvent(err) diff --git a/pkg/resource/obtenantoperation_manager.go b/pkg/resource/obtenantoperation_manager.go index f1e02a2be..22aa19c09 100644 --- a/pkg/resource/obtenantoperation_manager.go +++ b/pkg/resource/obtenantoperation_manager.go @@ -20,7 +20,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,12 +38,11 @@ import ( type ObTenantOperationManager struct { ResourceManager - Ctx context.Context - Resource *v1alpha1.OBTenantOperation - Client client.Client - Recorder record.EventRecorder - Telemetry telemetry.Telemetry - Logger *logr.Logger + Ctx context.Context + Resource *v1alpha1.OBTenantOperation + Client client.Client + Recorder telemetry.Recorder + Logger *logr.Logger con *operation.OceanbaseOperationManager } diff --git a/pkg/resource/obtenantrestore_manager.go b/pkg/resource/obtenantrestore_manager.go index e4d3c92e5..f2254d650 100644 --- a/pkg/resource/obtenantrestore_manager.go +++ b/pkg/resource/obtenantrestore_manager.go @@ -19,7 +19,6 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,12 +38,11 @@ import ( type ObTenantRestoreManager struct { ResourceManager - Ctx context.Context - Resource *v1alpha1.OBTenantRestore - Client client.Client - Recorder record.EventRecorder - Telemetry telemetry.Telemetry - Logger *logr.Logger + Ctx context.Context + Resource *v1alpha1.OBTenantRestore + Client client.Client + Recorder telemetry.Recorder + Logger *logr.Logger con *operation.OceanbaseOperationManager } diff --git a/pkg/resource/obtenantrestore_task.go b/pkg/resource/obtenantrestore_task.go index e729d2069..968e8122e 100644 --- a/pkg/resource/obtenantrestore_task.go +++ b/pkg/resource/obtenantrestore_task.go @@ -112,13 +112,13 @@ func (m *OBTenantManager) WatchRestoreJobToFinish() error { if runningRestore.Status.Status == constants.RestoreJobSuccessful { break } else if runningRestore.Status.Status == constants.RestoreJobFailed { - m.Telemetry.Event(m.OBTenant, "RestoreJobFailed", "", "restore job failed") + m.Recorder.Event(m.OBTenant, "RestoreJobFailed", "", "restore job failed") return errors.New("Restore job failed") } time.Sleep(5 * time.Second) } GlobalWhiteListMap[m.OBTenant.Spec.TenantName] = m.OBTenant.Spec.ConnectWhiteList - m.Telemetry.Event(m.OBTenant, "RestoreJobFinished", "", "restore job finished successfully") + m.Recorder.Event(m.OBTenant, "RestoreJobFinished", "", "restore job finished successfully") return nil } diff --git a/pkg/resource/obzone_manager.go b/pkg/resource/obzone_manager.go index b4a5a4de3..4207ee7e2 100644 --- a/pkg/resource/obzone_manager.go +++ b/pkg/resource/obzone_manager.go @@ -21,7 +21,6 @@ import ( "github.com/pkg/errors" kubeerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,12 +39,11 @@ import ( type OBZoneManager struct { ResourceManager - Ctx context.Context - OBZone *v1alpha1.OBZone - Client client.Client - Recorder record.EventRecorder - Telemetry telemetry.Telemetry - Logger *logr.Logger + Ctx context.Context + OBZone *v1alpha1.OBZone + Client client.Client + Recorder telemetry.Recorder + Logger *logr.Logger } func (m *OBZoneManager) IsNewResource() bool { diff --git a/pkg/resource/template_manager.go b/pkg/resource/template_manager.go index eb0453f41..cfc7f2992 100644 --- a/pkg/resource/template_manager.go +++ b/pkg/resource/template_manager.go @@ -17,7 +17,6 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" @@ -29,12 +28,11 @@ import ( type ObResourceManager[T client.Object] struct { ResourceManager - Ctx context.Context - Resource T - Client client.Client - Recorder record.EventRecorder - Logger *logr.Logger - Telemetry telemetry.Telemetry + Ctx context.Context + Resource T + Client client.Client + Recorder telemetry.Recorder + Logger *logr.Logger con *operation.OceanbaseOperationManager } diff --git a/pkg/telemetry/configs.go b/pkg/telemetry/configs.go index b721503fe..8dc20f365 100644 --- a/pkg/telemetry/configs.go +++ b/pkg/telemetry/configs.go @@ -12,14 +12,12 @@ See the Mulan PSL v2 for more details. package telemetry -import "os" - -var TelemetryRequestSignature = os.Getenv(TelemetrySignatureEnvName) - const ( DefaultThrottlerBufferSize = 30 DefaultThrottlerWorkerCount = 30 DefaultWaitThrottlerSeconds = 60 ) -const TelemetryReportHost = TelemetryReportProdHost +var TelemetryReportHost = TelemetryReportProdHost +var TelemetryReportScheme = SchemeHttps +var TelemetryDisabled = false diff --git a/pkg/telemetry/consts.go b/pkg/telemetry/consts.go index 1544efa9b..16f1bff7b 100644 --- a/pkg/telemetry/consts.go +++ b/pkg/telemetry/consts.go @@ -13,9 +13,7 @@ See the Mulan PSL v2 for more details. package telemetry const ( - TelemetryReportDevHost = "openwebapi.dev.alipay.net" // http - TelemetryReportTestHost = "openwebapi.test.alipay.net" // http - TelemetryReportProdHost = "openwebapi.ocenbase.com" // https + TelemetryReportProdHost = "openwebapi-pre.oceanbase.com" // https TelemetryReportPath = "/api/web/oceanbase/report" ) @@ -41,7 +39,12 @@ const ( ) const ( - DisableTelemetryEnvName = "DISABLE_TELEMETRY" - TelemetrySignatureEnvName = "TELEMETRY_SIGNATURE" - TelemetryDebugEnvName = "TELEMETRY_DEBUG" + DisableTelemetryEnvName = "DISABLE_TELEMETRY" + TelemetryDebugEnvName = "TELEMETRY_DEBUG" + TelemetryReportHostEnvName = "TELEMETRY_REPORT_HOST" +) + +const ( + ObjectTypeUnknown = "Unknown" + ObjectTypeOperator = "Operator" ) diff --git a/pkg/telemetry/init.go b/pkg/telemetry/init.go new file mode 100644 index 000000000..0cd7c5e9d --- /dev/null +++ b/pkg/telemetry/init.go @@ -0,0 +1,40 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "net/http" + "net/url" + "os" + "strings" + "time" +) + +func init() { + TelemetryDisabled = os.Getenv(DisableTelemetryEnvName) == "true" + if TelemetryDisabled { + return + } + if host, ok := os.LookupEnv(TelemetryReportHostEnvName); ok && host != "" && strings.HasPrefix(host, "http") { + if u, err := url.Parse(host); err == nil { + clt := http.Client{ + Timeout: time.Second, + } + _, err := clt.Head(u.String()) + if err == nil { + TelemetryReportScheme = u.Scheme + TelemetryReportHost = u.Host + } + } + } +} diff --git a/pkg/telemetry/logging.go b/pkg/telemetry/logging.go index 85ec7eead..b81bab5b3 100644 --- a/pkg/telemetry/logging.go +++ b/pkg/telemetry/logging.go @@ -27,7 +27,6 @@ func configLogger() { if debugMode { file, err := os.OpenFile("/tmp/telemetry.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { - // log.Println("Failed to open log file:", err) lg = log.New(io.Discard, "[Telemetry] ", log.LstdFlags|log.Lshortfile) } else { lg = log.New(file, "[Telemetry] ", log.LstdFlags|log.Lshortfile) diff --git a/pkg/telemetry/misc_test.go b/pkg/telemetry/misc_test.go new file mode 100644 index 000000000..fd161a365 --- /dev/null +++ b/pkg/telemetry/misc_test.go @@ -0,0 +1,75 @@ +/* +Copyright (c) 2023 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package telemetry + +import ( + "net/http" + "net/url" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Telemetry", Label("misc"), func() { + It("Test url parse 1", func() { + u, err := url.Parse("https://www.baidu.com") + Expect(err).ShouldNot(HaveOccurred()) + Expect(u.Scheme).Should(Equal("https")) + Expect(u.Host).Should(Equal("www.baidu.com")) + }) + + It("Test url parse 2", func() { + u, err := url.Parse("http://www.baidu.com") + Expect(err).ShouldNot(HaveOccurred()) + Expect(u.Scheme).Should(Equal("http")) + Expect(u.Host).Should(Equal("www.baidu.com")) + }) + + It("Test url parse 3", func() { + u, err := url.Parse("www.baidu.com") + Expect(err).ShouldNot(HaveOccurred()) + Expect(u.Scheme).Should(Equal("")) + Expect(u.Host).Should(Equal("")) + }) + + It("Test url parse and string()", func() { + urlStr := "https://www.baidu.com" + u, err := url.Parse(urlStr) + Expect(err).ShouldNot(HaveOccurred()) + Expect(u.String()).Should(Equal(urlStr)) + }) + + It("Test head request", func() { + _, err := http.DefaultClient.Head("https://www.baidu.com") + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("Test head request 2", func() { + _, err := http.DefaultClient.Head("http://www.baidu.com") + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("Test head request 3", func() { + _, err := http.DefaultClient.Head("www.baidu.com") + Expect(err).Should(HaveOccurred()) + }) + + It("Test head a not exist url with timeout 1s", func() { + clt := http.Client{ + Timeout: time.Second, + } + _, err := clt.Head("https://www.baidx.com/abc") + Expect(err).Should(HaveOccurred()) + }) +}) diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/recorder.go similarity index 67% rename from pkg/telemetry/telemetry.go rename to pkg/telemetry/recorder.go index 2aba1368e..20d0e8c09 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/recorder.go @@ -25,14 +25,14 @@ import ( "github.com/oceanbase/ob-operator/pkg/telemetry/models" ) -type Telemetry interface { +type Recorder interface { record.EventRecorder GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...models.ExtraField) GetHostMetrics() *hostMetrics Done() } -type telemetry struct { +type recorder struct { *throttler *hostMetrics record.EventRecorder @@ -41,13 +41,13 @@ type telemetry struct { telemetryDisabled bool } -func NewTelemetry(ctx context.Context, recorder record.EventRecorder) Telemetry { - clt := &telemetry{ +func NewRecorder(ctx context.Context, er record.EventRecorder) Recorder { + clt := &recorder{ ctx: ctx, - EventRecorder: recorder, + EventRecorder: er, } - // if telemetry is disabled, return a dummy telemetry as original event recorder + // if telemetry is disabled, return a dummy recorder as original event recorder if os.Getenv(DisableTelemetryEnvName) == "true" { clt.telemetryDisabled = true return clt @@ -58,25 +58,25 @@ func NewTelemetry(ctx context.Context, recorder record.EventRecorder) Telemetry } // Implement record.EventRecorder interface -func (t *telemetry) Event(object runtime.Object, eventType, reason, message string) { +func (t *recorder) Event(object runtime.Object, eventType, reason, message string) { t.EventRecorder.Event(object, t.transformEventType(eventType), reason, message) t.generateFromEvent(object, nil, eventType, reason, message) } // Implement record.EventRecorder interface -func (t *telemetry) Eventf(object runtime.Object, eventType, reason, messageFmt string, args ...any) { +func (t *recorder) Eventf(object runtime.Object, eventType, reason, messageFmt string, args ...any) { t.EventRecorder.Eventf(object, t.transformEventType(eventType), reason, messageFmt, args...) t.generateFromEvent(object, nil, eventType, reason, messageFmt, args...) } // Implement record.EventRecorder interface -func (t *telemetry) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...any) { +func (t *recorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...any) { t.EventRecorder.AnnotatedEventf(object, annotations, t.transformEventType(eventType), reason, messageFmt, args...) t.generateFromEvent(object, annotations, eventType, reason, messageFmt, args...) } // Use Event, Eventf, AnnotatedEventf first -func (t *telemetry) GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...models.ExtraField) { +func (t *recorder) GenerateTelemetryRecord(object any, objectType, eventType, reason, message string, annotations map[string]string, extra ...models.ExtraField) { if t.telemetryDisabled { return } @@ -84,38 +84,41 @@ func (t *telemetry) GenerateTelemetryRecord(object any, objectType, eventType, r objectSentry(object) record := newRecordFromEvent(object, objectType, eventType, reason, message, annotations, extra...) record.IpHashes = t.hostMetrics.IPHashes - if object == nil && objectType == "Operator" { + if object == nil && objectType == ObjectTypeOperator { record.Resource = t.hostMetrics } select { - case <-ctx.Done(): - return case ch <- record: + case <-ctx.Done(): case <-time.After(DefaultWaitThrottlerSeconds * time.Second): default: } }(t.ctx, t.chanIn()) } -func (t *telemetry) Done() { +func (t *recorder) Done() { t.throttler.close() } -func (t *telemetry) GetHostMetrics() *hostMetrics { +func (t *recorder) GetHostMetrics() *hostMetrics { return t.hostMetrics } -func (t *telemetry) generateFromEvent(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...any) { +func (t *recorder) generateFromEvent(object runtime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...any) { + if t.telemetryDisabled { + return + } if object == nil { - t.GenerateTelemetryRecord(nil, "Unknown", eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) + t.GenerateTelemetryRecord(nil, ObjectTypeUnknown, eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) } else { t.GenerateTelemetryRecord(object.DeepCopyObject(), object.GetObjectKind().GroupVersionKind().Kind, eventType, reason, fmt.Sprintf(messageFmt, args...), annotations) } } -func (t *telemetry) transformEventType(eventType string) string { +func (t *recorder) transformEventType(eventType string) string { + // k8s EventRecorder only accepts `Warning` and `Normal` as event type switch eventType { - case "Error", "Warning", "error": + case "Error", "Warning", "error", "warning": return corev1.EventTypeWarning default: return corev1.EventTypeNormal diff --git a/pkg/telemetry/telemetry_test.go b/pkg/telemetry/recorder_test.go similarity index 96% rename from pkg/telemetry/telemetry_test.go rename to pkg/telemetry/recorder_test.go index 338ca9ccc..2564bc5ca 100644 --- a/pkg/telemetry/telemetry_test.go +++ b/pkg/telemetry/recorder_test.go @@ -34,7 +34,7 @@ func (f *fakeEventRecorder) AnnotatedEventf(object runtime.Object, annotations m } var _ = Describe("Telemetry", Label("telemetry"), Ordered, func() { - var telemetry Telemetry + var telemetry Recorder tenant := &v1alpha1.OBTenant{ TypeMeta: metav1.TypeMeta{ Kind: "OBTenant", @@ -43,7 +43,7 @@ var _ = Describe("Telemetry", Label("telemetry"), Ordered, func() { } BeforeAll(func() { - telemetry = NewTelemetry(context.TODO(), &fakeEventRecorder{}) + telemetry = NewRecorder(context.TODO(), &fakeEventRecorder{}) Expect(telemetry).ShouldNot(BeNil()) }) diff --git a/pkg/telemetry/throttler.go b/pkg/telemetry/throttler.go index a4d1d627a..d935433c2 100644 --- a/pkg/telemetry/throttler.go +++ b/pkg/telemetry/throttler.go @@ -69,13 +69,12 @@ func (t *throttler) sendTelemetryRecord(record *models.TelemetryRecord) (*http.R req := &http.Request{ Method: http.MethodPost, URL: &url.URL{ - Scheme: SchemeHttp, - Host: TelemetryReportTestHost, + Scheme: TelemetryReportScheme, + Host: TelemetryReportHost, Path: TelemetryReportPath, }, Header: http.Header{ "content-type": []string{ContentTypeJson}, - "sig": []string{TelemetryRequestSignature}, }, Body: body, } From 8bb84c83d4084a9da6f97ad2c99c358fcbd999e3 Mon Sep 17 00:00:00 2001 From: yuyi Date: Tue, 7 Nov 2023 15:42:01 +0800 Subject: [PATCH 13/18] chore: update dev image and reportHost; fix Dockerfile USER --- Dockerfile | 2 +- config/manager/kustomization.yaml | 2 +- config/manager/manager.yaml | 2 ++ deploy/operator.yaml | 44 ++++++++++++++++++++++++++++++- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 60e8c61c4..c2da8e66e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,6 @@ FROM openanolis/anolisos:8.4-x86_64 WORKDIR / COPY --from=builder /workspace/manager . RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime -USER 65532:65532 +USER 65534:65534 ENTRYPOINT ["/manager"] diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 7f0a32564..47109d881 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: oceanbasedev/ob-operator - newTag: 2.0.1-alpha.2 + newTag: 2.1.0-alpha.1 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 701b4ed32..b3dd3425e 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -81,6 +81,8 @@ spec: env: - name: TELEMETRY_DEBUG value: "true" + - name: TELEMETRY_REPORT_HOST + value: "http://openwebapi.test.alipay.net" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 00a067ba0..4211cd4fa 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -12639,7 +12639,9 @@ spec: env: - name: TELEMETRY_DEBUG value: "true" - image: oceanbasedev/ob-operator:2.0.1-alpha.2 + - name: TELEMETRY_REPORT_HOST + value: http://openwebapi.test.alipay.net + image: oceanbasedev/ob-operator:2.1.0-alpha.1 livenessProbe: httpGet: path: /healthz @@ -12756,6 +12758,26 @@ metadata: app.kubernetes.io/part-of: ob-operator-generate name: oceanbase-mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: oceanbase-webhook-service + namespace: oceanbase-system + path: /mutate-oceanbase-oceanbase-com-v1alpha1-obcluster + failurePolicy: Fail + name: mobcluster.kb.io + rules: + - apiGroups: + - oceanbase.oceanbase.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - obclusters + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -12831,6 +12853,26 @@ metadata: app.kubernetes.io/part-of: ob-operator-generate name: oceanbase-validating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: oceanbase-webhook-service + namespace: oceanbase-system + path: /validate-oceanbase-oceanbase-com-v1alpha1-obcluster + failurePolicy: Fail + name: vobcluster.kb.io + rules: + - apiGroups: + - oceanbase.oceanbase.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - obclusters + sideEffects: None - admissionReviewVersions: - v1 clientConfig: From a1cc0e2edd5e4748bfe3c73fec630a731b61f89d Mon Sep 17 00:00:00 2001 From: yuyi Date: Tue, 7 Nov 2023 17:36:36 +0800 Subject: [PATCH 14/18] fix: telemetry recorder.Done() may panic if telemetry is disabled --- pkg/telemetry/recorder.go | 3 +++ pkg/telemetry/recorder_test.go | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/telemetry/recorder.go b/pkg/telemetry/recorder.go index 20d0e8c09..567b894f8 100644 --- a/pkg/telemetry/recorder.go +++ b/pkg/telemetry/recorder.go @@ -97,6 +97,9 @@ func (t *recorder) GenerateTelemetryRecord(object any, objectType, eventType, re } func (t *recorder) Done() { + if t.telemetryDisabled { + return + } t.throttler.close() } diff --git a/pkg/telemetry/recorder_test.go b/pkg/telemetry/recorder_test.go index 2564bc5ca..00db0aa49 100644 --- a/pkg/telemetry/recorder_test.go +++ b/pkg/telemetry/recorder_test.go @@ -27,10 +27,13 @@ import ( type fakeEventRecorder struct{} func (f *fakeEventRecorder) Event(object runtime.Object, eventtype, reason, message string) { + getLogger().Printf("Event: %+v, %s, %s, %s\n", object, eventtype, reason, message) } func (f *fakeEventRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...any) { + getLogger().Printf("Eventf: %+v, %s, %s, %s, %v\n", object, eventtype, reason, messageFmt, args) } func (f *fakeEventRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...any) { + getLogger().Printf("AnnotatedEventf: %+v, %+v, %s, %s, %s, %v\n", object, annotations, eventtype, reason, messageFmt, args) } var _ = Describe("Telemetry", Label("telemetry"), Ordered, func() { @@ -60,7 +63,11 @@ var _ = Describe("Telemetry", Label("telemetry"), Ordered, func() { It("GetHostMetrics", func() { metrics := telemetry.GetHostMetrics() - Expect(metrics).ShouldNot(BeNil()) + if TelemetryDisabled { + Expect(metrics).Should(BeNil()) + } else { + Expect(metrics).ShouldNot(BeNil()) + } }) It("Event", func() { From 0a1d30f6519d56cef3620edcc56fd6debc6048e5 Mon Sep 17 00:00:00 2001 From: yuyi Date: Wed, 8 Nov 2023 17:57:39 +0800 Subject: [PATCH 15/18] fix(manifests): update operator.yaml with obcluster webhook --- config/crd/kustomization.yaml | 4 ++-- deploy/crd.yaml | 13 ++++++++++++- deploy/operator.yaml | 10 ++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 6a0e7ad49..c39477abc 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -19,7 +19,7 @@ resources: patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD -# - patches/webhook_in_obclusters.yaml +- patches/webhook_in_obclusters.yaml # - patches/webhook_in_obzones.yaml # - patches/webhook_in_observers.yaml # - patches/webhook_in_obparameters.yaml @@ -35,7 +35,7 @@ patchesStrategicMerge: # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD -# - patches/cainjection_in_obclusters.yaml +- patches/cainjection_in_obclusters.yaml # - patches/cainjection_in_obzones.yaml # - patches/cainjection_in_observers.yaml # - patches/cainjection_in_obparameters.yaml diff --git a/deploy/crd.yaml b/deploy/crd.yaml index ae36e4616..170b90b47 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -100,9 +100,20 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME controller-gen.kubebuilder.io/version: v0.13.0 name: obclusters.oceanbase.oceanbase.com spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: webhook-service + namespace: system + path: /convert + conversionReviewVersions: + - v1 group: oceanbase.oceanbase.com names: kind: OBCluster @@ -6671,7 +6682,7 @@ spec: type: string create_time: type: string - gmt_create_time: + gmt_create: type: string in_recyclebin: type: string diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 4211cd4fa..229916996 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -119,6 +119,16 @@ metadata: controller-gen.kubebuilder.io/version: v0.13.0 name: obclusters.oceanbase.oceanbase.com spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: oceanbase-webhook-service + namespace: oceanbase-system + path: /convert + conversionReviewVersions: + - v1 group: oceanbase.oceanbase.com names: kind: OBCluster From 54fdf4b3397aab1887668fa6bcbbe3d6e0681b18 Mon Sep 17 00:00:00 2001 From: yuyi Date: Wed, 8 Nov 2023 17:58:20 +0800 Subject: [PATCH 16/18] fix(backup): skip StopBackup task when deleting backuppolicy --- pkg/resource/obtenantbackuppolicy_manager.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/resource/obtenantbackuppolicy_manager.go b/pkg/resource/obtenantbackuppolicy_manager.go index 4594b85cf..3fb96655f 100644 --- a/pkg/resource/obtenantbackuppolicy_manager.go +++ b/pkg/resource/obtenantbackuppolicy_manager.go @@ -78,17 +78,19 @@ func (m *ObTenantBackupPolicyManager) CheckAndUpdateFinalizers() error { } if !finalizerFinished { - tenant, err := m.getOBTenantCR() - if err != nil { - // the tenant is deleted, no need to wait finalizer - if kubeerrors.IsNotFound(err) { + if m.BackupPolicy.Spec.TenantCRName != "" { + tenant, err := m.getOBTenantCR() + if err != nil { + // the tenant is deleted, no need to wait finalizer + if kubeerrors.IsNotFound(err) { + finalizerFinished = true + } else { + return errors.Wrap(err, "Get obtenant failed") + } + } else if !tenant.GetDeletionTimestamp().IsZero() { + // the tenant is being deleted finalizerFinished = true - } else { - return errors.Wrap(err, "Get obtenant failed") } - } else if !tenant.GetDeletionTimestamp().IsZero() { - // the tenant is being deleted - finalizerFinished = true } else { err := m.StopBackup() // the policy is being deleted, connection still exists, stop backup From fdf791a2b28f6657d2e0b9914a9352ac0ccccd59 Mon Sep 17 00:00:00 2001 From: yuyi Date: Wed, 8 Nov 2023 20:07:00 +0800 Subject: [PATCH 17/18] fix: obtenantbackup_controller --- cmd/main.go | 2 +- pkg/controller/obtenantbackup_controller.go | 26 ++++++--------------- pkg/controller/suite_test.go | 4 +++- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5ec4147a0..d48d4ecbc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -149,7 +149,7 @@ func main() { if err = (&controller.OBTenantBackupReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor(config.OBTenantBackupControllerName), + Recorder: telemetry.NewRecorder(context.Background(), mgr.GetEventRecorderFor(config.OBTenantBackupControllerName)), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "OBTenantBackup") os.Exit(1) diff --git a/pkg/controller/obtenantbackup_controller.go b/pkg/controller/obtenantbackup_controller.go index adeb13c24..0104b34da 100644 --- a/pkg/controller/obtenantbackup_controller.go +++ b/pkg/controller/obtenantbackup_controller.go @@ -24,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -44,9 +43,8 @@ import ( // OBTenantBackupReconciler reconciles a OBTenantBackup object type OBTenantBackupReconciler struct { client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder - Telemetry telemetry.Recorder + Scheme *runtime.Scheme + Recorder telemetry.Recorder telemetryOnce sync.Once } @@ -128,19 +126,19 @@ func (r *OBTenantBackupReconciler) createBackupJobInOB(ctx context.Context, job password, err := resource.ReadPassword(r.Client, job.Namespace, job.Spec.EncryptionSecret) if err != nil { logger.Error(err, "failed to read backup encryption secret") - r.getTelemetry(ctx).Event(job, "Warning", "ReadBackupEncryptionSecretFailed", err.Error()) + r.Recorder.Event(job, "Warning", "ReadBackupEncryptionSecretFailed", err.Error()) } else if password != "" { err = con.SetBackupPassword(password) if err != nil { logger.Error(err, "failed to set backup password") - r.getTelemetry(ctx).Event(job, "Warning", "SetBackupPasswordFailed", err.Error()) + r.Recorder.Event(job, "Warning", "SetBackupPasswordFailed", err.Error()) } } } latest, err := con.CreateAndReturnBackupJob(job.Spec.Type) if err != nil { logger.Error(err, "failed to create and return backup job") - r.getTelemetry(ctx).Event(job, "Warning", "CreateAndReturnBackupJobFailed", err.Error()) + r.Recorder.Event(job, "Warning", "CreateAndReturnBackupJobFailed", err.Error()) return err } @@ -148,10 +146,10 @@ func (r *OBTenantBackupReconciler) createBackupJobInOB(ctx context.Context, job err = r.retryUpdateStatus(ctx, job) if err != nil { logger.Error(err, "failed to update status") - r.getTelemetry(ctx).Event(job, "Warning", "UpdateStatusFailed", err.Error()) + r.Recorder.Event(job, "Warning", "UpdateStatusFailed", err.Error()) return err } - r.getTelemetry(ctx).Event(job, "Create", "", "create backup job successfully") + r.Recorder.Event(job, "Create", "", "create backup job successfully") return nil } @@ -302,13 +300,3 @@ func (r *OBTenantBackupReconciler) retryUpdateStatus(ctx context.Context, job *v return r.Status().Update(ctx, newestJob) }) } - -func (r *OBTenantBackupReconciler) getTelemetry(ctx context.Context) telemetry.Recorder { - if r.Telemetry != nil { - return r.Telemetry - } - r.telemetryOnce.Do(func() { - r.Telemetry = telemetry.NewRecorder(ctx, r.Recorder) - }) - return r.Telemetry -} diff --git a/pkg/controller/suite_test.go b/pkg/controller/suite_test.go index 49a27bcc6..06388f551 100644 --- a/pkg/controller/suite_test.go +++ b/pkg/controller/suite_test.go @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "context" "os" "path/filepath" "testing" @@ -33,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/oceanbase/ob-operator/pkg/controller/config" + "github.com/oceanbase/ob-operator/pkg/telemetry" v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" //+kubebuilder:scaffold:imports @@ -134,7 +136,7 @@ var _ = BeforeSuite(func() { err = (&OBTenantBackupReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor(config.OBTenantBackupControllerName), + Recorder: telemetry.NewRecorder(context.Background(), k8sManager.GetEventRecorderFor(config.OBTenantBackupControllerName)), }).SetupWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) From cfc9d776490589deafb817a0793ee14dc548e672 Mon Sep 17 00:00:00 2001 From: yuyi Date: Wed, 8 Nov 2023 20:17:24 +0800 Subject: [PATCH 18/18] fix: remove sync.Once for telemetry initialization --- pkg/controller/obtenantbackup_controller.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/controller/obtenantbackup_controller.go b/pkg/controller/obtenantbackup_controller.go index 0104b34da..a3e428e9b 100644 --- a/pkg/controller/obtenantbackup_controller.go +++ b/pkg/controller/obtenantbackup_controller.go @@ -19,7 +19,6 @@ package controller import ( "context" "fmt" - "sync" "time" "k8s.io/apimachinery/pkg/runtime" @@ -45,8 +44,6 @@ type OBTenantBackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder telemetry.Recorder - - telemetryOnce sync.Once } //+kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obtenantbackups,verbs=get;list;watch;create;update;patch;delete