diff --git a/go.mod b/go.mod index 1df6145b..d422c59e 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-logr/logr v1.4.2 github.com/google/go-cmp v0.6.0 github.com/openshift/api v0.0.0-20240301093301-ce10821dc999 + github.com/openshift/library-go v0.0.0-20240216151214-738f3fa4ccf8 github.com/pkg/errors v0.9.1 github.com/prometheus/common v0.59.1 github.com/rhobs/obo-prometheus-operator v0.77.1-rhobs1 @@ -106,6 +107,7 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apiserver v0.31.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index d4f15965..a417ec0b 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20240301093301-ce10821dc999 h1:+S998xHiJApsJZjRAO8wyedU9GfqFd8mtwWly6LqHDo= github.com/openshift/api v0.0.0-20240301093301-ce10821dc999/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= +github.com/openshift/library-go v0.0.0-20240216151214-738f3fa4ccf8 h1:dKtHGYiOwl0DKZEWBW4MFWFS6IYW02AVD1WSuUAVwEo= +github.com/openshift/library-go v0.0.0-20240216151214-738f3fa4ccf8/go.mod h1:ePlaOqUiPplRc++6aYdMe+2FmXb2xTNS9Nz5laG2YmI= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -357,6 +359,8 @@ k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/ k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c= +k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM= k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= diff --git a/pkg/assets/certificate_generator.go b/pkg/assets/certificate_generator.go new file mode 100644 index 00000000..f91310c5 --- /dev/null +++ b/pkg/assets/certificate_generator.go @@ -0,0 +1,147 @@ +package assets + +import ( + "crypto/rand" + "crypto/x509" + "fmt" + "math/big" + "time" + + "github.com/go-logr/logr" + "github.com/openshift/library-go/pkg/crypto" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/authentication/user" +) + +const certificateLifetime = time.Duration(crypto.DefaultCertificateLifetimeInDays) * 24 * time.Hour +const GRPCSecretName = "thanos-grpc-secret" + +// Taken from +// https://github.com/openshift/library-go/blob/08c2fd1b452520da35ad210930ea9d100545589a/pkg/operator/certrotation/signer.go#L68-L86 +// without refresh time handling. We just take care of rotation if we reach 1/5 of the validity timespan before expiration. +func needsNewCert(notBefore, notAfter time.Time, now func() time.Time) bool { + maxWait := notAfter.Sub(notBefore) / 5 + latestTime := notAfter.Add(-maxWait) + return now().After(latestTime) +} + +// Taken from +// https://github.com/openshift/cluster-monitoring-operator/blob/765d0b0369b176a5997d787b6710783437172879/pkg/manifests/tls.go#L113 +func RotateGRPCSecret(s *v1.Secret, logger logr.Logger) (bool, error) { + var ( + curCA, newCA *crypto.CA + curCABytes, crtPresent = s.Data["ca.crt"] + curCAKeyBytes, keyPresent = s.Data["ca.key"] + rotate = !crtPresent || !keyPresent + ) + + if crtPresent && keyPresent { + var err error + curCA, err = crypto.GetCAFromBytes(curCABytes, curCAKeyBytes) + if err != nil { + logger.Info(fmt.Sprintf("generating a new CA due to error reading CA: %v", err)) + rotate = true + } else if needsNewCert(curCA.Config.Certs[0].NotBefore, curCA.Config.Certs[0].NotAfter, time.Now) { + logger.Info("generating new CA, because the current one is older than 1/5 of it validity timestamp") + rotate = true + } + } + + if !rotate { + return rotate, nil + } + + if curCA == nil { + newCAConfig, err := crypto.MakeSelfSignedCAConfig( + fmt.Sprintf("%s@%d", "openshift-cluster-monitoring", time.Now().Unix()), + crypto.DefaultCertificateLifetimeInDays, + ) + if err != nil { + return rotate, fmt.Errorf("error generating self signed CA: %w", err) + } + + newCA = &crypto.CA{ + SerialGenerator: &crypto.RandomSerialGenerator{}, + Config: newCAConfig, + } + } else { + template := curCA.Config.Certs[0] + now := time.Now() + template.NotBefore = now.Add(-1 * time.Second) + template.NotAfter = now.Add(certificateLifetime) + template.SerialNumber = template.SerialNumber.Add(template.SerialNumber, big.NewInt(1)) + + newCACert, err := createCertificate(template, template, template.PublicKey, curCA.Config.Key) + if err != nil { + return rotate, fmt.Errorf("error rotating CA: %w", err) + } + + newCA = &crypto.CA{ + SerialGenerator: &crypto.RandomSerialGenerator{}, + Config: &crypto.TLSCertificateConfig{ + Certs: []*x509.Certificate{newCACert}, + Key: curCA.Config.Key, + }, + } + } + + newCABytes, newCAKeyBytes, err := newCA.Config.GetPEMBytes() + if err != nil { + return rotate, fmt.Errorf("error getting PEM bytes from CA: %w", err) + } + + s.Data["ca.crt"] = newCABytes + s.Data["ca.key"] = newCAKeyBytes + + { + cfg, err := newCA.MakeClientCertificateForDuration( + &user.DefaultInfo{ + Name: "thanos-querier", + }, + time.Duration(crypto.DefaultCertificateLifetimeInDays)*24*time.Hour, + ) + if err != nil { + return rotate, fmt.Errorf("error making client certificate: %w", err) + } + + crt, key, err := cfg.GetPEMBytes() + if err != nil { + return rotate, fmt.Errorf("error getting PEM bytes for thanos querier client certificate: %w", err) + } + s.Data["thanos-querier-client.crt"] = crt + s.Data["thanos-querier-client.key"] = key + } + + { + cfg, err := newCA.MakeServerCert( + sets.NewString("prometheus-grpc"), + crypto.DefaultCertificateLifetimeInDays, + ) + if err != nil { + return rotate, fmt.Errorf("error making server certificate: %w", err) + } + + crt, key, err := cfg.GetPEMBytes() + if err != nil { + return rotate, fmt.Errorf("error getting PEM bytes for prometheus-k8s server certificate: %w", err) + } + s.Data["prometheus-server.crt"] = crt + s.Data["prometheus-server.key"] = key + } + + return rotate, nil +} + +// createCertificate creates a new certificate and returns it in x509.Certificate form. +func createCertificate(template, parent *x509.Certificate, pub, priv interface{}) (*x509.Certificate, error) { + rawCert, err := x509.CreateCertificate(rand.Reader, template, parent, pub, priv) + if err != nil { + return nil, fmt.Errorf("error creating certificate: %w", err) + } + parsedCerts, err := x509.ParseCertificates(rawCert) + if err != nil { + return nil, fmt.Errorf("error parsing certificate: %w", err) + } + return parsedCerts[0], nil +} diff --git a/pkg/controllers/monitoring/monitoring-stack/components.go b/pkg/controllers/monitoring/monitoring-stack/components.go index c2fd7914..4130384c 100644 --- a/pkg/controllers/monitoring/monitoring-stack/components.go +++ b/pkg/controllers/monitoring/monitoring-stack/components.go @@ -1,6 +1,7 @@ package monitoringstack import ( + "fmt" "reflect" monv1 "github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring/v1" @@ -12,6 +13,7 @@ import ( "k8s.io/utils/ptr" stack "github.com/rhobs/observability-operator/pkg/apis/monitoring/v1alpha1" + "github.com/rhobs/observability-operator/pkg/assets" "github.com/rhobs/observability-operator/pkg/reconciler" ) @@ -28,6 +30,7 @@ func stackComponentReconcilers( thanos ThanosConfiguration, prometheus PrometheusConfiguration, alertmanager AlertmanagerConfiguration, + tlsHashes map[string]string, ) []reconciler.Reconciler { prometheusName := ms.Name + "-prometheus" alertmanagerName := ms.Name + "-alertmanager" @@ -44,7 +47,7 @@ func stackComponentReconcilers( reconciler.NewUpdater(newPrometheus(ms, prometheusName, additionalScrapeConfigsSecretName, instanceSelectorKey, instanceSelectorValue, - thanos, prometheus), ms), + thanos, prometheus, tlsHashes), ms), reconciler.NewUpdater(newPrometheusService(ms, instanceSelectorKey, instanceSelectorValue), ms), reconciler.NewUpdater(newThanosSidecarService(ms, instanceSelectorKey, instanceSelectorValue), ms), reconciler.NewOptionalUpdater(newPrometheusPDB(ms, instanceSelectorKey, instanceSelectorValue), ms, @@ -115,6 +118,7 @@ func newPrometheus( instanceSelectorValue string, thanosCfg ThanosConfiguration, prometheusCfg PrometheusConfiguration, + tlsHashes map[string]string, ) *monv1.Prometheus { prometheusSelector := ms.Spec.ResourceSelector @@ -193,12 +197,33 @@ func newPrometheus( } return []monv1.EnableFeature{} }(), + Volumes: []corev1.Volume{ + { + Name: "thanos-tls-assets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: assets.GRPCSecretName, + }, + }, + }, + }, }, Retention: ms.Spec.Retention, RuleSelector: prometheusSelector, RuleNamespaceSelector: ms.Spec.NamespaceSelector, Thanos: &monv1.ThanosSpec{ Image: ptr.To(thanosCfg.Image), + GRPCServerTLSConfig: &monv1.TLSConfig{ + CAFile: "/etc/thanos/tls-assets/ca.crt", + CertFile: "/etc/thanos/tls-assets/prometheus-server.crt", + KeyFile: "/etc/thanos/tls-assets/prometheus-server.key", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "thanos-tls-assets", + MountPath: "/etc/thanos/tls-assets", + }, + }, }, }, } @@ -230,6 +255,14 @@ func newPrometheus( prometheus.Spec.Secrets = append(prometheus.Spec.Secrets, tlsConfig.CertificateAuthority.Name) } + if len(tlsHashes) > 0 { + tlsAnnotations := map[string]string{} + for name, hash := range tlsHashes { + tlsAnnotations[fmt.Sprintf("monitoring.openshift.io/%s-hash", name)] = hash + } + prometheus.Spec.CommonPrometheusFields.PodMetadata.Annotations = tlsAnnotations + } + if prometheusCfg.Image != "" { prometheus.Spec.CommonPrometheusFields.Image = ptr.To(prometheusCfg.Image) } diff --git a/pkg/controllers/monitoring/monitoring-stack/controller.go b/pkg/controllers/monitoring/monitoring-stack/controller.go index 52068406..84d13e50 100644 --- a/pkg/controllers/monitoring/monitoring-stack/controller.go +++ b/pkg/controllers/monitoring/monitoring-stack/controller.go @@ -28,7 +28,9 @@ import ( policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,6 +38,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" stack "github.com/rhobs/observability-operator/pkg/apis/monitoring/v1alpha1" + "github.com/rhobs/observability-operator/pkg/assets" + "github.com/rhobs/observability-operator/pkg/controllers/monitoring/utils" ) type resourceManager struct { @@ -133,6 +137,42 @@ func RegisterWithManager(mgr ctrl.Manager, opts Options) error { func (rm resourceManager) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := rm.logger.WithValues("stack", req.NamespacedName) logger.Info("Reconciling monitoring stack") + + gRPCSecret := v1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: assets.GRPCSecretName, + Namespace: req.Namespace, + }, + Data: map[string][]byte{}, + } + err := rm.k8sClient.Get(ctx, + types.NamespacedName{ + Name: assets.GRPCSecretName, + Namespace: req.Namespace, + }, + &gRPCSecret) + if client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + + rotate, err := assets.RotateGRPCSecret(&gRPCSecret, logger) + if err != nil { + return ctrl.Result{}, err + } + if rotate { + err = rm.k8sClient.Update(ctx, &gRPCSecret) + if errors.IsNotFound(err) { + err = rm.k8sClient.Create(ctx, &gRPCSecret) + } + if err != nil { + return ctrl.Result{}, err + } + } + ms, err := rm.getStack(ctx, req) if err != nil { // retry since some error has occured @@ -149,12 +189,24 @@ func (rm resourceManager) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{}, nil } + // querier <---> sidecar mTLS hashes + mTLSSecretKeys := []string{"prometheus-server.key", "prometheus-server.crt", "ca.crt"} + tlsHashes := map[string]string{} + for _, key := range mTLSSecretKeys { + hash, err := utils.HashOfTLSSecret(assets.GRPCSecretName, key, ms.Namespace, rm.k8sClient) + if err != nil { + return ctrl.Result{}, err + } + tlsHashes[fmt.Sprintf("%s-%s", assets.GRPCSecretName, key)] = hash + } + reconcilers := stackComponentReconcilers(ms, rm.instanceSelectorKey, rm.instanceSelectorValue, rm.thanos, rm.prometheus, rm.alertmanager, + tlsHashes, ) for _, reconciler := range reconcilers { err := reconciler.Reconcile(ctx, rm.k8sClient, rm.scheme) diff --git a/pkg/controllers/monitoring/thanos-querier/components.go b/pkg/controllers/monitoring/thanos-querier/components.go index 90553995..704e7f74 100644 --- a/pkg/controllers/monitoring/thanos-querier/components.go +++ b/pkg/controllers/monitoring/thanos-querier/components.go @@ -10,6 +10,7 @@ import ( "k8s.io/utils/ptr" msoapi "github.com/rhobs/observability-operator/pkg/apis/monitoring/v1alpha1" + "github.com/rhobs/observability-operator/pkg/assets" "github.com/rhobs/observability-operator/pkg/reconciler" ) @@ -29,6 +30,11 @@ func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecar "--log.format=logfmt", "--query.replica-label=prometheus_replica", "--query.auto-downsampling", + "--grpc-client-tls-secure", + "--grpc-client-server-name=prometheus-grpc", + "--grpc-client-tls-ca=/etc/thanos/tls-sidecar-assets/ca.crt", + "--grpc-client-tls-key=/etc/thanos/tls-sidecar-assets/thanos-querier-client.key", + "--grpc-client-tls-cert=/etc/thanos/tls-sidecar-assets/thanos-querier-client.crt", } for _, endpoint := range sidecarUrls { args = append(args, fmt.Sprintf("--endpoint=%s", endpoint)) @@ -86,6 +92,12 @@ func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecar Type: corev1.SeccompProfileTypeRuntimeDefault, }, }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "thanos-sidecar-tls-assets", + MountPath: "/etc/thanos/tls-sidecar-assets", + }, + }, }, }, NodeSelector: map[string]string{ @@ -97,11 +109,75 @@ func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecar Type: corev1.SeccompProfileTypeRuntimeDefault, }, }, + Volumes: []corev1.Volume{ + { + Name: "thanos-sidecar-tls-assets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: assets.GRPCSecretName, + }, + }, + }, + }, }, }, ProgressDeadlineSeconds: ptr.To(int32(300)), }, } + if spec.Spec.WebTLSConfig != nil { + thanos.Spec.Template.Spec.Volumes = append(thanos.Spec.Template.Spec.Volumes, []corev1.Volume{ + { + Name: "thanos-web-tls-key", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: spec.Spec.WebTLSConfig.PrivateKey.Name, + }, + }, + }, + { + Name: "thanos-web-tls-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: spec.Spec.WebTLSConfig.Certificate.Name, + }, + }, + }, + { + Name: "thanos-web-http-conf", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: httpConfCMName, + }, + }, + }, + }, + }...) + thanos.Spec.Template.Spec.Containers[0].VolumeMounts = append(thanos.Spec.Template.Spec.Containers[0].VolumeMounts, []corev1.VolumeMount{ + { + Name: "thanos-web-tls-key", + MountPath: "/etc/thanos/tls-assets/web-cert-secret", + ReadOnly: true, + }, + { + Name: "thanos-web-tls-cert", + MountPath: "/etc/thanos/tls-assets/web-key-secret", + ReadOnly: true, + }, + { + Name: "thanos-web-http-conf", + MountPath: "/etc/thanos/tls-assets/web-http-conf-cm", + ReadOnly: true, + }, + }...) + } + if len(tlsHashes) > 0 { + tlsAnnotations := map[string]string{} + for name, hash := range tlsHashes { + tlsAnnotations[fmt.Sprintf("monitoring.openshift.io/%s-hash", name)] = hash + } + thanos.Spec.Template.ObjectMeta.Annotations = tlsAnnotations + } return thanos } diff --git a/pkg/controllers/monitoring/thanos-querier/controller.go b/pkg/controllers/monitoring/thanos-querier/controller.go index 87275a99..b77285ac 100644 --- a/pkg/controllers/monitoring/thanos-querier/controller.go +++ b/pkg/controllers/monitoring/thanos-querier/controller.go @@ -33,6 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" msoapi "github.com/rhobs/observability-operator/pkg/apis/monitoring/v1alpha1" + "github.com/rhobs/observability-operator/pkg/assets" + "github.com/rhobs/observability-operator/pkg/controllers/monitoring/utils" ) type resourceManager struct { @@ -90,6 +92,16 @@ func RegisterWithManager(mgr ctrl.Manager, opts Options) error { handler.EnqueueRequestsFromMapFunc(rm.findQueriersForMonitoringStack), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(rm.findQueriersForTLSSecrets), + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(rm.findQueriersForGrpcTLSSecret), + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). Complete(rm) } @@ -113,7 +125,35 @@ func (rm resourceManager) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{RequeueAfter: 10 * time.Second}, err } - reconcilers := thanosComponentReconcilers(querier, sidecarServices, rm.thanos) + tlsHashes := map[string]string{} + + // Web endponint TLS hashes + if querier.Spec.WebTLSConfig != nil { + secretSelectors := []msoapi.SecretKeySelector{ + querier.Spec.WebTLSConfig.CertificateAuthority, + querier.Spec.WebTLSConfig.Certificate, + querier.Spec.WebTLSConfig.PrivateKey, + } + for _, secretSelector := range secretSelectors { + hash, err := utils.HashOfTLSSecret(secretSelector.Name, secretSelector.Key, querier.Namespace, rm) + if err != nil { + return ctrl.Result{}, err + } + tlsHashes[fmt.Sprintf("%s-%s", secretSelector.Name, secretSelector.Key)] = hash + } + } + + // querier <---> sidecar mTLS hashes + mTLSSecretKeys := []string{"thanos-querier-client.key", "thanos-querier-client.crt", "ca.crt"} + for _, key := range mTLSSecretKeys { + hash, err := utils.HashOfTLSSecret(assets.GRPCSecretName, key, querier.Namespace, rm) + if err != nil { + return ctrl.Result{}, err + } + tlsHashes[fmt.Sprintf("%s-%s", assets.GRPCSecretName, key)] = hash + } + + reconcilers := thanosComponentReconcilers(querier, sidecarServices, rm.thanos, tlsHashes) for _, reconciler := range reconcilers { err := reconciler.Reconcile(ctx, rm, rm.scheme) // handle creation / updation errors that can happen due to a stale cache by @@ -191,3 +231,75 @@ func (rm resourceManager) findQueriersForMonitoringStack(ctx context.Context, ms } return requests } + +// Find all ThanosQueriers, whose TLS secrets fit the given Secret and +// return a list of reconcile requests, one for each ThanosQuerier. +func (rm resourceManager) findQueriersForTLSSecrets(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + logger := rm.logger.WithValues("Secret", src.GetNamespace()+"/"+src.GetName()) + logger.Info("watched Secret changed, checking for matching querier") + + thanosWatchFields := []string{ + thanosTLSCertificateAuthoritySecretNameField, + thanosTLSCertificateSecretNameField, + thanosTLSPrivateKeySecretNameField, + } + + for _, field := range thanosWatchFields { + crList := &msoapi.ThanosQuerierList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := rm.Client.List(ctx, crList, listOps) + if err != nil { + logger.Error(err, "Failed to list Thanosqueriers") + return []reconcile.Request{} + } + + for _, item := range crList.Items { + logger.Info("Found querier, scheduling sync") + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }) + } + } + + return requests +} + +// Enqueue all ThanosQueriers from a namespace, where GRPC secret changed +func (rm resourceManager) findQueriersForGrpcTLSSecret(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + if src.GetName() == assets.GRPCSecretName { + logger := rm.logger.WithValues("Secret", src.GetNamespace()+"/"+src.GetName()) + logger.Info("watched Secret changed, checking for matching querier") + + crList := &msoapi.ThanosQuerierList{} + listOps := &client.ListOptions{ + Namespace: src.GetNamespace(), + } + err := rm.Client.List(ctx, crList, listOps) + if err != nil { + logger.Error(err, "Failed to list Thanosqueriers") + return []reconcile.Request{} + } + + for _, item := range crList.Items { + logger.Info("Found querier, scheduling sync") + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }) + } + } + + return requests +} diff --git a/pkg/controllers/monitoring/utils/utils.go b/pkg/controllers/monitoring/utils/utils.go new file mode 100644 index 00000000..bbc537c1 --- /dev/null +++ b/pkg/controllers/monitoring/utils/utils.go @@ -0,0 +1,47 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "crypto/sha256" + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func HashOfTLSSecret( + secretName string, + secretKey string, + namespace string, + client client.Client, +) (string, error) { + var secret v1.Secret + err := client.Get(context.Background(), types.NamespacedName{ + Name: secretName, + Namespace: namespace, + }, &secret) + if err != nil { + return "", fmt.Errorf("Couldn't get TLS secret %s: %s", secretName, err) + } + + hash := sha256.Sum256(secret.Data[secretKey]) + return rand.SafeEncodeString(fmt.Sprint(hash)), nil +} diff --git a/test/e2e/monitoring_stack_controller_test.go b/test/e2e/monitoring_stack_controller_test.go index b06c3370..edd8b845 100644 --- a/test/e2e/monitoring_stack_controller_test.go +++ b/test/e2e/monitoring_stack_controller_test.go @@ -959,6 +959,11 @@ const oboManagedFieldsJson = ` "f:image": {}, "f:logLevel": {}, "f:podMetadata": { + "f:annotations": { + "f:monitoring.openshift.io/thanos-grpc-secret-ca.crt-hash": {}, + "f:monitoring.openshift.io/thanos-grpc-secret-prometheus-server.crt-hash": {}, + "f:monitoring.openshift.io/thanos-grpc-secret-prometheus-server.key-hash": {} + }, "f:labels": { "f:app.kubernetes.io/component": {}, "f:app.kubernetes.io/part-of": {} @@ -1005,9 +1010,18 @@ const oboManagedFieldsJson = ` } }, "f:thanos": { + "f:grpcServerTlsConfig": { + "f:ca": {}, + "f:caFile": {}, + "f:cert": {}, + "f:certFile": {}, + "f:keyFile": {} + }, "f:image": {}, - "f:resources": {} + "f:resources": {}, + "f:volumeMounts": {} }, + "f:volumes": {}, "f:web": { "f:tlsConfig": { "f:cert": {