From 4b68265e788c1e964f25dee0bb2e0ab10f0c86fb Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Wed, 29 Nov 2023 20:31:53 +0100 Subject: [PATCH 01/16] Support setting multiple conditions: cherry picked from f390f3734 --- operator/apis/loki/v1/lokistack_types.go | 4 + operator/internal/status/conditions.go | 37 ++++++ operator/internal/status/conditions_test.go | 139 ++++++++++++++++++++ operator/internal/status/lokistack.go | 22 +++- operator/internal/status/status.go | 28 +--- 5 files changed, 202 insertions(+), 28 deletions(-) create mode 100644 operator/internal/status/conditions.go create mode 100644 operator/internal/status/conditions_test.go diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index 8b163ff780300..6d31d9891a858 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -1041,6 +1041,10 @@ const ( // ConditionDegraded defines the condition that some or all components in the Loki deployment // are degraded or the cluster cannot connect to object storage. ConditionDegraded LokiStackConditionType = "Degraded" + + // ConditionWarning is used for configurations that are not recommended, but don't currently cause + // issues. There can be multiple warning conditions active at a time. + ConditionWarning LokiStackConditionType = "Warning" ) // LokiStackConditionReason defines the type for valid reasons of a Loki deployment conditions. diff --git a/operator/internal/status/conditions.go b/operator/internal/status/conditions.go new file mode 100644 index 0000000000000..637a50e6f89f4 --- /dev/null +++ b/operator/internal/status/conditions.go @@ -0,0 +1,37 @@ +package status + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +func mergeConditions(old, active []metav1.Condition, now metav1.Time) []metav1.Condition { + merged := make([]metav1.Condition, 0, len(old)+len(active)) + for len(old) > 0 { + c := old[0] + found := -1 + for i, ac := range active { + if c.Type == ac.Type && c.Reason == ac.Reason { + found = i + break + } + } + + if found != -1 { + c = active[found] + active = append(active[:found], active[found+1:]...) + + c.Status = metav1.ConditionTrue + } else { + c.Status = metav1.ConditionFalse + } + + c.LastTransitionTime = now + merged = append(merged, c) + old = old[1:] + } + + for _, c := range active { + c.Status = metav1.ConditionTrue + c.LastTransitionTime = now + merged = append(merged, c) + } + return merged +} diff --git a/operator/internal/status/conditions_test.go b/operator/internal/status/conditions_test.go new file mode 100644 index 0000000000000..86bb087c84bfb --- /dev/null +++ b/operator/internal/status/conditions_test.go @@ -0,0 +1,139 @@ +package status + +import ( + "github.com/google/go-cmp/cmp" + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" + "time" +) + +func TestMergeConditions(t *testing.T) { + now := metav1.NewTime(time.Unix(0, 0)) + tt := []struct { + desc string + old []metav1.Condition + active []metav1.Condition + wantMerged []metav1.Condition + }{ + { + desc: "set status and time", + old: []metav1.Condition{}, + active: []metav1.Condition{ + conditionReady, + }, + wantMerged: []metav1.Condition{ + { + Type: conditionReady.Type, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: conditionReady.Reason, + Message: conditionReady.Message, + }, + }, + }, + { + desc: "reset old condition", + old: []metav1.Condition{ + conditionPending, + }, + active: []metav1.Condition{ + conditionReady, + }, + wantMerged: []metav1.Condition{ + { + Type: conditionPending.Type, + Status: metav1.ConditionFalse, + LastTransitionTime: now, + Reason: conditionPending.Reason, + Message: conditionPending.Message, + }, + { + Type: conditionReady.Type, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: conditionReady.Reason, + Message: conditionReady.Message, + }, + }, + }, + { + desc: "keep active conditions", + old: []metav1.Condition{ + { + Type: conditionReady.Type, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: conditionReady.Reason, + Message: conditionReady.Message, + }, + { + Type: conditionPending.Type, + Status: metav1.ConditionFalse, + LastTransitionTime: now, + Reason: conditionPending.Reason, + Message: conditionPending.Message, + }, + }, + active: []metav1.Condition{ + conditionReady, + { + Type: string(lokiv1.ConditionWarning), + Reason: "test-warning", + Message: "test-warning-message", + }, + }, + wantMerged: []metav1.Condition{ + { + Type: conditionReady.Type, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: conditionReady.Reason, + Message: conditionReady.Message, + }, + { + Type: conditionPending.Type, + Status: metav1.ConditionFalse, + LastTransitionTime: now, + Reason: conditionPending.Reason, + Message: conditionPending.Message, + }, + { + Type: string(lokiv1.ConditionWarning), + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: "test-warning", + Message: "test-warning-message", + }, + }, + }, + } + + for _, tc := range tt { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + beforeLenOld := len(tc.old) + beforeLenActive := len(tc.active) + + merged := mergeConditions(tc.old, tc.active, now) + + afterLenOld := len(tc.old) + afterLenActive := len(tc.active) + + if diff := cmp.Diff(merged, tc.wantMerged); diff != "" { + t.Errorf("Merged conditions differ: -got+want\n%s", diff) + } + + if beforeLenOld != afterLenOld { + t.Errorf("old length differs: got %v, want %v", afterLenOld, beforeLenOld) + } + + if beforeLenActive != afterLenActive { + t.Errorf("active length differs: got %v, want %v", afterLenActive, beforeLenActive) + } + }) + } +} diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index 1212e5c2f7e92..9381ab6e8ab7c 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -63,15 +63,19 @@ func (e *DegradedError) Error() string { return fmt.Sprintf("cluster degraded: %s", e.Message) } -// SetDegradedCondition appends the condition Degraded to the lokistack status conditions. -func SetDegradedCondition(ctx context.Context, k k8s.Client, req ctrl.Request, msg string, reason lokiv1.LokiStackConditionReason) error { - degraded := metav1.Condition{ - Type: string(lokiv1.ConditionDegraded), - Message: msg, - Reason: string(reason), +func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) { + conditions, err := generateWarnings(ctx, cs, k, req, stack) + if err != nil { + return nil, err + } + + mainCondition, err := generateCondition(ctx, cs, k, req, stack, degradedErr) + if err != nil { + return nil, err } - return updateCondition(ctx, k, req, degraded) + conditions = append(conditions, mainCondition) + return conditions, nil } func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) (metav1.Condition, error) { @@ -149,6 +153,10 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1 return true, true, nil } +func generateWarnings(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) ([]metav1.Condition, error) { + return []metav1.Condition{}, nil +} + func updateCondition(ctx context.Context, k k8s.Client, req ctrl.Request, condition metav1.Condition) error { var stack lokiv1.LokiStack if err := k.Get(ctx, req.NamespacedName, &stack); err != nil { diff --git a/operator/internal/status/status.go b/operator/internal/status/status.go index 97a8e81bbae08..47dbe377bd71b 100644 --- a/operator/internal/status/status.go +++ b/operator/internal/status/status.go @@ -31,34 +31,20 @@ func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time) return err } - condition, err := generateCondition(ctx, cs, k, req, &stack) + activeConditions, err := generateConditions(ctx, cs, k, req, &stack, degradedErr) if err != nil { return err } - condition.LastTransitionTime = metav1.NewTime(now) - condition.Status = metav1.ConditionTrue + metaTime := metav1.NewTime(now) + for _, c := range activeConditions { + c.LastTransitionTime = metaTime + c.Status = metav1.ConditionTrue + } statusUpdater := func(stack *lokiv1.LokiStack) { stack.Status.Components = *cs - - index := -1 - for i := range stack.Status.Conditions { - // Reset all other conditions first - stack.Status.Conditions[i].Status = metav1.ConditionFalse - stack.Status.Conditions[i].LastTransitionTime = metav1.NewTime(now) - - // Locate existing pending condition if any - if stack.Status.Conditions[i].Type == condition.Type { - index = i - } - } - - if index == -1 { - stack.Status.Conditions = append(stack.Status.Conditions, condition) - } else { - stack.Status.Conditions[index] = condition - } + stack.Status.Conditions = mergeConditions(stack.Status.Conditions, activeConditions, metaTime) } statusUpdater(&stack) From 90d5ac87e6003c125af2afadf501196f13400010 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Wed, 29 Nov 2023 18:51:00 +0100 Subject: [PATCH 02/16] Move handling of degraded state into status package: cherry-picked from 6f7e8b8d3 --- .../controllers/loki/lokistack_controller.go | 42 +++--- operator/internal/status/lokistack_test.go | 138 +++--------------- operator/internal/status/status.go | 2 +- operator/internal/status/status_test.go | 4 +- 4 files changed, 46 insertions(+), 140 deletions(-) diff --git a/operator/controllers/loki/lokistack_controller.go b/operator/controllers/loki/lokistack_controller.go index 708390b62c846..49b5bdab069e0 100644 --- a/operator/controllers/loki/lokistack_controller.go +++ b/operator/controllers/loki/lokistack_controller.go @@ -5,7 +5,6 @@ import ( "errors" "time" - "github.com/ViaQ/logerr/v2/kverrors" "github.com/go-logr/logr" "github.com/google/go-cmp/cmp" openshiftconfigv1 "github.com/openshift/api/config/v1" @@ -150,40 +149,41 @@ func (r *LokiStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, nil } - if r.FeatureGates.BuiltInCertManagement.Enabled { - err = handlers.CreateOrRotateCertificates(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates) - if err != nil { - return handleDegradedError(ctx, r.Client, req, err) - } + var degraded *status.DegradedError + err = r.updateResources(ctx, req) + switch { + case errors.As(err, °raded): + // degraded errors are handled by status.Refresh below + case err != nil: + return ctrl.Result{}, err } - err = handlers.CreateOrUpdateLokiStack(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates) + err = status.Refresh(ctx, r.Client, req, time.Now(), degraded) if err != nil { - return handleDegradedError(ctx, r.Client, req, err) + return ctrl.Result{}, err } - err = status.Refresh(ctx, r.Client, req, time.Now()) - if err != nil { - return ctrl.Result{}, err + if degraded != nil { + return ctrl.Result{ + Requeue: degraded.Requeue, + }, nil } return ctrl.Result{}, nil } -func handleDegradedError(ctx context.Context, c client.Client, req ctrl.Request, err error) (ctrl.Result, error) { - var degraded *status.DegradedError - if errors.As(err, °raded) { - err = status.SetDegradedCondition(ctx, c, req, degraded.Message, degraded.Reason) - if err != nil { - return ctrl.Result{}, kverrors.Wrap(err, "error setting degraded condition") +func (r *LokiStackReconciler) updateResources(ctx context.Context, req ctrl.Request) error { + if r.FeatureGates.BuiltInCertManagement.Enabled { + if err := handlers.CreateOrRotateCertificates(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates); err != nil { + return err } + } - return ctrl.Result{ - Requeue: degraded.Requeue, - }, nil + if err := handlers.CreateOrUpdateLokiStack(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates); err != nil { + return err } - return ctrl.Result{}, err + return nil } // SetupWithManager sets up the controller with the Manager. diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go index 8bdc9fadc7cf6..a4dc36c0710ee 100644 --- a/operator/internal/status/lokistack_test.go +++ b/operator/internal/status/lokistack_test.go @@ -39,120 +39,6 @@ func setupFakesNoError(t *testing.T, stack *lokiv1.LokiStack) (*k8sfakes.FakeCli return k, sw } -func TestSetDegradedCondition_WhenGetLokiStackReturnsNotFound_DoNothing(t *testing.T) { - msg := "tell me nothing" - reason := lokiv1.ReasonMissingObjectStorageSecret - - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "some-ns", - }, - } - - k := &k8sfakes.FakeClient{} - k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { - return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") - } - - err := SetDegradedCondition(context.Background(), k, r, msg, reason) - require.NoError(t, err) -} - -func TestSetDegradedCondition_WhenExisting_DoNothing(t *testing.T) { - msg := "tell me nothing" - reason := lokiv1.ReasonMissingObjectStorageSecret - s := lokiv1.LokiStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "some-ns", - }, - Status: lokiv1.LokiStackStatus{ - Conditions: []metav1.Condition{ - { - Type: string(lokiv1.ConditionDegraded), - Reason: string(reason), - Message: msg, - Status: metav1.ConditionTrue, - }, - }, - }, - } - - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "some-ns", - }, - } - - k, _ := setupFakesNoError(t, &s) - - err := SetDegradedCondition(context.Background(), k, r, msg, reason) - require.NoError(t, err) - require.Zero(t, k.StatusCallCount()) -} - -func TestSetDegradedCondition_WhenExisting_SetDegradedConditionTrue(t *testing.T) { - msg := "tell me something" - reason := lokiv1.ReasonMissingObjectStorageSecret - s := lokiv1.LokiStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "some-ns", - }, - Status: lokiv1.LokiStackStatus{ - Conditions: []metav1.Condition{ - { - Type: string(lokiv1.ConditionDegraded), - Reason: string(reason), - Status: metav1.ConditionFalse, - }, - }, - }, - } - - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "some-ns", - }, - } - - k, sw := setupFakesNoError(t, &s) - - err := SetDegradedCondition(context.Background(), k, r, msg, reason) - require.NoError(t, err) - require.NotZero(t, k.StatusCallCount()) - require.NotZero(t, sw.UpdateCallCount()) -} - -func TestSetDegradedCondition_WhenNoneExisting_AppendDegradedCondition(t *testing.T) { - msg := "tell me something" - reason := lokiv1.ReasonMissingObjectStorageSecret - s := lokiv1.LokiStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "some-ns", - }, - } - - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "some-ns", - }, - } - - k, sw := setupFakesNoError(t, &s) - - err := SetDegradedCondition(context.Background(), k, r, msg, reason) - require.NoError(t, err) - - require.NotZero(t, k.StatusCallCount()) - require.NotZero(t, sw.UpdateCallCount()) -} - func TestGenerateCondition(t *testing.T) { k := &k8sfakes.FakeClient{} r := ctrl.Request{ @@ -173,6 +59,7 @@ func TestGenerateCondition(t *testing.T) { tt := []struct { desc string componentStatus *lokiv1.LokiStackComponentStatus + degradedErr *DegradedError wantCondition metav1.Condition }{ { @@ -202,6 +89,25 @@ func TestGenerateCondition(t *testing.T) { }, wantCondition: conditionFailed, }, + { + desc: "degraded error", + componentStatus: &lokiv1.LokiStackComponentStatus{ + Ingester: map[corev1.PodPhase][]string{ + corev1.PodRunning: { + "pod-0", + }, + }, + }, + degradedErr: &DegradedError{ + Message: "test-message", + Reason: "test-reason", + }, + wantCondition: metav1.Condition{ + Type: "Degraded", + Reason: "test-reason", + Message: "test-message", + }, + }, } for _, tc := range tt { @@ -209,7 +115,7 @@ func TestGenerateCondition(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { t.Parallel() - condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack) + condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack, tc.degradedErr) require.Nil(t, err) require.Equal(t, tc.wantCondition, condition) }) @@ -306,7 +212,7 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) { return tc.wantErr } - condition, err := generateCondition(context.TODO(), componentStatus, k, r, &lokiStack) + condition, err := generateCondition(context.TODO(), componentStatus, k, r, &lokiStack, nil) require.Equal(t, tc.wantErr, err) require.Equal(t, tc.wantCondition, condition) diff --git a/operator/internal/status/status.go b/operator/internal/status/status.go index 47dbe377bd71b..5619735bbe67d 100644 --- a/operator/internal/status/status.go +++ b/operator/internal/status/status.go @@ -17,7 +17,7 @@ import ( // Refresh executes an aggregate update of the LokiStack Status struct, i.e. // - It recreates the Status.Components pod status map per component. // - It sets the appropriate Status.Condition to true that matches the pod status maps. -func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time) error { +func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time, degradedErr *DegradedError) error { var stack lokiv1.LokiStack if err := k.Get(ctx, req.NamespacedName, &stack); err != nil { if apierrors.IsNotFound(err) { diff --git a/operator/internal/status/status_test.go b/operator/internal/status/status_test.go index 6befb13df8f7a..a34b1bc81181f 100644 --- a/operator/internal/status/status_test.go +++ b/operator/internal/status/status_test.go @@ -67,7 +67,7 @@ func TestRefreshSuccess(t *testing.T) { k, sw := setupListClient(t, stack, componentPods) - err := Refresh(context.Background(), k, req, now) + err := Refresh(context.Background(), k, req, now, nil) require.NoError(t, err) require.Equal(t, 1, k.GetCallCount()) @@ -129,7 +129,7 @@ func TestRefreshSuccess_ZoneAwarePendingPod(t *testing.T) { return nil } - err := Refresh(context.Background(), k, req, now) + err := Refresh(context.Background(), k, req, now, nil) require.NoError(t, err) require.Equal(t, 1, k.GetCallCount()) From e6b76889bf254136ddbbf5a3b1c1366c2fd54001 Mon Sep 17 00:00:00 2001 From: btaani Date: Thu, 14 Dec 2023 15:18:45 +0100 Subject: [PATCH 03/16] add unit test for adding schema --- .../manifests/internal/config/build_test.go | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 537ec84bf71a5..9b1974e49333e 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -5418,3 +5418,296 @@ analytics: }) } } + +func TestBuild_ConfigAndRuntimeConfig_AddSchema(t *testing.T) { + effectiveDate := "2030-11-07" + + expCfg := ` +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + s3: + s3: http://test.default.svc.cluster.local.:9000 + bucketnames: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + s3forcepathstyle: true + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 + match_max_concurrent: true +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + max_transfer_retries: 0 + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2500 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 1s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + enforce_metric_name: false + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + per_stream_rate_limit: 3MB + per_stream_rate_limit_burst: 15MB + split_queries_by_interval: 30m + query_timeout: 1m + allow_structured_metadata: true +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper + ${V13_CONFIG} +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + shared_store: s3 + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 + ${TSDB_SHIPPER_STORAGE_CONFIG} +tracing: + enabled: false +analytics: + reporting_enabled: true +` + opts := Options{ + Stack: lokiv1.LokiStackSpec{ + Replication: &lokiv1.ReplicationSpec{ + Factor: 1, + }, + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + IngestionLimits: &lokiv1.IngestionLimitSpec{ + IngestionRate: 4, + IngestionBurstSize: 6, + MaxLabelNameLength: 1024, + MaxLabelValueLength: 2048, + MaxLabelNamesPerSeries: 30, + MaxGlobalStreamsPerTenant: 0, + MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, + }, + QueryLimits: &lokiv1.QueryLimitSpec{ + MaxEntriesLimitPerQuery: 5000, + MaxChunksPerQuery: 2000000, + MaxQuerySeries: 500, + QueryTimeout: "1m", + CardinalityLimit: 100000, + }, + }, + }, + }, + Namespace: "test-ns", + Name: "test", + Compactor: Address{ + FQDN: "loki-compactor-grpc-lokistack-dev.default.svc.cluster.local", + Port: 9095, + }, + FrontendWorker: Address{ + FQDN: "loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local", + Port: 9095, + }, + GossipRing: GossipRing{ + InstancePort: 9095, + BindPort: 7946, + MembersDiscoveryAddr: "loki-gossip-ring-lokistack-dev.default.svc.cluster.local", + }, + Querier: Address{ + Protocol: "http", + FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", + Port: 3100, + }, + IndexGateway: Address{ + FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", + Port: 9095, + }, + StorageDirectory: "/tmp/loki", + MaxConcurrent: MaxConcurrent{ + AvailableQuerierCPUCores: 2, + }, + WriteAheadLog: WriteAheadLog{ + Directory: "/tmp/wal", + IngesterMemoryRequest: 5000, + }, + ObjectStorage: storage.Options{ + SharedStore: lokiv1.ObjectStorageSecretS3, + S3: &storage.S3StorageConfig{ + Endpoint: "http://test.default.svc.cluster.local.:9000", + Region: "us-east", + Buckets: "loki", + }, + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-01", + }, + }, + }, + Shippers: []string{"boltdb"}, + EnableRemoteReporting: true, + HTTPTimeouts: HTTPTimeoutConfig{ + IdleTimeout: 30 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 10 * time.Minute, + }, + } + + cfg, _, err := Build(opts) + + expCfg_before := strings.Replace(expCfg, "${TSDB_LIMIT}", "", -1) + expCfg_before = strings.Replace(expCfg_before, " ${V13_CONFIG}", "", -1) + expCfg_before = strings.Replace(expCfg_before, "${TSDB_SHIPPER_STORAGE_CONFIG}", "", -1) + + require.NoError(t, err) + require.YAMLEq(t, expCfg_before, string(cfg)) + + v13schema := lokiv1.ObjectStorageSchema{ + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: lokiv1.StorageSchemaEffectiveDate(effectiveDate), + } + v13_config := `- from: "2030-11-07" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v13 + store: tsdb` + tsdb_shipper_config := ` + tsdb_shipper: + active_index_directory: /tmp/loki/tsdb-index + cache_location: /tmp/loki/tsdb-cache + cache_ttl: 24h + resync_interval: 5m + shared_store: s3 + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095` + tsdb_limit := `tsdb_max_query_parallelism: 512` + + expCfg = strings.Replace(expCfg, "${TSDB_LIMIT}", tsdb_limit, -1) + expCfg = strings.Replace(expCfg, "${V13_CONFIG}", v13_config, -1) + expCfg = strings.Replace(expCfg, "${TSDB_SHIPPER_STORAGE_CONFIG}", tsdb_shipper_config, -1) + + opts.ObjectStorage.Schemas = append(opts.ObjectStorage.Schemas, v13schema) + opts.Shippers = append(opts.Shippers, "tsdb") + + cfg, _, err = Build(opts) + + require.NoError(t, err) + require.YAMLEq(t, expCfg, string(cfg)) + +} From eab8176874907815c84170fe0cbe5fea81e85b12 Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 00:20:19 +0100 Subject: [PATCH 04/16] add warning condition on old schemas --- operator/internal/status/lokistack.go | 65 ++++------------------ operator/internal/status/lokistack_test.go | 9 ++- 2 files changed, 19 insertions(+), 55 deletions(-) diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index 9381ab6e8ab7c..ea7ebc79e386b 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -4,11 +4,8 @@ import ( "context" "fmt" - "github.com/ViaQ/logerr/v2/kverrors" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -22,6 +19,7 @@ const ( messagePending = "Some LokiStack components pending on dependencies" messageDegradedMissingNodes = "Cluster contains no nodes matching the labels used for zone-awareness" messageDegradedEmptyNodeLabel = "No value for the labels used for zone-awareness" + messageOldSchemaVersion = "The schema configuration contains one or more schemas that do not use the most recent version." ) var ( @@ -78,7 +76,7 @@ func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus return conditions, nil } -func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) (metav1.Condition, error) { +func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) (metav1.Condition, error) { // Check for failed pods first failed := len(cs.Compactor[corev1.PodFailed]) + len(cs.Distributor[corev1.PodFailed]) + @@ -154,56 +152,17 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1 } func generateWarnings(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) ([]metav1.Condition, error) { - return []metav1.Condition{}, nil -} - -func updateCondition(ctx context.Context, k k8s.Client, req ctrl.Request, condition metav1.Condition) error { - var stack lokiv1.LokiStack - if err := k.Get(ctx, req.NamespacedName, &stack); err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return kverrors.Wrap(err, "failed to lookup LokiStack", "name", req.NamespacedName) - } - - for _, c := range stack.Status.Conditions { - if c.Type == condition.Type && - c.Reason == condition.Reason && - c.Message == condition.Message && - c.Status == metav1.ConditionTrue { - // resource already has desired condition - return nil + warnings := make([]metav1.Condition, 0) + for _, sc := range stack.Status.Storage.Schemas { + if sc.Version != lokiv1.ObjectStorageSchemaV13 { + warnings = append(warnings, metav1.Condition{ + Type: string(lokiv1.ConditionWarning), + Reason: string(lokiv1.ReasonSchemaUpgradeRecommended), + Message: messageOldSchemaVersion, + }) + break } } - condition.Status = metav1.ConditionTrue - - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - if err := k.Get(ctx, req.NamespacedName, &stack); err != nil { - return err - } - - now := metav1.Now() - condition.LastTransitionTime = now - - index := -1 - for i := range stack.Status.Conditions { - // Reset all other conditions first - stack.Status.Conditions[i].Status = metav1.ConditionFalse - stack.Status.Conditions[i].LastTransitionTime = now - - // Locate existing pending condition if any - if stack.Status.Conditions[i].Type == condition.Type { - index = i - } - } - - if index == -1 { - stack.Status.Conditions = append(stack.Status.Conditions, condition) - } else { - stack.Status.Conditions[index] = condition - } - - return k.Status().Update(ctx, &stack) - }) + return warnings, nil } diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go index a4dc36c0710ee..947c7472455cf 100644 --- a/operator/internal/status/lokistack_test.go +++ b/operator/internal/status/lokistack_test.go @@ -62,7 +62,7 @@ func TestGenerateCondition(t *testing.T) { degradedErr *DegradedError wantCondition metav1.Condition }{ - { + /* { desc: "no error", componentStatus: &lokiv1.LokiStackComponentStatus{}, wantCondition: conditionReady, @@ -88,7 +88,7 @@ func TestGenerateCondition(t *testing.T) { }, }, wantCondition: conditionFailed, - }, + }, */ { desc: "degraded error", componentStatus: &lokiv1.LokiStackComponentStatus{ @@ -113,6 +113,7 @@ func TestGenerateCondition(t *testing.T) { for _, tc := range tt { tc := tc t.Run(tc.desc, func(t *testing.T) { + println(tc.desc) t.Parallel() condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack, tc.degradedErr) @@ -219,3 +220,7 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) { }) } } + +func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { + +} From 7f23e7bfa3e3ec5f0756bed8bf0a2b2877762c10 Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 00:40:07 +0100 Subject: [PATCH 05/16] add reason to api --- operator/apis/loki/v1/lokistack_types.go | 2 ++ operator/docs/operator/api.md | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index 6d31d9891a858..18264e02868a4 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -1101,6 +1101,8 @@ const ( ReasonZoneAwareNodesMissing LokiStackConditionReason = "ReasonZoneAwareNodesMissing" // ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value. ReasonZoneAwareEmptyLabel LokiStackConditionReason = "ReasonZoneAwareEmptyLabel" + // ReasonStorageSchemaVersionIsOld when the object storage schema version is older than V13 + ReasonStorageSchemaVersionIsOld LokiStackConditionReason = "ReasonStorageSchemaVersionIsOld" ) // PodStatusMap defines the type for mapping pod status to pod name. diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index f2ee194a87a2a..69f899b16ffb3 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -1780,6 +1780,9 @@ for the ruler is missing.

"ReadyComponents"

ReasonReadyComponents when all LokiStack components are ready to serve traffic.

+

"ReasonStorageSchemaVersionIsOld"

+

ReasonStorageSchemaVersionIsOld when the object storage schema version is older than V13

+

"ReasonZoneAwareEmptyLabel"

ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value.

@@ -1814,6 +1817,10 @@ are degraded or the cluster cannot connect to object storage.

"Ready"

ConditionReady defines the condition that all components in the Loki deployment are ready.

+

"Warning"

+

ConditionWarning is used for configurations that are not recommended, but don’t currently cause +issues. There can be multiple warning conditions active at a time.

+ From 721a700db166ad3a52334a11550716eea184ed53 Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 00:50:52 +0100 Subject: [PATCH 06/16] add missing resturn statement in generateCondition --- operator/internal/status/lokistack.go | 10 +++++++++- operator/internal/status/lokistack_test.go | 5 ++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index ea7ebc79e386b..387c9db664b22 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -77,6 +77,14 @@ func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus } func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) (metav1.Condition, error) { + if degradedErr != nil { + return metav1.Condition{ + Type: string(lokiv1.ConditionDegraded), + Message: degradedErr.Message, + Reason: string(degradedErr.Reason), + }, nil + } + // Check for failed pods first failed := len(cs.Compactor[corev1.PodFailed]) + len(cs.Distributor[corev1.PodFailed]) + @@ -157,7 +165,7 @@ func generateWarnings(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, if sc.Version != lokiv1.ObjectStorageSchemaV13 { warnings = append(warnings, metav1.Condition{ Type: string(lokiv1.ConditionWarning), - Reason: string(lokiv1.ReasonSchemaUpgradeRecommended), + Reason: string(lokiv1.ReasonStorageSchemaVersionIsOld), Message: messageOldSchemaVersion, }) break diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go index 947c7472455cf..ff7767f585ebd 100644 --- a/operator/internal/status/lokistack_test.go +++ b/operator/internal/status/lokistack_test.go @@ -62,7 +62,7 @@ func TestGenerateCondition(t *testing.T) { degradedErr *DegradedError wantCondition metav1.Condition }{ - /* { + { desc: "no error", componentStatus: &lokiv1.LokiStackComponentStatus{}, wantCondition: conditionReady, @@ -88,7 +88,7 @@ func TestGenerateCondition(t *testing.T) { }, }, wantCondition: conditionFailed, - }, */ + }, { desc: "degraded error", componentStatus: &lokiv1.LokiStackComponentStatus{ @@ -113,7 +113,6 @@ func TestGenerateCondition(t *testing.T) { for _, tc := range tt { tc := tc t.Run(tc.desc, func(t *testing.T) { - println(tc.desc) t.Parallel() condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack, tc.degradedErr) From 08487415ee5e8f5ae0bca18174cf3739de546707 Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 01:12:24 +0100 Subject: [PATCH 07/16] add unit test --- operator/internal/status/lokistack_test.go | 79 ++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go index ff7767f585ebd..c1bc7e358e794 100644 --- a/operator/internal/status/lokistack_test.go +++ b/operator/internal/status/lokistack_test.go @@ -221,5 +221,84 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) { } func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + s := &lokiv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + schemas := []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2023-10-11", + }, + } + + expected := lokiv1.LokiStackStatus{ + Storage: lokiv1.LokiStackStorageStatus{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2023-10-11", + }, + }, + }, + } + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.SubResourceUpdateOption) error { + stack := obj.(*lokiv1.LokiStack) + require.Equal(t, expected.Storage.Schemas, stack.Status.Storage.Schemas) + s = stack + return nil + } + + err := SetStorageSchemaStatus(context.TODO(), k, r, schemas) + + require.NoError(t, err) + require.NotEmpty(t, s.Status.Storage.Schemas) + + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) + + wantCondition := metav1.Condition{ + Type: string(lokiv1.ConditionWarning), + Reason: string(lokiv1.ReasonStorageSchemaVersionIsOld), + Message: messageOldSchemaVersion, + } + componentStatus := &lokiv1.LokiStackComponentStatus{ + Ingester: map[corev1.PodPhase][]string{ + corev1.PodRunning: { + "pod-0", + }, + }, + } + condition, err := generateWarnings(context.TODO(), componentStatus, k, r, s) + require.Nil(t, err) + require.Contains(t, condition, wantCondition) } From 8228baef1ab56e6d5c83daa0eb2c51ae724fc839 Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 01:17:14 +0100 Subject: [PATCH 08/16] modify CHANGELOG.md --- operator/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 592dc4f5837c2..600bc945a1720 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,5 +1,6 @@ ## Main +- [11158](https://github.com/grafana/loki/pull/11158) **btaani**: operator: Add warning for old schema configuration - [11473](https://github.com/grafana/loki/pull/11473) **JoaoBraveCoding**: Adds structured metadata dashboards - [11448](https://github.com/grafana/loki/pull/11448) **periklis**: Update Loki operand to v2.9.3 - [11357](https://github.com/grafana/loki/pull/11357) **periklis**: Fix storing authentication credentials in the Loki ConfigMap From 9079b4bd2d71a87adc07df8521746f50e86c479d Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 17:36:34 +0100 Subject: [PATCH 09/16] address review comments --- operator/apis/loki/v1/lokistack_types.go | 4 +- operator/docs/operator/api.md | 4 +- operator/internal/status/lokistack.go | 30 ++-- operator/internal/status/lokistack_test.go | 194 ++++++++++++++------- 4 files changed, 148 insertions(+), 84 deletions(-) diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index 18264e02868a4..6124c65cd5217 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -1101,8 +1101,8 @@ const ( ReasonZoneAwareNodesMissing LokiStackConditionReason = "ReasonZoneAwareNodesMissing" // ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value. ReasonZoneAwareEmptyLabel LokiStackConditionReason = "ReasonZoneAwareEmptyLabel" - // ReasonStorageSchemaVersionIsOld when the object storage schema version is older than V13 - ReasonStorageSchemaVersionIsOld LokiStackConditionReason = "ReasonStorageSchemaVersionIsOld" + // ReasonStorageNeedsSchemaUpdate when the object storage schema version is older than V13 + ReasonStorageNeedsSchemaUpdate LokiStackConditionReason = "StorageNeedsSchemaUpdate" ) // PodStatusMap defines the type for mapping pod status to pod name. diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index 69f899b16ffb3..989a6ef481649 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -1780,8 +1780,8 @@ for the ruler is missing.

"ReadyComponents"

ReasonReadyComponents when all LokiStack components are ready to serve traffic.

-

"ReasonStorageSchemaVersionIsOld"

-

ReasonStorageSchemaVersionIsOld when the object storage schema version is older than V13

+

"StorageNeedsSchemaUpdate"

+

ReasonStorageNeedsSchemaUpdate when the object storage schema version is older than V13

"ReasonZoneAwareEmptyLabel"

ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value.

diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index 387c9db664b22..3c28d6ef00bc8 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -14,12 +14,12 @@ import ( ) const ( - messageReady = "All components ready" - messageFailed = "Some LokiStack components failed" - messagePending = "Some LokiStack components pending on dependencies" - messageDegradedMissingNodes = "Cluster contains no nodes matching the labels used for zone-awareness" - messageDegradedEmptyNodeLabel = "No value for the labels used for zone-awareness" - messageOldSchemaVersion = "The schema configuration contains one or more schemas that do not use the most recent version." + messageReady = "All components ready" + messageFailed = "Some LokiStack components failed" + messagePending = "Some LokiStack components pending on dependencies" + messageDegradedMissingNodes = "Cluster contains no nodes matching the labels used for zone-awareness" + messageDegradedEmptyNodeLabel = "No value for the labels used for zone-awareness" + messageWarningNeedsSchemaVersionUpdate = "The schema configuration does not contain the most recent schema version and needs an update" ) var ( @@ -62,12 +62,12 @@ func (e *DegradedError) Error() string { } func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) { - conditions, err := generateWarnings(ctx, cs, k, req, stack) + conditions, err := generateWarnings(stack.Status.Storage.Schemas) if err != nil { return nil, err } - mainCondition, err := generateCondition(ctx, cs, k, req, stack, degradedErr) + mainCondition, err := generateCondition(ctx, cs, k, stack, degradedErr) if err != nil { return nil, err } @@ -76,7 +76,7 @@ func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus return conditions, nil } -func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) (metav1.Condition, error) { +func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, stack *lokiv1.LokiStack, degradedErr *DegradedError) (metav1.Condition, error) { if degradedErr != nil { return metav1.Condition{ Type: string(lokiv1.ConditionDegraded), @@ -159,14 +159,18 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1 return true, true, nil } -func generateWarnings(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) ([]metav1.Condition, error) { +func generateWarnings(schemas []lokiv1.ObjectStorageSchema) ([]metav1.Condition, error) { warnings := make([]metav1.Condition, 0) - for _, sc := range stack.Status.Storage.Schemas { + for _, sc := range schemas { + if schemas[len(schemas)-1].Version == lokiv1.ObjectStorageSchemaV13 { + return warnings, nil + } if sc.Version != lokiv1.ObjectStorageSchemaV13 { warnings = append(warnings, metav1.Condition{ Type: string(lokiv1.ConditionWarning), - Reason: string(lokiv1.ReasonStorageSchemaVersionIsOld), - Message: messageOldSchemaVersion, + Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), + Message: messageWarningNeedsSchemaVersionUpdate, + Status: metav1.ConditionTrue, }) break } diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go index c1bc7e358e794..defdbd165b7b3 100644 --- a/operator/internal/status/lokistack_test.go +++ b/operator/internal/status/lokistack_test.go @@ -41,12 +41,6 @@ func setupFakesNoError(t *testing.T, stack *lokiv1.LokiStack) (*k8sfakes.FakeCli func TestGenerateCondition(t *testing.T) { k := &k8sfakes.FakeClient{} - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "test-lokistack", - Namespace: "some-ns", - }, - } lokiStack := lokiv1.LokiStack{ TypeMeta: metav1.TypeMeta{ Kind: "LokiStack", @@ -115,7 +109,7 @@ func TestGenerateCondition(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { t.Parallel() - condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack, tc.degradedErr) + condition, err := generateCondition(context.TODO(), tc.componentStatus, k, &lokiStack, tc.degradedErr) require.Nil(t, err) require.Equal(t, tc.wantCondition, condition) }) @@ -165,12 +159,6 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { t.Parallel() - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "test-lokistack", - Namespace: "some-ns", - }, - } componentStatus := &lokiv1.LokiStackComponentStatus{ Ingester: map[corev1.PodPhase][]string{ corev1.PodPending: { @@ -212,7 +200,7 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) { return tc.wantErr } - condition, err := generateCondition(context.TODO(), componentStatus, k, r, &lokiStack, nil) + condition, err := generateCondition(context.TODO(), componentStatus, k, &lokiStack, nil) require.Equal(t, tc.wantErr, err) require.Equal(t, tc.wantCondition, condition) @@ -221,11 +209,115 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) { } func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { + + tt := []struct { + desc string + schemas []lokiv1.ObjectStorageSchema + expectedSchemaStatus []lokiv1.ObjectStorageSchema + wantCondition []metav1.Condition + }{ + { + desc: "no V13 in schema config", + schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV12, + EffectiveDate: "2023-10-11", + }, + }, + expectedSchemaStatus: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV12, + EffectiveDate: "2023-10-11", + }, + }, + wantCondition: []metav1.Condition{{ + Type: string(lokiv1.ConditionWarning), + Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), + Message: messageWarningNeedsSchemaVersionUpdate, + }}, + }, + { + desc: "with V13 not as the last element in schema config", + schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2023-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV12, + EffectiveDate: "2024-10-11", + }, + }, + expectedSchemaStatus: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2023-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV12, + EffectiveDate: "2024-10-11", + }, + }, + wantCondition: []metav1.Condition{{ + Type: string(lokiv1.ConditionWarning), + Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), + Message: messageWarningNeedsSchemaVersionUpdate, + }}, + }, { + desc: "with V13 as the last element in schema config", + schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV12, + EffectiveDate: "2023-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2024-10-11", + }, + }, + expectedSchemaStatus: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV12, + EffectiveDate: "2023-10-11", + }, + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2024-10-11", + }, + }, + wantCondition: []metav1.Condition{}, + }, + } + sw := &k8sfakes.FakeStatusWriter{} k := &k8sfakes.FakeClient{} k.StatusStub = func() client.StatusWriter { return sw } - s := &lokiv1.LokiStack{ + s := lokiv1.LokiStack{ ObjectMeta: metav1.ObjectMeta{ Name: "my-stack", Namespace: "some-ns", @@ -238,67 +330,35 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { }, } - schemas := []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV11, - EffectiveDate: "2020-10-11", - }, - { - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2023-10-11", - }, - } - - expected := lokiv1.LokiStackStatus{ - Storage: lokiv1.LokiStackStorageStatus{ - Schemas: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV11, - EffectiveDate: "2020-10-11", - }, - { - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2023-10-11", - }, - }, - }, - } k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { if r.Name == name.Name && r.Namespace == name.Namespace { - k.SetClientObject(object, s) + k.SetClientObject(object, &s) return nil } return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") } - sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.SubResourceUpdateOption) error { - stack := obj.(*lokiv1.LokiStack) - require.Equal(t, expected.Storage.Schemas, stack.Status.Storage.Schemas) - s = stack - return nil - } - err := SetStorageSchemaStatus(context.TODO(), k, r, schemas) + for _, tc := range tt { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.SubResourceUpdateOption) error { + stack := obj.(*lokiv1.LokiStack) + require.Equal(t, tc.schemas, stack.Status.Storage.Schemas) + s = *stack + return nil + } + err := SetStorageSchemaStatus(context.TODO(), k, r, tc.schemas) - require.NoError(t, err) - require.NotEmpty(t, s.Status.Storage.Schemas) + require.NoError(t, err) + require.NotEmpty(t, s.Status.Storage.Schemas) - require.NotZero(t, k.StatusCallCount()) - require.NotZero(t, sw.UpdateCallCount()) + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) - wantCondition := metav1.Condition{ - Type: string(lokiv1.ConditionWarning), - Reason: string(lokiv1.ReasonStorageSchemaVersionIsOld), - Message: messageOldSchemaVersion, - } - componentStatus := &lokiv1.LokiStackComponentStatus{ - Ingester: map[corev1.PodPhase][]string{ - corev1.PodRunning: { - "pod-0", - }, - }, - } + condition, err := generateWarnings(s.Status.Storage.Schemas) + require.Nil(t, err) + require.Equal(t, condition, tc.wantCondition) - condition, err := generateWarnings(context.TODO(), componentStatus, k, r, s) - require.Nil(t, err) - require.Contains(t, condition, wantCondition) + }) + } } From 017d250f3cbe357eb5615b256d4c31ca70df8477 Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 17:47:14 +0100 Subject: [PATCH 10/16] remove setting condition status --- operator/internal/status/lokistack.go | 1 - 1 file changed, 1 deletion(-) diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index 3c28d6ef00bc8..24bf1fce7804e 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -170,7 +170,6 @@ func generateWarnings(schemas []lokiv1.ObjectStorageSchema) ([]metav1.Condition, Type: string(lokiv1.ConditionWarning), Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), Message: messageWarningNeedsSchemaVersionUpdate, - Status: metav1.ConditionTrue, }) break } From 1b5a5e179ff60b75adf141669dc47d980b52b22b Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 17:59:23 +0100 Subject: [PATCH 11/16] minor refactoring --- operator/internal/status/lokistack.go | 33 +++++++++------------- operator/internal/status/lokistack_test.go | 5 ++-- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index 24bf1fce7804e..bf236834c90c2 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -62,10 +62,7 @@ func (e *DegradedError) Error() string { } func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) { - conditions, err := generateWarnings(stack.Status.Storage.Schemas) - if err != nil { - return nil, err - } + conditions := generateWarnings(stack.Status.Storage.Schemas) mainCondition, err := generateCondition(ctx, cs, k, stack, degradedErr) if err != nil { @@ -159,21 +156,19 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1 return true, true, nil } -func generateWarnings(schemas []lokiv1.ObjectStorageSchema) ([]metav1.Condition, error) { +func generateWarnings(schemas []lokiv1.ObjectStorageSchema) []metav1.Condition { warnings := make([]metav1.Condition, 0) - for _, sc := range schemas { - if schemas[len(schemas)-1].Version == lokiv1.ObjectStorageSchemaV13 { - return warnings, nil - } - if sc.Version != lokiv1.ObjectStorageSchemaV13 { - warnings = append(warnings, metav1.Condition{ - Type: string(lokiv1.ConditionWarning), - Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), - Message: messageWarningNeedsSchemaVersionUpdate, - }) - break - } + if len(schemas) == 0 { + return nil + } + if schemas[len(schemas)-1].Version == lokiv1.ObjectStorageSchemaV13 { + return nil + } else { + warnings = append(warnings, metav1.Condition{ + Type: string(lokiv1.ConditionWarning), + Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), + Message: messageWarningNeedsSchemaVersionUpdate, + }) + return warnings } - - return warnings, nil } diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go index defdbd165b7b3..12d2bc443a4ea 100644 --- a/operator/internal/status/lokistack_test.go +++ b/operator/internal/status/lokistack_test.go @@ -309,7 +309,7 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { EffectiveDate: "2024-10-11", }, }, - wantCondition: []metav1.Condition{}, + wantCondition: nil, }, } @@ -355,8 +355,7 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { require.NotZero(t, k.StatusCallCount()) require.NotZero(t, sw.UpdateCallCount()) - condition, err := generateWarnings(s.Status.Storage.Schemas) - require.Nil(t, err) + condition := generateWarnings(s.Status.Storage.Schemas) require.Equal(t, condition, tc.wantCondition) }) From 7e8415e65d948f06cdd04e9ca504047209347975 Mon Sep 17 00:00:00 2001 From: btaani Date: Fri, 15 Dec 2023 20:45:43 +0100 Subject: [PATCH 12/16] another refactor --- .../manifests/internal/config/build_test.go | 293 ------------------ operator/internal/status/lokistack.go | 13 +- operator/internal/status/lokistack_test.go | 93 +----- 3 files changed, 11 insertions(+), 388 deletions(-) diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 9b1974e49333e..537ec84bf71a5 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -5418,296 +5418,3 @@ analytics: }) } } - -func TestBuild_ConfigAndRuntimeConfig_AddSchema(t *testing.T) { - effectiveDate := "2030-11-07" - - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - s3: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 - match_max_concurrent: true -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - max_transfer_retries: 0 - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2500 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 1s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - enforce_metric_name: false - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - per_stream_rate_limit: 3MB - per_stream_rate_limit_burst: 15MB - split_queries_by_interval: 30m - query_timeout: 1m - allow_structured_metadata: true -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper - ${V13_CONFIG} -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - shared_store: s3 - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 - ${TSDB_SHIPPER_STORAGE_CONFIG} -tracing: - enabled: false -analytics: - reporting_enabled: true -` - opts := Options{ - Stack: lokiv1.LokiStackSpec{ - Replication: &lokiv1.ReplicationSpec{ - Factor: 1, - }, - Limits: &lokiv1.LimitsSpec{ - Global: &lokiv1.LimitsTemplateSpec{ - IngestionLimits: &lokiv1.IngestionLimitSpec{ - IngestionRate: 4, - IngestionBurstSize: 6, - MaxLabelNameLength: 1024, - MaxLabelValueLength: 2048, - MaxLabelNamesPerSeries: 30, - MaxGlobalStreamsPerTenant: 0, - MaxLineSize: 256000, - PerStreamRateLimit: 3, - PerStreamRateLimitBurst: 15, - }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxEntriesLimitPerQuery: 5000, - MaxChunksPerQuery: 2000000, - MaxQuerySeries: 500, - QueryTimeout: "1m", - CardinalityLimit: 100000, - }, - }, - }, - }, - Namespace: "test-ns", - Name: "test", - Compactor: Address{ - FQDN: "loki-compactor-grpc-lokistack-dev.default.svc.cluster.local", - Port: 9095, - }, - FrontendWorker: Address{ - FQDN: "loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local", - Port: 9095, - }, - GossipRing: GossipRing{ - InstancePort: 9095, - BindPort: 7946, - MembersDiscoveryAddr: "loki-gossip-ring-lokistack-dev.default.svc.cluster.local", - }, - Querier: Address{ - Protocol: "http", - FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", - Port: 3100, - }, - IndexGateway: Address{ - FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", - Port: 9095, - }, - StorageDirectory: "/tmp/loki", - MaxConcurrent: MaxConcurrent{ - AvailableQuerierCPUCores: 2, - }, - WriteAheadLog: WriteAheadLog{ - Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, - }, - ObjectStorage: storage.Options{ - SharedStore: lokiv1.ObjectStorageSecretS3, - S3: &storage.S3StorageConfig{ - Endpoint: "http://test.default.svc.cluster.local.:9000", - Region: "us-east", - Buckets: "loki", - }, - Schemas: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV11, - EffectiveDate: "2020-10-01", - }, - }, - }, - Shippers: []string{"boltdb"}, - EnableRemoteReporting: true, - HTTPTimeouts: HTTPTimeoutConfig{ - IdleTimeout: 30 * time.Second, - ReadTimeout: 30 * time.Second, - WriteTimeout: 10 * time.Minute, - }, - } - - cfg, _, err := Build(opts) - - expCfg_before := strings.Replace(expCfg, "${TSDB_LIMIT}", "", -1) - expCfg_before = strings.Replace(expCfg_before, " ${V13_CONFIG}", "", -1) - expCfg_before = strings.Replace(expCfg_before, "${TSDB_SHIPPER_STORAGE_CONFIG}", "", -1) - - require.NoError(t, err) - require.YAMLEq(t, expCfg_before, string(cfg)) - - v13schema := lokiv1.ObjectStorageSchema{ - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: lokiv1.StorageSchemaEffectiveDate(effectiveDate), - } - v13_config := `- from: "2030-11-07" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v13 - store: tsdb` - tsdb_shipper_config := ` - tsdb_shipper: - active_index_directory: /tmp/loki/tsdb-index - cache_location: /tmp/loki/tsdb-cache - cache_ttl: 24h - resync_interval: 5m - shared_store: s3 - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095` - tsdb_limit := `tsdb_max_query_parallelism: 512` - - expCfg = strings.Replace(expCfg, "${TSDB_LIMIT}", tsdb_limit, -1) - expCfg = strings.Replace(expCfg, "${V13_CONFIG}", v13_config, -1) - expCfg = strings.Replace(expCfg, "${TSDB_SHIPPER_STORAGE_CONFIG}", tsdb_shipper_config, -1) - - opts.ObjectStorage.Schemas = append(opts.ObjectStorage.Schemas, v13schema) - opts.Shippers = append(opts.Shippers, "tsdb") - - cfg, _, err = Build(opts) - - require.NoError(t, err) - require.YAMLEq(t, expCfg, string(cfg)) - -} diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index bf236834c90c2..cdb8226f2687b 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -157,18 +157,15 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1 } func generateWarnings(schemas []lokiv1.ObjectStorageSchema) []metav1.Condition { - warnings := make([]metav1.Condition, 0) - if len(schemas) == 0 { - return nil - } - if schemas[len(schemas)-1].Version == lokiv1.ObjectStorageSchemaV13 { - return nil - } else { + warnings := make([]metav1.Condition, 0, 2) + + if len(schemas) > 0 && schemas[len(schemas)-1].Version != lokiv1.ObjectStorageSchemaV13 { warnings = append(warnings, metav1.Condition{ Type: string(lokiv1.ConditionWarning), Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), Message: messageWarningNeedsSchemaVersionUpdate, }) - return warnings } + + return warnings } diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go index 12d2bc443a4ea..dd4f45dcaa31e 100644 --- a/operator/internal/status/lokistack_test.go +++ b/operator/internal/status/lokistack_test.go @@ -13,7 +13,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -209,12 +208,10 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) { } func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { - tt := []struct { - desc string - schemas []lokiv1.ObjectStorageSchema - expectedSchemaStatus []lokiv1.ObjectStorageSchema - wantCondition []metav1.Condition + desc string + schemas []lokiv1.ObjectStorageSchema + wantCondition []metav1.Condition }{ { desc: "no V13 in schema config", @@ -228,16 +225,6 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { EffectiveDate: "2023-10-11", }, }, - expectedSchemaStatus: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV11, - EffectiveDate: "2020-10-11", - }, - { - Version: lokiv1.ObjectStorageSchemaV12, - EffectiveDate: "2023-10-11", - }, - }, wantCondition: []metav1.Condition{{ Type: string(lokiv1.ConditionWarning), Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), @@ -260,20 +247,6 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { EffectiveDate: "2024-10-11", }, }, - expectedSchemaStatus: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV11, - EffectiveDate: "2020-10-11", - }, - { - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2023-10-11", - }, - { - Version: lokiv1.ObjectStorageSchemaV12, - EffectiveDate: "2024-10-11", - }, - }, wantCondition: []metav1.Condition{{ Type: string(lokiv1.ConditionWarning), Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate), @@ -295,69 +268,15 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) { EffectiveDate: "2024-10-11", }, }, - expectedSchemaStatus: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV11, - EffectiveDate: "2020-10-11", - }, - { - Version: lokiv1.ObjectStorageSchemaV12, - EffectiveDate: "2023-10-11", - }, - { - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2024-10-11", - }, - }, - wantCondition: nil, - }, - } - - sw := &k8sfakes.FakeStatusWriter{} - k := &k8sfakes.FakeClient{} - - k.StatusStub = func() client.StatusWriter { return sw } - s := lokiv1.LokiStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "some-ns", + wantCondition: []metav1.Condition{}, }, } - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "some-ns", - }, - } - - k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { - if r.Name == name.Name && r.Namespace == name.Namespace { - k.SetClientObject(object, &s) - return nil - } - return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") - } - for _, tc := range tt { tc := tc t.Run(tc.desc, func(t *testing.T) { - sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.SubResourceUpdateOption) error { - stack := obj.(*lokiv1.LokiStack) - require.Equal(t, tc.schemas, stack.Status.Storage.Schemas) - s = *stack - return nil - } - err := SetStorageSchemaStatus(context.TODO(), k, r, tc.schemas) - - require.NoError(t, err) - require.NotEmpty(t, s.Status.Storage.Schemas) - - require.NotZero(t, k.StatusCallCount()) - require.NotZero(t, sw.UpdateCallCount()) - - condition := generateWarnings(s.Status.Storage.Schemas) + t.Parallel() + condition := generateWarnings(tc.schemas) require.Equal(t, condition, tc.wantCondition) - }) } } From 459b75a61081d4f2575030f9eb024c12b0e33716 Mon Sep 17 00:00:00 2001 From: Bayan Taani <86984560+btaani@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:47:16 +0100 Subject: [PATCH 13/16] Update operator/internal/status/lokistack.go Co-authored-by: Periklis Tsirakidis --- operator/internal/status/lokistack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index cdb8226f2687b..db9c8af4cd8f6 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -61,7 +61,7 @@ func (e *DegradedError) Error() string { return fmt.Sprintf("cluster degraded: %s", e.Message) } -func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) { +func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) { conditions := generateWarnings(stack.Status.Storage.Schemas) mainCondition, err := generateCondition(ctx, cs, k, stack, degradedErr) From 68fac33dc401f6a417d465363eb74920faa97d5f Mon Sep 17 00:00:00 2001 From: Bayan Taani <86984560+btaani@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:48:17 +0100 Subject: [PATCH 14/16] Update operator/internal/status/lokistack.go Co-authored-by: Periklis Tsirakidis --- operator/internal/status/lokistack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index db9c8af4cd8f6..3040243a0f116 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -157,7 +157,7 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1 } func generateWarnings(schemas []lokiv1.ObjectStorageSchema) []metav1.Condition { - warnings := make([]metav1.Condition, 0, 2) + warnings := make([]metav1.Condition, 0, 1) if len(schemas) > 0 && schemas[len(schemas)-1].Version != lokiv1.ObjectStorageSchemaV13 { warnings = append(warnings, metav1.Condition{ From cd7f71f56140037c3f2d4e11fda69acc5c3a6793 Mon Sep 17 00:00:00 2001 From: btaani Date: Mon, 18 Dec 2023 12:39:00 +0100 Subject: [PATCH 15/16] undo change --- operator/internal/status/lokistack.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index 3040243a0f116..467fef398ea77 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -6,7 +6,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" @@ -157,7 +156,7 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1 } func generateWarnings(schemas []lokiv1.ObjectStorageSchema) []metav1.Condition { - warnings := make([]metav1.Condition, 0, 1) + warnings := make([]metav1.Condition, 0, 2) if len(schemas) > 0 && schemas[len(schemas)-1].Version != lokiv1.ObjectStorageSchemaV13 { warnings = append(warnings, metav1.Condition{ From 9fad7712a3107ca7c7dab35997bd48f141945aea Mon Sep 17 00:00:00 2001 From: btaani Date: Mon, 18 Dec 2023 12:52:44 +0100 Subject: [PATCH 16/16] remove extra parameter in function call --- operator/internal/status/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/internal/status/status.go b/operator/internal/status/status.go index 5619735bbe67d..281a167355c37 100644 --- a/operator/internal/status/status.go +++ b/operator/internal/status/status.go @@ -31,7 +31,7 @@ func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time, return err } - activeConditions, err := generateConditions(ctx, cs, k, req, &stack, degradedErr) + activeConditions, err := generateConditions(ctx, cs, k, &stack, degradedErr) if err != nil { return err }