From 4b68265e788c1e964f25dee0bb2e0ab10f0c86fb Mon Sep 17 00:00:00 2001
From: Robert Jacob
Date: Wed, 29 Nov 2023 20:31:53 +0100
Subject: [PATCH 01/16] Support setting multiple conditions: cherry picked from
f390f3734
---
operator/apis/loki/v1/lokistack_types.go | 4 +
operator/internal/status/conditions.go | 37 ++++++
operator/internal/status/conditions_test.go | 139 ++++++++++++++++++++
operator/internal/status/lokistack.go | 22 +++-
operator/internal/status/status.go | 28 +---
5 files changed, 202 insertions(+), 28 deletions(-)
create mode 100644 operator/internal/status/conditions.go
create mode 100644 operator/internal/status/conditions_test.go
diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go
index 8b163ff780300..6d31d9891a858 100644
--- a/operator/apis/loki/v1/lokistack_types.go
+++ b/operator/apis/loki/v1/lokistack_types.go
@@ -1041,6 +1041,10 @@ const (
// ConditionDegraded defines the condition that some or all components in the Loki deployment
// are degraded or the cluster cannot connect to object storage.
ConditionDegraded LokiStackConditionType = "Degraded"
+
+ // ConditionWarning is used for configurations that are not recommended, but don't currently cause
+ // issues. There can be multiple warning conditions active at a time.
+ ConditionWarning LokiStackConditionType = "Warning"
)
// LokiStackConditionReason defines the type for valid reasons of a Loki deployment conditions.
diff --git a/operator/internal/status/conditions.go b/operator/internal/status/conditions.go
new file mode 100644
index 0000000000000..637a50e6f89f4
--- /dev/null
+++ b/operator/internal/status/conditions.go
@@ -0,0 +1,37 @@
+package status
+
+import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+func mergeConditions(old, active []metav1.Condition, now metav1.Time) []metav1.Condition {
+ merged := make([]metav1.Condition, 0, len(old)+len(active))
+ for len(old) > 0 {
+ c := old[0]
+ found := -1
+ for i, ac := range active {
+ if c.Type == ac.Type && c.Reason == ac.Reason {
+ found = i
+ break
+ }
+ }
+
+ if found != -1 {
+ c = active[found]
+ active = append(active[:found], active[found+1:]...)
+
+ c.Status = metav1.ConditionTrue
+ } else {
+ c.Status = metav1.ConditionFalse
+ }
+
+ c.LastTransitionTime = now
+ merged = append(merged, c)
+ old = old[1:]
+ }
+
+ for _, c := range active {
+ c.Status = metav1.ConditionTrue
+ c.LastTransitionTime = now
+ merged = append(merged, c)
+ }
+ return merged
+}
diff --git a/operator/internal/status/conditions_test.go b/operator/internal/status/conditions_test.go
new file mode 100644
index 0000000000000..86bb087c84bfb
--- /dev/null
+++ b/operator/internal/status/conditions_test.go
@@ -0,0 +1,139 @@
+package status
+
+import (
+ "github.com/google/go-cmp/cmp"
+ lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "testing"
+ "time"
+)
+
+func TestMergeConditions(t *testing.T) {
+ now := metav1.NewTime(time.Unix(0, 0))
+ tt := []struct {
+ desc string
+ old []metav1.Condition
+ active []metav1.Condition
+ wantMerged []metav1.Condition
+ }{
+ {
+ desc: "set status and time",
+ old: []metav1.Condition{},
+ active: []metav1.Condition{
+ conditionReady,
+ },
+ wantMerged: []metav1.Condition{
+ {
+ Type: conditionReady.Type,
+ Status: metav1.ConditionTrue,
+ LastTransitionTime: now,
+ Reason: conditionReady.Reason,
+ Message: conditionReady.Message,
+ },
+ },
+ },
+ {
+ desc: "reset old condition",
+ old: []metav1.Condition{
+ conditionPending,
+ },
+ active: []metav1.Condition{
+ conditionReady,
+ },
+ wantMerged: []metav1.Condition{
+ {
+ Type: conditionPending.Type,
+ Status: metav1.ConditionFalse,
+ LastTransitionTime: now,
+ Reason: conditionPending.Reason,
+ Message: conditionPending.Message,
+ },
+ {
+ Type: conditionReady.Type,
+ Status: metav1.ConditionTrue,
+ LastTransitionTime: now,
+ Reason: conditionReady.Reason,
+ Message: conditionReady.Message,
+ },
+ },
+ },
+ {
+ desc: "keep active conditions",
+ old: []metav1.Condition{
+ {
+ Type: conditionReady.Type,
+ Status: metav1.ConditionTrue,
+ LastTransitionTime: now,
+ Reason: conditionReady.Reason,
+ Message: conditionReady.Message,
+ },
+ {
+ Type: conditionPending.Type,
+ Status: metav1.ConditionFalse,
+ LastTransitionTime: now,
+ Reason: conditionPending.Reason,
+ Message: conditionPending.Message,
+ },
+ },
+ active: []metav1.Condition{
+ conditionReady,
+ {
+ Type: string(lokiv1.ConditionWarning),
+ Reason: "test-warning",
+ Message: "test-warning-message",
+ },
+ },
+ wantMerged: []metav1.Condition{
+ {
+ Type: conditionReady.Type,
+ Status: metav1.ConditionTrue,
+ LastTransitionTime: now,
+ Reason: conditionReady.Reason,
+ Message: conditionReady.Message,
+ },
+ {
+ Type: conditionPending.Type,
+ Status: metav1.ConditionFalse,
+ LastTransitionTime: now,
+ Reason: conditionPending.Reason,
+ Message: conditionPending.Message,
+ },
+ {
+ Type: string(lokiv1.ConditionWarning),
+ Status: metav1.ConditionTrue,
+ LastTransitionTime: now,
+ Reason: "test-warning",
+ Message: "test-warning-message",
+ },
+ },
+ },
+ }
+
+ for _, tc := range tt {
+ tc := tc
+
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ beforeLenOld := len(tc.old)
+ beforeLenActive := len(tc.active)
+
+ merged := mergeConditions(tc.old, tc.active, now)
+
+ afterLenOld := len(tc.old)
+ afterLenActive := len(tc.active)
+
+ if diff := cmp.Diff(merged, tc.wantMerged); diff != "" {
+ t.Errorf("Merged conditions differ: -got+want\n%s", diff)
+ }
+
+ if beforeLenOld != afterLenOld {
+ t.Errorf("old length differs: got %v, want %v", afterLenOld, beforeLenOld)
+ }
+
+ if beforeLenActive != afterLenActive {
+ t.Errorf("active length differs: got %v, want %v", afterLenActive, beforeLenActive)
+ }
+ })
+ }
+}
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index 1212e5c2f7e92..9381ab6e8ab7c 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -63,15 +63,19 @@ func (e *DegradedError) Error() string {
return fmt.Sprintf("cluster degraded: %s", e.Message)
}
-// SetDegradedCondition appends the condition Degraded to the lokistack status conditions.
-func SetDegradedCondition(ctx context.Context, k k8s.Client, req ctrl.Request, msg string, reason lokiv1.LokiStackConditionReason) error {
- degraded := metav1.Condition{
- Type: string(lokiv1.ConditionDegraded),
- Message: msg,
- Reason: string(reason),
+func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) {
+ conditions, err := generateWarnings(ctx, cs, k, req, stack)
+ if err != nil {
+ return nil, err
+ }
+
+ mainCondition, err := generateCondition(ctx, cs, k, req, stack, degradedErr)
+ if err != nil {
+ return nil, err
}
- return updateCondition(ctx, k, req, degraded)
+ conditions = append(conditions, mainCondition)
+ return conditions, nil
}
func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) (metav1.Condition, error) {
@@ -149,6 +153,10 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1
return true, true, nil
}
+func generateWarnings(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) ([]metav1.Condition, error) {
+ return []metav1.Condition{}, nil
+}
+
func updateCondition(ctx context.Context, k k8s.Client, req ctrl.Request, condition metav1.Condition) error {
var stack lokiv1.LokiStack
if err := k.Get(ctx, req.NamespacedName, &stack); err != nil {
diff --git a/operator/internal/status/status.go b/operator/internal/status/status.go
index 97a8e81bbae08..47dbe377bd71b 100644
--- a/operator/internal/status/status.go
+++ b/operator/internal/status/status.go
@@ -31,34 +31,20 @@ func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time)
return err
}
- condition, err := generateCondition(ctx, cs, k, req, &stack)
+ activeConditions, err := generateConditions(ctx, cs, k, req, &stack, degradedErr)
if err != nil {
return err
}
- condition.LastTransitionTime = metav1.NewTime(now)
- condition.Status = metav1.ConditionTrue
+ metaTime := metav1.NewTime(now)
+ for _, c := range activeConditions {
+ c.LastTransitionTime = metaTime
+ c.Status = metav1.ConditionTrue
+ }
statusUpdater := func(stack *lokiv1.LokiStack) {
stack.Status.Components = *cs
-
- index := -1
- for i := range stack.Status.Conditions {
- // Reset all other conditions first
- stack.Status.Conditions[i].Status = metav1.ConditionFalse
- stack.Status.Conditions[i].LastTransitionTime = metav1.NewTime(now)
-
- // Locate existing pending condition if any
- if stack.Status.Conditions[i].Type == condition.Type {
- index = i
- }
- }
-
- if index == -1 {
- stack.Status.Conditions = append(stack.Status.Conditions, condition)
- } else {
- stack.Status.Conditions[index] = condition
- }
+ stack.Status.Conditions = mergeConditions(stack.Status.Conditions, activeConditions, metaTime)
}
statusUpdater(&stack)
From 90d5ac87e6003c125af2afadf501196f13400010 Mon Sep 17 00:00:00 2001
From: Robert Jacob
Date: Wed, 29 Nov 2023 18:51:00 +0100
Subject: [PATCH 02/16] Move handling of degraded state into status package:
cherry-picked from 6f7e8b8d3
---
.../controllers/loki/lokistack_controller.go | 42 +++---
operator/internal/status/lokistack_test.go | 138 +++---------------
operator/internal/status/status.go | 2 +-
operator/internal/status/status_test.go | 4 +-
4 files changed, 46 insertions(+), 140 deletions(-)
diff --git a/operator/controllers/loki/lokistack_controller.go b/operator/controllers/loki/lokistack_controller.go
index 708390b62c846..49b5bdab069e0 100644
--- a/operator/controllers/loki/lokistack_controller.go
+++ b/operator/controllers/loki/lokistack_controller.go
@@ -5,7 +5,6 @@ import (
"errors"
"time"
- "github.com/ViaQ/logerr/v2/kverrors"
"github.com/go-logr/logr"
"github.com/google/go-cmp/cmp"
openshiftconfigv1 "github.com/openshift/api/config/v1"
@@ -150,40 +149,41 @@ func (r *LokiStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
return ctrl.Result{}, nil
}
- if r.FeatureGates.BuiltInCertManagement.Enabled {
- err = handlers.CreateOrRotateCertificates(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates)
- if err != nil {
- return handleDegradedError(ctx, r.Client, req, err)
- }
+ var degraded *status.DegradedError
+ err = r.updateResources(ctx, req)
+ switch {
+ case errors.As(err, °raded):
+ // degraded errors are handled by status.Refresh below
+ case err != nil:
+ return ctrl.Result{}, err
}
- err = handlers.CreateOrUpdateLokiStack(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates)
+ err = status.Refresh(ctx, r.Client, req, time.Now(), degraded)
if err != nil {
- return handleDegradedError(ctx, r.Client, req, err)
+ return ctrl.Result{}, err
}
- err = status.Refresh(ctx, r.Client, req, time.Now())
- if err != nil {
- return ctrl.Result{}, err
+ if degraded != nil {
+ return ctrl.Result{
+ Requeue: degraded.Requeue,
+ }, nil
}
return ctrl.Result{}, nil
}
-func handleDegradedError(ctx context.Context, c client.Client, req ctrl.Request, err error) (ctrl.Result, error) {
- var degraded *status.DegradedError
- if errors.As(err, °raded) {
- err = status.SetDegradedCondition(ctx, c, req, degraded.Message, degraded.Reason)
- if err != nil {
- return ctrl.Result{}, kverrors.Wrap(err, "error setting degraded condition")
+func (r *LokiStackReconciler) updateResources(ctx context.Context, req ctrl.Request) error {
+ if r.FeatureGates.BuiltInCertManagement.Enabled {
+ if err := handlers.CreateOrRotateCertificates(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates); err != nil {
+ return err
}
+ }
- return ctrl.Result{
- Requeue: degraded.Requeue,
- }, nil
+ if err := handlers.CreateOrUpdateLokiStack(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates); err != nil {
+ return err
}
- return ctrl.Result{}, err
+ return nil
}
// SetupWithManager sets up the controller with the Manager.
diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go
index 8bdc9fadc7cf6..a4dc36c0710ee 100644
--- a/operator/internal/status/lokistack_test.go
+++ b/operator/internal/status/lokistack_test.go
@@ -39,120 +39,6 @@ func setupFakesNoError(t *testing.T, stack *lokiv1.LokiStack) (*k8sfakes.FakeCli
return k, sw
}
-func TestSetDegradedCondition_WhenGetLokiStackReturnsNotFound_DoNothing(t *testing.T) {
- msg := "tell me nothing"
- reason := lokiv1.ReasonMissingObjectStorageSecret
-
- r := ctrl.Request{
- NamespacedName: types.NamespacedName{
- Name: "my-stack",
- Namespace: "some-ns",
- },
- }
-
- k := &k8sfakes.FakeClient{}
- k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error {
- return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found")
- }
-
- err := SetDegradedCondition(context.Background(), k, r, msg, reason)
- require.NoError(t, err)
-}
-
-func TestSetDegradedCondition_WhenExisting_DoNothing(t *testing.T) {
- msg := "tell me nothing"
- reason := lokiv1.ReasonMissingObjectStorageSecret
- s := lokiv1.LokiStack{
- ObjectMeta: metav1.ObjectMeta{
- Name: "my-stack",
- Namespace: "some-ns",
- },
- Status: lokiv1.LokiStackStatus{
- Conditions: []metav1.Condition{
- {
- Type: string(lokiv1.ConditionDegraded),
- Reason: string(reason),
- Message: msg,
- Status: metav1.ConditionTrue,
- },
- },
- },
- }
-
- r := ctrl.Request{
- NamespacedName: types.NamespacedName{
- Name: "my-stack",
- Namespace: "some-ns",
- },
- }
-
- k, _ := setupFakesNoError(t, &s)
-
- err := SetDegradedCondition(context.Background(), k, r, msg, reason)
- require.NoError(t, err)
- require.Zero(t, k.StatusCallCount())
-}
-
-func TestSetDegradedCondition_WhenExisting_SetDegradedConditionTrue(t *testing.T) {
- msg := "tell me something"
- reason := lokiv1.ReasonMissingObjectStorageSecret
- s := lokiv1.LokiStack{
- ObjectMeta: metav1.ObjectMeta{
- Name: "my-stack",
- Namespace: "some-ns",
- },
- Status: lokiv1.LokiStackStatus{
- Conditions: []metav1.Condition{
- {
- Type: string(lokiv1.ConditionDegraded),
- Reason: string(reason),
- Status: metav1.ConditionFalse,
- },
- },
- },
- }
-
- r := ctrl.Request{
- NamespacedName: types.NamespacedName{
- Name: "my-stack",
- Namespace: "some-ns",
- },
- }
-
- k, sw := setupFakesNoError(t, &s)
-
- err := SetDegradedCondition(context.Background(), k, r, msg, reason)
- require.NoError(t, err)
- require.NotZero(t, k.StatusCallCount())
- require.NotZero(t, sw.UpdateCallCount())
-}
-
-func TestSetDegradedCondition_WhenNoneExisting_AppendDegradedCondition(t *testing.T) {
- msg := "tell me something"
- reason := lokiv1.ReasonMissingObjectStorageSecret
- s := lokiv1.LokiStack{
- ObjectMeta: metav1.ObjectMeta{
- Name: "my-stack",
- Namespace: "some-ns",
- },
- }
-
- r := ctrl.Request{
- NamespacedName: types.NamespacedName{
- Name: "my-stack",
- Namespace: "some-ns",
- },
- }
-
- k, sw := setupFakesNoError(t, &s)
-
- err := SetDegradedCondition(context.Background(), k, r, msg, reason)
- require.NoError(t, err)
-
- require.NotZero(t, k.StatusCallCount())
- require.NotZero(t, sw.UpdateCallCount())
-}
-
func TestGenerateCondition(t *testing.T) {
k := &k8sfakes.FakeClient{}
r := ctrl.Request{
@@ -173,6 +59,7 @@ func TestGenerateCondition(t *testing.T) {
tt := []struct {
desc string
componentStatus *lokiv1.LokiStackComponentStatus
+ degradedErr *DegradedError
wantCondition metav1.Condition
}{
{
@@ -202,6 +89,25 @@ func TestGenerateCondition(t *testing.T) {
},
wantCondition: conditionFailed,
},
+ {
+ desc: "degraded error",
+ componentStatus: &lokiv1.LokiStackComponentStatus{
+ Ingester: map[corev1.PodPhase][]string{
+ corev1.PodRunning: {
+ "pod-0",
+ },
+ },
+ },
+ degradedErr: &DegradedError{
+ Message: "test-message",
+ Reason: "test-reason",
+ },
+ wantCondition: metav1.Condition{
+ Type: "Degraded",
+ Reason: "test-reason",
+ Message: "test-message",
+ },
+ },
}
for _, tc := range tt {
@@ -209,7 +115,7 @@ func TestGenerateCondition(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
- condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack)
+ condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack, tc.degradedErr)
require.Nil(t, err)
require.Equal(t, tc.wantCondition, condition)
})
@@ -306,7 +212,7 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) {
return tc.wantErr
}
- condition, err := generateCondition(context.TODO(), componentStatus, k, r, &lokiStack)
+ condition, err := generateCondition(context.TODO(), componentStatus, k, r, &lokiStack, nil)
require.Equal(t, tc.wantErr, err)
require.Equal(t, tc.wantCondition, condition)
diff --git a/operator/internal/status/status.go b/operator/internal/status/status.go
index 47dbe377bd71b..5619735bbe67d 100644
--- a/operator/internal/status/status.go
+++ b/operator/internal/status/status.go
@@ -17,7 +17,7 @@ import (
// Refresh executes an aggregate update of the LokiStack Status struct, i.e.
// - It recreates the Status.Components pod status map per component.
// - It sets the appropriate Status.Condition to true that matches the pod status maps.
-func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time) error {
+func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time, degradedErr *DegradedError) error {
var stack lokiv1.LokiStack
if err := k.Get(ctx, req.NamespacedName, &stack); err != nil {
if apierrors.IsNotFound(err) {
diff --git a/operator/internal/status/status_test.go b/operator/internal/status/status_test.go
index 6befb13df8f7a..a34b1bc81181f 100644
--- a/operator/internal/status/status_test.go
+++ b/operator/internal/status/status_test.go
@@ -67,7 +67,7 @@ func TestRefreshSuccess(t *testing.T) {
k, sw := setupListClient(t, stack, componentPods)
- err := Refresh(context.Background(), k, req, now)
+ err := Refresh(context.Background(), k, req, now, nil)
require.NoError(t, err)
require.Equal(t, 1, k.GetCallCount())
@@ -129,7 +129,7 @@ func TestRefreshSuccess_ZoneAwarePendingPod(t *testing.T) {
return nil
}
- err := Refresh(context.Background(), k, req, now)
+ err := Refresh(context.Background(), k, req, now, nil)
require.NoError(t, err)
require.Equal(t, 1, k.GetCallCount())
From e6b76889bf254136ddbbf5a3b1c1366c2fd54001 Mon Sep 17 00:00:00 2001
From: btaani
Date: Thu, 14 Dec 2023 15:18:45 +0100
Subject: [PATCH 03/16] add unit test for adding schema
---
.../manifests/internal/config/build_test.go | 293 ++++++++++++++++++
1 file changed, 293 insertions(+)
diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go
index 537ec84bf71a5..9b1974e49333e 100644
--- a/operator/internal/manifests/internal/config/build_test.go
+++ b/operator/internal/manifests/internal/config/build_test.go
@@ -5418,3 +5418,296 @@ analytics:
})
}
}
+
+func TestBuild_ConfigAndRuntimeConfig_AddSchema(t *testing.T) {
+ effectiveDate := "2030-11-07"
+
+ expCfg := `
+---
+auth_enabled: true
+chunk_store_config:
+ chunk_cache_config:
+ embedded_cache:
+ enabled: true
+ max_size_mb: 500
+common:
+ storage:
+ s3:
+ s3: http://test.default.svc.cluster.local.:9000
+ bucketnames: loki
+ region: us-east
+ access_key_id: ${AWS_ACCESS_KEY_ID}
+ secret_access_key: ${AWS_ACCESS_KEY_SECRET}
+ s3forcepathstyle: true
+ compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095
+ ring:
+ kvstore:
+ store: memberlist
+ heartbeat_period: 5s
+ heartbeat_timeout: 1m
+ instance_port: 9095
+compactor:
+ compaction_interval: 2h
+ working_directory: /tmp/loki/compactor
+frontend:
+ tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100
+ compress_responses: true
+ max_outstanding_per_tenant: 4096
+ log_queries_longer_than: 5s
+frontend_worker:
+ frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095
+ grpc_client_config:
+ max_send_msg_size: 104857600
+ match_max_concurrent: true
+ingester:
+ chunk_block_size: 262144
+ chunk_encoding: snappy
+ chunk_idle_period: 1h
+ chunk_retain_period: 5m
+ chunk_target_size: 2097152
+ flush_op_timeout: 10m
+ lifecycler:
+ final_sleep: 0s
+ join_after: 30s
+ num_tokens: 512
+ ring:
+ replication_factor: 1
+ max_chunk_age: 2h
+ max_transfer_retries: 0
+ wal:
+ enabled: true
+ dir: /tmp/wal
+ replay_memory_ceiling: 2500
+ingester_client:
+ grpc_client_config:
+ max_recv_msg_size: 67108864
+ remote_timeout: 1s
+# NOTE: Keep the order of keys as in Loki docs
+# to enable easy diffs when vendoring newer
+# Loki releases.
+# (See https://grafana.com/docs/loki/latest/configuration/#limits_config)
+#
+# Values for not exposed fields are taken from the grafana/loki production
+# configuration manifests.
+# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet)
+limits_config:
+ ingestion_rate_strategy: global
+ ingestion_rate_mb: 4
+ ingestion_burst_size_mb: 6
+ max_label_name_length: 1024
+ max_label_value_length: 2048
+ max_label_names_per_series: 30
+ reject_old_samples: true
+ reject_old_samples_max_age: 168h
+ creation_grace_period: 10m
+ enforce_metric_name: false
+ # Keep max_streams_per_user always to 0 to default
+ # using max_global_streams_per_user always.
+ # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73)
+ max_streams_per_user: 0
+ max_line_size: 256000
+ max_entries_limit_per_query: 5000
+ max_global_streams_per_user: 0
+ max_chunks_per_query: 2000000
+ max_query_length: 721h
+ max_query_parallelism: 32
+ tsdb_max_query_parallelism: 512
+ max_query_series: 500
+ cardinality_limit: 100000
+ max_streams_matchers_per_query: 1000
+ max_cache_freshness_per_query: 10m
+ per_stream_rate_limit: 3MB
+ per_stream_rate_limit_burst: 15MB
+ split_queries_by_interval: 30m
+ query_timeout: 1m
+ allow_structured_metadata: true
+memberlist:
+ abort_if_cluster_join_fails: true
+ advertise_port: 7946
+ bind_port: 7946
+ join_members:
+ - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946
+ max_join_backoff: 1m
+ max_join_retries: 10
+ min_join_backoff: 1s
+querier:
+ engine:
+ max_look_back_period: 30s
+ extra_query_delay: 0s
+ max_concurrent: 2
+ query_ingesters_within: 3h
+ tail_max_duration: 1h
+query_range:
+ align_queries_with_step: true
+ cache_results: true
+ max_retries: 5
+ results_cache:
+ cache:
+ embedded_cache:
+ enabled: true
+ max_size_mb: 500
+ parallelise_shardable_queries: true
+schema_config:
+ configs:
+ - from: "2020-10-01"
+ index:
+ period: 24h
+ prefix: index_
+ object_store: s3
+ schema: v11
+ store: boltdb-shipper
+ ${V13_CONFIG}
+server:
+ graceful_shutdown_timeout: 5s
+ grpc_server_min_time_between_pings: '10s'
+ grpc_server_ping_without_stream_allowed: true
+ grpc_server_max_concurrent_streams: 1000
+ grpc_server_max_recv_msg_size: 104857600
+ grpc_server_max_send_msg_size: 104857600
+ http_listen_port: 3100
+ http_server_idle_timeout: 30s
+ http_server_read_timeout: 30s
+ http_server_write_timeout: 10m0s
+ log_level: info
+storage_config:
+ boltdb_shipper:
+ active_index_directory: /tmp/loki/index
+ cache_location: /tmp/loki/index_cache
+ cache_ttl: 24h
+ resync_interval: 5m
+ shared_store: s3
+ index_gateway_client:
+ server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095
+ ${TSDB_SHIPPER_STORAGE_CONFIG}
+tracing:
+ enabled: false
+analytics:
+ reporting_enabled: true
+`
+ opts := Options{
+ Stack: lokiv1.LokiStackSpec{
+ Replication: &lokiv1.ReplicationSpec{
+ Factor: 1,
+ },
+ Limits: &lokiv1.LimitsSpec{
+ Global: &lokiv1.LimitsTemplateSpec{
+ IngestionLimits: &lokiv1.IngestionLimitSpec{
+ IngestionRate: 4,
+ IngestionBurstSize: 6,
+ MaxLabelNameLength: 1024,
+ MaxLabelValueLength: 2048,
+ MaxLabelNamesPerSeries: 30,
+ MaxGlobalStreamsPerTenant: 0,
+ MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
+ },
+ QueryLimits: &lokiv1.QueryLimitSpec{
+ MaxEntriesLimitPerQuery: 5000,
+ MaxChunksPerQuery: 2000000,
+ MaxQuerySeries: 500,
+ QueryTimeout: "1m",
+ CardinalityLimit: 100000,
+ },
+ },
+ },
+ },
+ Namespace: "test-ns",
+ Name: "test",
+ Compactor: Address{
+ FQDN: "loki-compactor-grpc-lokistack-dev.default.svc.cluster.local",
+ Port: 9095,
+ },
+ FrontendWorker: Address{
+ FQDN: "loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local",
+ Port: 9095,
+ },
+ GossipRing: GossipRing{
+ InstancePort: 9095,
+ BindPort: 7946,
+ MembersDiscoveryAddr: "loki-gossip-ring-lokistack-dev.default.svc.cluster.local",
+ },
+ Querier: Address{
+ Protocol: "http",
+ FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
+ Port: 3100,
+ },
+ IndexGateway: Address{
+ FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
+ Port: 9095,
+ },
+ StorageDirectory: "/tmp/loki",
+ MaxConcurrent: MaxConcurrent{
+ AvailableQuerierCPUCores: 2,
+ },
+ WriteAheadLog: WriteAheadLog{
+ Directory: "/tmp/wal",
+ IngesterMemoryRequest: 5000,
+ },
+ ObjectStorage: storage.Options{
+ SharedStore: lokiv1.ObjectStorageSecretS3,
+ S3: &storage.S3StorageConfig{
+ Endpoint: "http://test.default.svc.cluster.local.:9000",
+ Region: "us-east",
+ Buckets: "loki",
+ },
+ Schemas: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-01",
+ },
+ },
+ },
+ Shippers: []string{"boltdb"},
+ EnableRemoteReporting: true,
+ HTTPTimeouts: HTTPTimeoutConfig{
+ IdleTimeout: 30 * time.Second,
+ ReadTimeout: 30 * time.Second,
+ WriteTimeout: 10 * time.Minute,
+ },
+ }
+
+ cfg, _, err := Build(opts)
+
+ expCfg_before := strings.Replace(expCfg, "${TSDB_LIMIT}", "", -1)
+ expCfg_before = strings.Replace(expCfg_before, " ${V13_CONFIG}", "", -1)
+ expCfg_before = strings.Replace(expCfg_before, "${TSDB_SHIPPER_STORAGE_CONFIG}", "", -1)
+
+ require.NoError(t, err)
+ require.YAMLEq(t, expCfg_before, string(cfg))
+
+ v13schema := lokiv1.ObjectStorageSchema{
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: lokiv1.StorageSchemaEffectiveDate(effectiveDate),
+ }
+ v13_config := `- from: "2030-11-07"
+ index:
+ period: 24h
+ prefix: index_
+ object_store: s3
+ schema: v13
+ store: tsdb`
+ tsdb_shipper_config := `
+ tsdb_shipper:
+ active_index_directory: /tmp/loki/tsdb-index
+ cache_location: /tmp/loki/tsdb-cache
+ cache_ttl: 24h
+ resync_interval: 5m
+ shared_store: s3
+ index_gateway_client:
+ server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095`
+ tsdb_limit := `tsdb_max_query_parallelism: 512`
+
+ expCfg = strings.Replace(expCfg, "${TSDB_LIMIT}", tsdb_limit, -1)
+ expCfg = strings.Replace(expCfg, "${V13_CONFIG}", v13_config, -1)
+ expCfg = strings.Replace(expCfg, "${TSDB_SHIPPER_STORAGE_CONFIG}", tsdb_shipper_config, -1)
+
+ opts.ObjectStorage.Schemas = append(opts.ObjectStorage.Schemas, v13schema)
+ opts.Shippers = append(opts.Shippers, "tsdb")
+
+ cfg, _, err = Build(opts)
+
+ require.NoError(t, err)
+ require.YAMLEq(t, expCfg, string(cfg))
+
+}
From eab8176874907815c84170fe0cbe5fea81e85b12 Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 00:20:19 +0100
Subject: [PATCH 04/16] add warning condition on old schemas
---
operator/internal/status/lokistack.go | 65 ++++------------------
operator/internal/status/lokistack_test.go | 9 ++-
2 files changed, 19 insertions(+), 55 deletions(-)
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index 9381ab6e8ab7c..ea7ebc79e386b 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -4,11 +4,8 @@ import (
"context"
"fmt"
- "github.com/ViaQ/logerr/v2/kverrors"
corev1 "k8s.io/api/core/v1"
- apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -22,6 +19,7 @@ const (
messagePending = "Some LokiStack components pending on dependencies"
messageDegradedMissingNodes = "Cluster contains no nodes matching the labels used for zone-awareness"
messageDegradedEmptyNodeLabel = "No value for the labels used for zone-awareness"
+ messageOldSchemaVersion = "The schema configuration contains one or more schemas that do not use the most recent version."
)
var (
@@ -78,7 +76,7 @@ func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus
return conditions, nil
}
-func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) (metav1.Condition, error) {
+func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) (metav1.Condition, error) {
// Check for failed pods first
failed := len(cs.Compactor[corev1.PodFailed]) +
len(cs.Distributor[corev1.PodFailed]) +
@@ -154,56 +152,17 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1
}
func generateWarnings(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) ([]metav1.Condition, error) {
- return []metav1.Condition{}, nil
-}
-
-func updateCondition(ctx context.Context, k k8s.Client, req ctrl.Request, condition metav1.Condition) error {
- var stack lokiv1.LokiStack
- if err := k.Get(ctx, req.NamespacedName, &stack); err != nil {
- if apierrors.IsNotFound(err) {
- return nil
- }
- return kverrors.Wrap(err, "failed to lookup LokiStack", "name", req.NamespacedName)
- }
-
- for _, c := range stack.Status.Conditions {
- if c.Type == condition.Type &&
- c.Reason == condition.Reason &&
- c.Message == condition.Message &&
- c.Status == metav1.ConditionTrue {
- // resource already has desired condition
- return nil
+ warnings := make([]metav1.Condition, 0)
+ for _, sc := range stack.Status.Storage.Schemas {
+ if sc.Version != lokiv1.ObjectStorageSchemaV13 {
+ warnings = append(warnings, metav1.Condition{
+ Type: string(lokiv1.ConditionWarning),
+ Reason: string(lokiv1.ReasonSchemaUpgradeRecommended),
+ Message: messageOldSchemaVersion,
+ })
+ break
}
}
- condition.Status = metav1.ConditionTrue
-
- return retry.RetryOnConflict(retry.DefaultRetry, func() error {
- if err := k.Get(ctx, req.NamespacedName, &stack); err != nil {
- return err
- }
-
- now := metav1.Now()
- condition.LastTransitionTime = now
-
- index := -1
- for i := range stack.Status.Conditions {
- // Reset all other conditions first
- stack.Status.Conditions[i].Status = metav1.ConditionFalse
- stack.Status.Conditions[i].LastTransitionTime = now
-
- // Locate existing pending condition if any
- if stack.Status.Conditions[i].Type == condition.Type {
- index = i
- }
- }
-
- if index == -1 {
- stack.Status.Conditions = append(stack.Status.Conditions, condition)
- } else {
- stack.Status.Conditions[index] = condition
- }
-
- return k.Status().Update(ctx, &stack)
- })
+ return warnings, nil
}
diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go
index a4dc36c0710ee..947c7472455cf 100644
--- a/operator/internal/status/lokistack_test.go
+++ b/operator/internal/status/lokistack_test.go
@@ -62,7 +62,7 @@ func TestGenerateCondition(t *testing.T) {
degradedErr *DegradedError
wantCondition metav1.Condition
}{
- {
+ /* {
desc: "no error",
componentStatus: &lokiv1.LokiStackComponentStatus{},
wantCondition: conditionReady,
@@ -88,7 +88,7 @@ func TestGenerateCondition(t *testing.T) {
},
},
wantCondition: conditionFailed,
- },
+ }, */
{
desc: "degraded error",
componentStatus: &lokiv1.LokiStackComponentStatus{
@@ -113,6 +113,7 @@ func TestGenerateCondition(t *testing.T) {
for _, tc := range tt {
tc := tc
t.Run(tc.desc, func(t *testing.T) {
+ println(tc.desc)
t.Parallel()
condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack, tc.degradedErr)
@@ -219,3 +220,7 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) {
})
}
}
+
+func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
+
+}
From 7f23e7bfa3e3ec5f0756bed8bf0a2b2877762c10 Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 00:40:07 +0100
Subject: [PATCH 05/16] add reason to api
---
operator/apis/loki/v1/lokistack_types.go | 2 ++
operator/docs/operator/api.md | 7 +++++++
2 files changed, 9 insertions(+)
diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go
index 6d31d9891a858..18264e02868a4 100644
--- a/operator/apis/loki/v1/lokistack_types.go
+++ b/operator/apis/loki/v1/lokistack_types.go
@@ -1101,6 +1101,8 @@ const (
ReasonZoneAwareNodesMissing LokiStackConditionReason = "ReasonZoneAwareNodesMissing"
// ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value.
ReasonZoneAwareEmptyLabel LokiStackConditionReason = "ReasonZoneAwareEmptyLabel"
+ // ReasonStorageSchemaVersionIsOld when the object storage schema version is older than V13
+ ReasonStorageSchemaVersionIsOld LokiStackConditionReason = "ReasonStorageSchemaVersionIsOld"
)
// PodStatusMap defines the type for mapping pod status to pod name.
diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md
index f2ee194a87a2a..69f899b16ffb3 100644
--- a/operator/docs/operator/api.md
+++ b/operator/docs/operator/api.md
@@ -1780,6 +1780,9 @@ for the ruler is missing.
"ReadyComponents" |
ReasonReadyComponents when all LokiStack components are ready to serve traffic.
|
+
"ReasonStorageSchemaVersionIsOld" |
+ReasonStorageSchemaVersionIsOld when the object storage schema version is older than V13
+ |
"ReasonZoneAwareEmptyLabel" |
ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value.
|
@@ -1814,6 +1817,10 @@ are degraded or the cluster cannot connect to object storage.
"Ready" |
ConditionReady defines the condition that all components in the Loki deployment are ready.
|
+
"Warning" |
+ConditionWarning is used for configurations that are not recommended, but don’t currently cause
+issues. There can be multiple warning conditions active at a time.
+ |
From 721a700db166ad3a52334a11550716eea184ed53 Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 00:50:52 +0100
Subject: [PATCH 06/16] add missing resturn statement in generateCondition
---
operator/internal/status/lokistack.go | 10 +++++++++-
operator/internal/status/lokistack_test.go | 5 ++---
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index ea7ebc79e386b..387c9db664b22 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -77,6 +77,14 @@ func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus
}
func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) (metav1.Condition, error) {
+ if degradedErr != nil {
+ return metav1.Condition{
+ Type: string(lokiv1.ConditionDegraded),
+ Message: degradedErr.Message,
+ Reason: string(degradedErr.Reason),
+ }, nil
+ }
+
// Check for failed pods first
failed := len(cs.Compactor[corev1.PodFailed]) +
len(cs.Distributor[corev1.PodFailed]) +
@@ -157,7 +165,7 @@ func generateWarnings(ctx context.Context, cs *lokiv1.LokiStackComponentStatus,
if sc.Version != lokiv1.ObjectStorageSchemaV13 {
warnings = append(warnings, metav1.Condition{
Type: string(lokiv1.ConditionWarning),
- Reason: string(lokiv1.ReasonSchemaUpgradeRecommended),
+ Reason: string(lokiv1.ReasonStorageSchemaVersionIsOld),
Message: messageOldSchemaVersion,
})
break
diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go
index 947c7472455cf..ff7767f585ebd 100644
--- a/operator/internal/status/lokistack_test.go
+++ b/operator/internal/status/lokistack_test.go
@@ -62,7 +62,7 @@ func TestGenerateCondition(t *testing.T) {
degradedErr *DegradedError
wantCondition metav1.Condition
}{
- /* {
+ {
desc: "no error",
componentStatus: &lokiv1.LokiStackComponentStatus{},
wantCondition: conditionReady,
@@ -88,7 +88,7 @@ func TestGenerateCondition(t *testing.T) {
},
},
wantCondition: conditionFailed,
- }, */
+ },
{
desc: "degraded error",
componentStatus: &lokiv1.LokiStackComponentStatus{
@@ -113,7 +113,6 @@ func TestGenerateCondition(t *testing.T) {
for _, tc := range tt {
tc := tc
t.Run(tc.desc, func(t *testing.T) {
- println(tc.desc)
t.Parallel()
condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack, tc.degradedErr)
From 08487415ee5e8f5ae0bca18174cf3739de546707 Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 01:12:24 +0100
Subject: [PATCH 07/16] add unit test
---
operator/internal/status/lokistack_test.go | 79 ++++++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go
index ff7767f585ebd..c1bc7e358e794 100644
--- a/operator/internal/status/lokistack_test.go
+++ b/operator/internal/status/lokistack_test.go
@@ -221,5 +221,84 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) {
}
func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
+ sw := &k8sfakes.FakeStatusWriter{}
+ k := &k8sfakes.FakeClient{}
+
+ k.StatusStub = func() client.StatusWriter { return sw }
+ s := &lokiv1.LokiStack{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-stack",
+ Namespace: "some-ns",
+ },
+ }
+ r := ctrl.Request{
+ NamespacedName: types.NamespacedName{
+ Name: "my-stack",
+ Namespace: "some-ns",
+ },
+ }
+
+ schemas := []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: "2023-10-11",
+ },
+ }
+
+ expected := lokiv1.LokiStackStatus{
+ Storage: lokiv1.LokiStackStorageStatus{
+ Schemas: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: "2023-10-11",
+ },
+ },
+ },
+ }
+ k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error {
+ if r.Name == name.Name && r.Namespace == name.Namespace {
+ k.SetClientObject(object, s)
+ return nil
+ }
+ return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found")
+ }
+ sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.SubResourceUpdateOption) error {
+ stack := obj.(*lokiv1.LokiStack)
+ require.Equal(t, expected.Storage.Schemas, stack.Status.Storage.Schemas)
+ s = stack
+ return nil
+ }
+
+ err := SetStorageSchemaStatus(context.TODO(), k, r, schemas)
+
+ require.NoError(t, err)
+ require.NotEmpty(t, s.Status.Storage.Schemas)
+
+ require.NotZero(t, k.StatusCallCount())
+ require.NotZero(t, sw.UpdateCallCount())
+
+ wantCondition := metav1.Condition{
+ Type: string(lokiv1.ConditionWarning),
+ Reason: string(lokiv1.ReasonStorageSchemaVersionIsOld),
+ Message: messageOldSchemaVersion,
+ }
+ componentStatus := &lokiv1.LokiStackComponentStatus{
+ Ingester: map[corev1.PodPhase][]string{
+ corev1.PodRunning: {
+ "pod-0",
+ },
+ },
+ }
+ condition, err := generateWarnings(context.TODO(), componentStatus, k, r, s)
+ require.Nil(t, err)
+ require.Contains(t, condition, wantCondition)
}
From 8228baef1ab56e6d5c83daa0eb2c51ae724fc839 Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 01:17:14 +0100
Subject: [PATCH 08/16] modify CHANGELOG.md
---
operator/CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
index 592dc4f5837c2..600bc945a1720 100644
--- a/operator/CHANGELOG.md
+++ b/operator/CHANGELOG.md
@@ -1,5 +1,6 @@
## Main
+- [11158](https://github.com/grafana/loki/pull/11158) **btaani**: operator: Add warning for old schema configuration
- [11473](https://github.com/grafana/loki/pull/11473) **JoaoBraveCoding**: Adds structured metadata dashboards
- [11448](https://github.com/grafana/loki/pull/11448) **periklis**: Update Loki operand to v2.9.3
- [11357](https://github.com/grafana/loki/pull/11357) **periklis**: Fix storing authentication credentials in the Loki ConfigMap
From 9079b4bd2d71a87adc07df8521746f50e86c479d Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 17:36:34 +0100
Subject: [PATCH 09/16] address review comments
---
operator/apis/loki/v1/lokistack_types.go | 4 +-
operator/docs/operator/api.md | 4 +-
operator/internal/status/lokistack.go | 30 ++--
operator/internal/status/lokistack_test.go | 194 ++++++++++++++-------
4 files changed, 148 insertions(+), 84 deletions(-)
diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go
index 18264e02868a4..6124c65cd5217 100644
--- a/operator/apis/loki/v1/lokistack_types.go
+++ b/operator/apis/loki/v1/lokistack_types.go
@@ -1101,8 +1101,8 @@ const (
ReasonZoneAwareNodesMissing LokiStackConditionReason = "ReasonZoneAwareNodesMissing"
// ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value.
ReasonZoneAwareEmptyLabel LokiStackConditionReason = "ReasonZoneAwareEmptyLabel"
- // ReasonStorageSchemaVersionIsOld when the object storage schema version is older than V13
- ReasonStorageSchemaVersionIsOld LokiStackConditionReason = "ReasonStorageSchemaVersionIsOld"
+ // ReasonStorageNeedsSchemaUpdate when the object storage schema version is older than V13
+ ReasonStorageNeedsSchemaUpdate LokiStackConditionReason = "StorageNeedsSchemaUpdate"
)
// PodStatusMap defines the type for mapping pod status to pod name.
diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md
index 69f899b16ffb3..989a6ef481649 100644
--- a/operator/docs/operator/api.md
+++ b/operator/docs/operator/api.md
@@ -1780,8 +1780,8 @@ for the ruler is missing.
"ReadyComponents" |
ReasonReadyComponents when all LokiStack components are ready to serve traffic.
|
-
"ReasonStorageSchemaVersionIsOld" |
-ReasonStorageSchemaVersionIsOld when the object storage schema version is older than V13
+ |
"StorageNeedsSchemaUpdate" |
+ReasonStorageNeedsSchemaUpdate when the object storage schema version is older than V13
|
"ReasonZoneAwareEmptyLabel" |
ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value.
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index 387c9db664b22..3c28d6ef00bc8 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -14,12 +14,12 @@ import (
)
const (
- messageReady = "All components ready"
- messageFailed = "Some LokiStack components failed"
- messagePending = "Some LokiStack components pending on dependencies"
- messageDegradedMissingNodes = "Cluster contains no nodes matching the labels used for zone-awareness"
- messageDegradedEmptyNodeLabel = "No value for the labels used for zone-awareness"
- messageOldSchemaVersion = "The schema configuration contains one or more schemas that do not use the most recent version."
+ messageReady = "All components ready"
+ messageFailed = "Some LokiStack components failed"
+ messagePending = "Some LokiStack components pending on dependencies"
+ messageDegradedMissingNodes = "Cluster contains no nodes matching the labels used for zone-awareness"
+ messageDegradedEmptyNodeLabel = "No value for the labels used for zone-awareness"
+ messageWarningNeedsSchemaVersionUpdate = "The schema configuration does not contain the most recent schema version and needs an update"
)
var (
@@ -62,12 +62,12 @@ func (e *DegradedError) Error() string {
}
func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) {
- conditions, err := generateWarnings(ctx, cs, k, req, stack)
+ conditions, err := generateWarnings(stack.Status.Storage.Schemas)
if err != nil {
return nil, err
}
- mainCondition, err := generateCondition(ctx, cs, k, req, stack, degradedErr)
+ mainCondition, err := generateCondition(ctx, cs, k, stack, degradedErr)
if err != nil {
return nil, err
}
@@ -76,7 +76,7 @@ func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus
return conditions, nil
}
-func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) (metav1.Condition, error) {
+func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, stack *lokiv1.LokiStack, degradedErr *DegradedError) (metav1.Condition, error) {
if degradedErr != nil {
return metav1.Condition{
Type: string(lokiv1.ConditionDegraded),
@@ -159,14 +159,18 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1
return true, true, nil
}
-func generateWarnings(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) ([]metav1.Condition, error) {
+func generateWarnings(schemas []lokiv1.ObjectStorageSchema) ([]metav1.Condition, error) {
warnings := make([]metav1.Condition, 0)
- for _, sc := range stack.Status.Storage.Schemas {
+ for _, sc := range schemas {
+ if schemas[len(schemas)-1].Version == lokiv1.ObjectStorageSchemaV13 {
+ return warnings, nil
+ }
if sc.Version != lokiv1.ObjectStorageSchemaV13 {
warnings = append(warnings, metav1.Condition{
Type: string(lokiv1.ConditionWarning),
- Reason: string(lokiv1.ReasonStorageSchemaVersionIsOld),
- Message: messageOldSchemaVersion,
+ Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
+ Message: messageWarningNeedsSchemaVersionUpdate,
+ Status: metav1.ConditionTrue,
})
break
}
diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go
index c1bc7e358e794..defdbd165b7b3 100644
--- a/operator/internal/status/lokistack_test.go
+++ b/operator/internal/status/lokistack_test.go
@@ -41,12 +41,6 @@ func setupFakesNoError(t *testing.T, stack *lokiv1.LokiStack) (*k8sfakes.FakeCli
func TestGenerateCondition(t *testing.T) {
k := &k8sfakes.FakeClient{}
- r := ctrl.Request{
- NamespacedName: types.NamespacedName{
- Name: "test-lokistack",
- Namespace: "some-ns",
- },
- }
lokiStack := lokiv1.LokiStack{
TypeMeta: metav1.TypeMeta{
Kind: "LokiStack",
@@ -115,7 +109,7 @@ func TestGenerateCondition(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
- condition, err := generateCondition(context.TODO(), tc.componentStatus, k, r, &lokiStack, tc.degradedErr)
+ condition, err := generateCondition(context.TODO(), tc.componentStatus, k, &lokiStack, tc.degradedErr)
require.Nil(t, err)
require.Equal(t, tc.wantCondition, condition)
})
@@ -165,12 +159,6 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
- r := ctrl.Request{
- NamespacedName: types.NamespacedName{
- Name: "test-lokistack",
- Namespace: "some-ns",
- },
- }
componentStatus := &lokiv1.LokiStackComponentStatus{
Ingester: map[corev1.PodPhase][]string{
corev1.PodPending: {
@@ -212,7 +200,7 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) {
return tc.wantErr
}
- condition, err := generateCondition(context.TODO(), componentStatus, k, r, &lokiStack, nil)
+ condition, err := generateCondition(context.TODO(), componentStatus, k, &lokiStack, nil)
require.Equal(t, tc.wantErr, err)
require.Equal(t, tc.wantCondition, condition)
@@ -221,11 +209,115 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) {
}
func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
+
+ tt := []struct {
+ desc string
+ schemas []lokiv1.ObjectStorageSchema
+ expectedSchemaStatus []lokiv1.ObjectStorageSchema
+ wantCondition []metav1.Condition
+ }{
+ {
+ desc: "no V13 in schema config",
+ schemas: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV12,
+ EffectiveDate: "2023-10-11",
+ },
+ },
+ expectedSchemaStatus: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV12,
+ EffectiveDate: "2023-10-11",
+ },
+ },
+ wantCondition: []metav1.Condition{{
+ Type: string(lokiv1.ConditionWarning),
+ Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
+ Message: messageWarningNeedsSchemaVersionUpdate,
+ }},
+ },
+ {
+ desc: "with V13 not as the last element in schema config",
+ schemas: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: "2023-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV12,
+ EffectiveDate: "2024-10-11",
+ },
+ },
+ expectedSchemaStatus: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: "2023-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV12,
+ EffectiveDate: "2024-10-11",
+ },
+ },
+ wantCondition: []metav1.Condition{{
+ Type: string(lokiv1.ConditionWarning),
+ Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
+ Message: messageWarningNeedsSchemaVersionUpdate,
+ }},
+ }, {
+ desc: "with V13 as the last element in schema config",
+ schemas: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV12,
+ EffectiveDate: "2023-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: "2024-10-11",
+ },
+ },
+ expectedSchemaStatus: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV11,
+ EffectiveDate: "2020-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV12,
+ EffectiveDate: "2023-10-11",
+ },
+ {
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: "2024-10-11",
+ },
+ },
+ wantCondition: []metav1.Condition{},
+ },
+ }
+
sw := &k8sfakes.FakeStatusWriter{}
k := &k8sfakes.FakeClient{}
k.StatusStub = func() client.StatusWriter { return sw }
- s := &lokiv1.LokiStack{
+ s := lokiv1.LokiStack{
ObjectMeta: metav1.ObjectMeta{
Name: "my-stack",
Namespace: "some-ns",
@@ -238,67 +330,35 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
},
}
- schemas := []lokiv1.ObjectStorageSchema{
- {
- Version: lokiv1.ObjectStorageSchemaV11,
- EffectiveDate: "2020-10-11",
- },
- {
- Version: lokiv1.ObjectStorageSchemaV13,
- EffectiveDate: "2023-10-11",
- },
- }
-
- expected := lokiv1.LokiStackStatus{
- Storage: lokiv1.LokiStackStorageStatus{
- Schemas: []lokiv1.ObjectStorageSchema{
- {
- Version: lokiv1.ObjectStorageSchemaV11,
- EffectiveDate: "2020-10-11",
- },
- {
- Version: lokiv1.ObjectStorageSchemaV13,
- EffectiveDate: "2023-10-11",
- },
- },
- },
- }
k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error {
if r.Name == name.Name && r.Namespace == name.Namespace {
- k.SetClientObject(object, s)
+ k.SetClientObject(object, &s)
return nil
}
return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found")
}
- sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.SubResourceUpdateOption) error {
- stack := obj.(*lokiv1.LokiStack)
- require.Equal(t, expected.Storage.Schemas, stack.Status.Storage.Schemas)
- s = stack
- return nil
- }
- err := SetStorageSchemaStatus(context.TODO(), k, r, schemas)
+ for _, tc := range tt {
+ tc := tc
+ t.Run(tc.desc, func(t *testing.T) {
+ sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.SubResourceUpdateOption) error {
+ stack := obj.(*lokiv1.LokiStack)
+ require.Equal(t, tc.schemas, stack.Status.Storage.Schemas)
+ s = *stack
+ return nil
+ }
+ err := SetStorageSchemaStatus(context.TODO(), k, r, tc.schemas)
- require.NoError(t, err)
- require.NotEmpty(t, s.Status.Storage.Schemas)
+ require.NoError(t, err)
+ require.NotEmpty(t, s.Status.Storage.Schemas)
- require.NotZero(t, k.StatusCallCount())
- require.NotZero(t, sw.UpdateCallCount())
+ require.NotZero(t, k.StatusCallCount())
+ require.NotZero(t, sw.UpdateCallCount())
- wantCondition := metav1.Condition{
- Type: string(lokiv1.ConditionWarning),
- Reason: string(lokiv1.ReasonStorageSchemaVersionIsOld),
- Message: messageOldSchemaVersion,
- }
- componentStatus := &lokiv1.LokiStackComponentStatus{
- Ingester: map[corev1.PodPhase][]string{
- corev1.PodRunning: {
- "pod-0",
- },
- },
- }
+ condition, err := generateWarnings(s.Status.Storage.Schemas)
+ require.Nil(t, err)
+ require.Equal(t, condition, tc.wantCondition)
- condition, err := generateWarnings(context.TODO(), componentStatus, k, r, s)
- require.Nil(t, err)
- require.Contains(t, condition, wantCondition)
+ })
+ }
}
From 017d250f3cbe357eb5615b256d4c31ca70df8477 Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 17:47:14 +0100
Subject: [PATCH 10/16] remove setting condition status
---
operator/internal/status/lokistack.go | 1 -
1 file changed, 1 deletion(-)
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index 3c28d6ef00bc8..24bf1fce7804e 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -170,7 +170,6 @@ func generateWarnings(schemas []lokiv1.ObjectStorageSchema) ([]metav1.Condition,
Type: string(lokiv1.ConditionWarning),
Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
Message: messageWarningNeedsSchemaVersionUpdate,
- Status: metav1.ConditionTrue,
})
break
}
From 1b5a5e179ff60b75adf141669dc47d980b52b22b Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 17:59:23 +0100
Subject: [PATCH 11/16] minor refactoring
---
operator/internal/status/lokistack.go | 33 +++++++++-------------
operator/internal/status/lokistack_test.go | 5 ++--
2 files changed, 16 insertions(+), 22 deletions(-)
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index 24bf1fce7804e..bf236834c90c2 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -62,10 +62,7 @@ func (e *DegradedError) Error() string {
}
func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) {
- conditions, err := generateWarnings(stack.Status.Storage.Schemas)
- if err != nil {
- return nil, err
- }
+ conditions := generateWarnings(stack.Status.Storage.Schemas)
mainCondition, err := generateCondition(ctx, cs, k, stack, degradedErr)
if err != nil {
@@ -159,21 +156,19 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1
return true, true, nil
}
-func generateWarnings(schemas []lokiv1.ObjectStorageSchema) ([]metav1.Condition, error) {
+func generateWarnings(schemas []lokiv1.ObjectStorageSchema) []metav1.Condition {
warnings := make([]metav1.Condition, 0)
- for _, sc := range schemas {
- if schemas[len(schemas)-1].Version == lokiv1.ObjectStorageSchemaV13 {
- return warnings, nil
- }
- if sc.Version != lokiv1.ObjectStorageSchemaV13 {
- warnings = append(warnings, metav1.Condition{
- Type: string(lokiv1.ConditionWarning),
- Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
- Message: messageWarningNeedsSchemaVersionUpdate,
- })
- break
- }
+ if len(schemas) == 0 {
+ return nil
+ }
+ if schemas[len(schemas)-1].Version == lokiv1.ObjectStorageSchemaV13 {
+ return nil
+ } else {
+ warnings = append(warnings, metav1.Condition{
+ Type: string(lokiv1.ConditionWarning),
+ Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
+ Message: messageWarningNeedsSchemaVersionUpdate,
+ })
+ return warnings
}
-
- return warnings, nil
}
diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go
index defdbd165b7b3..12d2bc443a4ea 100644
--- a/operator/internal/status/lokistack_test.go
+++ b/operator/internal/status/lokistack_test.go
@@ -309,7 +309,7 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
EffectiveDate: "2024-10-11",
},
},
- wantCondition: []metav1.Condition{},
+ wantCondition: nil,
},
}
@@ -355,8 +355,7 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
require.NotZero(t, k.StatusCallCount())
require.NotZero(t, sw.UpdateCallCount())
- condition, err := generateWarnings(s.Status.Storage.Schemas)
- require.Nil(t, err)
+ condition := generateWarnings(s.Status.Storage.Schemas)
require.Equal(t, condition, tc.wantCondition)
})
From 7e8415e65d948f06cdd04e9ca504047209347975 Mon Sep 17 00:00:00 2001
From: btaani
Date: Fri, 15 Dec 2023 20:45:43 +0100
Subject: [PATCH 12/16] another refactor
---
.../manifests/internal/config/build_test.go | 293 ------------------
operator/internal/status/lokistack.go | 13 +-
operator/internal/status/lokistack_test.go | 93 +-----
3 files changed, 11 insertions(+), 388 deletions(-)
diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go
index 9b1974e49333e..537ec84bf71a5 100644
--- a/operator/internal/manifests/internal/config/build_test.go
+++ b/operator/internal/manifests/internal/config/build_test.go
@@ -5418,296 +5418,3 @@ analytics:
})
}
}
-
-func TestBuild_ConfigAndRuntimeConfig_AddSchema(t *testing.T) {
- effectiveDate := "2030-11-07"
-
- expCfg := `
----
-auth_enabled: true
-chunk_store_config:
- chunk_cache_config:
- embedded_cache:
- enabled: true
- max_size_mb: 500
-common:
- storage:
- s3:
- s3: http://test.default.svc.cluster.local.:9000
- bucketnames: loki
- region: us-east
- access_key_id: ${AWS_ACCESS_KEY_ID}
- secret_access_key: ${AWS_ACCESS_KEY_SECRET}
- s3forcepathstyle: true
- compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095
- ring:
- kvstore:
- store: memberlist
- heartbeat_period: 5s
- heartbeat_timeout: 1m
- instance_port: 9095
-compactor:
- compaction_interval: 2h
- working_directory: /tmp/loki/compactor
-frontend:
- tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100
- compress_responses: true
- max_outstanding_per_tenant: 4096
- log_queries_longer_than: 5s
-frontend_worker:
- frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095
- grpc_client_config:
- max_send_msg_size: 104857600
- match_max_concurrent: true
-ingester:
- chunk_block_size: 262144
- chunk_encoding: snappy
- chunk_idle_period: 1h
- chunk_retain_period: 5m
- chunk_target_size: 2097152
- flush_op_timeout: 10m
- lifecycler:
- final_sleep: 0s
- join_after: 30s
- num_tokens: 512
- ring:
- replication_factor: 1
- max_chunk_age: 2h
- max_transfer_retries: 0
- wal:
- enabled: true
- dir: /tmp/wal
- replay_memory_ceiling: 2500
-ingester_client:
- grpc_client_config:
- max_recv_msg_size: 67108864
- remote_timeout: 1s
-# NOTE: Keep the order of keys as in Loki docs
-# to enable easy diffs when vendoring newer
-# Loki releases.
-# (See https://grafana.com/docs/loki/latest/configuration/#limits_config)
-#
-# Values for not exposed fields are taken from the grafana/loki production
-# configuration manifests.
-# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet)
-limits_config:
- ingestion_rate_strategy: global
- ingestion_rate_mb: 4
- ingestion_burst_size_mb: 6
- max_label_name_length: 1024
- max_label_value_length: 2048
- max_label_names_per_series: 30
- reject_old_samples: true
- reject_old_samples_max_age: 168h
- creation_grace_period: 10m
- enforce_metric_name: false
- # Keep max_streams_per_user always to 0 to default
- # using max_global_streams_per_user always.
- # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73)
- max_streams_per_user: 0
- max_line_size: 256000
- max_entries_limit_per_query: 5000
- max_global_streams_per_user: 0
- max_chunks_per_query: 2000000
- max_query_length: 721h
- max_query_parallelism: 32
- tsdb_max_query_parallelism: 512
- max_query_series: 500
- cardinality_limit: 100000
- max_streams_matchers_per_query: 1000
- max_cache_freshness_per_query: 10m
- per_stream_rate_limit: 3MB
- per_stream_rate_limit_burst: 15MB
- split_queries_by_interval: 30m
- query_timeout: 1m
- allow_structured_metadata: true
-memberlist:
- abort_if_cluster_join_fails: true
- advertise_port: 7946
- bind_port: 7946
- join_members:
- - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946
- max_join_backoff: 1m
- max_join_retries: 10
- min_join_backoff: 1s
-querier:
- engine:
- max_look_back_period: 30s
- extra_query_delay: 0s
- max_concurrent: 2
- query_ingesters_within: 3h
- tail_max_duration: 1h
-query_range:
- align_queries_with_step: true
- cache_results: true
- max_retries: 5
- results_cache:
- cache:
- embedded_cache:
- enabled: true
- max_size_mb: 500
- parallelise_shardable_queries: true
-schema_config:
- configs:
- - from: "2020-10-01"
- index:
- period: 24h
- prefix: index_
- object_store: s3
- schema: v11
- store: boltdb-shipper
- ${V13_CONFIG}
-server:
- graceful_shutdown_timeout: 5s
- grpc_server_min_time_between_pings: '10s'
- grpc_server_ping_without_stream_allowed: true
- grpc_server_max_concurrent_streams: 1000
- grpc_server_max_recv_msg_size: 104857600
- grpc_server_max_send_msg_size: 104857600
- http_listen_port: 3100
- http_server_idle_timeout: 30s
- http_server_read_timeout: 30s
- http_server_write_timeout: 10m0s
- log_level: info
-storage_config:
- boltdb_shipper:
- active_index_directory: /tmp/loki/index
- cache_location: /tmp/loki/index_cache
- cache_ttl: 24h
- resync_interval: 5m
- shared_store: s3
- index_gateway_client:
- server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095
- ${TSDB_SHIPPER_STORAGE_CONFIG}
-tracing:
- enabled: false
-analytics:
- reporting_enabled: true
-`
- opts := Options{
- Stack: lokiv1.LokiStackSpec{
- Replication: &lokiv1.ReplicationSpec{
- Factor: 1,
- },
- Limits: &lokiv1.LimitsSpec{
- Global: &lokiv1.LimitsTemplateSpec{
- IngestionLimits: &lokiv1.IngestionLimitSpec{
- IngestionRate: 4,
- IngestionBurstSize: 6,
- MaxLabelNameLength: 1024,
- MaxLabelValueLength: 2048,
- MaxLabelNamesPerSeries: 30,
- MaxGlobalStreamsPerTenant: 0,
- MaxLineSize: 256000,
- PerStreamRateLimit: 3,
- PerStreamRateLimitBurst: 15,
- },
- QueryLimits: &lokiv1.QueryLimitSpec{
- MaxEntriesLimitPerQuery: 5000,
- MaxChunksPerQuery: 2000000,
- MaxQuerySeries: 500,
- QueryTimeout: "1m",
- CardinalityLimit: 100000,
- },
- },
- },
- },
- Namespace: "test-ns",
- Name: "test",
- Compactor: Address{
- FQDN: "loki-compactor-grpc-lokistack-dev.default.svc.cluster.local",
- Port: 9095,
- },
- FrontendWorker: Address{
- FQDN: "loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local",
- Port: 9095,
- },
- GossipRing: GossipRing{
- InstancePort: 9095,
- BindPort: 7946,
- MembersDiscoveryAddr: "loki-gossip-ring-lokistack-dev.default.svc.cluster.local",
- },
- Querier: Address{
- Protocol: "http",
- FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
- Port: 3100,
- },
- IndexGateway: Address{
- FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
- Port: 9095,
- },
- StorageDirectory: "/tmp/loki",
- MaxConcurrent: MaxConcurrent{
- AvailableQuerierCPUCores: 2,
- },
- WriteAheadLog: WriteAheadLog{
- Directory: "/tmp/wal",
- IngesterMemoryRequest: 5000,
- },
- ObjectStorage: storage.Options{
- SharedStore: lokiv1.ObjectStorageSecretS3,
- S3: &storage.S3StorageConfig{
- Endpoint: "http://test.default.svc.cluster.local.:9000",
- Region: "us-east",
- Buckets: "loki",
- },
- Schemas: []lokiv1.ObjectStorageSchema{
- {
- Version: lokiv1.ObjectStorageSchemaV11,
- EffectiveDate: "2020-10-01",
- },
- },
- },
- Shippers: []string{"boltdb"},
- EnableRemoteReporting: true,
- HTTPTimeouts: HTTPTimeoutConfig{
- IdleTimeout: 30 * time.Second,
- ReadTimeout: 30 * time.Second,
- WriteTimeout: 10 * time.Minute,
- },
- }
-
- cfg, _, err := Build(opts)
-
- expCfg_before := strings.Replace(expCfg, "${TSDB_LIMIT}", "", -1)
- expCfg_before = strings.Replace(expCfg_before, " ${V13_CONFIG}", "", -1)
- expCfg_before = strings.Replace(expCfg_before, "${TSDB_SHIPPER_STORAGE_CONFIG}", "", -1)
-
- require.NoError(t, err)
- require.YAMLEq(t, expCfg_before, string(cfg))
-
- v13schema := lokiv1.ObjectStorageSchema{
- Version: lokiv1.ObjectStorageSchemaV13,
- EffectiveDate: lokiv1.StorageSchemaEffectiveDate(effectiveDate),
- }
- v13_config := `- from: "2030-11-07"
- index:
- period: 24h
- prefix: index_
- object_store: s3
- schema: v13
- store: tsdb`
- tsdb_shipper_config := `
- tsdb_shipper:
- active_index_directory: /tmp/loki/tsdb-index
- cache_location: /tmp/loki/tsdb-cache
- cache_ttl: 24h
- resync_interval: 5m
- shared_store: s3
- index_gateway_client:
- server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095`
- tsdb_limit := `tsdb_max_query_parallelism: 512`
-
- expCfg = strings.Replace(expCfg, "${TSDB_LIMIT}", tsdb_limit, -1)
- expCfg = strings.Replace(expCfg, "${V13_CONFIG}", v13_config, -1)
- expCfg = strings.Replace(expCfg, "${TSDB_SHIPPER_STORAGE_CONFIG}", tsdb_shipper_config, -1)
-
- opts.ObjectStorage.Schemas = append(opts.ObjectStorage.Schemas, v13schema)
- opts.Shippers = append(opts.Shippers, "tsdb")
-
- cfg, _, err = Build(opts)
-
- require.NoError(t, err)
- require.YAMLEq(t, expCfg, string(cfg))
-
-}
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index bf236834c90c2..cdb8226f2687b 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -157,18 +157,15 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1
}
func generateWarnings(schemas []lokiv1.ObjectStorageSchema) []metav1.Condition {
- warnings := make([]metav1.Condition, 0)
- if len(schemas) == 0 {
- return nil
- }
- if schemas[len(schemas)-1].Version == lokiv1.ObjectStorageSchemaV13 {
- return nil
- } else {
+ warnings := make([]metav1.Condition, 0, 2)
+
+ if len(schemas) > 0 && schemas[len(schemas)-1].Version != lokiv1.ObjectStorageSchemaV13 {
warnings = append(warnings, metav1.Condition{
Type: string(lokiv1.ConditionWarning),
Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
Message: messageWarningNeedsSchemaVersionUpdate,
})
- return warnings
}
+
+ return warnings
}
diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go
index 12d2bc443a4ea..dd4f45dcaa31e 100644
--- a/operator/internal/status/lokistack_test.go
+++ b/operator/internal/status/lokistack_test.go
@@ -13,7 +13,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
- ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -209,12 +208,10 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) {
}
func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
-
tt := []struct {
- desc string
- schemas []lokiv1.ObjectStorageSchema
- expectedSchemaStatus []lokiv1.ObjectStorageSchema
- wantCondition []metav1.Condition
+ desc string
+ schemas []lokiv1.ObjectStorageSchema
+ wantCondition []metav1.Condition
}{
{
desc: "no V13 in schema config",
@@ -228,16 +225,6 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
EffectiveDate: "2023-10-11",
},
},
- expectedSchemaStatus: []lokiv1.ObjectStorageSchema{
- {
- Version: lokiv1.ObjectStorageSchemaV11,
- EffectiveDate: "2020-10-11",
- },
- {
- Version: lokiv1.ObjectStorageSchemaV12,
- EffectiveDate: "2023-10-11",
- },
- },
wantCondition: []metav1.Condition{{
Type: string(lokiv1.ConditionWarning),
Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
@@ -260,20 +247,6 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
EffectiveDate: "2024-10-11",
},
},
- expectedSchemaStatus: []lokiv1.ObjectStorageSchema{
- {
- Version: lokiv1.ObjectStorageSchemaV11,
- EffectiveDate: "2020-10-11",
- },
- {
- Version: lokiv1.ObjectStorageSchemaV13,
- EffectiveDate: "2023-10-11",
- },
- {
- Version: lokiv1.ObjectStorageSchemaV12,
- EffectiveDate: "2024-10-11",
- },
- },
wantCondition: []metav1.Condition{{
Type: string(lokiv1.ConditionWarning),
Reason: string(lokiv1.ReasonStorageNeedsSchemaUpdate),
@@ -295,69 +268,15 @@ func TestGenerateWarningCondition_WhenStorageSchemaIsOld(t *testing.T) {
EffectiveDate: "2024-10-11",
},
},
- expectedSchemaStatus: []lokiv1.ObjectStorageSchema{
- {
- Version: lokiv1.ObjectStorageSchemaV11,
- EffectiveDate: "2020-10-11",
- },
- {
- Version: lokiv1.ObjectStorageSchemaV12,
- EffectiveDate: "2023-10-11",
- },
- {
- Version: lokiv1.ObjectStorageSchemaV13,
- EffectiveDate: "2024-10-11",
- },
- },
- wantCondition: nil,
- },
- }
-
- sw := &k8sfakes.FakeStatusWriter{}
- k := &k8sfakes.FakeClient{}
-
- k.StatusStub = func() client.StatusWriter { return sw }
- s := lokiv1.LokiStack{
- ObjectMeta: metav1.ObjectMeta{
- Name: "my-stack",
- Namespace: "some-ns",
+ wantCondition: []metav1.Condition{},
},
}
- r := ctrl.Request{
- NamespacedName: types.NamespacedName{
- Name: "my-stack",
- Namespace: "some-ns",
- },
- }
-
- k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error {
- if r.Name == name.Name && r.Namespace == name.Namespace {
- k.SetClientObject(object, &s)
- return nil
- }
- return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found")
- }
-
for _, tc := range tt {
tc := tc
t.Run(tc.desc, func(t *testing.T) {
- sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.SubResourceUpdateOption) error {
- stack := obj.(*lokiv1.LokiStack)
- require.Equal(t, tc.schemas, stack.Status.Storage.Schemas)
- s = *stack
- return nil
- }
- err := SetStorageSchemaStatus(context.TODO(), k, r, tc.schemas)
-
- require.NoError(t, err)
- require.NotEmpty(t, s.Status.Storage.Schemas)
-
- require.NotZero(t, k.StatusCallCount())
- require.NotZero(t, sw.UpdateCallCount())
-
- condition := generateWarnings(s.Status.Storage.Schemas)
+ t.Parallel()
+ condition := generateWarnings(tc.schemas)
require.Equal(t, condition, tc.wantCondition)
-
})
}
}
From 459b75a61081d4f2575030f9eb024c12b0e33716 Mon Sep 17 00:00:00 2001
From: Bayan Taani <86984560+btaani@users.noreply.github.com>
Date: Mon, 18 Dec 2023 11:47:16 +0100
Subject: [PATCH 13/16] Update operator/internal/status/lokistack.go
Co-authored-by: Periklis Tsirakidis
---
operator/internal/status/lokistack.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index cdb8226f2687b..db9c8af4cd8f6 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -61,7 +61,7 @@ func (e *DegradedError) Error() string {
return fmt.Sprintf("cluster degraded: %s", e.Message)
}
-func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) {
+func generateConditions(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, stack *lokiv1.LokiStack, degradedErr *DegradedError) ([]metav1.Condition, error) {
conditions := generateWarnings(stack.Status.Storage.Schemas)
mainCondition, err := generateCondition(ctx, cs, k, stack, degradedErr)
From 68fac33dc401f6a417d465363eb74920faa97d5f Mon Sep 17 00:00:00 2001
From: Bayan Taani <86984560+btaani@users.noreply.github.com>
Date: Mon, 18 Dec 2023 11:48:17 +0100
Subject: [PATCH 14/16] Update operator/internal/status/lokistack.go
Co-authored-by: Periklis Tsirakidis
---
operator/internal/status/lokistack.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index db9c8af4cd8f6..3040243a0f116 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -157,7 +157,7 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1
}
func generateWarnings(schemas []lokiv1.ObjectStorageSchema) []metav1.Condition {
- warnings := make([]metav1.Condition, 0, 2)
+ warnings := make([]metav1.Condition, 0, 1)
if len(schemas) > 0 && schemas[len(schemas)-1].Version != lokiv1.ObjectStorageSchemaV13 {
warnings = append(warnings, metav1.Condition{
From cd7f71f56140037c3f2d4e11fda69acc5c3a6793 Mon Sep 17 00:00:00 2001
From: btaani
Date: Mon, 18 Dec 2023 12:39:00 +0100
Subject: [PATCH 15/16] undo change
---
operator/internal/status/lokistack.go | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go
index 3040243a0f116..467fef398ea77 100644
--- a/operator/internal/status/lokistack.go
+++ b/operator/internal/status/lokistack.go
@@ -6,7 +6,6 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
@@ -157,7 +156,7 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1
}
func generateWarnings(schemas []lokiv1.ObjectStorageSchema) []metav1.Condition {
- warnings := make([]metav1.Condition, 0, 1)
+ warnings := make([]metav1.Condition, 0, 2)
if len(schemas) > 0 && schemas[len(schemas)-1].Version != lokiv1.ObjectStorageSchemaV13 {
warnings = append(warnings, metav1.Condition{
From 9fad7712a3107ca7c7dab35997bd48f141945aea Mon Sep 17 00:00:00 2001
From: btaani
Date: Mon, 18 Dec 2023 12:52:44 +0100
Subject: [PATCH 16/16] remove extra parameter in function call
---
operator/internal/status/status.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/operator/internal/status/status.go b/operator/internal/status/status.go
index 5619735bbe67d..281a167355c37 100644
--- a/operator/internal/status/status.go
+++ b/operator/internal/status/status.go
@@ -31,7 +31,7 @@ func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time,
return err
}
- activeConditions, err := generateConditions(ctx, cs, k, req, &stack, degradedErr)
+ activeConditions, err := generateConditions(ctx, cs, k, &stack, degradedErr)
if err != nil {
return err
}
|