refactor!: Merge operational_config into limits_config configuration block

cmd/loki/main.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
-	"runtime"
+	rt "runtime"
 	"time"
 
 	"github.com/go-kit/log/level"
@@ -17,12 +17,11 @@ import (
 	"github.com/prometheus/common/version"
 
 	"github.com/grafana/loki/v3/pkg/loki"
-	loki_runtime "github.com/grafana/loki/v3/pkg/runtime"
+	"github.com/grafana/loki/v3/pkg/runtime"
 	"github.com/grafana/loki/v3/pkg/util"
 	_ "github.com/grafana/loki/v3/pkg/util/build"
 	"github.com/grafana/loki/v3/pkg/util/cfg"
 	util_log "github.com/grafana/loki/v3/pkg/util/log"
-	"github.com/grafana/loki/v3/pkg/validation"
 )
 
 func exit(code int) {
@@ -49,8 +48,7 @@ func main() {
 	// This global is set to the config passed into the last call to `NewOverrides`. If we don't
 	// call it atleast once, the defaults are set to an empty struct.
 	// We call it with the flag values so that the config file unmarshalling only overrides the values set in the config.
-	validation.SetDefaultLimitsForYAMLUnmarshalling(config.LimitsConfig)
-	loki_runtime.SetDefaultLimitsForYAMLUnmarshalling(config.OperationalConfig)
+	runtime.SetDefaultLimitsForYAMLUnmarshalling(config.LimitsConfig)
 
 	// Init the logger which will honor the log level set in config.Server
 	if reflect.DeepEqual(&config.Server.LogLevel, &log.Level{}) {
@@ -112,7 +110,7 @@ func main() {
 	// The larger the ballast, the lower the garbage collection frequency.
 	// https://github.com/grafana/loki/issues/781
 	ballast := make([]byte, config.BallastBytes)
-	runtime.KeepAlive(ballast)
+	rt.KeepAlive(ballast)
 
 	// Start Loki
 	t, err := loki.New(config.Config)
@@ -132,12 +130,12 @@ func main() {
 
 func setProfilingOptions(cfg loki.ProfilingConfig) {
 	if cfg.BlockProfileRate > 0 {
-		runtime.SetBlockProfileRate(cfg.BlockProfileRate)
+		rt.SetBlockProfileRate(cfg.BlockProfileRate)
 	}
 	if cfg.CPUProfileRate > 0 {
-		runtime.SetCPUProfileRate(cfg.CPUProfileRate)
+		rt.SetCPUProfileRate(cfg.CPUProfileRate)
 	}
 	if cfg.MutexProfileFraction > 0 {
-		runtime.SetMutexProfileFraction(cfg.MutexProfileFraction)
+		rt.SetMutexProfileFraction(cfg.MutexProfileFraction)
 	}
 }

cmd/migrate/main.go

@@ -19,14 +19,14 @@ import (
 
 	"github.com/grafana/loki/v3/pkg/logql/syntax"
 	"github.com/grafana/loki/v3/pkg/loki"
+	"github.com/grafana/loki/v3/pkg/runtime"
 	"github.com/grafana/loki/v3/pkg/storage"
 	"github.com/grafana/loki/v3/pkg/storage/chunk"
 	"github.com/grafana/loki/v3/pkg/storage/config"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper"
 	"github.com/grafana/loki/v3/pkg/util/cfg"
 	"github.com/grafana/loki/v3/pkg/util/constants"
 	util_log "github.com/grafana/loki/v3/pkg/util/log"
-	"github.com/grafana/loki/v3/pkg/validation"
 )
 
 type syncRange struct {
@@ -111,7 +111,7 @@ func main() {
 	// The long nature of queries requires stretching out the cardinality limit some and removing the query length limit
 	sourceConfig.LimitsConfig.CardinalityLimit = 1e9
 	sourceConfig.LimitsConfig.MaxQueryLength = 0
-	limits, err := validation.NewOverrides(sourceConfig.LimitsConfig, nil)
+	limits, err := runtime.NewOverrides(sourceConfig.LimitsConfig, nil)
 	if err != nil {
 		log.Println("Failed to create limit overrides:", err)
 		os.Exit(1)

docs/sources/setup/upgrade/_index.md

@@ -45,6 +45,10 @@ parameter contains a log selector query instead of returning inconsistent result
 
 Loki changes the default value of `-ruler.alertmanager-use-v2` from `false` to `true`. Alertmanager APIv1 was deprecated in Alertmanager 0.16.0 and is removed as of 0.27.0.
 
+#### `operational_config`
+
+The settings from the `operational_config` configuration block moved into the `limits_config`. The per-tenant configuration of these settings also moved into the `overrides` section in the runtime configuration file.
+
 ### Experimental Bloom Filters
 
 {{< admonition type="note" >}}

docs/sources/shared/configuration.md

@@ -792,11 +792,6 @@ compactor_grpc_client:
   # CLI flag: -compactor.grpc-client.connect-backoff-max-delay
   [connect_backoff_max_delay: <duration> | default = 5s]
 
-# The limits_config block configures global and per-tenant limits in Loki. The
-# values here can be overridden in the `overrides` section of the runtime_config
-# file
-[limits_config: <limits_config>]
-
 # The frontend_worker configures the worker - running within the Loki querier -
 # picking up and executing queries enqueued by the query-frontend.
 [frontend_worker: <frontend_worker>]
@@ -895,15 +890,15 @@ kafka_config:
   # CLI flag: -kafka.max-consumer-lag-at-startup
   [max_consumer_lag_at_startup: <duration> | default = 15s]
 
+# The limits_config block configures global and per-tenant limits in Loki. The
+# values here can be overridden in the `overrides` section of the runtime_config
+# file
+[limits_config: <limits_config>]
+
 # Configuration for 'runtime config' module, responsible for reloading runtime
 # configuration file.
 [runtime_config: <runtime_config>]
 
-# These are values which allow you to control aspects of Loki's operation, most
-# commonly used for controlling types of higher verbosity logging, the values
-# here can be overridden in the `configs` section of the `runtime_config` file.
-[operational_config: <operational_config>]
-
 # Configuration for tracing.
 [tracing: <tracing>]
 
@@ -3976,6 +3971,34 @@ otlp_config:
 # override is set, the encryption context will not be provided to S3. Ignored if
 # the SSE type override is not set.
 [s3_sse_kms_encryption_context: <string> | default = ""]
+
+# Log every new stream created by a push request (very verbose, recommend to
+# enable via runtime config only).
+# CLI flag: -operation-config.log-stream-creation
+[log_stream_creation: <boolean> | default = false]
+
+# Log every push request (very verbose, recommend to enable via runtime config
+# only).
+# CLI flag: -operation-config.log-push-request
+[log_push_request: <boolean> | default = false]
+
+# Log every stream in a push request (very verbose, recommend to enable via
+# runtime config only).
+# CLI flag: -operation-config.log-push-request-streams
+[log_push_request_streams: <boolean> | default = false]
+
+# Log metrics for duplicate lines received.
+# CLI flag: -operation-config.log-duplicate-metrics
+[log_duplicate_metrics: <boolean> | default = false]
+
+# Log stream info for duplicate lines received
+# CLI flag: -operation-config.log-duplicate-stream-info
+[log_duplicate_stream_info: <boolean> | default = false]
+
+# Log push errors with a rate limited logger, will show client push errors
+# without overly spamming logs.
+# CLI flag: -operation-config.limited-log-push-errors
+[limited_log_push_errors: <boolean> | default = true]
 ```
 
 ### local_storage_config
@@ -4229,40 +4252,6 @@ Named store from this example can be used by setting object_store to store-1 in
 [cos: <map of string to cos_storage_config>]
 ```
 
-### operational_config
-
-These are values which allow you to control aspects of Loki's operation, most commonly used for controlling types of higher verbosity logging, the values here can be overridden in the `configs` section of the `runtime_config` file.
-
-```yaml
-# Log every new stream created by a push request (very verbose, recommend to
-# enable via runtime config only).
-# CLI flag: -operation-config.log-stream-creation
-[log_stream_creation: <boolean> | default = false]
-
-# Log every push request (very verbose, recommend to enable via runtime config
-# only).
-# CLI flag: -operation-config.log-push-request
-[log_push_request: <boolean> | default = false]
-
-# Log every stream in a push request (very verbose, recommend to enable via
-# runtime config only).
-# CLI flag: -operation-config.log-push-request-streams
-[log_push_request_streams: <boolean> | default = false]
-
-# Log metrics for duplicate lines received.
-# CLI flag: -operation-config.log-duplicate-metrics
-[log_duplicate_metrics: <boolean> | default = false]
-
-# Log stream info for duplicate lines received
-# CLI flag: -operation-config.log-duplicate-stream-info
-[log_duplicate_stream_info: <boolean> | default = false]
-
-# Log push errors with a rate limited logger, will show client push errors
-# without overly spamming logs.
-# CLI flag: -operation-config.limited-log-push-errors
-[limited_log_push_errors: <boolean> | default = true]
-```
-
 ### period_config
 
 The `period_config` block configures what index schemas should be used for from specific time periods.

integration/cluster/cluster.go

@@ -26,11 +26,11 @@ import (
 	"github.com/grafana/loki/v3/integration/util"
 
 	"github.com/grafana/loki/v3/pkg/loki"
+	"github.com/grafana/loki/v3/pkg/runtime"
 	"github.com/grafana/loki/v3/pkg/storage"
 	"github.com/grafana/loki/v3/pkg/storage/config"
 	"github.com/grafana/loki/v3/pkg/util/cfg"
 	util_log "github.com/grafana/loki/v3/pkg/util/log"
-	"github.com/grafana/loki/v3/pkg/validation"
 )
 
 var configTemplate = template.Must(template.New("").Parse(`
@@ -509,14 +509,14 @@ func (c *Component) Restart() error {
 }
 
 type runtimeConfigValues struct {
-	TenantLimits map[string]*validation.Limits `yaml:"overrides"`
+	TenantLimits map[string]*runtime.Limits `yaml:"overrides"`
 }
 
-func (c *Component) SetTenantLimits(tenant string, limits validation.Limits) error {
+func (c *Component) SetTenantLimits(tenant string, limits runtime.Limits) error {
 	rcv := runtimeConfigValues{}
 	rcv.TenantLimits = c.loki.TenantLimits.AllByUserID()
 	if rcv.TenantLimits == nil {
-		rcv.TenantLimits = map[string]*validation.Limits{}
+		rcv.TenantLimits = map[string]*runtime.Limits{}
 	}
 	rcv.TenantLimits[tenant] = &limits
 
@@ -528,7 +528,7 @@ func (c *Component) SetTenantLimits(tenant string, limits validation.Limits) err
 	return os.WriteFile(c.overridesFile, config, 0640) // #nosec G306 -- this is fencing off the "other" permissions
 }
 
-func (c *Component) GetTenantLimits(tenant string) validation.Limits {
+func (c *Component) GetTenantLimits(tenant string) runtime.Limits {
 	limits := c.loki.TenantLimits.TenantLimits(tenant)
 	if limits == nil {
 		return c.loki.Cfg.LimitsConfig

pkg/bloombuild/planner/config.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/grafana/loki/v3/pkg/bloombuild/planner/queue"
 	"github.com/grafana/loki/v3/pkg/bloombuild/planner/strategies"
+	"github.com/grafana/loki/v3/pkg/compactor/retention"
 )
 
 // Config configures the bloom-planner component.
@@ -50,7 +51,7 @@ func (cfg *Config) Validate() error {
 }
 
 type Limits interface {
-	RetentionLimits
+	retention.Limits
 	strategies.Limits
 	BloomCreationEnabled(tenantID string) bool
 	BloomBuildMaxBuilders(tenantID string) int

pkg/bloombuild/planner/retention.go

@@ -12,10 +12,11 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
 
+	"github.com/grafana/loki/v3/pkg/compactor/retention"
+	"github.com/grafana/loki/v3/pkg/runtime"
 	"github.com/grafana/loki/v3/pkg/storage/chunk/client"
 	storageconfig "github.com/grafana/loki/v3/pkg/storage/config"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper"
-	"github.com/grafana/loki/v3/pkg/validation"
 )
 
 type RetentionConfig struct {
@@ -39,16 +40,9 @@ func (cfg *RetentionConfig) Validate() error {
 	return nil
 }
 
-type RetentionLimits interface {
-	RetentionPeriod(userID string) time.Duration
-	StreamRetention(userID string) []validation.StreamRetention
-	AllByUserID() map[string]*validation.Limits
-	DefaultLimits() *validation.Limits
-}
-
 type RetentionManager struct {
 	cfg        RetentionConfig
-	limits     RetentionLimits
+	limits     retention.Limits
 	bloomStore bloomshipper.StoreBase
 	metrics    *Metrics
 	logger     log.Logger
@@ -60,7 +54,7 @@ type RetentionManager struct {
 
 func NewRetentionManager(
 	cfg RetentionConfig,
-	limits RetentionLimits,
+	limits retention.Limits,
 	bloomStore bloomshipper.StoreBase,
 	metrics *Metrics,
 	logger log.Logger,
@@ -198,12 +192,12 @@ func (r *RetentionManager) reportTenantsExceedingLookback(retentionByTenant map[
 	r.metrics.retentionTenantsExceedingLookback.Set(float64(tenantsExceedingLookback))
 }
 
-func findLongestRetention(globalRetention time.Duration, streamRetention []validation.StreamRetention) time.Duration {
+func findLongestRetention(globalRetention time.Duration, streamRetention []runtime.StreamRetention) time.Duration {
 	if len(streamRetention) == 0 {
 		return globalRetention
 	}
 
-	maxStreamRetention := slices.MaxFunc(streamRetention, func(a, b validation.StreamRetention) int {
+	maxStreamRetention := slices.MaxFunc(streamRetention, func(a, b runtime.StreamRetention) int {
 		return int(a.Period - b.Period)
 	})
 
@@ -213,7 +207,7 @@ func findLongestRetention(globalRetention time.Duration, streamRetention []valid
 	return globalRetention
 }
 
-func retentionByTenant(limits RetentionLimits) map[string]time.Duration {
+func retentionByTenant(limits retention.Limits) map[string]time.Duration {
 	all := limits.AllByUserID()
 	if len(all) == 0 {
 		return nil