diff --git a/cmd/loki/main.go b/cmd/loki/main.go index bb839c6cf3ec8..7dbae1c45e755 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -5,7 +5,7 @@ import ( "fmt" "os" "reflect" - "runtime" + rt "runtime" "time" "github.com/go-kit/log/level" @@ -17,12 +17,11 @@ import ( "github.com/prometheus/common/version" "github.com/grafana/loki/v3/pkg/loki" - loki_runtime "github.com/grafana/loki/v3/pkg/runtime" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util" _ "github.com/grafana/loki/v3/pkg/util/build" "github.com/grafana/loki/v3/pkg/util/cfg" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) func exit(code int) { @@ -49,8 +48,7 @@ func main() { // This global is set to the config passed into the last call to `NewOverrides`. If we don't // call it atleast once, the defaults are set to an empty struct. // We call it with the flag values so that the config file unmarshalling only overrides the values set in the config. - validation.SetDefaultLimitsForYAMLUnmarshalling(config.LimitsConfig) - loki_runtime.SetDefaultLimitsForYAMLUnmarshalling(config.OperationalConfig) + runtime.SetDefaultLimitsForYAMLUnmarshalling(config.LimitsConfig) // Init the logger which will honor the log level set in config.Server if reflect.DeepEqual(&config.Server.LogLevel, &log.Level{}) { @@ -112,7 +110,7 @@ func main() { // The larger the ballast, the lower the garbage collection frequency. // https://github.com/grafana/loki/issues/781 ballast := make([]byte, config.BallastBytes) - runtime.KeepAlive(ballast) + rt.KeepAlive(ballast) // Start Loki t, err := loki.New(config.Config) @@ -132,12 +130,12 @@ func main() { func setProfilingOptions(cfg loki.ProfilingConfig) { if cfg.BlockProfileRate > 0 { - runtime.SetBlockProfileRate(cfg.BlockProfileRate) + rt.SetBlockProfileRate(cfg.BlockProfileRate) } if cfg.CPUProfileRate > 0 { - runtime.SetCPUProfileRate(cfg.CPUProfileRate) + rt.SetCPUProfileRate(cfg.CPUProfileRate) } if cfg.MutexProfileFraction > 0 { - runtime.SetMutexProfileFraction(cfg.MutexProfileFraction) + rt.SetMutexProfileFraction(cfg.MutexProfileFraction) } } diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index 6ff24256afac1..1aa9244c1cbb2 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/loki" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/config" @@ -26,7 +27,6 @@ import ( "github.com/grafana/loki/v3/pkg/util/cfg" "github.com/grafana/loki/v3/pkg/util/constants" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) type syncRange struct { @@ -111,7 +111,7 @@ func main() { // The long nature of queries requires stretching out the cardinality limit some and removing the query length limit sourceConfig.LimitsConfig.CardinalityLimit = 1e9 sourceConfig.LimitsConfig.MaxQueryLength = 0 - limits, err := validation.NewOverrides(sourceConfig.LimitsConfig, nil) + limits, err := runtime.NewOverrides(sourceConfig.LimitsConfig, nil) if err != nil { log.Println("Failed to create limit overrides:", err) os.Exit(1) diff --git a/docs/sources/setup/upgrade/_index.md b/docs/sources/setup/upgrade/_index.md index 3ecf9107f91ac..f86e21dba042d 100644 --- a/docs/sources/setup/upgrade/_index.md +++ b/docs/sources/setup/upgrade/_index.md @@ -45,6 +45,10 @@ parameter contains a log selector query instead of returning inconsistent result Loki changes the default value of `-ruler.alertmanager-use-v2` from `false` to `true`. Alertmanager APIv1 was deprecated in Alertmanager 0.16.0 and is removed as of 0.27.0. +#### `operational_config` + +The settings from the `operational_config` configuration block moved into the `limits_config`. The per-tenant configuration of these settings also moved into the `overrides` section in the runtime configuration file. + ### Experimental Bloom Filters {{< admonition type="note" >}} diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index d602c48e058ec..b0c9bb409ba10 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -792,11 +792,6 @@ compactor_grpc_client: # CLI flag: -compactor.grpc-client.connect-backoff-max-delay [connect_backoff_max_delay: | default = 5s] -# The limits_config block configures global and per-tenant limits in Loki. The -# values here can be overridden in the `overrides` section of the runtime_config -# file -[limits_config: ] - # The frontend_worker configures the worker - running within the Loki querier - # picking up and executing queries enqueued by the query-frontend. [frontend_worker: ] @@ -895,15 +890,15 @@ kafka_config: # CLI flag: -kafka.max-consumer-lag-at-startup [max_consumer_lag_at_startup: | default = 15s] +# The limits_config block configures global and per-tenant limits in Loki. The +# values here can be overridden in the `overrides` section of the runtime_config +# file +[limits_config: ] + # Configuration for 'runtime config' module, responsible for reloading runtime # configuration file. [runtime_config: ] -# These are values which allow you to control aspects of Loki's operation, most -# commonly used for controlling types of higher verbosity logging, the values -# here can be overridden in the `configs` section of the `runtime_config` file. -[operational_config: ] - # Configuration for tracing. [tracing: ] @@ -3976,6 +3971,34 @@ otlp_config: # override is set, the encryption context will not be provided to S3. Ignored if # the SSE type override is not set. [s3_sse_kms_encryption_context: | default = ""] + +# Log every new stream created by a push request (very verbose, recommend to +# enable via runtime config only). +# CLI flag: -operation-config.log-stream-creation +[log_stream_creation: | default = false] + +# Log every push request (very verbose, recommend to enable via runtime config +# only). +# CLI flag: -operation-config.log-push-request +[log_push_request: | default = false] + +# Log every stream in a push request (very verbose, recommend to enable via +# runtime config only). +# CLI flag: -operation-config.log-push-request-streams +[log_push_request_streams: | default = false] + +# Log metrics for duplicate lines received. +# CLI flag: -operation-config.log-duplicate-metrics +[log_duplicate_metrics: | default = false] + +# Log stream info for duplicate lines received +# CLI flag: -operation-config.log-duplicate-stream-info +[log_duplicate_stream_info: | default = false] + +# Log push errors with a rate limited logger, will show client push errors +# without overly spamming logs. +# CLI flag: -operation-config.limited-log-push-errors +[limited_log_push_errors: | default = true] ``` ### local_storage_config @@ -4229,40 +4252,6 @@ Named store from this example can be used by setting object_store to store-1 in [cos: ] ``` -### operational_config - -These are values which allow you to control aspects of Loki's operation, most commonly used for controlling types of higher verbosity logging, the values here can be overridden in the `configs` section of the `runtime_config` file. - -```yaml -# Log every new stream created by a push request (very verbose, recommend to -# enable via runtime config only). -# CLI flag: -operation-config.log-stream-creation -[log_stream_creation: | default = false] - -# Log every push request (very verbose, recommend to enable via runtime config -# only). -# CLI flag: -operation-config.log-push-request -[log_push_request: | default = false] - -# Log every stream in a push request (very verbose, recommend to enable via -# runtime config only). -# CLI flag: -operation-config.log-push-request-streams -[log_push_request_streams: | default = false] - -# Log metrics for duplicate lines received. -# CLI flag: -operation-config.log-duplicate-metrics -[log_duplicate_metrics: | default = false] - -# Log stream info for duplicate lines received -# CLI flag: -operation-config.log-duplicate-stream-info -[log_duplicate_stream_info: | default = false] - -# Log push errors with a rate limited logger, will show client push errors -# without overly spamming logs. -# CLI flag: -operation-config.limited-log-push-errors -[limited_log_push_errors: | default = true] -``` - ### period_config The `period_config` block configures what index schemas should be used for from specific time periods. diff --git a/integration/cluster/cluster.go b/integration/cluster/cluster.go index 2d1c92037d05a..32483f5e8913f 100644 --- a/integration/cluster/cluster.go +++ b/integration/cluster/cluster.go @@ -26,11 +26,11 @@ import ( "github.com/grafana/loki/v3/integration/util" "github.com/grafana/loki/v3/pkg/loki" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/util/cfg" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) var configTemplate = template.Must(template.New("").Parse(` @@ -509,14 +509,14 @@ func (c *Component) Restart() error { } type runtimeConfigValues struct { - TenantLimits map[string]*validation.Limits `yaml:"overrides"` + TenantLimits map[string]*runtime.Limits `yaml:"overrides"` } -func (c *Component) SetTenantLimits(tenant string, limits validation.Limits) error { +func (c *Component) SetTenantLimits(tenant string, limits runtime.Limits) error { rcv := runtimeConfigValues{} rcv.TenantLimits = c.loki.TenantLimits.AllByUserID() if rcv.TenantLimits == nil { - rcv.TenantLimits = map[string]*validation.Limits{} + rcv.TenantLimits = map[string]*runtime.Limits{} } rcv.TenantLimits[tenant] = &limits @@ -528,7 +528,7 @@ func (c *Component) SetTenantLimits(tenant string, limits validation.Limits) err return os.WriteFile(c.overridesFile, config, 0640) // #nosec G306 -- this is fencing off the "other" permissions } -func (c *Component) GetTenantLimits(tenant string) validation.Limits { +func (c *Component) GetTenantLimits(tenant string) runtime.Limits { limits := c.loki.TenantLimits.TenantLimits(tenant) if limits == nil { return c.loki.Cfg.LimitsConfig diff --git a/pkg/bloombuild/planner/config.go b/pkg/bloombuild/planner/config.go index f6be8322f74d5..db202f8458d08 100644 --- a/pkg/bloombuild/planner/config.go +++ b/pkg/bloombuild/planner/config.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/loki/v3/pkg/bloombuild/planner/queue" "github.com/grafana/loki/v3/pkg/bloombuild/planner/strategies" + "github.com/grafana/loki/v3/pkg/compactor/retention" ) // Config configures the bloom-planner component. @@ -50,7 +51,7 @@ func (cfg *Config) Validate() error { } type Limits interface { - RetentionLimits + retention.Limits strategies.Limits BloomCreationEnabled(tenantID string) bool BloomBuildMaxBuilders(tenantID string) int diff --git a/pkg/bloombuild/planner/retention.go b/pkg/bloombuild/planner/retention.go index 8a937d332a42f..376fac35bf33f 100644 --- a/pkg/bloombuild/planner/retention.go +++ b/pkg/bloombuild/planner/retention.go @@ -12,10 +12,11 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/grafana/loki/v3/pkg/compactor/retention" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/client" storageconfig "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" - "github.com/grafana/loki/v3/pkg/validation" ) type RetentionConfig struct { @@ -39,16 +40,9 @@ func (cfg *RetentionConfig) Validate() error { return nil } -type RetentionLimits interface { - RetentionPeriod(userID string) time.Duration - StreamRetention(userID string) []validation.StreamRetention - AllByUserID() map[string]*validation.Limits - DefaultLimits() *validation.Limits -} - type RetentionManager struct { cfg RetentionConfig - limits RetentionLimits + limits retention.Limits bloomStore bloomshipper.StoreBase metrics *Metrics logger log.Logger @@ -60,7 +54,7 @@ type RetentionManager struct { func NewRetentionManager( cfg RetentionConfig, - limits RetentionLimits, + limits retention.Limits, bloomStore bloomshipper.StoreBase, metrics *Metrics, logger log.Logger, @@ -198,12 +192,12 @@ func (r *RetentionManager) reportTenantsExceedingLookback(retentionByTenant map[ r.metrics.retentionTenantsExceedingLookback.Set(float64(tenantsExceedingLookback)) } -func findLongestRetention(globalRetention time.Duration, streamRetention []validation.StreamRetention) time.Duration { +func findLongestRetention(globalRetention time.Duration, streamRetention []runtime.StreamRetention) time.Duration { if len(streamRetention) == 0 { return globalRetention } - maxStreamRetention := slices.MaxFunc(streamRetention, func(a, b validation.StreamRetention) int { + maxStreamRetention := slices.MaxFunc(streamRetention, func(a, b runtime.StreamRetention) int { return int(a.Period - b.Period) }) @@ -213,7 +207,7 @@ func findLongestRetention(globalRetention time.Duration, streamRetention []valid return globalRetention } -func retentionByTenant(limits RetentionLimits) map[string]time.Duration { +func retentionByTenant(limits retention.Limits) map[string]time.Duration { all := limits.AllByUserID() if len(all) == 0 { return nil diff --git a/pkg/bloombuild/planner/retention_test.go b/pkg/bloombuild/planner/retention_test.go index a309a7fc53013..67a469773b73b 100644 --- a/pkg/bloombuild/planner/retention_test.go +++ b/pkg/bloombuild/planner/retention_test.go @@ -12,6 +12,8 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/v3/pkg/bloombuild/planner/plannertest" + "github.com/grafana/loki/v3/pkg/compactor/retention" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" "github.com/grafana/loki/v3/pkg/storage/chunk/cache" @@ -21,7 +23,6 @@ import ( "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper/config" "github.com/grafana/loki/v3/pkg/storage/types" "github.com/grafana/loki/v3/pkg/util/mempool" - "github.com/grafana/loki/v3/pkg/validation" ) var testTime = plannertest.ParseDayTime("2024-12-31").ModelTime() @@ -116,7 +117,7 @@ func TestRetention(t *testing.T) { "3": 200 * 24 * time.Hour, "4": 400 * 24 * time.Hour, }, - streamRetention: map[string][]validation.StreamRetention{ + streamRetention: map[string][]runtime.StreamRetention{ "1": { { Period: model.Duration(30 * 24 * time.Hour), @@ -178,7 +179,7 @@ func TestRetention(t *testing.T) { "3": 200 * 24 * time.Hour, "4": 400 * 24 * time.Hour, }, - streamRetention: map[string][]validation.StreamRetention{ + streamRetention: map[string][]runtime.StreamRetention{ "1": { { Period: model.Duration(30 * 24 * time.Hour), @@ -350,7 +351,7 @@ func TestFindLongestRetention(t *testing.T) { for _, tc := range []struct { name string globalRetention time.Duration - streamRetention []validation.StreamRetention + streamRetention []runtime.StreamRetention expectedRetention time.Duration }{ { @@ -364,7 +365,7 @@ func TestFindLongestRetention(t *testing.T) { }, { name: "stream retention", - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(30 * 24 * time.Hour), }, @@ -373,7 +374,7 @@ func TestFindLongestRetention(t *testing.T) { }, { name: "two stream retention", - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(30 * 24 * time.Hour), }, @@ -386,7 +387,7 @@ func TestFindLongestRetention(t *testing.T) { { name: "stream retention bigger than global", globalRetention: 20 * 24 * time.Hour, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(30 * 24 * time.Hour), }, @@ -399,7 +400,7 @@ func TestFindLongestRetention(t *testing.T) { { name: "global retention bigger than stream", globalRetention: 40 * 24 * time.Hour, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(20 * 24 * time.Hour), }, @@ -420,7 +421,7 @@ func TestFindLongestRetention(t *testing.T) { func TestSmallestRetention(t *testing.T) { for _, tc := range []struct { name string - limits RetentionLimits + limits retention.Limits expectedRetention time.Duration expectedHasRetention bool }{ @@ -439,7 +440,7 @@ func TestSmallestRetention(t *testing.T) { { name: "default stream retention", limits: mockRetentionLimits{ - defaultStreamRetention: []validation.StreamRetention{ + defaultStreamRetention: []runtime.StreamRetention{ { Period: model.Duration(30 * 24 * time.Hour), }, @@ -463,7 +464,7 @@ func TestSmallestRetention(t *testing.T) { retention: map[string]time.Duration{ "1": 30 * 24 * time.Hour, }, - streamRetention: map[string][]validation.StreamRetention{ + streamRetention: map[string][]runtime.StreamRetention{ "1": { { Period: model.Duration(40 * 24 * time.Hour), @@ -480,7 +481,7 @@ func TestSmallestRetention(t *testing.T) { "1": 30 * 24 * time.Hour, "2": 20 * 24 * time.Hour, }, - streamRetention: map[string][]validation.StreamRetention{ + streamRetention: map[string][]runtime.StreamRetention{ "1": { { Period: model.Duration(40 * 24 * time.Hour), @@ -501,7 +502,7 @@ func TestSmallestRetention(t *testing.T) { retention: map[string]time.Duration{ "1": 10 * 24 * time.Hour, }, - streamRetention: map[string][]validation.StreamRetention{ + streamRetention: map[string][]runtime.StreamRetention{ "1": { { Period: model.Duration(20 * 24 * time.Hour), @@ -509,7 +510,7 @@ func TestSmallestRetention(t *testing.T) { }, }, defaultRetention: 40 * 24 * time.Hour, - defaultStreamRetention: []validation.StreamRetention{ + defaultStreamRetention: []runtime.StreamRetention{ { Period: model.Duration(30 * 24 * time.Hour), }, @@ -523,7 +524,7 @@ func TestSmallestRetention(t *testing.T) { retention: map[string]time.Duration{ "1": 30 * 24 * time.Hour, }, - streamRetention: map[string][]validation.StreamRetention{ + streamRetention: map[string][]runtime.StreamRetention{ "1": { { Period: model.Duration(40 * 24 * time.Hour), @@ -531,7 +532,7 @@ func TestSmallestRetention(t *testing.T) { }, }, defaultRetention: 10 * 24 * time.Hour, - defaultStreamRetention: []validation.StreamRetention{ + defaultStreamRetention: []runtime.StreamRetention{ { Period: model.Duration(20 * 24 * time.Hour), }, @@ -722,32 +723,32 @@ func NewMockBloomStoreWithWorkDir(t *testing.T, workDir string, logger log.Logge type mockRetentionLimits struct { retention map[string]time.Duration - streamRetention map[string][]validation.StreamRetention + streamRetention map[string][]runtime.StreamRetention defaultRetention time.Duration - defaultStreamRetention []validation.StreamRetention + defaultStreamRetention []runtime.StreamRetention } func (m mockRetentionLimits) RetentionPeriod(tenant string) time.Duration { return m.retention[tenant] } -func (m mockRetentionLimits) StreamRetention(tenant string) []validation.StreamRetention { +func (m mockRetentionLimits) StreamRetention(tenant string) []runtime.StreamRetention { return m.streamRetention[tenant] } -func (m mockRetentionLimits) AllByUserID() map[string]*validation.Limits { - tenants := make(map[string]*validation.Limits, len(m.retention)) +func (m mockRetentionLimits) AllByUserID() map[string]*runtime.Limits { + tenants := make(map[string]*runtime.Limits, len(m.retention)) for tenant, retention := range m.retention { if _, ok := tenants[tenant]; !ok { - tenants[tenant] = &validation.Limits{} + tenants[tenant] = &runtime.Limits{} } tenants[tenant].RetentionPeriod = model.Duration(retention) } for tenant, streamRetention := range m.streamRetention { if _, ok := tenants[tenant]; !ok { - tenants[tenant] = &validation.Limits{} + tenants[tenant] = &runtime.Limits{} } tenants[tenant].StreamRetention = streamRetention } @@ -755,8 +756,8 @@ func (m mockRetentionLimits) AllByUserID() map[string]*validation.Limits { return tenants } -func (m mockRetentionLimits) DefaultLimits() *validation.Limits { - return &validation.Limits{ +func (m mockRetentionLimits) DefaultLimits() *runtime.Limits { + return &runtime.Limits{ RetentionPeriod: model.Duration(m.defaultRetention), StreamRetention: m.defaultStreamRetention, } diff --git a/pkg/bloomgateway/bloomgateway_test.go b/pkg/bloomgateway/bloomgateway_test.go index 63657ce5b5e4d..1a88107382168 100644 --- a/pkg/bloomgateway/bloomgateway_test.go +++ b/pkg/bloomgateway/bloomgateway_test.go @@ -21,6 +21,7 @@ import ( "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/querier/plan" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" @@ -29,7 +30,6 @@ import ( bloomshipperconfig "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper/config" "github.com/grafana/loki/v3/pkg/storage/types" "github.com/grafana/loki/v3/pkg/util/mempool" - "github.com/grafana/loki/v3/pkg/validation" ) func stringSlice[T fmt.Stringer](s []T) []string { @@ -53,13 +53,13 @@ func groupRefs(t *testing.T, chunkRefs []*logproto.ChunkRef) []*logproto.Grouped return grouped } -func newLimits() *validation.Overrides { - limits := validation.Limits{} +func newLimits() *runtime.Overrides { + limits := runtime.Limits{} flagext.DefaultValues(&limits) limits.BloomGatewayEnabled = true limits.BloomGatewayShardSize = 1 - overrides, _ := validation.NewOverrides(limits, nil) + overrides, _ := runtime.NewOverrides(limits, nil) return overrides } diff --git a/pkg/compactor/compactor.go b/pkg/compactor/compactor.go index 908152a3edbbe..e76cb16c03c05 100644 --- a/pkg/compactor/compactor.go +++ b/pkg/compactor/compactor.go @@ -24,6 +24,7 @@ import ( "github.com/grafana/loki/v3/pkg/analytics" "github.com/grafana/loki/v3/pkg/compactor/deletion" "github.com/grafana/loki/v3/pkg/compactor/retention" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/client" "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" chunk_util "github.com/grafana/loki/v3/pkg/storage/chunk/client/util" @@ -32,7 +33,6 @@ import ( "github.com/grafana/loki/v3/pkg/util/filter" util_log "github.com/grafana/loki/v3/pkg/util/log" lokiring "github.com/grafana/loki/v3/pkg/util/ring" - "github.com/grafana/loki/v3/pkg/validation" ) // Here is how the generic compactor works: @@ -199,9 +199,9 @@ type storeContainer struct { } type Limits interface { + runtime.ExportedLimits deletion.Limits retention.Limits - DefaultLimits() *validation.Limits } func NewCompactor(cfg Config, objectStoreClients map[config.DayTime]client.ObjectClient, deleteStoreClient client.ObjectClient, schemaConfig config.SchemaConfig, limits Limits, r prometheus.Registerer, metricsNamespace string) (*Compactor, error) { diff --git a/pkg/compactor/compactor_test.go b/pkg/compactor/compactor_test.go index 3fccbb237b78f..4e640b2c06399 100644 --- a/pkg/compactor/compactor_test.go +++ b/pkg/compactor/compactor_test.go @@ -15,12 +15,12 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/client" "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/util/constants" loki_net "github.com/grafana/loki/v3/pkg/util/net" - "github.com/grafana/loki/v3/pkg/validation" ) const indexTablePrefix = "table_" @@ -54,11 +54,11 @@ func setupTestCompactor(t *testing.T, objectClients map[config.DayTime]client.Ob require.NoError(t, cfg.Validate()) - defaultLimits := validation.Limits{} + defaultLimits := runtime.Limits{} flagext.DefaultValues(&defaultLimits) require.NoError(t, defaultLimits.RetentionPeriod.Set("30d")) - overrides, err := validation.NewOverrides(defaultLimits, nil) + overrides, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) c, err := NewCompactor(cfg, objectClients, objectClients[periodConfigs[len(periodConfigs)-1].From], config.SchemaConfig{ diff --git a/pkg/compactor/deletion/tenant_delete_requests_client.go b/pkg/compactor/deletion/tenant_delete_requests_client.go index 495ece96e181c..4fd3de488d9f9 100644 --- a/pkg/compactor/deletion/tenant_delete_requests_client.go +++ b/pkg/compactor/deletion/tenant_delete_requests_client.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) const deletionNotAvailableMsg = "deletion is not available for this tenant" @@ -12,7 +12,7 @@ const deletionNotAvailableMsg = "deletion is not available for this tenant" type Limits interface { DeletionMode(userID string) string RetentionPeriod(userID string) time.Duration - StreamRetention(userID string) []validation.StreamRetention + StreamRetention(userID string) []runtime.StreamRetention } type perTenantDeleteRequestsClient struct { diff --git a/pkg/compactor/deletion/tenant_request_handler_test.go b/pkg/compactor/deletion/tenant_request_handler_test.go index cca06f4c18cfe..e9987b4b1aab7 100644 --- a/pkg/compactor/deletion/tenant_request_handler_test.go +++ b/pkg/compactor/deletion/tenant_request_handler_test.go @@ -9,7 +9,7 @@ import ( "github.com/grafana/dskit/user" "github.com/stretchr/testify/require" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) func TestDeleteRequestHandlerDeletionMiddleware(t *testing.T) { @@ -53,7 +53,7 @@ func TestDeleteRequestHandlerDeletionMiddleware(t *testing.T) { type limit struct { deletionMode string retentionPeriod time.Duration - streamRetention []validation.StreamRetention + streamRetention []runtime.StreamRetention } type fakeLimits struct { @@ -78,6 +78,6 @@ func (f *fakeLimits) RetentionPeriod(userID string) time.Duration { return f.getLimitForUser(userID).retentionPeriod } -func (f *fakeLimits) StreamRetention(userID string) []validation.StreamRetention { +func (f *fakeLimits) StreamRetention(userID string) []runtime.StreamRetention { return f.getLimitForUser(userID).streamRetention } diff --git a/pkg/compactor/retention/expiration.go b/pkg/compactor/retention/expiration.go index 45029f9652c5a..b4c3f8a615064 100644 --- a/pkg/compactor/retention/expiration.go +++ b/pkg/compactor/retention/expiration.go @@ -8,9 +8,9 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util/filter" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) // IntervalFilter contains the interval to delete @@ -37,10 +37,9 @@ type expirationChecker struct { } type Limits interface { + runtime.ExportedLimits RetentionPeriod(userID string) time.Duration - StreamRetention(userID string) []validation.StreamRetention - AllByUserID() map[string]*validation.Limits - DefaultLimits() *validation.Limits + StreamRetention(userID string) []runtime.StreamRetention } func NewExpirationChecker(limits Limits) ExpirationChecker { @@ -135,7 +134,7 @@ func (tr *TenantsRetention) RetentionPeriodFor(userID string, lbs labels.Labels) streamRetentions := tr.limits.StreamRetention(userID) globalRetention := tr.limits.RetentionPeriod(userID) var ( - matchedRule validation.StreamRetention + matchedRule runtime.StreamRetention found bool ) Outer: diff --git a/pkg/compactor/retention/expiration_test.go b/pkg/compactor/retention/expiration_test.go index 3cc69f88ae613..38e84f24f4cef 100644 --- a/pkg/compactor/retention/expiration_test.go +++ b/pkg/compactor/retention/expiration_test.go @@ -9,16 +9,16 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) type retentionLimit struct { retentionPeriod time.Duration - streamRetention []validation.StreamRetention + streamRetention []runtime.StreamRetention } -func (r retentionLimit) convertToValidationLimit() *validation.Limits { - return &validation.Limits{ +func (r retentionLimit) convertToValidationLimit() *runtime.Limits { + return &runtime.Limits{ RetentionPeriod: model.Duration(r.retentionPeriod), StreamRetention: r.streamRetention, } @@ -33,41 +33,41 @@ func (f fakeLimits) RetentionPeriod(userID string) time.Duration { return f.perTenant[userID].retentionPeriod } -func (f fakeLimits) StreamRetention(userID string) []validation.StreamRetention { +func (f fakeLimits) StreamRetention(userID string) []runtime.StreamRetention { return f.perTenant[userID].streamRetention } -func (f fakeLimits) DefaultLimits() *validation.Limits { +func (f fakeLimits) DefaultLimits() *runtime.Limits { return f.defaultLimit.convertToValidationLimit() } -func (f fakeLimits) AllByUserID() map[string]*validation.Limits { - res := make(map[string]*validation.Limits) +func (f fakeLimits) AllByUserID() map[string]*runtime.Limits { + res := make(map[string]*runtime.Limits) for userID, ret := range f.perTenant { res[userID] = ret.convertToValidationLimit() } return res } -func defaultLimitsTestConfig() validation.Limits { - limits := validation.Limits{} +func defaultLimitsTestConfig() runtime.Limits { + limits := runtime.Limits{} flagext.DefaultValues(&limits) return limits } -func overridesTestConfig(defaultLimits validation.Limits, tenantLimits validation.TenantLimits) (*validation.Overrides, error) { - return validation.NewOverrides(defaultLimits, tenantLimits) +func overridesTestConfig(defaultLimits runtime.Limits, tenantLimits runtime.TenantLimits) (*runtime.Overrides, error) { + return runtime.NewOverrides(defaultLimits, tenantLimits) } type fakeOverrides struct { - tenantLimits map[string]*validation.Limits + tenantLimits map[string]*runtime.Limits } -func (f fakeOverrides) TenantLimits(userID string) *validation.Limits { +func (f fakeOverrides) TenantLimits(userID string) *runtime.Limits { return f.tenantLimits[userID] } -func (f fakeOverrides) AllByUserID() map[string]*validation.Limits { +func (f fakeOverrides) AllByUserID() map[string]*runtime.Limits { //TODO implement me panic("implement me") } @@ -81,19 +81,19 @@ func Test_expirationChecker_Expired(t *testing.T) { // Override tenant 1 and tenant 2 t1 := defaultLimitsTestConfig() t1.RetentionPeriod = model.Duration(time.Hour) - t1.StreamRetention = []validation.StreamRetention{ + t1.StreamRetention = []runtime.StreamRetention{ {Period: model.Duration(2 * time.Hour), Priority: 10, Matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}}, {Period: model.Duration(2 * time.Hour), Priority: 1, Matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "foo", "ba.+")}}, } t2 := defaultLimitsTestConfig() t2.RetentionPeriod = model.Duration(24 * time.Hour) - t2.StreamRetention = []validation.StreamRetention{ + t2.StreamRetention = []runtime.StreamRetention{ {Period: model.Duration(1 * time.Hour), Matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}}, {Period: model.Duration(2 * time.Hour), Matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "foo", "ba.")}}, } f := fakeOverrides{ - tenantLimits: map[string]*validation.Limits{ + tenantLimits: map[string]*runtime.Limits{ "1": &t1, "2": &t2, }, @@ -133,7 +133,7 @@ func Test_expirationChecker_Expired_zeroValue(t *testing.T) { dur, _ := model.ParseDuration("24h") tl.RetentionPeriod = dur f := fakeOverrides{ - tenantLimits: map[string]*validation.Limits{ + tenantLimits: map[string]*runtime.Limits{ "2": &tl, }, } @@ -176,7 +176,7 @@ func Test_expirationChecker_Expired_zeroValueOverride(t *testing.T) { t3.RetentionPeriod = dur f := fakeOverrides{ - tenantLimits: map[string]*validation.Limits{ + tenantLimits: map[string]*runtime.Limits{ "2": &t2, "3": &t3, }, @@ -213,7 +213,7 @@ func Test_expirationChecker_DropFromIndex_zeroValue(t *testing.T) { dur, _ := model.ParseDuration("24h") tl.RetentionPeriod = dur f := fakeOverrides{ - tenantLimits: map[string]*validation.Limits{ + tenantLimits: map[string]*runtime.Limits{ "2": &tl, }, } @@ -267,7 +267,7 @@ func TestFindLatestRetentionStartTime(t *testing.T) { limit: fakeLimits{ defaultLimit: retentionLimit{ retentionPeriod: 7 * dayDuration, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(10 * dayDuration), }, @@ -292,7 +292,7 @@ func TestFindLatestRetentionStartTime(t *testing.T) { limit: fakeLimits{ defaultLimit: retentionLimit{ retentionPeriod: 7 * dayDuration, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(3 * dayDuration), }, @@ -317,7 +317,7 @@ func TestFindLatestRetentionStartTime(t *testing.T) { limit: fakeLimits{ defaultLimit: retentionLimit{ retentionPeriod: 7 * dayDuration, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(10 * dayDuration), }, @@ -326,7 +326,7 @@ func TestFindLatestRetentionStartTime(t *testing.T) { perTenant: map[string]retentionLimit{ "0": { retentionPeriod: 20 * dayDuration, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(10 * dayDuration), }, @@ -334,7 +334,7 @@ func TestFindLatestRetentionStartTime(t *testing.T) { }, "1": { retentionPeriod: 5 * dayDuration, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(15 * dayDuration), }, @@ -356,7 +356,7 @@ func TestFindLatestRetentionStartTime(t *testing.T) { limit: fakeLimits{ defaultLimit: retentionLimit{ retentionPeriod: 7 * dayDuration, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(10 * dayDuration), }, @@ -365,7 +365,7 @@ func TestFindLatestRetentionStartTime(t *testing.T) { perTenant: map[string]retentionLimit{ "0": { retentionPeriod: 20 * dayDuration, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(10 * dayDuration), }, @@ -373,7 +373,7 @@ func TestFindLatestRetentionStartTime(t *testing.T) { }, "1": { retentionPeriod: 15 * dayDuration, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ { Period: model.Duration(2 * dayDuration), }, diff --git a/pkg/compactor/retention/retention_test.go b/pkg/compactor/retention/retention_test.go index cdc7ef61dbc9c..8c62728aed655 100644 --- a/pkg/compactor/retention/retention_test.go +++ b/pkg/compactor/retention/retention_test.go @@ -27,10 +27,10 @@ import ( ingesterclient "github.com/grafana/loki/v3/pkg/ingester/client" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql/log" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/util/filter" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) type mockChunkClient struct { @@ -133,7 +133,7 @@ func Test_Retention(t *testing.T) { perTenant: map[string]retentionLimit{ "1": { retentionPeriod: 10 * time.Hour, - streamRetention: []validation.StreamRetention{ + streamRetention: []runtime.StreamRetention{ {Period: model.Duration(5 * time.Hour), Matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "buzz")}}, }, }, diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 6ede42aab1c20..6fdba1ffb626b 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -137,7 +137,6 @@ type Distributor struct { ingesterCfg ingester.Config logger log.Logger clientCfg client.Config - tenantConfigs *runtime.TenantConfigs tenantsRetention *retention.TenantsRetention ingestersRing ring.ReadRing validator *Validator @@ -193,7 +192,6 @@ func New( cfg Config, ingesterCfg ingester.Config, clientCfg client.Config, - configs *runtime.TenantConfigs, ingestersRing ring.ReadRing, partitionRing ring.PartitionRingReader, overrides Limits, @@ -228,7 +226,7 @@ func New( var servs []services.Service - rateLimitStrat := validation.LocalIngestionRateStrategy + rateLimitStrat := runtime.LocalIngestionRateStrategy labelCache, err := lru.New[string, labelData](maxLabelCacheSize) if err != nil { return nil, err @@ -253,7 +251,6 @@ func New( ingesterCfg: ingesterCfg, logger: logger, clientCfg: clientCfg, - tenantConfigs: configs, tenantsRetention: retention.NewTenantsRetention(overrides), ingestersRing: ingestersRing, validator: validator, @@ -315,13 +312,13 @@ func New( Help: "The number of records a single per-partition write request has been split into.", Buckets: prometheus.ExponentialBuckets(1, 2, 8), }), - writeFailuresManager: writefailures.NewManager(logger, registerer, cfg.WriteFailuresLogging, configs, "distributor"), + writeFailuresManager: writefailures.NewManager(logger, registerer, cfg.WriteFailuresLogging, overrides, "distributor"), kafkaWriter: kafkaWriter, partitionRing: partitionRing, } - if overrides.IngestionRateStrategy() == validation.GlobalIngestionRateStrategy { - d.rateLimitStrat = validation.GlobalIngestionRateStrategy + if overrides.IngestionRateStrategy() == runtime.GlobalIngestionRateStrategy { + d.rateLimitStrat = runtime.GlobalIngestionRateStrategy distributorsRing, distributorsLifecycler, err = newRingAndLifecycler(cfg.DistributorRing, d.healthyInstancesCount, logger, registerer, metricsNamespace) if err != nil { diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 91d3fcdf1367b..3eb5b2150d2cb 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -116,7 +116,7 @@ func TestDistributor(t *testing.T) { }, } { t.Run(fmt.Sprintf("[%d](lines=%v)", i, tc.lines), func(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.IngestionRateMB = ingestionRateLimitMB limits.IngestionBurstSizeMB = ingestionRateLimitMB @@ -152,23 +152,23 @@ func TestDistributor(t *testing.T) { } func Test_IncrementTimestamp(t *testing.T) { - incrementingDisabled := &validation.Limits{} + incrementingDisabled := &runtime.Limits{} flagext.DefaultValues(incrementingDisabled) incrementingDisabled.RejectOldSamples = false incrementingDisabled.DiscoverLogLevels = false - incrementingEnabled := &validation.Limits{} + incrementingEnabled := &runtime.Limits{} flagext.DefaultValues(incrementingEnabled) incrementingEnabled.RejectOldSamples = false incrementingEnabled.IncrementDuplicateTimestamp = true incrementingEnabled.DiscoverLogLevels = false - defaultLimits := &validation.Limits{} + defaultLimits := &runtime.Limits{} flagext.DefaultValues(defaultLimits) defaultLimits.DiscoverLogLevels = false tests := map[string]struct { - limits *validation.Limits + limits *runtime.Limits push *logproto.PushRequest expectedPush *logproto.PushRequest }{ @@ -427,7 +427,7 @@ func Test_IncrementTimestamp(t *testing.T) { } func Test_MissingEnforcedLabels(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.EnforcedLabels = []string{"app", "env"} @@ -454,7 +454,7 @@ func Test_MissingEnforcedLabels(t *testing.T) { } func Test_PushWithEnforcedLabels(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) // makeWriteRequest only contains a `{foo="bar"}` label. @@ -480,7 +480,7 @@ func Test_PushWithEnforcedLabels(t *testing.T) { } func TestDistributorPushConcurrently(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) distributors, ingesters := prepare(t, 1, 5, limits, nil) @@ -533,7 +533,7 @@ func TestDistributorPushConcurrently(t *testing.T) { } func TestDistributorPushErrors(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) t.Run("with RF=3 a single push can fail", func(t *testing.T) { @@ -572,7 +572,7 @@ func TestDistributorPushErrors(t *testing.T) { } func TestDistributorPushToKafka(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) t.Run("with kafka, any failure fails the request", func(t *testing.T) { @@ -645,7 +645,7 @@ func TestDistributorPushToKafka(t *testing.T) { func Test_SortLabelsOnPush(t *testing.T) { t.Run("with service_name already present in labels", func(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) ingester := &mockIngester{} distributors, _ := prepare(t, 1, 5, limits, func(_ string) (ring_client.PoolClient, error) { return ingester, nil }) @@ -660,8 +660,8 @@ func Test_SortLabelsOnPush(t *testing.T) { } func Test_TruncateLogLines(t *testing.T) { - setup := func() (*validation.Limits, *mockIngester) { - limits := &validation.Limits{} + setup := func() (*runtime.Limits, *mockIngester) { + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.MaxLineSize = 5 @@ -681,8 +681,8 @@ func Test_TruncateLogLines(t *testing.T) { } func Test_DiscardEmptyStreamsAfterValidation(t *testing.T) { - setup := func() (*validation.Limits, *mockIngester) { - limits := &validation.Limits{} + setup := func() (*runtime.Limits, *mockIngester) { + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.MaxLineSize = 5 @@ -786,11 +786,11 @@ func TestStreamShard(t *testing.T) { t.Run(tc.name, func(t *testing.T) { baseStream.Entries = tc.entries - distributorLimits := &validation.Limits{} + distributorLimits := &runtime.Limits{} flagext.DefaultValues(distributorLimits) distributorLimits.ShardStreams.DesiredRate = desiredRate - overrides, err := validation.NewOverrides(*distributorLimits, nil) + overrides, err := runtime.NewOverrides(*distributorLimits, nil) require.NoError(t, err) validator, err := NewValidator(overrides, nil) @@ -830,11 +830,11 @@ func TestStreamShardAcrossCalls(t *testing.T) { streamRate := loki_flagext.ByteSize(400).Val() - distributorLimits := &validation.Limits{} + distributorLimits := &runtime.Limits{} flagext.DefaultValues(distributorLimits) distributorLimits.ShardStreams.DesiredRate = loki_flagext.ByteSize(100) - overrides, err := validation.NewOverrides(*distributorLimits, nil) + overrides, err := runtime.NewOverrides(*distributorLimits, nil) require.NoError(t, err) validator, err := NewValidator(overrides, nil) @@ -1160,11 +1160,11 @@ func BenchmarkShardStream(b *testing.B) { desiredRate := 3000 - distributorLimits := &validation.Limits{} + distributorLimits := &runtime.Limits{} flagext.DefaultValues(distributorLimits) distributorLimits.ShardStreams.DesiredRate = loki_flagext.ByteSize(desiredRate) - overrides, err := validation.NewOverrides(*distributorLimits, nil) + overrides, err := runtime.NewOverrides(*distributorLimits, nil) require.NoError(b, err) validator, err := NewValidator(overrides, nil) @@ -1224,7 +1224,7 @@ func BenchmarkShardStream(b *testing.B) { } func Benchmark_SortLabelsOnPush(b *testing.B) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) distributors, _ := prepare(&testing.T{}, 1, 5, limits, nil) d := distributors[0] @@ -1241,7 +1241,7 @@ func Benchmark_SortLabelsOnPush(b *testing.B) { } func TestParseStreamLabels(t *testing.T) { - defaultLimit := &validation.Limits{} + defaultLimit := &runtime.Limits{} flagext.DefaultValues(defaultLimit) for _, tc := range []struct { @@ -1249,13 +1249,13 @@ func TestParseStreamLabels(t *testing.T) { origLabels string expectedLabels labels.Labels expectedErr error - generateLimits func() *validation.Limits + generateLimits func() *runtime.Limits }{ { name: "service name label should not get counted against max labels count", origLabels: `{foo="bar", service_name="unknown_service"}`, - generateLimits: func() *validation.Limits { - limits := &validation.Limits{} + generateLimits: func() *runtime.Limits { + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.MaxLabelNamesPerSeries = 1 return limits @@ -1295,7 +1295,7 @@ func TestParseStreamLabels(t *testing.T) { } func Benchmark_Push(b *testing.B) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.IngestionBurstSizeMB = math.MaxInt32 limits.CardinalityLimit = math.MaxInt32 @@ -1511,7 +1511,7 @@ func TestShardCountFor(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.ShardStreams.DesiredRate = tc.desiredRate @@ -1525,7 +1525,7 @@ func TestShardCountFor(t *testing.T) { } func Benchmark_PushWithLineTruncation(b *testing.B) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.IngestionRateMB = math.MaxInt32 @@ -1562,7 +1562,7 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { }{ "local strategy: limit should be set to each distributor": { distributors: 2, - ingestionRateStrategy: validation.LocalIngestionRateStrategy, + ingestionRateStrategy: runtime.LocalIngestionRateStrategy, ingestionRateMB: datasize.ByteSize(100).MBytes(), ingestionBurstSizeMB: datasize.ByteSize(100).MBytes(), pushes: []testPush{ @@ -1574,7 +1574,7 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { }, "global strategy: limit should be evenly shared across distributors": { distributors: 2, - ingestionRateStrategy: validation.GlobalIngestionRateStrategy, + ingestionRateStrategy: runtime.GlobalIngestionRateStrategy, ingestionRateMB: datasize.ByteSize(200).MBytes(), ingestionBurstSizeMB: datasize.ByteSize(100).MBytes(), pushes: []testPush{ @@ -1586,7 +1586,7 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { }, "global strategy: burst should set to each distributor": { distributors: 2, - ingestionRateStrategy: validation.GlobalIngestionRateStrategy, + ingestionRateStrategy: runtime.GlobalIngestionRateStrategy, ingestionRateMB: datasize.ByteSize(100).MBytes(), ingestionBurstSizeMB: datasize.ByteSize(200).MBytes(), pushes: []testPush{ @@ -1600,7 +1600,7 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { for testName, testData := range tests { t.Run(testName, func(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.IngestionRateStrategy = testData.ingestionRateStrategy limits.IngestionRateMB = testData.ingestionRateMB @@ -1656,7 +1656,7 @@ func TestDistributor_PushIngestionBlocked(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.BlockIngestionUntil = flagext.Time(tc.blockUntil) limits.BlockIngestionStatusCode = tc.blockStatusCode @@ -1677,7 +1677,7 @@ func TestDistributor_PushIngestionBlocked(t *testing.T) { } } -func prepare(t *testing.T, numDistributors, numIngesters int, limits *validation.Limits, factory func(addr string) (ring_client.PoolClient, error)) ([]*Distributor, []mockIngester) { +func prepare(t *testing.T, numDistributors, numIngesters int, limits *runtime.Limits, factory func(addr string) (ring_client.PoolClient, error)) ([]*Distributor, []mockIngester) { t.Helper() ingesters := make([]mockIngester, numIngesters) @@ -1765,12 +1765,12 @@ func prepare(t *testing.T, numDistributors, numIngesters int, limits *validation }) } - overrides, err := validation.NewOverrides(*limits, nil) + overrides, err := runtime.NewOverrides(*limits, nil) require.NoError(t, err) ingesterConfig := ingester.Config{MaxChunkAge: 2 * time.Hour} - d, err := New(distributorConfig, ingesterConfig, clientConfig, runtime.DefaultTenantConfigs(), ingestersRing, partitionRingReader, overrides, prometheus.NewPedanticRegistry(), constants.Loki, nil, nil, log.NewNopLogger()) + d, err := New(distributorConfig, ingesterConfig, clientConfig, ingestersRing, partitionRingReader, overrides, prometheus.NewPedanticRegistry(), constants.Loki, nil, nil, log.NewNopLogger()) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), d)) distributors[i] = d @@ -2002,7 +2002,7 @@ func TestDistributorTee(t *testing.T) { }, } - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.RejectOldSamples = false distributors, _ := prepare(t, 1, 3, limits, nil) @@ -2023,7 +2023,7 @@ func TestDistributorTee(t *testing.T) { } func TestDistributor_StructuredMetadataSanitization(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) for _, tc := range []struct { req *logproto.PushRequest diff --git a/pkg/distributor/field_detection_test.go b/pkg/distributor/field_detection_test.go index b053ab0a76071..88ea5cb1c7351 100644 --- a/pkg/distributor/field_detection_test.go +++ b/pkg/distributor/field_detection_test.go @@ -10,17 +10,16 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/plog" + "github.com/grafana/loki/pkg/push" loghttp_push "github.com/grafana/loki/v3/pkg/loghttp/push" "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" - - "github.com/grafana/loki/pkg/push" ) func Test_DetectLogLevels(t *testing.T) { - setup := func(discoverLogLevels bool) (*validation.Limits, *mockIngester) { - limits := &validation.Limits{} + setup := func(discoverLogLevels bool) (*runtime.Limits, *mockIngester) { + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.DiscoverLogLevels = discoverLogLevels diff --git a/pkg/distributor/http.go b/pkg/distributor/http.go index 7337ce16209c4..a73a33a0fd64e 100644 --- a/pkg/distributor/http.go +++ b/pkg/distributor/http.go @@ -8,13 +8,13 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/dskit/httpgrpc" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util" "github.com/grafana/dskit/tenant" "github.com/grafana/loki/v3/pkg/loghttp/push" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) // PushHandler reads a snappy-compressed proto from the HTTP body. @@ -39,10 +39,10 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe pushRequestParser = d.RequestParserWrapper(pushRequestParser) } - logPushRequestStreams := d.tenantConfigs.LogPushRequestStreams(tenantID) + logPushRequestStreams := d.validator.LogPushRequestStreams(tenantID) req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser, d.usageTracker, logPushRequestStreams) if err != nil { - if d.tenantConfigs.LogPushRequest(tenantID) { + if d.validator.LogPushRequest(tenantID) { level.Debug(logger).Log( "msg", "push request failed", "code", http.StatusBadRequest, @@ -68,7 +68,7 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe _, err = d.Push(r.Context(), req) if err == nil { - if d.tenantConfigs.LogPushRequest(tenantID) { + if d.validator.LogPushRequest(tenantID) { level.Debug(logger).Log( "msg", "push request successful", ) @@ -80,7 +80,7 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe resp, ok := httpgrpc.HTTPResponseFromError(err) if ok { body := string(resp.Body) - if d.tenantConfigs.LogPushRequest(tenantID) { + if d.validator.LogPushRequest(tenantID) { level.Debug(logger).Log( "msg", "push request failed", "code", resp.Code, @@ -89,7 +89,7 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe } errorWriter(w, body, int(resp.Code), logger) } else { - if d.tenantConfigs.LogPushRequest(tenantID) { + if d.validator.LogPushRequest(tenantID) { level.Debug(logger).Log( "msg", "push request failed", "code", http.StatusInternalServerError, @@ -105,7 +105,7 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe // If the rate limiting strategy is local instead of global, no ring is used by // the distributor and as such, no ring status is returned from this function. func (d *Distributor) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if d.rateLimitStrat == validation.GlobalIngestionRateStrategy { + if d.rateLimitStrat == runtime.GlobalIngestionRateStrategy { d.distributorsLifecycler.ServeHTTP(w, r) return } diff --git a/pkg/distributor/http_test.go b/pkg/distributor/http_test.go index 8da8fc608fa98..3c1aa4279b222 100644 --- a/pkg/distributor/http_test.go +++ b/pkg/distributor/http_test.go @@ -8,19 +8,17 @@ import ( "testing" "github.com/go-kit/log" + "github.com/grafana/dskit/flagext" "github.com/grafana/dskit/user" + "github.com/stretchr/testify/require" "github.com/grafana/loki/v3/pkg/loghttp/push" "github.com/grafana/loki/v3/pkg/logproto" - - "github.com/grafana/dskit/flagext" - "github.com/stretchr/testify/require" - - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) func TestDistributorRingHandler(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) runServer := func() *httptest.Server { @@ -32,7 +30,7 @@ func TestDistributorRingHandler(t *testing.T) { } t.Run("renders ring status for global rate limiting", func(t *testing.T) { - limits.IngestionRateStrategy = validation.GlobalIngestionRateStrategy + limits.IngestionRateStrategy = runtime.GlobalIngestionRateStrategy svr := runServer() defer svr.Close() @@ -47,7 +45,7 @@ func TestDistributorRingHandler(t *testing.T) { }) t.Run("doesn't return ring status for local rate limiting", func(t *testing.T) { - limits.IngestionRateStrategy = validation.LocalIngestionRateStrategy + limits.IngestionRateStrategy = runtime.LocalIngestionRateStrategy svr := runServer() defer svr.Close() @@ -63,7 +61,7 @@ func TestDistributorRingHandler(t *testing.T) { } func TestRequestParserWrapping(t *testing.T) { - limits := &validation.Limits{} + limits := &runtime.Limits{} flagext.DefaultValues(limits) limits.RejectOldSamples = false distributors, _ := prepare(t, 1, 3, limits, nil) diff --git a/pkg/distributor/ingestion_rate_strategy_test.go b/pkg/distributor/ingestion_rate_strategy_test.go index 02463264cd0dd..60a3faaab9701 100644 --- a/pkg/distributor/ingestion_rate_strategy_test.go +++ b/pkg/distributor/ingestion_rate_strategy_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) const ( @@ -17,14 +17,14 @@ const ( func TestIngestionRateStrategy(t *testing.T) { tests := map[string]struct { - limits validation.Limits + limits runtime.Limits ring ReadLifecycler expectedLimit float64 expectedBurst int }{ "local rate limiter should just return configured limits": { - limits: validation.Limits{ - IngestionRateStrategy: validation.LocalIngestionRateStrategy, + limits: runtime.Limits{ + IngestionRateStrategy: runtime.LocalIngestionRateStrategy, IngestionRateMB: 1.0, IngestionBurstSizeMB: 2.0, }, @@ -33,8 +33,8 @@ func TestIngestionRateStrategy(t *testing.T) { expectedBurst: int(2.0 * float64(bytesInMB)), }, "global rate limiter should share the limit across the number of distributors": { - limits: validation.Limits{ - IngestionRateStrategy: validation.GlobalIngestionRateStrategy, + limits: runtime.Limits{ + IngestionRateStrategy: runtime.GlobalIngestionRateStrategy, IngestionRateMB: 1.0, IngestionBurstSizeMB: 2.0, }, @@ -47,8 +47,8 @@ func TestIngestionRateStrategy(t *testing.T) { expectedBurst: int(2.0 * float64(bytesInMB)), }, "global rate limiter should share nothing when there aren't any distributors": { - limits: validation.Limits{ - IngestionRateStrategy: validation.GlobalIngestionRateStrategy, + limits: runtime.Limits{ + IngestionRateStrategy: runtime.GlobalIngestionRateStrategy, IngestionRateMB: 1.0, IngestionBurstSizeMB: 2.0, }, @@ -67,14 +67,14 @@ func TestIngestionRateStrategy(t *testing.T) { var strategy limiter.RateLimiterStrategy // Init limits overrides - overrides, err := validation.NewOverrides(testData.limits, nil) + overrides, err := runtime.NewOverrides(testData.limits, nil) require.NoError(t, err) // Instance the strategy switch testData.limits.IngestionRateStrategy { - case validation.LocalIngestionRateStrategy: + case runtime.LocalIngestionRateStrategy: strategy = newLocalIngestionRateStrategy(overrides) - case validation.GlobalIngestionRateStrategy: + case runtime.GlobalIngestionRateStrategy: strategy = newGlobalIngestionRateStrategy(overrides, testData.ring) default: require.Fail(t, "Unknown strategy") diff --git a/pkg/distributor/limits.go b/pkg/distributor/limits.go index 62098dac6d96f..50e9c41e6c57d 100644 --- a/pkg/distributor/limits.go +++ b/pkg/distributor/limits.go @@ -5,12 +5,15 @@ import ( "github.com/grafana/loki/v3/pkg/compactor/retention" "github.com/grafana/loki/v3/pkg/distributor/shardstreams" + "github.com/grafana/loki/v3/pkg/distributor/writefailures" "github.com/grafana/loki/v3/pkg/loghttp/push" ) // Limits is an interface for distributor limits/related configs type Limits interface { retention.Limits + writefailures.Limits + MaxLineSize(userID string) int MaxLineSizeTruncate(userID string) bool MaxLabelNamesPerSeries(userID string) int diff --git a/pkg/distributor/ratestore_test.go b/pkg/distributor/ratestore_test.go index 5bfacf96ebd46..339061f437306 100644 --- a/pkg/distributor/ratestore_test.go +++ b/pkg/distributor/ratestore_test.go @@ -8,7 +8,7 @@ import ( "time" "github.com/grafana/loki/v3/pkg/distributor/shardstreams" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/stretchr/testify/require" @@ -338,8 +338,8 @@ type fakeOverrides struct { enabled bool } -func (c *fakeOverrides) AllByUserID() map[string]*validation.Limits { - return map[string]*validation.Limits{ +func (c *fakeOverrides) AllByUserID() map[string]*runtime.Limits { + return map[string]*runtime.Limits{ "ingester0": { ShardStreams: shardstreams.Config{ Enabled: c.enabled, diff --git a/pkg/distributor/validator_test.go b/pkg/distributor/validator_test.go index 9e51099dfad38..f03ee80f9cdd9 100644 --- a/pkg/distributor/validator_test.go +++ b/pkg/distributor/validator_test.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql/syntax" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/validation" ) @@ -25,15 +26,15 @@ var ( ) type fakeLimits struct { - limits *validation.Limits + limits *runtime.Limits } -func (f fakeLimits) TenantLimits(_ string) *validation.Limits { +func (f fakeLimits) TenantLimits(_ string) *runtime.Limits { return f.limits } // unused, but satisfies interface -func (f fakeLimits) AllByUserID() map[string]*validation.Limits { +func (f fakeLimits) AllByUserID() map[string]*runtime.Limits { return nil } @@ -41,7 +42,7 @@ func TestValidator_ValidateEntry(t *testing.T) { tests := []struct { name string userID string - overrides validation.TenantLimits + overrides runtime.TenantLimits entry logproto.Entry expected error }{ @@ -56,7 +57,7 @@ func TestValidator_ValidateEntry(t *testing.T) { "test too old", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ RejectOldSamples: true, RejectOldSamplesMaxAge: model.Duration(1 * time.Hour), }, @@ -79,7 +80,7 @@ func TestValidator_ValidateEntry(t *testing.T) { "line too long", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ MaxLineSize: 10, }, }, @@ -90,7 +91,7 @@ func TestValidator_ValidateEntry(t *testing.T) { "disallowed structured metadata", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ AllowStructuredMetadata: false, }, }, @@ -101,7 +102,7 @@ func TestValidator_ValidateEntry(t *testing.T) { "structured metadata too big", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ AllowStructuredMetadata: true, MaxStructuredMetadataSize: 4, }, @@ -113,7 +114,7 @@ func TestValidator_ValidateEntry(t *testing.T) { "structured metadata too many", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ AllowStructuredMetadata: true, MaxStructuredMetadataEntriesCount: 1, }, @@ -124,9 +125,9 @@ func TestValidator_ValidateEntry(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l := &validation.Limits{} + l := &runtime.Limits{} flagext.DefaultValues(l) - o, err := validation.NewOverrides(*l, tt.overrides) + o, err := runtime.NewOverrides(*l, tt.overrides) assert.NoError(t, err) v, err := NewValidator(o, nil) assert.NoError(t, err) @@ -141,7 +142,7 @@ func TestValidator_ValidateLabels(t *testing.T) { tests := []struct { name string userID string - overrides validation.TenantLimits + overrides runtime.TenantLimits labels string expected error }{ @@ -163,7 +164,7 @@ func TestValidator_ValidateLabels(t *testing.T) { "test too many labels", "test", fakeLimits{ - &validation.Limits{MaxLabelNamesPerSeries: 2}, + &runtime.Limits{MaxLabelNamesPerSeries: 2}, }, "{foo=\"bar\",food=\"bars\",fed=\"bears\"}", fmt.Errorf(validation.MaxLabelNamesPerSeriesErrorMsg, "{foo=\"bar\",food=\"bars\",fed=\"bears\"}", 3, 2), @@ -172,7 +173,7 @@ func TestValidator_ValidateLabels(t *testing.T) { "label name too long", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ MaxLabelNamesPerSeries: 2, MaxLabelNameLength: 5, }, @@ -184,7 +185,7 @@ func TestValidator_ValidateLabels(t *testing.T) { "label value too long", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ MaxLabelNamesPerSeries: 2, MaxLabelNameLength: 5, MaxLabelValueLength: 5, @@ -197,7 +198,7 @@ func TestValidator_ValidateLabels(t *testing.T) { "duplicate label", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ MaxLabelNamesPerSeries: 2, MaxLabelNameLength: 5, MaxLabelValueLength: 5, @@ -210,7 +211,7 @@ func TestValidator_ValidateLabels(t *testing.T) { "label value contains %", "test", fakeLimits{ - &validation.Limits{ + &runtime.Limits{ MaxLabelNamesPerSeries: 2, MaxLabelNameLength: 5, MaxLabelValueLength: 5, @@ -222,9 +223,9 @@ func TestValidator_ValidateLabels(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l := &validation.Limits{} + l := &runtime.Limits{} flagext.DefaultValues(l) - o, err := validation.NewOverrides(*l, tt.overrides) + o, err := runtime.NewOverrides(*l, tt.overrides) assert.NoError(t, err) v, err := NewValidator(o, nil) assert.NoError(t, err) diff --git a/pkg/distributor/writefailures/manager.go b/pkg/distributor/writefailures/manager.go index 5a02a7f2c2226..5a145f5406721 100644 --- a/pkg/distributor/writefailures/manager.go +++ b/pkg/distributor/writefailures/manager.go @@ -7,18 +7,25 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/dskit/limiter" "github.com/prometheus/client_golang/prometheus" - - "github.com/grafana/loki/v3/pkg/runtime" ) +type Limits interface { + LogStreamCreation(userID string) bool + LogPushRequest(userID string) bool + LogPushRequestStreams(userID string) bool + LogDuplicateMetrics(userID string) bool + LogDuplicateStreamInfo(userID string) bool + LimitedLogPushErrors(userID string) bool +} + type Manager struct { - limiter *limiter.RateLimiter - logger log.Logger - tenantCfgs *runtime.TenantConfigs - metrics *metrics + Limits + limiter *limiter.RateLimiter + logger log.Logger + metrics *metrics } -func NewManager(logger log.Logger, reg prometheus.Registerer, cfg Cfg, tenants *runtime.TenantConfigs, subsystem string) *Manager { +func NewManager(logger log.Logger, reg prometheus.Registerer, cfg Cfg, overrides Limits, subsystem string) *Manager { logger = log.With(logger, "path", "write") if cfg.AddInsightsLabel { logger = log.With(logger, "insight", "true") @@ -27,10 +34,10 @@ func NewManager(logger log.Logger, reg prometheus.Registerer, cfg Cfg, tenants * strategy := newStrategy(cfg.LogRate.Val(), float64(cfg.LogRate.Val())) return &Manager{ - limiter: limiter.NewRateLimiter(strategy, time.Minute), - logger: logger, - tenantCfgs: tenants, - metrics: newMetrics(reg, subsystem), + Limits: overrides, + limiter: limiter.NewRateLimiter(strategy, time.Minute), + logger: logger, + metrics: newMetrics(reg, subsystem), } } @@ -39,8 +46,8 @@ func (m *Manager) Log(tenantID string, err error) { return } - if !(m.tenantCfgs.LimitedLogPushErrors(tenantID) || - m.tenantCfgs.LogDuplicateStreamInfo(tenantID)) { + if !(m.LimitedLogPushErrors(tenantID) || + m.LogDuplicateStreamInfo(tenantID)) { return } diff --git a/pkg/distributor/writefailures/manager_test.go b/pkg/distributor/writefailures/manager_test.go index 1618e3f048e9c..6dc7aeda02293 100644 --- a/pkg/distributor/writefailures/manager_test.go +++ b/pkg/distributor/writefailures/manager_test.go @@ -15,28 +15,45 @@ import ( "github.com/grafana/loki/v3/pkg/util/flagext" ) +type mockTenantLimits struct { + providerFn func(string) *runtime.Limits +} + +// AllByUserID implements runtime.TenantLimits. +func (m mockTenantLimits) AllByUserID() map[string]*runtime.Limits { + panic("unimplemented") +} + +// TenantLimits implements runtime.TenantLimits. +func (m mockTenantLimits) TenantLimits(userID string) *runtime.Limits { + return m.providerFn(userID) +} + +var _ runtime.TenantLimits = mockTenantLimits{} + func TestWriteFailuresLogging(t *testing.T) { t.Run("it only logs for the configured tenants and is disabled by default", func(t *testing.T) { buf := bytes.NewBuffer(nil) logger := log.NewLogfmtLogger(buf) - f := &providerMock{ - tenantConfig: func(tenantID string) *runtime.Config { - if tenantID == "good-tenant" { - return &runtime.Config{ + tenantLimits := &mockTenantLimits{ + providerFn: func(tenantID string) *runtime.Limits { + switch tenantID { + case "good-tenant": + return &runtime.Limits{ LimitedLogPushErrors: true, } - } - if tenantID == "bad-tenant" { - return &runtime.Config{ + case "bad-tenant": + return &runtime.Limits{ LimitedLogPushErrors: false, } + default: + return &runtime.Limits{} } - return &runtime.Config{} }, } - runtimeCfg, err := runtime.NewTenantConfigs(f) + runtimeCfg, err := runtime.NewOverrides(runtime.Limits{}, tenantLimits) require.NoError(t, err) manager := NewManager(logger, prometheus.NewRegistry(), Cfg{LogRate: flagext.ByteSize(1000)}, runtimeCfg, "ingester") @@ -57,14 +74,15 @@ func TestWriteFailuresRateLimiting(t *testing.T) { buf := bytes.NewBuffer(nil) logger := log.NewLogfmtLogger(buf) - provider := &providerMock{ - tenantConfig: func(_ string) *runtime.Config { - return &runtime.Config{ + tenantLimits := &mockTenantLimits{ + providerFn: func(_ string) *runtime.Limits { + return &runtime.Limits{ LimitedLogPushErrors: true, } }, } - runtimeCfg, err := runtime.NewTenantConfigs(provider) + + runtimeCfg, err := runtime.NewOverrides(runtime.Limits{}, tenantLimits) require.NoError(t, err) t.Run("with zero rate limiting", func(t *testing.T) { @@ -130,7 +148,7 @@ func TestWriteFailuresRateLimiting(t *testing.T) { }) t.Run("limit is per-tenant", func(t *testing.T) { - runtimeCfg, err := runtime.NewTenantConfigs(provider) + runtimeCfg, err := runtime.NewOverrides(runtime.Limits{}, tenantLimits) require.NoError(t, err) manager := NewManager(logger, prometheus.NewRegistry(), Cfg{LogRate: flagext.ByteSize(1000)}, runtimeCfg, "ingester") @@ -165,11 +183,3 @@ func TestWriteFailuresRateLimiting(t *testing.T) { require.Contains(t, content, "3z") }) } - -type providerMock struct { - tenantConfig func(string) *runtime.Config -} - -func (m *providerMock) TenantConfig(userID string) *runtime.Config { - return m.tenantConfig(userID) -} diff --git a/pkg/indexgateway/client_test.go b/pkg/indexgateway/client_test.go index 91005a591eb15..74747a7e88fdc 100644 --- a/pkg/indexgateway/client_test.go +++ b/pkg/indexgateway/client_test.go @@ -22,9 +22,9 @@ import ( "github.com/grafana/loki/v3/pkg/distributor/clientpool" "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/stores/series/index" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" ) // const ( @@ -103,13 +103,13 @@ func createTestGrpcServer(t *testing.T) (func(), string) { return s.GracefulStop, lis.Addr().String() } -type mockTenantLimits map[string]*validation.Limits +type mockTenantLimits map[string]*runtime.Limits -func (tl mockTenantLimits) TenantLimits(userID string) *validation.Limits { +func (tl mockTenantLimits) TenantLimits(userID string) *runtime.Limits { return tl[userID] } -func (tl mockTenantLimits) AllByUserID() map[string]*validation.Limits { +func (tl mockTenantLimits) AllByUserID() map[string]*runtime.Limits { return tl } @@ -172,7 +172,7 @@ func TestGatewayClient_RingMode(t *testing.T) { }) t.Run("global shard size", func(t *testing.T) { - o, err := validation.NewOverrides(validation.Limits{IndexGatewayShardSize: s}, nil) + o, err := runtime.NewOverrides(runtime.Limits{IndexGatewayShardSize: s}, nil) require.NoError(t, err) cfg := ClientConfig{} @@ -200,10 +200,10 @@ func TestGatewayClient_RingMode(t *testing.T) { t.Run("per tenant shard size", func(t *testing.T) { tl := mockTenantLimits{ - "12345": &validation.Limits{IndexGatewayShardSize: 1}, + "12345": &runtime.Limits{IndexGatewayShardSize: 1}, // tenant 67890 has not tenant specific overrides } - o, err := validation.NewOverrides(validation.Limits{IndexGatewayShardSize: s}, tl) + o, err := runtime.NewOverrides(runtime.Limits{IndexGatewayShardSize: s}, tl) require.NoError(t, err) cfg := ClientConfig{} @@ -241,7 +241,7 @@ func TestGatewayClient(t *testing.T) { cfg.Address = storeAddress cfg.PoolConfig = clientpool.PoolConfig{ClientCleanupPeriod: 500 * time.Millisecond} - overrides, _ := validation.NewOverrides(validation.Limits{}, nil) + overrides, _ := runtime.NewOverrides(runtime.Limits{}, nil) gatewayClient, err := NewGatewayClient(cfg, prometheus.DefaultRegisterer, overrides, logger, constants.Loki) require.NoError(t, err) @@ -285,12 +285,12 @@ func buildTableName(i int) string { type mockLimits struct{} -func (m mockLimits) AllByUserID() map[string]*validation.Limits { - return map[string]*validation.Limits{} +func (m mockLimits) AllByUserID() map[string]*runtime.Limits { + return map[string]*runtime.Limits{} } -func (m mockLimits) DefaultLimits() *validation.Limits { - return &validation.Limits{} +func (m mockLimits) DefaultLimits() *runtime.Limits { + return &runtime.Limits{} } func benchmarkIndexQueries(b *testing.B, queries []index.Query) { @@ -423,7 +423,7 @@ func Benchmark_QueriesMatchingLargeNumOfRows(b *testing.B) { func TestDoubleRegistration(t *testing.T) { logger := log.NewNopLogger() r := prometheus.NewRegistry() - o, _ := validation.NewOverrides(validation.Limits{}, nil) + o, _ := runtime.NewOverrides(runtime.Limits{}, nil) clientCfg := ClientConfig{ Address: "my-store-address:1234", diff --git a/pkg/ingester/checkpoint_test.go b/pkg/ingester/checkpoint_test.go index 7f30898bd7b7c..05de421b7d47b 100644 --- a/pkg/ingester/checkpoint_test.go +++ b/pkg/ingester/checkpoint_test.go @@ -24,7 +24,6 @@ import ( "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" ) // small util for ensuring data exists as we expect @@ -62,7 +61,7 @@ func TestIngesterWAL(t *testing.T) { ingesterConfig := defaultIngesterTestConfigWithWAL(t, walDir) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) newStore := func() *mockStore { @@ -73,7 +72,7 @@ func TestIngesterWAL(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -116,7 +115,7 @@ func TestIngesterWAL(t *testing.T) { expectCheckpoint(t, walDir, false, time.Second) // restart the ingester - i, err = New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err = New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) @@ -130,7 +129,7 @@ func TestIngesterWAL(t *testing.T) { require.Nil(t, services.StopAndAwaitTerminated(context.Background(), i)) // restart the ingester - i, err = New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err = New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) @@ -144,7 +143,7 @@ func TestIngesterWALIgnoresStreamLimits(t *testing.T) { ingesterConfig := defaultIngesterTestConfigWithWAL(t, walDir) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) newStore := func() *mockStore { @@ -155,7 +154,7 @@ func TestIngesterWALIgnoresStreamLimits(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -197,11 +196,11 @@ func TestIngesterWALIgnoresStreamLimits(t *testing.T) { // Limit all streams except those written during WAL recovery. limitCfg := defaultLimitsTestConfig() limitCfg.MaxLocalStreamsPerUser = -1 - limits, err = validation.NewOverrides(limitCfg, nil) + limits, err = runtime.NewOverrides(limitCfg, nil) require.NoError(t, err) // restart the ingester - i, err = New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err = New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) @@ -249,7 +248,7 @@ func TestIngesterWALBackpressureSegments(t *testing.T) { ingesterConfig := defaultIngesterTestConfigWithWAL(t, walDir) ingesterConfig.WAL.ReplayMemoryCeiling = 1000 - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) newStore := func() *mockStore { @@ -260,7 +259,7 @@ func TestIngesterWALBackpressureSegments(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -281,7 +280,7 @@ func TestIngesterWALBackpressureSegments(t *testing.T) { expectCheckpoint(t, walDir, false, time.Second) // restart the ingester, ensuring we replayed from WAL. - i, err = New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err = New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) @@ -293,7 +292,7 @@ func TestIngesterWALBackpressureCheckpoint(t *testing.T) { ingesterConfig := defaultIngesterTestConfigWithWAL(t, walDir) ingesterConfig.WAL.ReplayMemoryCeiling = 1000 - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) newStore := func() *mockStore { @@ -304,7 +303,7 @@ func TestIngesterWALBackpressureCheckpoint(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -325,7 +324,7 @@ func TestIngesterWALBackpressureCheckpoint(t *testing.T) { require.Nil(t, services.StopAndAwaitTerminated(context.Background(), i)) // restart the ingester, ensuring we can replay from the checkpoint as well. - i, err = New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err = New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) @@ -456,13 +455,14 @@ func Test_SeriesIterator(t *testing.T) { l.IngestionRateMB = 1e4 l.IngestionBurstSizeMB = 1e4 - limits, err := validation.NewOverrides(l, nil) + limits, err := runtime.NewOverrides(l, nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) for i := 0; i < 3; i++ { - inst, err := newInstance(defaultConfig(), defaultPeriodConfigs, fmt.Sprintf("%d", i), limiter, runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, nil, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + inst, err := newInstance(defaultConfig(), defaultPeriodConfigs, fmt.Sprintf("%d", i), limiter, noopWAL{}, NilMetrics, nil, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) require.Nil(t, err) require.NoError(t, inst.Push(context.Background(), &logproto.PushRequest{Streams: []logproto.Stream{stream1}})) require.NoError(t, inst.Push(context.Background(), &logproto.PushRequest{Streams: []logproto.Stream{stream2}})) @@ -504,13 +504,14 @@ func Benchmark_SeriesIterator(b *testing.B) { streams := buildStreams() instances := make([]*instance, 10) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(b, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) for i := range instances { - inst, _ := newInstance(defaultConfig(), defaultPeriodConfigs, fmt.Sprintf("instance %d", i), limiter, runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, nil, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + inst, _ := newInstance(defaultConfig(), defaultPeriodConfigs, fmt.Sprintf("instance %d", i), limiter, noopWAL{}, NilMetrics, nil, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) require.NoError(b, inst.Push(context.Background(), &logproto.PushRequest{ @@ -593,7 +594,7 @@ func TestIngesterWALReplaysUnorderedToOrdered(t *testing.T) { // First launch the ingester with unordered writes enabled dft := defaultLimitsTestConfig() dft.UnorderedWrites = true - limits, err := validation.NewOverrides(dft, nil) + limits, err := runtime.NewOverrides(dft, nil) require.NoError(t, err) newStore := func() *mockStore { @@ -604,7 +605,7 @@ func TestIngesterWALReplaysUnorderedToOrdered(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -672,11 +673,11 @@ func TestIngesterWALReplaysUnorderedToOrdered(t *testing.T) { // Now disable unordered writes limitCfg := defaultLimitsTestConfig() limitCfg.UnorderedWrites = false - limits, err = validation.NewOverrides(limitCfg, nil) + limits, err = runtime.NewOverrides(limitCfg, nil) require.NoError(t, err) // restart the ingester - i, err = New(ingesterConfig, client.Config{}, newStore(), limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) + i, err = New(ingesterConfig, client.Config{}, newStore(), limits, nil, writefailures.Cfg{}, constants.Loki, gokit_log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck require.Nil(t, services.StartAndAwaitRunning(context.Background(), i)) diff --git a/pkg/ingester/flush_test.go b/pkg/ingester/flush_test.go index f4251747115a2..e5fcece82a8f3 100644 --- a/pkg/ingester/flush_test.go +++ b/pkg/ingester/flush_test.go @@ -39,7 +39,6 @@ import ( "github.com/grafana/loki/v3/pkg/storage/stores/index/stats" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" ) const ( @@ -390,10 +389,10 @@ func newTestStore(t require.TestingT, cfg Config, walOverride WAL) (*testStore, readRingMock := mockReadRingWithOneActiveIngester() - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) - ing, err := New(cfg, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, gokitlog.NewNopLogger(), nil, readRingMock, nil) + ing, err := New(cfg, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, gokitlog.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index d0da49d359ddf..47a0979ec45e9 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -48,7 +48,6 @@ import ( "github.com/grafana/loki/v3/pkg/logqlmodel/metadata" "github.com/grafana/loki/v3/pkg/logqlmodel/stats" "github.com/grafana/loki/v3/pkg/querier/plan" - "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/config" @@ -237,8 +236,7 @@ type Ingester struct { cfg Config logger log.Logger - clientConfig client.Config - tenantConfigs *runtime.TenantConfigs + clientConfig client.Config shutdownMtx sync.Mutex // Allows processes to grab a lock and prevent a shutdown instancesMtx sync.RWMutex @@ -300,7 +298,7 @@ type Ingester struct { } // New makes a new Ingester. -func New(cfg Config, clientConfig client.Config, store Store, limits Limits, configs *runtime.TenantConfigs, registerer prometheus.Registerer, writeFailuresCfg writefailures.Cfg, metricsNamespace string, logger log.Logger, customStreamsTracker push.UsageTracker, readRing ring.ReadRing, partitionRingWatcher ring.PartitionRingReader) (*Ingester, error) { +func New(cfg Config, clientConfig client.Config, store Store, limits Limits, registerer prometheus.Registerer, writeFailuresCfg writefailures.Cfg, metricsNamespace string, logger log.Logger, customStreamsTracker push.UsageTracker, readRing ring.ReadRing, partitionRingWatcher ring.PartitionRingReader) (*Ingester, error) { if cfg.ingesterClientFactory == nil { cfg.ingesterClientFactory = client.New } @@ -316,7 +314,6 @@ func New(cfg Config, clientConfig client.Config, store Store, limits Limits, con cfg: cfg, logger: logger, clientConfig: clientConfig, - tenantConfigs: configs, instances: map[string]*instance{}, store: store, periodicConfigs: store.GetSchemaConfigs(), @@ -328,7 +325,7 @@ func New(cfg Config, clientConfig client.Config, store Store, limits Limits, con flushOnShutdownSwitch: &OnceSwitch{}, terminateOnShutdown: false, streamRateCalculator: NewStreamRateCalculator(), - writeLogManager: writefailures.NewManager(logger, registerer, writeFailuresCfg, configs, "ingester"), + writeLogManager: writefailures.NewManager(logger, registerer, writeFailuresCfg, limits, "ingester"), customStreamsTracker: customStreamsTracker, readRing: readRing, } @@ -1038,7 +1035,7 @@ func (i *Ingester) GetOrCreateInstance(instanceID string) (*instance, error) { / inst, ok = i.instances[instanceID] if !ok { var err error - inst, err = newInstance(&i.cfg, i.periodicConfigs, instanceID, i.limiter, i.tenantConfigs, i.wal, i.metrics, i.flushOnShutdownSwitch, i.chunkFilter, i.pipelineWrapper, i.extractorWrapper, i.streamRateCalculator, i.writeLogManager, i.customStreamsTracker) + inst, err = newInstance(&i.cfg, i.periodicConfigs, instanceID, i.limiter, i.wal, i.metrics, i.flushOnShutdownSwitch, i.chunkFilter, i.pipelineWrapper, i.extractorWrapper, i.streamRateCalculator, i.writeLogManager, i.customStreamsTracker) if err != nil { return nil, err } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index ebaa4b1904dc8..845a7b907301d 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -51,12 +51,11 @@ import ( "github.com/grafana/loki/v3/pkg/storage/stores/index/stats" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" ) func TestPrepareShutdownMarkerPathNotSet(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ @@ -65,7 +64,7 @@ func TestPrepareShutdownMarkerPathNotSet(t *testing.T) { mockRing := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, mockRing, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, mockRing, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -81,7 +80,7 @@ func TestPrepareShutdown(t *testing.T) { ingesterConfig.WAL.Enabled = true ingesterConfig.WAL.Dir = tempDir ingesterConfig.LifecyclerConfig.UnregisterOnShutdown = false - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ @@ -90,7 +89,7 @@ func TestPrepareShutdown(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -144,7 +143,7 @@ func TestPrepareShutdown(t *testing.T) { func TestIngester_GetStreamRates_Correctness(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ @@ -153,7 +152,7 @@ func TestIngester_GetStreamRates_Correctness(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -178,7 +177,7 @@ func TestIngester_GetStreamRates_Correctness(t *testing.T) { func BenchmarkGetStreamRatesAllocs(b *testing.B) { ingesterConfig := defaultIngesterTestConfig(b) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(b, err) store := &mockStore{ @@ -186,7 +185,7 @@ func BenchmarkGetStreamRatesAllocs(b *testing.B) { } readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(b, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -203,7 +202,7 @@ func BenchmarkGetStreamRatesAllocs(b *testing.B) { func TestIngester(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ @@ -212,7 +211,7 @@ func TestIngester(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -389,7 +388,7 @@ func TestIngesterStreamLimitExceeded(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) defaultLimits := defaultLimitsTestConfig() defaultLimits.MaxLocalStreamsPerUser = 1 - overrides, err := validation.NewOverrides(defaultLimits, nil) + overrides, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) @@ -399,7 +398,7 @@ func TestIngesterStreamLimitExceeded(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, overrides, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, overrides, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -535,8 +534,8 @@ func (m *mockQuerierServer) Context() context.Context { return m.ctx } -func defaultLimitsTestConfig() validation.Limits { - limits := validation.Limits{} +func defaultLimitsTestConfig() runtime.Limits { + limits := runtime.Limits{} flagext.DefaultValues(&limits) return limits } @@ -812,7 +811,7 @@ func TestValidate(t *testing.T) { func Test_InMemoryLabels(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ @@ -821,7 +820,7 @@ func Test_InMemoryLabels(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -869,14 +868,14 @@ func TestIngester_GetDetectedLabels(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "test") ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ chunks: map[string][]chunk.Chunk{}, } readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -933,14 +932,14 @@ func TestIngester_GetDetectedLabelsWithQuery(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "test") ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ chunks: map[string][]chunk.Chunk{}, } readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -1304,11 +1303,11 @@ func Test_DedupeIngesterParser(t *testing.T) { func TestStats(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, &mockStore{}, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, &mockStore{}, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) i.instances["test"] = defaultInstance(t) @@ -1332,11 +1331,11 @@ func TestStats(t *testing.T) { func TestVolume(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, &mockStore{}, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, &mockStore{}, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) i.instances["test"] = defaultInstance(t) @@ -1412,11 +1411,11 @@ func createIngesterSets(t *testing.T, config Config, count int) ([]ingesterClien func createIngesterServer(t *testing.T, ingesterConfig Config) (ingesterClient, func()) { t.Helper() - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) readRingMock := mockReadRingWithOneActiveIngester() - ing, err := New(ingesterConfig, client.Config{}, &mockStore{}, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + ing, err := New(ingesterConfig, client.Config{}, &mockStore{}, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) listener := bufconn.Listen(1024 * 1024) @@ -1629,11 +1628,11 @@ func mockReadRingWithOneActiveIngester() *readRingMock { func TestUpdateOwnedStreams(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, &mockStore{}, limits, runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, &mockStore{}, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) i.instances["test"] = defaultInstance(t) diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index c6afcacfbdfde..aa44125d413fc 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -36,7 +36,6 @@ import ( "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/logqlmodel/metadata" "github.com/grafana/loki/v3/pkg/logqlmodel/stats" - "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/index/seriesvolume" @@ -106,8 +105,6 @@ type instance struct { streamCountLimiter *streamCountLimiter ownedStreamsSvc *ownedStreamService - configs *runtime.TenantConfigs - wal WAL // Denotes whether the ingester should flush on shutdown. @@ -133,7 +130,6 @@ func newInstance( periodConfigs []config.PeriodConfig, instanceID string, limiter *Limiter, - configs *runtime.TenantConfigs, wal WAL, metrics *ingesterMetrics, flushOnShutdownSwitch *OnceSwitch, @@ -165,7 +161,6 @@ func newInstance( limiter: limiter, streamCountLimiter: newStreamCountLimiter(instanceID, streams.Len, limiter, ownedStreamsSvc), ownedStreamsSvc: ownedStreamsSvc, - configs: configs, wal: wal, metrics: metrics, @@ -279,7 +274,7 @@ func (i *instance) createStream(ctx context.Context, pushReqStream logproto.Stre labels, err := syntax.ParseLabels(pushReqStream.Labels) if err != nil { - if i.configs.LogStreamCreation(i.instanceID) { + if i.writeFailures.LogStreamCreation(i.instanceID) { level.Debug(util_log.Logger).Log( "msg", "failed to create stream, failed to parse labels", "org_id", i.instanceID, @@ -307,7 +302,7 @@ func (i *instance) createStream(ctx context.Context, pushReqStream logproto.Stre return nil, fmt.Errorf("failed to create stream: %w", err) } - s := newStream(chunkfmt, headfmt, i.cfg, i.limiter.rateLimitStrategy, i.instanceID, fp, sortedLabels, i.limiter.UnorderedWrites(i.instanceID), i.streamRateCalculator, i.metrics, i.writeFailures, i.configs) + s := newStream(chunkfmt, headfmt, i.cfg, i.limiter.rateLimitStrategy, i.instanceID, fp, sortedLabels, i.limiter.UnorderedWrites(i.instanceID), i.streamRateCalculator, i.metrics, i.writeFailures) // record will be nil when replaying the wal (we don't want to rewrite wal entries as we replay them). if record != nil { @@ -326,7 +321,7 @@ func (i *instance) createStream(ctx context.Context, pushReqStream logproto.Stre } func (i *instance) onStreamCreationError(ctx context.Context, pushReqStream logproto.Stream, err error, labels labels.Labels) (*stream, error) { - if i.configs.LogStreamCreation(i.instanceID) || i.cfg.KafkaIngestion.Enabled { + if i.writeFailures.LogStreamCreation(i.instanceID) || i.cfg.KafkaIngestion.Enabled { l := level.Debug(util_log.Logger) if i.cfg.KafkaIngestion.Enabled { @@ -358,7 +353,7 @@ func (i *instance) onStreamCreated(s *stream) { streamsCountStats.Add(1) // we count newly created stream as owned i.ownedStreamsSvc.trackStreamOwnership(s.fp, true) - if i.configs.LogStreamCreation(i.instanceID) { + if i.writeFailures.LogStreamCreation(i.instanceID) { level.Debug(util_log.Logger).Log( "msg", "successfully created stream", "org_id", i.instanceID, @@ -375,7 +370,7 @@ func (i *instance) createStreamByFP(ls labels.Labels, fp model.Fingerprint) (*st return nil, fmt.Errorf("failed to create stream for fingerprint: %w", err) } - s := newStream(chunkfmt, headfmt, i.cfg, i.limiter.rateLimitStrategy, i.instanceID, fp, sortedLabels, i.limiter.UnorderedWrites(i.instanceID), i.streamRateCalculator, i.metrics, i.writeFailures, i.configs) + s := newStream(chunkfmt, headfmt, i.cfg, i.limiter.rateLimitStrategy, i.instanceID, fp, sortedLabels, i.limiter.UnorderedWrites(i.instanceID), i.streamRateCalculator, i.metrics, i.writeFailures) i.onStreamCreated(s) diff --git a/pkg/ingester/instance_test.go b/pkg/ingester/instance_test.go index 369d0ab2d7469..cf3b51c60a7c2 100644 --- a/pkg/ingester/instance_test.go +++ b/pkg/ingester/instance_test.go @@ -4,38 +4,37 @@ import ( "context" "fmt" "math/rand" - "runtime" + rt "runtime" "sort" "sync" "testing" "time" - "github.com/grafana/loki/v3/pkg/storage/types" - "github.com/grafana/loki/v3/pkg/util/httpreq" - - "github.com/grafana/dskit/tenant" - "github.com/grafana/dskit/user" - - "github.com/grafana/loki/v3/pkg/logql/log" - + gokitlog "github.com/go-kit/log" "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/flagext" + "github.com/grafana/dskit/tenant" + "github.com/grafana/dskit/user" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" "github.com/grafana/loki/v3/pkg/distributor/shardstreams" + "github.com/grafana/loki/v3/pkg/distributor/writefailures" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql" + "github.com/grafana/loki/v3/pkg/logql/log" "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/querier/astmapper" "github.com/grafana/loki/v3/pkg/querier/plan" - loki_runtime "github.com/grafana/loki/v3/pkg/runtime" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/index/seriesvolume" + "github.com/grafana/loki/v3/pkg/storage/types" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/util/httpreq" ) func defaultConfig() *Config { @@ -76,11 +75,12 @@ var defaultPeriodConfigs = []config.PeriodConfig{ var NilMetrics = newIngesterMetrics(nil, constants.Loki) func TestLabelsCollisions(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) - i, err := newInstance(defaultConfig(), defaultPeriodConfigs, "test", limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + i, err := newInstance(defaultConfig(), defaultPeriodConfigs, "test", limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) require.Nil(t, err) // avoid entries from the future. @@ -104,11 +104,12 @@ func TestLabelsCollisions(t *testing.T) { } func TestConcurrentPushes(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) - inst, err := newInstance(defaultConfig(), defaultPeriodConfigs, "test", limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + inst, err := newInstance(defaultConfig(), defaultPeriodConfigs, "test", limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) require.Nil(t, err) const ( @@ -156,11 +157,12 @@ func TestConcurrentPushes(t *testing.T) { } func TestGetStreamRates(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) - inst, err := newInstance(defaultConfig(), defaultPeriodConfigs, "test", limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + inst, err := newInstance(defaultConfig(), defaultPeriodConfigs, "test", limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) require.NoError(t, err) const ( @@ -243,9 +245,10 @@ func labelHashNoShard(l labels.Labels) uint64 { } func TestSyncPeriod(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) const ( syncPeriod = 1 * time.Minute @@ -254,7 +257,7 @@ func TestSyncPeriod(t *testing.T) { minUtil = 0.20 ) - inst, err := newInstance(defaultConfig(), defaultPeriodConfigs, "test", limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + inst, err := newInstance(defaultConfig(), defaultPeriodConfigs, "test", limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) require.Nil(t, err) lbls := makeRandomLabels() @@ -288,9 +291,11 @@ func TestSyncPeriod(t *testing.T) { func setupTestStreams(t *testing.T) (*instance, time.Time, int) { t.Helper() - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) + indexShards := 2 // just some random values @@ -299,7 +304,7 @@ func setupTestStreams(t *testing.T) (*instance, time.Time, int) { cfg.SyncMinUtilization = 0.20 cfg.IndexShards = indexShards - instance, err := newInstance(cfg, defaultPeriodConfigs, "test", limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + instance, err := newInstance(cfg, defaultPeriodConfigs, "test", limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) require.Nil(t, err) currentTime := time.Now() @@ -315,7 +320,7 @@ func setupTestStreams(t *testing.T) (*instance, time.Time, int) { require.NoError(t, err) chunkfmt, headfmt, err := instance.chunkFormatAt(minTs(&testStream)) require.NoError(t, err) - chunk := newStream(chunkfmt, headfmt, cfg, limiter.rateLimitStrategy, "fake", 0, nil, true, NewStreamRateCalculator(), NilMetrics, nil, nil).NewChunk() + chunk := newStream(chunkfmt, headfmt, cfg, limiter.rateLimitStrategy, "fake", 0, nil, true, NewStreamRateCalculator(), NilMetrics, nil).NewChunk() for _, entry := range testStream.Entries { dup, err := chunk.Append(&entry) require.False(t, dup) @@ -505,11 +510,12 @@ func makeRandomLabels() labels.Labels { } func Benchmark_PushInstance(b *testing.B) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(b, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) - i, _ := newInstance(&Config{IndexShards: 1}, defaultPeriodConfigs, "test", limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + i, _ := newInstance(&Config{IndexShards: 1}, defaultPeriodConfigs, "test", limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) ctx := context.Background() for n := 0; n < b.N; n++ { @@ -547,13 +553,14 @@ func Benchmark_PushInstance(b *testing.B) { func Benchmark_instance_addNewTailer(b *testing.B) { l := defaultLimitsTestConfig() l.MaxLocalStreamsPerUser = 100000 - limits, err := validation.NewOverrides(l, nil) + limits, err := runtime.NewOverrides(l, nil) require.NoError(b, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) ctx := context.Background() - inst, _ := newInstance(&Config{}, defaultPeriodConfigs, "test", limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, nil) + inst, _ := newInstance(&Config{}, defaultPeriodConfigs, "test", limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) expr, err := syntax.ParseLogSelector(`{namespace="foo",pod="bar",instance=~"10.*"}`, true) require.NoError(b, err) t, err := newTailer("foo", expr, nil, 10) @@ -575,13 +582,13 @@ func Benchmark_instance_addNewTailer(b *testing.B) { b.Run("addTailersToNewStream", func(b *testing.B) { for n := 0; n < b.N; n++ { - inst.addTailersToNewStream(newStream(chunkfmt, headfmt, nil, limiter.rateLimitStrategy, "fake", 0, lbs, true, NewStreamRateCalculator(), NilMetrics, nil, nil)) + inst.addTailersToNewStream(newStream(chunkfmt, headfmt, nil, limiter.rateLimitStrategy, "fake", 0, lbs, true, NewStreamRateCalculator(), NilMetrics, nil)) } }) } func Benchmark_OnceSwitch(b *testing.B) { - threads := runtime.GOMAXPROCS(0) + threads := rt.GOMAXPROCS(0) // limit threads if threads > 4 { @@ -1043,10 +1050,10 @@ func Test_QuerySampleWithDelete(t *testing.T) { } type fakeLimits struct { - limits map[string]*validation.Limits + limits map[string]*runtime.Limits } -func (f fakeLimits) TenantLimits(userID string) *validation.Limits { +func (f fakeLimits) TenantLimits(userID string) *runtime.Limits { limits, ok := f.limits[userID] if !ok { return nil @@ -1055,16 +1062,16 @@ func (f fakeLimits) TenantLimits(userID string) *validation.Limits { return limits } -func (f fakeLimits) AllByUserID() map[string]*validation.Limits { +func (f fakeLimits) AllByUserID() map[string]*runtime.Limits { return f.limits } func TestStreamShardingUsage(t *testing.T) { - setupCustomTenantLimit := func(perStreamLimit string) *validation.Limits { + setupCustomTenantLimit := func(perStreamLimit string) *runtime.Limits { shardStreamsCfg := shardstreams.Config{Enabled: true, LoggingEnabled: true} shardStreamsCfg.DesiredRate.Set("6MB") //nolint:errcheck - customTenantLimits := &validation.Limits{} + customTenantLimits := &runtime.Limits{} flagext.DefaultValues(customTenantLimits) customTenantLimits.PerStreamRateLimit.Set(perStreamLimit) //nolint:errcheck @@ -1078,7 +1085,7 @@ func TestStreamShardingUsage(t *testing.T) { customTenant2 := "my-org2" limitsDefinition := &fakeLimits{ - limits: make(map[string]*validation.Limits), + limits: make(map[string]*runtime.Limits), } // testing with 1 because although 1 is enough to accept at least the // first line entry, because per-stream sharding is enabled, @@ -1086,10 +1093,11 @@ func TestStreamShardingUsage(t *testing.T) { limitsDefinition.limits[customTenant1] = setupCustomTenantLimit("1") limitsDefinition.limits[customTenant2] = setupCustomTenantLimit("4") - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), limitsDefinition) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), limitsDefinition) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) + wfm := newWriteFailuresManager(limits) defaultShardStreamsCfg := limiter.limits.ShardStreams("fake") tenantShardStreamsCfg := limiter.limits.ShardStreams(customTenant1) @@ -1108,10 +1116,9 @@ func TestStreamShardingUsage(t *testing.T) { t.Run("invalid push returns error", func(t *testing.T) { tracker := &mockUsageTracker{} + i, _ := newInstance(&Config{IndexShards: 1, OwnedStreamsCheckInterval: 1 * time.Second}, defaultPeriodConfigs, customTenant1, limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, tracker) - i, _ := newInstance(&Config{IndexShards: 1, OwnedStreamsCheckInterval: 1 * time.Second}, defaultPeriodConfigs, customTenant1, limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, tracker) ctx := context.Background() - err = i.Push(ctx, &logproto.PushRequest{ Streams: []logproto.Stream{ { @@ -1129,9 +1136,9 @@ func TestStreamShardingUsage(t *testing.T) { }) t.Run("valid push returns no error", func(t *testing.T) { - i, _ := newInstance(&Config{IndexShards: 1, OwnedStreamsCheckInterval: 1 * time.Second}, defaultPeriodConfigs, customTenant2, limiter, loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), nil, nil) - ctx := context.Background() + i, _ := newInstance(&Config{IndexShards: 1, OwnedStreamsCheckInterval: 1 * time.Second}, defaultPeriodConfigs, customTenant2, limiter, noopWAL{}, NilMetrics, &OnceSwitch{}, nil, nil, nil, NewStreamRateCalculator(), wfm, nil) + ctx := context.Background() err = i.Push(ctx, &logproto.PushRequest{ Streams: []logproto.Stream{ { @@ -1448,14 +1455,13 @@ func TestGetStats(t *testing.T) { func defaultInstance(t *testing.T) *instance { ingesterConfig := defaultIngesterTestConfig(t) defaultLimits := defaultLimitsTestConfig() - overrides, err := validation.NewOverrides(defaultLimits, nil) + overrides, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) instance, err := newInstance( &ingesterConfig, defaultPeriodConfigs, "fake", NewLimiter(overrides, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: overrides}), - loki_runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, nil, @@ -1463,7 +1469,7 @@ func defaultInstance(t *testing.T) *instance { nil, nil, NewStreamRateCalculator(), - nil, + newWriteFailuresManager(overrides), nil, ) require.Nil(t, err) @@ -1564,3 +1570,7 @@ func (m *mockUsageTracker) DiscardedBytesAdd(_ context.Context, _ string, _ stri // ReceivedBytesAdd implements push.UsageTracker. func (*mockUsageTracker) ReceivedBytesAdd(_ context.Context, _ string, _ time.Duration, _ labels.Labels, _ float64) { } + +func newWriteFailuresManager(limits *runtime.Overrides) *writefailures.Manager { + return writefailures.NewManager(gokitlog.NewNopLogger(), prometheus.NewPedanticRegistry(), writefailures.Cfg{}, limits, "ingester") +} diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index c4e64149f1658..eab4352859915 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -10,7 +10,8 @@ import ( "golang.org/x/time/rate" "github.com/grafana/loki/v3/pkg/distributor/shardstreams" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/distributor/writefailures" + "github.com/grafana/loki/v3/pkg/runtime" ) const ( @@ -26,11 +27,12 @@ type RingCount interface { } type Limits interface { + writefailures.Limits UnorderedWrites(userID string) bool UseOwnedStreamCount(userID string) bool MaxLocalStreamsPerUser(userID string) int MaxGlobalStreamsPerUser(userID string) int - PerStreamRateLimit(userID string) validation.RateLimit + PerStreamRateLimit(userID string) runtime.RateLimit ShardStreams(userID string) shardstreams.Config IngestionPartitionsTenantShardSize(userID string) int } @@ -234,7 +236,7 @@ func (l *streamCountLimiter) getSuppliers(tenant string) (streamCountSupplier, f } type RateLimiterStrategy interface { - RateLimit(tenant string) validation.RateLimit + RateLimit(tenant string) runtime.RateLimit SetDisabled(bool) } @@ -243,9 +245,9 @@ type TenantBasedStrategy struct { limits Limits } -func (l *TenantBasedStrategy) RateLimit(tenant string) validation.RateLimit { +func (l *TenantBasedStrategy) RateLimit(tenant string) runtime.RateLimit { if l.disabled { - return validation.Unlimited + return runtime.Unlimited } return l.limits.PerStreamRateLimit(tenant) @@ -257,8 +259,8 @@ func (l *TenantBasedStrategy) SetDisabled(disabled bool) { type NoLimitsStrategy struct{} -func (l *NoLimitsStrategy) RateLimit(_ string) validation.RateLimit { - return validation.Unlimited +func (l *NoLimitsStrategy) RateLimit(_ string) runtime.RateLimit { + return runtime.Unlimited } func (l *NoLimitsStrategy) SetDisabled(_ bool) { diff --git a/pkg/ingester/limiter_test.go b/pkg/ingester/limiter_test.go index b611db4d109e1..1c44d142c01f4 100644 --- a/pkg/ingester/limiter_test.go +++ b/pkg/ingester/limiter_test.go @@ -12,7 +12,7 @@ import ( "go.uber.org/atomic" "golang.org/x/time/rate" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) type fixedStrategy struct { @@ -121,7 +121,7 @@ func TestStreamCountLimiter_AssertNewStreamAllowed(t *testing.T) { for testName, testData := range tests { t.Run(testName, func(t *testing.T) { // Mock limits - limits, err := validation.NewOverrides(validation.Limits{ + limits, err := runtime.NewOverrides(runtime.Limits{ MaxLocalStreamsPerUser: testData.maxLocalStreamsPerUser, MaxGlobalStreamsPerUser: testData.maxGlobalStreamsPerUser, UseOwnedStreamCount: testData.useOwnedStreamService, diff --git a/pkg/ingester/owned_streams_test.go b/pkg/ingester/owned_streams_test.go index 1f197d580e04c..5c6fa8b9fd569 100644 --- a/pkg/ingester/owned_streams_test.go +++ b/pkg/ingester/owned_streams_test.go @@ -7,11 +7,11 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) func Test_OwnedStreamService(t *testing.T) { - limits, err := validation.NewOverrides(validation.Limits{ + limits, err := runtime.NewOverrides(runtime.Limits{ MaxGlobalStreamsPerUser: 100, }, nil) require.NoError(t, err) diff --git a/pkg/ingester/recalculate_owned_streams_test.go b/pkg/ingester/recalculate_owned_streams_test.go index 3e531dcdef66f..e89a593bd880b 100644 --- a/pkg/ingester/recalculate_owned_streams_test.go +++ b/pkg/ingester/recalculate_owned_streams_test.go @@ -15,7 +15,6 @@ import ( "github.com/grafana/loki/v3/pkg/runtime" lokiring "github.com/grafana/loki/v3/pkg/util/ring" - "github.com/grafana/loki/v3/pkg/validation" ) func Test_recalculateOwnedStreams_newRecalculateOwnedStreamsIngester(t *testing.T) { @@ -65,7 +64,7 @@ func Test_recalculateOwnedStreams_recalculateWithIngesterStrategy(t *testing.T) }, } - limits, err := validation.NewOverrides(validation.Limits{ + limits, err := runtime.NewOverrides(runtime.Limits{ MaxGlobalStreamsPerUser: 100, UseOwnedStreamCount: testData.featureEnabled, }, nil) @@ -77,7 +76,6 @@ func Test_recalculateOwnedStreams_recalculateWithIngesterStrategy(t *testing.T) defaultPeriodConfigs, tenantName, limiter, - runtime.DefaultTenantConfigs(), noopWAL{}, NilMetrics, nil, @@ -85,7 +83,7 @@ func Test_recalculateOwnedStreams_recalculateWithIngesterStrategy(t *testing.T) nil, nil, NewStreamRateCalculator(), - nil, + newWriteFailuresManager(limits), nil, ) require.NoError(t, err) diff --git a/pkg/ingester/recovery_test.go b/pkg/ingester/recovery_test.go index 5979293aae3f0..684ac24b297ca 100644 --- a/pkg/ingester/recovery_test.go +++ b/pkg/ingester/recovery_test.go @@ -3,7 +3,7 @@ package ingester import ( "context" "fmt" - "runtime" + rt "runtime" "strings" "sync" "testing" @@ -21,10 +21,9 @@ import ( "github.com/grafana/loki/v3/pkg/ingester/client" "github.com/grafana/loki/v3/pkg/ingester/wal" "github.com/grafana/loki/v3/pkg/logproto" - loki_runtime "github.com/grafana/loki/v3/pkg/runtime" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" ) type MemoryWALReader struct { @@ -127,7 +126,7 @@ func NewMemRecoverer() *MemRecoverer { } } -func (r *MemRecoverer) NumWorkers() int { return runtime.GOMAXPROCS(0) } +func (r *MemRecoverer) NumWorkers() int { return rt.GOMAXPROCS(0) } func (r *MemRecoverer) Series(_ *Series) error { return nil } @@ -221,7 +220,7 @@ func Test_InMemorySegmentRecover(t *testing.T) { func TestSeriesRecoveryNoDuplicates(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ @@ -230,7 +229,7 @@ func TestSeriesRecoveryNoDuplicates(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, loki_runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) mkSample := func(i int) *logproto.PushRequest { @@ -264,7 +263,7 @@ func TestSeriesRecoveryNoDuplicates(t *testing.T) { require.Equal(t, false, iter.Next()) // create a new ingester now - i, err = New(ingesterConfig, client.Config{}, store, limits, loki_runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err = New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) // recover the checkpointed series @@ -306,7 +305,7 @@ func TestSeriesRecoveryNoDuplicates(t *testing.T) { func TestRecoveryWritesContinuesEntryCountAfterWALReplay(t *testing.T) { ingesterConfig := defaultIngesterTestConfig(t) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) store := &mockStore{ @@ -315,7 +314,7 @@ func TestRecoveryWritesContinuesEntryCountAfterWALReplay(t *testing.T) { readRingMock := mockReadRingWithOneActiveIngester() - i, err := New(ingesterConfig, client.Config{}, store, limits, loki_runtime.DefaultTenantConfigs(), nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) + i, err := New(ingesterConfig, client.Config{}, store, limits, nil, writefailures.Cfg{}, constants.Loki, log.NewNopLogger(), nil, readRingMock, nil) require.NoError(t, err) var ( diff --git a/pkg/ingester/stream.go b/pkg/ingester/stream.go index b36cbb290db7e..077ccb53aef26 100644 --- a/pkg/ingester/stream.go +++ b/pkg/ingester/stream.go @@ -23,7 +23,6 @@ import ( "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql/log" "github.com/grafana/loki/v3/pkg/logqlmodel/stats" - "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util" "github.com/grafana/loki/v3/pkg/util/flagext" util_log "github.com/grafana/loki/v3/pkg/util/log" @@ -80,8 +79,6 @@ type stream struct { chunkFormat byte chunkHeadBlockFormat chunkenc.HeadBlockFmt - - configs *runtime.TenantConfigs } type chunkDesc struct { @@ -111,7 +108,6 @@ func newStream( streamRateCalculator *StreamRateCalculator, metrics *ingesterMetrics, writeFailures *writefailures.Manager, - configs *runtime.TenantConfigs, ) *stream { hashNoShard, _ := labels.HashWithoutLabels(make([]byte, 0, 1024), ShardLbName) return &stream{ @@ -131,8 +127,6 @@ func newStream( writeFailures: writeFailures, chunkFormat: chunkFormat, chunkHeadBlockFormat: headBlockFmt, - - configs: configs, } } @@ -369,13 +363,13 @@ func (s *stream) storeEntries(ctx context.Context, entries []logproto.Entry, usa } func (s *stream) handleLoggingOfDuplicateEntry(entry logproto.Entry) { - if s.configs == nil { + if s.writeFailures == nil { return } - if s.configs.LogDuplicateMetrics(s.tenant) { + if s.writeFailures.LogDuplicateMetrics(s.tenant) { s.metrics.duplicateLogBytesTotal.WithLabelValues(s.tenant).Add(float64(len(entry.Line))) } - if s.configs.LogDuplicateStreamInfo(s.tenant) { + if s.writeFailures.LogDuplicateStreamInfo(s.tenant) { errMsg := fmt.Sprintf("duplicate log entry with size=%d at timestamp %s for stream %s", len(entry.Line), entry.Timestamp.Format(time.RFC3339), s.labelsString) dupErr := errors.New(errMsg) s.writeFailures.Log(s.tenant, dupErr) diff --git a/pkg/ingester/stream_test.go b/pkg/ingester/stream_test.go index 8bf7bfaf4ce98..c1d1bdc5800ff 100644 --- a/pkg/ingester/stream_test.go +++ b/pkg/ingester/stream_test.go @@ -9,25 +9,22 @@ import ( "testing" "time" - "github.com/prometheus/client_golang/prometheus/testutil" - gokitlog "github.com/go-kit/log" - "github.com/prometheus/client_golang/prometheus" - - "github.com/grafana/loki/v3/pkg/compression" - "github.com/grafana/loki/v3/pkg/runtime" - "github.com/grafana/dskit/httpgrpc" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" "github.com/grafana/loki/v3/pkg/chunkenc" + "github.com/grafana/loki/v3/pkg/compression" "github.com/grafana/loki/v3/pkg/distributor/writefailures" "github.com/grafana/loki/v3/pkg/iter" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql/log" "github.com/grafana/loki/v3/pkg/logql/syntax" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util/flagext" "github.com/grafana/loki/v3/pkg/validation" ) @@ -54,7 +51,7 @@ func TestMaxReturnedStreamsErrors(t *testing.T) { {"unlimited", 0, numLogs}, } - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -78,7 +75,6 @@ func TestMaxReturnedStreamsErrors(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ) _, err := s.Push(context.Background(), []logproto.Entry{ @@ -112,7 +108,7 @@ func TestMaxReturnedStreamsErrors(t *testing.T) { } func TestPushDeduplication(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -132,7 +128,6 @@ func TestPushDeduplication(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ) written, err := s.Push(context.Background(), []logproto.Entry{ @@ -148,7 +143,7 @@ func TestPushDeduplication(t *testing.T) { } func TestPushDeduplicationExtraMetrics(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -157,20 +152,20 @@ func TestPushDeduplicationExtraMetrics(t *testing.T) { buf := bytes.NewBuffer(nil) logger := gokitlog.NewLogfmtLogger(buf) - provider := &providerMock{ - tenantConfig: func(tenantID string) *runtime.Config { + tenantLimits := &mockTenantLimits{ + providerFn: func(tenantID string) *runtime.Limits { if tenantID == "fake" { - return &runtime.Config{ + return &runtime.Limits{ LogDuplicateMetrics: true, LogDuplicateStreamInfo: true, } } - - return &runtime.Config{} + return &runtime.Limits{} }, } - runtimeCfg, err := runtime.NewTenantConfigs(provider) + runtimeCfg, err := runtime.NewOverrides(runtime.Limits{}, tenantLimits) + require.NoError(t, err) registry := prometheus.NewRegistry() manager := writefailures.NewManager(logger, registry, writefailures.Cfg{LogRate: flagext.ByteSize(1000), AddInsightsLabel: true}, runtimeCfg, "ingester") @@ -192,7 +187,6 @@ func TestPushDeduplicationExtraMetrics(t *testing.T) { NewStreamRateCalculator(), metrics, manager, - runtimeCfg, ) _, err = s.Push(context.Background(), []logproto.Entry{ @@ -218,7 +212,7 @@ func TestPushDeduplicationExtraMetrics(t *testing.T) { } func TestPushRejectOldCounter(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -238,7 +232,6 @@ func TestPushRejectOldCounter(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ) // counter should be 2 now since the first line will be deduped @@ -322,11 +315,11 @@ func TestStreamIterator(t *testing.T) { func TestEntryErrorCorrectlyReported(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.MaxChunkAge = time.Minute - l := validation.Limits{ + l := runtime.Limits{ PerStreamRateLimit: 15, PerStreamRateLimitBurst: 15, } - limits, err := validation.NewOverrides(l, nil) + limits, err := runtime.NewOverrides(l, nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -346,7 +339,6 @@ func TestEntryErrorCorrectlyReported(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ) s.highestTs = time.Now() @@ -365,7 +357,7 @@ func TestEntryErrorCorrectlyReported(t *testing.T) { func TestUnorderedPush(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.MaxChunkAge = 10 * time.Second - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -385,7 +377,6 @@ func TestUnorderedPush(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ) for _, x := range []struct { @@ -464,11 +455,11 @@ func TestUnorderedPush(t *testing.T) { } func TestPushRateLimit(t *testing.T) { - l := validation.Limits{ + l := runtime.Limits{ PerStreamRateLimit: 10, PerStreamRateLimitBurst: 10, } - limits, err := validation.NewOverrides(l, nil) + limits, err := runtime.NewOverrides(l, nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -488,7 +479,6 @@ func TestPushRateLimit(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ) entries := []logproto.Entry{ @@ -504,11 +494,11 @@ func TestPushRateLimit(t *testing.T) { } func TestPushRateLimitAllOrNothing(t *testing.T) { - l := validation.Limits{ + l := runtime.Limits{ PerStreamRateLimit: 10, PerStreamRateLimitBurst: 10, } - limits, err := validation.NewOverrides(l, nil) + limits, err := runtime.NewOverrides(l, nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -529,7 +519,6 @@ func TestPushRateLimitAllOrNothing(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ) entries := []logproto.Entry{ @@ -547,7 +536,7 @@ func TestPushRateLimitAllOrNothing(t *testing.T) { } func TestReplayAppendIgnoresValidityWindow(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) @@ -569,7 +558,6 @@ func TestReplayAppendIgnoresValidityWindow(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ) base := time.Now() @@ -615,12 +603,12 @@ func Benchmark_PushStream(b *testing.B) { labels.Label{Name: "container", Value: "ingester"}, } - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(b, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) chunkfmt, headfmt := defaultChunkFormat(b) - s := newStream(chunkfmt, headfmt, &Config{MaxChunkAge: 24 * time.Hour}, limiter.rateLimitStrategy, "fake", model.Fingerprint(0), ls, true, NewStreamRateCalculator(), NilMetrics, nil, nil) + s := newStream(chunkfmt, headfmt, &Config{MaxChunkAge: 24 * time.Hour}, limiter.rateLimitStrategy, "fake", model.Fingerprint(0), ls, true, NewStreamRateCalculator(), NilMetrics, nil) expr, err := syntax.ParseLogSelector(`{namespace="loki-dev"}`, true) require.NoError(b, err) t, err := newTailer("foo", expr, &fakeTailServer{}, 10) @@ -655,10 +643,18 @@ func defaultChunkFormat(t testing.TB) (byte, chunkenc.HeadBlockFmt) { return chunkfmt, headfmt } -type providerMock struct { - tenantConfig func(string) *runtime.Config +type mockTenantLimits struct { + providerFn func(string) *runtime.Limits } -func (m *providerMock) TenantConfig(userID string) *runtime.Config { - return m.tenantConfig(userID) +// AllByUserID implements runtime.TenantLimits. +func (m mockTenantLimits) AllByUserID() map[string]*runtime.Limits { + panic("unimplemented") } + +// TenantLimits implements runtime.TenantLimits. +func (m mockTenantLimits) TenantLimits(userID string) *runtime.Limits { + return m.providerFn(userID) +} + +var _ runtime.TenantLimits = mockTenantLimits{} diff --git a/pkg/ingester/streams_map_test.go b/pkg/ingester/streams_map_test.go index 273c489d34d4a..2b209d9634625 100644 --- a/pkg/ingester/streams_map_test.go +++ b/pkg/ingester/streams_map_test.go @@ -7,11 +7,11 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" - "github.com/grafana/loki/v3/pkg/validation" + runtime "github.com/grafana/loki/v3/pkg/runtime" ) func TestStreamsMap(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) limiter := NewLimiter(limits, NilMetrics, newIngesterRingLimiterStrategy(&ringCountMock{count: 1}, 1), &TenantBasedStrategy{limits: limits}) chunkfmt, headfmt := defaultChunkFormat(t) @@ -31,7 +31,6 @@ func TestStreamsMap(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ), newStream( chunkfmt, @@ -47,7 +46,6 @@ func TestStreamsMap(t *testing.T) { NewStreamRateCalculator(), NilMetrics, nil, - nil, ), } var s *stream diff --git a/pkg/logcli/query/query.go b/pkg/logcli/query/query.go index 0551e076f742e..090e137c6f89e 100644 --- a/pkg/logcli/query/query.go +++ b/pkg/logcli/query/query.go @@ -23,6 +23,7 @@ import ( "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql" "github.com/grafana/loki/v3/pkg/loki" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" chunk "github.com/grafana/loki/v3/pkg/storage/chunk/client" "github.com/grafana/loki/v3/pkg/storage/config" @@ -31,7 +32,6 @@ import ( "github.com/grafana/loki/v3/pkg/util/constants" util_log "github.com/grafana/loki/v3/pkg/util/log" "github.com/grafana/loki/v3/pkg/util/marshal" - "github.com/grafana/loki/v3/pkg/validation" ) const schemaConfigFilename = "schemaconfig" @@ -464,7 +464,7 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string return err } - limits, err := validation.NewOverrides(conf.LimitsConfig, nil) + limits, err := runtime.NewOverrides(conf.LimitsConfig, nil) if err != nil { return err } diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 14963c09940c4..9669a51345f4f 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -70,7 +70,6 @@ import ( util_log "github.com/grafana/loki/v3/pkg/util/log" lokiring "github.com/grafana/loki/v3/pkg/util/ring" serverutil "github.com/grafana/loki/v3/pkg/util/server" - "github.com/grafana/loki/v3/pkg/validation" ) // Config is the root config for Loki. @@ -103,25 +102,22 @@ type Config struct { CompactorConfig compactor.Config `yaml:"compactor,omitempty"` CompactorHTTPClient compactorclient.HTTPConfig `yaml:"compactor_client,omitempty" doc:"hidden"` CompactorGRPCClient compactorclient.GRPCConfig `yaml:"compactor_grpc_client,omitempty"` - LimitsConfig validation.Limits `yaml:"limits_config"` Worker worker.Config `yaml:"frontend_worker,omitempty"` TableManager index.TableManagerConfig `yaml:"table_manager,omitempty"` MemberlistKV memberlist.KVConfig `yaml:"memberlist"` KafkaConfig kafka.Config `yaml:"kafka_config,omitempty" category:"experimental"` - RuntimeConfig runtimeconfig.Config `yaml:"runtime_config,omitempty"` - OperationalConfig runtime.Config `yaml:"operational_config,omitempty"` - Tracing tracing.Config `yaml:"tracing"` - Analytics analytics.Config `yaml:"analytics"` - Profiling ProfilingConfig `yaml:"profiling,omitempty"` + LimitsConfig runtime.Limits `yaml:"limits_config"` + RuntimeConfig runtimeconfig.Config `yaml:"runtime_config,omitempty"` - LegacyReadTarget bool `yaml:"legacy_read_target,omitempty" doc:"hidden|deprecated"` + Tracing tracing.Config `yaml:"tracing"` + Analytics analytics.Config `yaml:"analytics"` + Profiling ProfilingConfig `yaml:"profiling,omitempty"` - Common common.Config `yaml:"common,omitempty"` - - ShutdownDelay time.Duration `yaml:"shutdown_delay"` - - MetricsNamespace string `yaml:"metrics_namespace"` + LegacyReadTarget bool `yaml:"legacy_read_target,omitempty" doc:"hidden|deprecated"` + Common common.Config `yaml:"common,omitempty"` + ShutdownDelay time.Duration `yaml:"shutdown_delay"` + MetricsNamespace string `yaml:"metrics_namespace"` } // RegisterFlags registers flag. @@ -184,7 +180,6 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.BloomBuild.RegisterFlags(f) c.QueryScheduler.RegisterFlags(f) c.Analytics.RegisterFlags(f) - c.OperationalConfig.RegisterFlags(f) c.Profiling.RegisterFlags(f) c.KafkaConfig.RegisterFlags(f) c.BlockBuilder.RegisterFlags(f) @@ -355,8 +350,7 @@ type Loki struct { InternalServer *server.Server ring *ring.Ring Overrides limiter.CombinedLimits - tenantConfigs *runtime.TenantConfigs - TenantLimits validation.TenantLimits + TenantLimits runtime.TenantLimits distributor *distributor.Distributor Ingester ingester.Interface PatternIngester *pattern.Ingester @@ -670,7 +664,6 @@ func (t *Loki) setupModuleManager() error { mm.RegisterModule(Ring, t.initRing, modules.UserInvisibleModule) mm.RegisterModule(Overrides, t.initOverrides, modules.UserInvisibleModule) mm.RegisterModule(OverridesExporter, t.initOverridesExporter) - mm.RegisterModule(TenantConfigs, t.initTenantConfigs, modules.UserInvisibleModule) mm.RegisterModule(Distributor, t.initDistributor) mm.RegisterModule(Store, t.initStore, modules.UserInvisibleModule) mm.RegisterModule(Querier, t.initQuerier) @@ -713,16 +706,15 @@ func (t *Loki) setupModuleManager() error { Analytics: {}, Overrides: {RuntimeConfig}, OverridesExporter: {Overrides, Server}, - TenantConfigs: {RuntimeConfig}, - Distributor: {Ring, Server, Overrides, TenantConfigs, PatternRingClient, PatternIngesterTee, Analytics, PartitionRing}, + Distributor: {Ring, Server, Overrides, PatternRingClient, PatternIngesterTee, Analytics, PartitionRing}, Store: {Overrides, IndexGatewayRing}, - Ingester: {Store, Server, MemberlistKV, TenantConfigs, Analytics, PartitionRing}, + Ingester: {Store, Server, MemberlistKV, Analytics, PartitionRing}, Querier: {Store, Ring, Server, IngesterQuerier, PatternRingClient, Overrides, Analytics, CacheGenerationLoader, QuerySchedulerRing}, - QueryFrontendTripperware: {Server, Overrides, TenantConfigs}, + QueryFrontendTripperware: {Server, Overrides}, QueryFrontend: {QueryFrontendTripperware, Analytics, CacheGenerationLoader, QuerySchedulerRing}, QueryScheduler: {Server, Overrides, MemberlistKV, Analytics, QuerySchedulerRing}, - Ruler: {Ring, Server, RulerStorage, RuleEvaluator, Overrides, TenantConfigs, Analytics}, - RuleEvaluator: {Ring, Server, Store, IngesterQuerier, Overrides, TenantConfigs, Analytics}, + Ruler: {Ring, Server, RulerStorage, RuleEvaluator, Overrides, Analytics}, + RuleEvaluator: {Ring, Server, Store, IngesterQuerier, Overrides, Analytics}, TableManager: {Server, Analytics}, Compactor: {Server, Overrides, MemberlistKV, Analytics}, IndexGateway: {Server, Store, BloomStore, IndexGatewayRing, IndexGatewayInterceptors, Analytics}, diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index bdd498b32a2d4..b463c2f285761 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -100,7 +100,6 @@ const ( RuntimeConfig string = "runtime-config" Overrides string = "overrides" OverridesExporter string = "overrides-exporter" - TenantConfigs string = "tenant-configs" Server string = "server" InternalServer string = "internal-server" Distributor string = "distributor" @@ -274,8 +273,7 @@ func (t *Loki) initRuntimeConfig() (services.Service, error) { t.Cfg.RuntimeConfig.Loader = loadRuntimeConfig // make sure to set default limits before we start loading configuration into memory - validation.SetDefaultLimitsForYAMLUnmarshalling(t.Cfg.LimitsConfig) - runtime.SetDefaultLimitsForYAMLUnmarshalling(t.Cfg.OperationalConfig) + runtime.SetDefaultLimitsForYAMLUnmarshalling(t.Cfg.LimitsConfig) var err error t.runtimeConfig, err = runtimeconfig.New(t.Cfg.RuntimeConfig, "loki", prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer), util_log.Logger) @@ -300,7 +298,7 @@ func (t *Loki) initOverrides() (_ services.Service, err error) { if t.Cfg.LimitsConfig.IndexGatewayShardSize == 0 { t.Cfg.LimitsConfig.IndexGatewayShardSize = t.Cfg.IndexGateway.Ring.ReplicationFactor } - t.Overrides, err = validation.NewOverrides(t.Cfg.LimitsConfig, t.TenantLimits) + t.Overrides, err = runtime.NewOverrides(t.Cfg.LimitsConfig, t.TenantLimits) // overrides are not a service, since they don't have any operational state. return nil, err } @@ -319,12 +317,6 @@ func (t *Loki) initOverridesExporter() (services.Service, error) { return nil, nil } -func (t *Loki) initTenantConfigs() (_ services.Service, err error) { - t.tenantConfigs, err = runtime.NewTenantConfigs(newTenantConfigProvider(t.runtimeConfig)) - // tenantConfigs are not a service, since they don't have any operational state. - return nil, err -} - func (t *Loki) initDistributor() (services.Service, error) { t.Cfg.Distributor.KafkaConfig = t.Cfg.KafkaConfig @@ -338,7 +330,6 @@ func (t *Loki) initDistributor() (services.Service, error) { t.Cfg.Distributor, t.Cfg.Ingester, t.Cfg.IngesterClient, - t.tenantConfigs, t.ring, t.partitionRing, t.Overrides, @@ -595,7 +586,7 @@ func (t *Loki) initIngester() (_ services.Service, err error) { level.Warn(util_log.Logger).Log("msg", "The config setting shutdown marker path is not set. The /ingester/prepare_shutdown endpoint won't work") } - t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Store, t.Overrides, t.tenantConfigs, prometheus.DefaultRegisterer, t.Cfg.Distributor.WriteFailuresLogging, t.Cfg.MetricsNamespace, logger, t.UsageTracker, t.ring, t.partitionRingWatcher) + t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Store, t.Overrides, prometheus.DefaultRegisterer, t.Cfg.Distributor.WriteFailuresLogging, t.Cfg.MetricsNamespace, logger, t.UsageTracker, t.ring, t.partitionRingWatcher) if err != nil { return } @@ -677,7 +668,6 @@ func (t *Loki) initPatternIngesterTee() (services.Service, error) { t.Cfg.Pattern, t.Overrides, t.PatternRingClient, - t.tenantConfigs, t.Cfg.MetricsNamespace, prometheus.DefaultRegisterer, logger, diff --git a/pkg/loki/runtime_config.go b/pkg/loki/runtime_config.go index c2e72fbd4b78f..5e82a3958ac0e 100644 --- a/pkg/loki/runtime_config.go +++ b/pkg/loki/runtime_config.go @@ -11,15 +11,13 @@ import ( "github.com/grafana/loki/v3/pkg/runtime" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) // runtimeConfigValues are values that can be reloaded from configuration file while Loki is running. // Reloading is done by runtimeconfig.Manager, which also keeps the currently loaded config. // These values are then pushed to the components that are interested in them. type runtimeConfigValues struct { - TenantLimits map[string]*validation.Limits `yaml:"overrides"` - TenantConfig map[string]*runtime.Config `yaml:"configs"` + TenantLimits map[string]*runtime.Limits `yaml:"overrides"` Multi kv.MultiRuntimeConfig `yaml:"multi_kv_config"` } @@ -57,7 +55,7 @@ type tenantLimitsFromRuntimeConfig struct { c *runtimeconfig.Manager } -func (t *tenantLimitsFromRuntimeConfig) AllByUserID() map[string]*validation.Limits { +func (t *tenantLimitsFromRuntimeConfig) AllByUserID() map[string]*runtime.Limits { if t.c == nil { return nil } @@ -70,7 +68,7 @@ func (t *tenantLimitsFromRuntimeConfig) AllByUserID() map[string]*validation.Lim return nil } -func (t *tenantLimitsFromRuntimeConfig) TenantLimits(userID string) *validation.Limits { +func (t *tenantLimitsFromRuntimeConfig) TenantLimits(userID string) *runtime.Limits { allByUserID := t.AllByUserID() if allByUserID == nil { return nil @@ -79,34 +77,10 @@ func (t *tenantLimitsFromRuntimeConfig) TenantLimits(userID string) *validation. return allByUserID[userID] } -func newtenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) validation.TenantLimits { +func newtenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) runtime.TenantLimits { return &tenantLimitsFromRuntimeConfig{c: c} } -type tenantConfigProvider struct { - c *runtimeconfig.Manager -} - -func newTenantConfigProvider(c *runtimeconfig.Manager) runtime.TenantConfigProvider { - return &tenantConfigProvider{c: c} -} - -// TenantConfig returns the user config or default config if none was defined. -func (t *tenantConfigProvider) TenantConfig(userID string) *runtime.Config { - if t.c == nil { - return nil - } - - cfg, ok := t.c.GetConfig().(*runtimeConfigValues) - if !ok || cfg == nil { - return nil - } - if tenantCfg, ok := cfg.TenantConfig[userID]; ok { - return tenantCfg - } - return nil -} - func multiClientRuntimeConfigChannel(manager *runtimeconfig.Manager) func() <-chan kv.MultiRuntimeConfig { if manager == nil { return nil diff --git a/pkg/loki/runtime_config_test.go b/pkg/loki/runtime_config_test.go index ee55dd55b5421..db955f672b5b7 100644 --- a/pkg/loki/runtime_config_test.go +++ b/pkg/loki/runtime_config_test.go @@ -18,7 +18,6 @@ import ( "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/runtime" - "github.com/grafana/loki/v3/pkg/validation" ) func Test_LoadRetentionRules(t *testing.T) { @@ -47,10 +46,10 @@ overrides: `) require.Equal(t, time.Duration(0), overrides.RetentionPeriod("1")) // default require.Equal(t, 2*30*24*time.Hour, overrides.RetentionPeriod("29")) // overrides - require.Equal(t, []validation.StreamRetention(nil), overrides.StreamRetention("1")) + require.Equal(t, []runtime.StreamRetention(nil), overrides.StreamRetention("1")) actual := overrides.StreamRetention("29") - expected := []validation.StreamRetention{ + expected := []runtime.StreamRetention{ {Period: model.Duration(48 * time.Hour), Priority: 10, Selector: `{app="foo"}`, Matchers: []*labels.Matcher{ labels.MustNewMatcher(labels.MatchEqual, "app", "foo"), }}, @@ -63,8 +62,8 @@ overrides: require.Equal(t, removeFastRegexMatcher(expected), removeFastRegexMatcher(actual)) } -func removeFastRegexMatcher(configs []validation.StreamRetention) []validation.StreamRetention { - result := make([]validation.StreamRetention, 0, len(configs)) +func removeFastRegexMatcher(configs []runtime.StreamRetention) []runtime.StreamRetention { + result := make([]runtime.StreamRetention, 0, len(configs)) for _, config := range configs { config.Matchers = syntax.RemoveFastRegexMatchers(config.Matchers) } @@ -97,75 +96,7 @@ overrides: require.Equal(t, "invalid override for tenant 29: retention period must be >= 24h was 5h", err.Error()) } -func Test_DefaultConfig(t *testing.T) { - runtimeGetter := newTestRuntimeconfig(t, - ` -configs: - "1": - log_push_request: false - limited_log_push_errors: false - log_duplicate_metrics: false - log_duplicate_stream_info: false - "2": - log_push_request: true - log_duplicate_metrics: true - log_duplicate_stream_info: true -`) - - tenantConfigs, err := runtime.NewTenantConfigs(runtimeGetter) - require.NoError(t, err) - - require.Equal(t, false, tenantConfigs.LogPushRequest("1")) - require.Equal(t, false, tenantConfigs.LimitedLogPushErrors("1")) - require.Equal(t, false, tenantConfigs.LimitedLogPushErrors("2")) - require.Equal(t, true, tenantConfigs.LogPushRequest("2")) - require.Equal(t, true, tenantConfigs.LimitedLogPushErrors("3")) - require.Equal(t, false, tenantConfigs.LogPushRequest("3")) - require.Equal(t, false, tenantConfigs.LogDuplicateMetrics("1")) - require.Equal(t, true, tenantConfigs.LogDuplicateMetrics("2")) - require.Equal(t, false, tenantConfigs.LogDuplicateMetrics("3")) - require.Equal(t, false, tenantConfigs.LogDuplicateStreamInfo("1")) - require.Equal(t, true, tenantConfigs.LogDuplicateStreamInfo("2")) - require.Equal(t, false, tenantConfigs.LogDuplicateStreamInfo("3")) -} - -func newTestRuntimeconfig(t *testing.T, yaml string) runtime.TenantConfigProvider { - t.Helper() - f, err := os.CreateTemp(t.TempDir(), "bar") - require.NoError(t, err) - path := f.Name() - // fake loader to load from string instead of file. - loader := func(_ io.Reader) (interface{}, error) { - return loadRuntimeConfig(strings.NewReader(yaml)) - } - cfg := runtimeconfig.Config{ - ReloadPeriod: 1 * time.Second, - Loader: loader, - LoadPath: []string{path}, - } - flagset := flag.NewFlagSet("", flag.PanicOnError) - var defaults validation.Limits - var operations runtime.Config - defaults.RegisterFlags(flagset) - operations.RegisterFlags(flagset) - runtime.SetDefaultLimitsForYAMLUnmarshalling(operations) - require.NoError(t, flagset.Parse(nil)) - - reg := prometheus.NewPedanticRegistry() - runtimeConfig, err := runtimeconfig.New(cfg, "test", prometheus.WrapRegistererWithPrefix("loki_", reg), log.NewNopLogger()) - require.NoError(t, err) - - require.NoError(t, runtimeConfig.StartAsync(context.Background())) - require.NoError(t, runtimeConfig.AwaitRunning(context.Background())) - defer func() { - runtimeConfig.StopAsync() - require.NoError(t, runtimeConfig.AwaitTerminated(context.Background())) - }() - - return newTenantConfigProvider(runtimeConfig) -} - -func newTestOverrides(t *testing.T, yaml string) *validation.Overrides { +func newTestOverrides(t *testing.T, yaml string) *runtime.Overrides { t.Helper() f, err := os.CreateTemp(t.TempDir(), "bar") require.NoError(t, err) @@ -180,10 +111,10 @@ func newTestOverrides(t *testing.T, yaml string) *validation.Overrides { LoadPath: []string{path}, } flagset := flag.NewFlagSet("", flag.PanicOnError) - var defaults validation.Limits + var defaults runtime.Limits defaults.RegisterFlags(flagset) require.NoError(t, flagset.Parse(nil)) - validation.SetDefaultLimitsForYAMLUnmarshalling(defaults) + runtime.SetDefaultLimitsForYAMLUnmarshalling(defaults) reg := prometheus.NewPedanticRegistry() runtimeConfig, err := runtimeconfig.New(cfg, "test", prometheus.WrapRegistererWithPrefix("loki_", reg), log.NewNopLogger()) @@ -196,7 +127,7 @@ func newTestOverrides(t *testing.T, yaml string) *validation.Overrides { require.NoError(t, runtimeConfig.AwaitTerminated(context.Background())) }() - overrides, err := validation.NewOverrides(defaults, newtenantLimitsFromRuntimeConfig(runtimeConfig)) + overrides, err := runtime.NewOverrides(defaults, newtenantLimitsFromRuntimeConfig(runtimeConfig)) require.NoError(t, err) return overrides } @@ -204,11 +135,11 @@ func newTestOverrides(t *testing.T, yaml string) *validation.Overrides { func Test_NoOverrides(t *testing.T) { flagset := flag.NewFlagSet("", flag.PanicOnError) - var defaults validation.Limits + var defaults runtime.Limits defaults.RegisterFlags(flagset) require.NoError(t, flagset.Parse(nil)) - validation.SetDefaultLimitsForYAMLUnmarshalling(defaults) - overrides, err := validation.NewOverrides(defaults, newtenantLimitsFromRuntimeConfig(nil)) + runtime.SetDefaultLimitsForYAMLUnmarshalling(defaults) + overrides, err := runtime.NewOverrides(defaults, newtenantLimitsFromRuntimeConfig(nil)) require.NoError(t, err) require.Equal(t, time.Duration(defaults.QuerySplitDuration), overrides.QuerySplitDuration("foo")) } diff --git a/pkg/pattern/ingester.go b/pkg/pattern/ingester.go index 0e2e1ad1432a0..16f8dde92dc09 100644 --- a/pkg/pattern/ingester.go +++ b/pkg/pattern/ingester.go @@ -21,6 +21,7 @@ import ( ring_client "github.com/grafana/dskit/ring/client" + "github.com/grafana/loki/v3/pkg/distributor/writefailures" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/pattern/aggregation" "github.com/grafana/loki/v3/pkg/pattern/clientpool" @@ -151,6 +152,7 @@ func (cfg *Config) Validate() error { type Limits interface { drain.Limits aggregation.Limits + writefailures.Limits } type Ingester struct { diff --git a/pkg/pattern/ingester_test.go b/pkg/pattern/ingester_test.go index 0b1404fe544a8..96ff088d65d4e 100644 --- a/pkg/pattern/ingester_test.go +++ b/pkg/pattern/ingester_test.go @@ -344,6 +344,10 @@ type fakeLimits struct { metricAggregationEnabled bool } +func (f *fakeLimits) LogPushRequestStreams(_ string) bool { + return false +} + func (f *fakeLimits) PatternIngesterTokenizableJSONFields(_ string) []string { return []string{"log", "message", "msg", "msg_", "_msg", "content"} } diff --git a/pkg/pattern/tee_service.go b/pkg/pattern/tee_service.go index 19584276f39ef..5ed775d05767a 100644 --- a/pkg/pattern/tee_service.go +++ b/pkg/pattern/tee_service.go @@ -21,7 +21,6 @@ import ( "github.com/grafana/loki/v3/pkg/loghttp/push" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql/syntax" - "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util/spanlogger" ring_client "github.com/grafana/dskit/ring/client" @@ -30,7 +29,6 @@ import ( type TeeService struct { cfg Config limits Limits - tenantCfgs *runtime.TenantConfigs logger log.Logger ringClient RingClient wg *sync.WaitGroup @@ -54,7 +52,6 @@ func NewTeeService( cfg Config, limits Limits, ringClient RingClient, - tenantCfgs *runtime.TenantConfigs, metricsNamespace string, registerer prometheus.Registerer, logger log.Logger, @@ -90,7 +87,6 @@ func NewTeeService( ), cfg: cfg, limits: limits, - tenantCfgs: tenantCfgs, ringClient: ringClient, wg: &sync.WaitGroup{}, @@ -344,7 +340,7 @@ func (ts *TeeService) sendBatch(ctx context.Context, clientRequest clientRequest // this is basically the same as logging push request streams, // so put it behind the same flag - if ts.tenantCfgs.LogPushRequestStreams(clientRequest.tenant) { + if ts.limits.LogPushRequestStreams(clientRequest.tenant) { level.Debug(ts.logger). Log( "msg", "forwarded push request to pattern ingester", diff --git a/pkg/pattern/tee_service_test.go b/pkg/pattern/tee_service_test.go index ed6de3c90ce1a..0fb8a032f062a 100644 --- a/pkg/pattern/tee_service_test.go +++ b/pkg/pattern/tee_service_test.go @@ -15,7 +15,6 @@ import ( "github.com/grafana/loki/v3/pkg/distributor" "github.com/grafana/loki/v3/pkg/logproto" - "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/pkg/push" ) @@ -52,7 +51,6 @@ func getTestTee(t *testing.T) (*TeeService, *mockPoolClient) { metricAggregationEnabled: true, }, ringClient, - runtime.DefaultTenantConfigs(), "test", nil, log.NewNopLogger(), diff --git a/pkg/querier/http_test.go b/pkg/querier/http_test.go index d568b7b9934b4..c35d1525d1e22 100644 --- a/pkg/querier/http_test.go +++ b/pkg/querier/http_test.go @@ -8,23 +8,22 @@ import ( "testing" "time" + "github.com/go-kit/log" + "github.com/grafana/dskit/user" "github.com/pkg/errors" + "github.com/prometheus/common/model" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/grafana/loki/v3/pkg/loghttp" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logqlmodel" - "github.com/grafana/loki/v3/pkg/validation" - - "github.com/go-kit/log" - "github.com/grafana/dskit/user" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" + "github.com/grafana/loki/v3/pkg/runtime" ) func TestInstantQueryHandler(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) t.Run("log selector expression not allowed for instant queries", func(t *testing.T) { @@ -53,7 +52,7 @@ func TestInstantQueryHandler(t *testing.T) { func TestTailHandler(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) api := NewQuerierAPI(mockQuerierConfig(), nil, limits, log.NewNopLogger()) @@ -111,7 +110,7 @@ func TestQueryWrapperMiddleware(t *testing.T) { defaultLimits := defaultLimitsTestConfig() defaultLimits.QueryTimeout = model.Duration(time.Millisecond * 10) - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) // request timeout is 5ms but it sleeps for 100ms, so timeout injected in the request is expected. @@ -150,7 +149,7 @@ func TestQueryWrapperMiddleware(t *testing.T) { defaultLimits := defaultLimitsTestConfig() defaultLimits.QueryTimeout = model.Duration(shortestTimeout) - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) connSimulator := &slowConnectionSimulator{ diff --git a/pkg/querier/querier_mock_test.go b/pkg/querier/querier_mock_test.go index 0fd9b421de000..f719cffe49429 100644 --- a/pkg/querier/querier_mock_test.go +++ b/pkg/querier/querier_mock_test.go @@ -9,10 +9,7 @@ import ( "github.com/grafana/loki/v3/pkg/logql/log" "github.com/grafana/loki/v3/pkg/logql/syntax" - - "github.com/grafana/loki/pkg/push" - - "github.com/grafana/loki/v3/pkg/loghttp" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/dskit/grpcclient" "github.com/grafana/dskit/ring" @@ -24,13 +21,14 @@ import ( "google.golang.org/grpc/health/grpc_health_v1" grpc_metadata "google.golang.org/grpc/metadata" - logql_log "github.com/grafana/loki/v3/pkg/logql/log" - + "github.com/grafana/loki/pkg/push" "github.com/grafana/loki/v3/pkg/distributor/clientpool" "github.com/grafana/loki/v3/pkg/ingester/client" "github.com/grafana/loki/v3/pkg/iter" + "github.com/grafana/loki/v3/pkg/loghttp" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql" + logql_log "github.com/grafana/loki/v3/pkg/logql/log" "github.com/grafana/loki/v3/pkg/logqlmodel" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/chunk/fetcher" @@ -38,7 +36,6 @@ import ( "github.com/grafana/loki/v3/pkg/storage/stores/index/stats" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding" "github.com/grafana/loki/v3/pkg/util" - "github.com/grafana/loki/v3/pkg/validation" ) // querierClientMock is a mockable version of QuerierClient, used in querier @@ -856,17 +853,17 @@ func (q queryMock) Exec(_ context.Context) (logqlmodel.Result, error) { return q.result, nil } -type mockTenantLimits map[string]*validation.Limits +type mockTenantLimits map[string]*runtime.Limits -func (tl mockTenantLimits) TenantLimits(userID string) *validation.Limits { +func (tl mockTenantLimits) TenantLimits(userID string) *runtime.Limits { limits, ok := tl[userID] if !ok { - return &validation.Limits{} + return &runtime.Limits{} } return limits } -func (tl mockTenantLimits) AllByUserID() map[string]*validation.Limits { +func (tl mockTenantLimits) AllByUserID() map[string]*runtime.Limits { return tl } diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index 41265e00df59f..39a5d2dc72ef8 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -20,17 +20,16 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/compactor/deletion" "github.com/grafana/loki/v3/pkg/ingester/client" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql" "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/querier/plan" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" + util_log "github.com/grafana/loki/v3/pkg/util/log" ) const ( @@ -56,7 +55,7 @@ func TestQuerier_Label_QueryTimeoutConfigFlag(t *testing.T) { store.On("LabelValuesForMetricName", mock.Anything, "test", model.TimeFromUnixNano(startTime.UnixNano()), model.TimeFromUnixNano(endTime.UnixNano()), "logs", "test").Return([]string{"foo", "bar"}, nil) limitsCfg := defaultLimitsTestConfig() limitsCfg.QueryTimeout = model.Duration(queryTimeout) - limits, err := validation.NewOverrides(limitsCfg, nil) + limits, err := runtime.NewOverrides(limitsCfg, nil) require.NoError(t, err) q, err := newQuerier( @@ -114,7 +113,7 @@ func TestQuerier_Tail_QueryTimeoutConfigFlag(t *testing.T) { limitsCfg := defaultLimitsTestConfig() limitsCfg.QueryTimeout = model.Duration(queryTimeout) - limits, err := validation.NewOverrides(limitsCfg, nil) + limits, err := runtime.NewOverrides(limitsCfg, nil) require.NoError(t, err) q, err := newQuerier( @@ -168,8 +167,8 @@ func mockLabelResponse(values []string) *logproto.LabelResponse { } } -func defaultLimitsTestConfig() validation.Limits { - limits := validation.Limits{} +func defaultLimitsTestConfig() runtime.Limits { + limits := runtime.Limits{} flagext.DefaultValues(&limits) return limits } @@ -199,7 +198,7 @@ func TestQuerier_validateQueryRequest(t *testing.T) { defaultLimits.MaxStreamsMatchersPerQuery = 1 defaultLimits.MaxQueryLength = model.Duration(2 * time.Minute) - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) q, err := newQuerier( @@ -248,13 +247,13 @@ func TestQuerier_SeriesAPI(t *testing.T) { for _, tc := range []struct { desc string req *logproto.SeriesRequest - setup func(*storeMock, *queryClientMock, *querierClientMock, validation.Limits, *logproto.SeriesRequest) + setup func(*storeMock, *queryClientMock, *querierClientMock, runtime.Limits, *logproto.SeriesRequest) run func(*testing.T, *SingleTenantQuerier, *logproto.SeriesRequest) }{ { "ingester error", mkReq([]string{`{a="1"}`}), - func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ validation.Limits, req *logproto.SeriesRequest) { + func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ runtime.Limits, req *logproto.SeriesRequest) { ingester.On("Series", mock.Anything, req, mock.Anything).Return(nil, errors.New("tst-err")) store.On("SelectSeries", mock.Anything, mock.Anything).Return(nil, nil) @@ -268,7 +267,7 @@ func TestQuerier_SeriesAPI(t *testing.T) { { "store error", mkReq([]string{`{a="1"}`}), - func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ validation.Limits, req *logproto.SeriesRequest) { + func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ runtime.Limits, req *logproto.SeriesRequest) { ingester.On("Series", mock.Anything, req, mock.Anything).Return(mockSeriesResponse([]map[string]string{ {"a": "1"}, }), nil) @@ -284,7 +283,7 @@ func TestQuerier_SeriesAPI(t *testing.T) { { "no matches", mkReq([]string{`{a="1"}`}), - func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ validation.Limits, req *logproto.SeriesRequest) { + func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ runtime.Limits, req *logproto.SeriesRequest) { ingester.On("Series", mock.Anything, req, mock.Anything).Return(mockSeriesResponse(nil), nil) store.On("SelectSeries", mock.Anything, mock.Anything).Return(nil, nil) }, @@ -298,7 +297,7 @@ func TestQuerier_SeriesAPI(t *testing.T) { { "returns series", mkReq([]string{`{a="1"}`}), - func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ validation.Limits, req *logproto.SeriesRequest) { + func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ runtime.Limits, req *logproto.SeriesRequest) { ingester.On("Series", mock.Anything, req, mock.Anything).Return(mockSeriesResponse([]map[string]string{ {"a": "1", "b": "2"}, {"a": "1", "b": "3"}, @@ -344,7 +343,7 @@ func TestQuerier_SeriesAPI(t *testing.T) { { "dedupes", mkReq([]string{`{a="1"}`}), - func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ validation.Limits, req *logproto.SeriesRequest) { + func(store *storeMock, _ *queryClientMock, ingester *querierClientMock, _ runtime.Limits, req *logproto.SeriesRequest) { ingester.On("Series", mock.Anything, req, mock.Anything).Return(mockSeriesResponse([]map[string]string{ {"a": "1", "b": "2"}, }), nil) @@ -386,7 +385,7 @@ func TestQuerier_SeriesAPI(t *testing.T) { tc.setup(store, queryClient, ingesterClient, defaultLimits, tc.req) } - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) q, err := newQuerier( @@ -404,7 +403,7 @@ func TestQuerier_SeriesAPI(t *testing.T) { } func TestQuerier_IngesterMaxQueryLookback(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) for _, tc := range []struct { @@ -549,7 +548,7 @@ func TestQuerier_concurrentTailLimits(t *testing.T) { defaultLimits := defaultLimitsTestConfig() defaultLimits.MaxConcurrentTailRequests = 5 - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) q, err := newQuerier( @@ -1000,7 +999,7 @@ func TestQuerier_RequestingIngesters(t *testing.T) { conf.IngesterQueryStoreMaxLookback = conf.QueryIngestersWithin } - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) for _, request := range requests { @@ -1028,7 +1027,7 @@ func TestQuerier_Volumes(t *testing.T) { {Name: "foo", Volume: 38}, }} - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) ingesterClient := newQuerierClientMock() @@ -1063,7 +1062,7 @@ func TestQuerier_Volumes(t *testing.T) { {Name: "foo", Volume: 38}, }} - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) ingesterClient := newQuerierClientMock() @@ -1099,7 +1098,7 @@ func TestQuerier_Volumes(t *testing.T) { {Name: "foo", Volume: 38}, }} - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) ingesterClient := newQuerierClientMock() @@ -1132,7 +1131,7 @@ func TestQuerier_Volumes(t *testing.T) { }) } -func setupIngesterQuerierMocks(conf Config, limits *validation.Overrides) (*querierClientMock, *storeMock, *SingleTenantQuerier, error) { +func setupIngesterQuerierMocks(conf Config, limits *runtime.Overrides) (*querierClientMock, *storeMock, *SingleTenantQuerier, error) { queryClient := newQueryClientMock() queryClient.On("Recv").Return(mockQueryResponse([]logproto.Stream{mockStream(1, 1)}), nil) @@ -1235,7 +1234,7 @@ func TestQuerier_SelectLogWithDeletes(t *testing.T) { ingesterClient := newQuerierClientMock() ingesterClient.On("Query", mock.Anything, mock.Anything, mock.Anything).Return(queryClient, nil) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) delGetter := &mockDeleteGettter{ @@ -1303,7 +1302,7 @@ func TestQuerier_SelectSamplesWithDeletes(t *testing.T) { ingesterClient := newQuerierClientMock() ingesterClient.On("QuerySample", mock.Anything, mock.Anything, mock.Anything).Return(queryClient, nil) - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) delGetter := &mockDeleteGettter{ @@ -1359,7 +1358,7 @@ func TestQuerier_SelectSamplesWithDeletes(t *testing.T) { require.Equal(t, "test", delGetter.user) } -func newQuerier(cfg Config, clientCfg client.Config, clientFactory ring_client.PoolFactory, ring ring.ReadRing, dg *mockDeleteGettter, store storage.Store, limits *validation.Overrides) (*SingleTenantQuerier, error) { +func newQuerier(cfg Config, clientCfg client.Config, clientFactory ring_client.PoolFactory, ring ring.ReadRing, dg *mockDeleteGettter, store storage.Store, limits *runtime.Overrides) (*SingleTenantQuerier, error) { iq, err := newIngesterQuerier(cfg, clientCfg, ring, nil, nil, clientFactory, constants.Loki, util_log.Logger) if err != nil { return nil, err @@ -1385,7 +1384,7 @@ func TestQuerier_DetectedLabels(t *testing.T) { manyValues = append(manyValues, "a"+strconv.Itoa(i)) } - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, err := runtime.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) ctx := user.InjectOrgID(context.Background(), "test") @@ -1688,7 +1687,7 @@ func TestQuerier_DetectedLabels(t *testing.T) { func BenchmarkQuerierDetectedLabels(b *testing.B) { now := time.Now() - limits, _ := validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits, _ := runtime.NewOverrides(defaultLimitsTestConfig(), nil) ctx := user.InjectOrgID(context.Background(), "test") conf := mockQuerierConfig() diff --git a/pkg/querier/queryrange/roundtrip_test.go b/pkg/querier/queryrange/roundtrip_test.go index ba46bce9c32f3..c84912599c9a5 100644 --- a/pkg/querier/queryrange/roundtrip_test.go +++ b/pkg/querier/queryrange/roundtrip_test.go @@ -28,6 +28,7 @@ import ( "github.com/grafana/loki/v3/pkg/logqlmodel/stats" "github.com/grafana/loki/v3/pkg/querier/plan" base "github.com/grafana/loki/v3/pkg/querier/queryrange/queryrangebase" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/cache" "github.com/grafana/loki/v3/pkg/storage/chunk/cache/resultscache" "github.com/grafana/loki/v3/pkg/storage/config" @@ -36,7 +37,6 @@ import ( "github.com/grafana/loki/v3/pkg/util/constants" util_log "github.com/grafana/loki/v3/pkg/util/log" "github.com/grafana/loki/v3/pkg/util/validation" - valid "github.com/grafana/loki/v3/pkg/validation" ) var ( @@ -1529,7 +1529,7 @@ func (f fakeLimits) VolumeEnabled(_ string) bool { } func (f fakeLimits) TSDBMaxBytesPerShard(_ string) int { - return valid.DefaultTSDBMaxBytesPerShard + return runtime.DefaultTSDBMaxBytesPerShard } func (f fakeLimits) TSDBShardingStrategy(string) string { diff --git a/pkg/querier/testutils.go b/pkg/querier/testutils.go index 34cae5f70580d..eba840ad26174 100644 --- a/pkg/querier/testutils.go +++ b/pkg/querier/testutils.go @@ -3,11 +3,11 @@ package querier import ( "github.com/grafana/dskit/flagext" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) -func DefaultLimitsConfig() validation.Limits { - limits := validation.Limits{} +func DefaultLimitsConfig() runtime.Limits { + limits := runtime.Limits{} flagext.DefaultValues(&limits) return limits } diff --git a/pkg/ruler/base/error_translate_queryable.go b/pkg/ruler/base/error_translate_queryable.go index 1ac65d79cbd37..b2b289ec2814f 100644 --- a/pkg/ruler/base/error_translate_queryable.go +++ b/pkg/ruler/base/error_translate_queryable.go @@ -10,8 +10,8 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" + "github.com/grafana/loki/v3/pkg/runtime" storage_errors "github.com/grafana/loki/v3/pkg/storage/errors" - "github.com/grafana/loki/v3/pkg/validation" ) // TranslateToPromqlAPIError converts error to one of promql.Errors for consumption in PromQL API. @@ -38,7 +38,7 @@ func TranslateToPromqlAPIError(err error) error { case promql.ErrStorage, promql.ErrTooManySamples, promql.ErrQueryCanceled, promql.ErrQueryTimeout: // Don't translate those, just in case we use them internally. return err - case storage_errors.QueryError, validation.LimitError: + case storage_errors.QueryError, runtime.LimitError: // This will be returned with status code 422 by Prometheus API. return err default: diff --git a/pkg/ruler/compat_test.go b/pkg/ruler/compat_test.go index a699da506276d..08b1aa1f5c86b 100644 --- a/pkg/ruler/compat_test.go +++ b/pkg/ruler/compat_test.go @@ -15,8 +15,8 @@ import ( "github.com/grafana/loki/v3/pkg/iter" "github.com/grafana/loki/v3/pkg/logql" rulerbase "github.com/grafana/loki/v3/pkg/ruler/base" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) // TestInvalidRuleGroup tests that a validation error is raised when rule group is invalid @@ -101,7 +101,7 @@ func TestInvalidRemoteWriteConfig(t *testing.T) { // TestNonMetricQuery tests that only metric queries can be executed in the query function, // as both alert and recording rules rely on metric queries being run func TestNonMetricQuery(t *testing.T) { - overrides, err := validation.NewOverrides(validation.Limits{}, nil) + overrides, err := runtime.NewOverrides(runtime.Limits{}, nil) require.Nil(t, err) log := log.Logger diff --git a/pkg/ruler/evaluator_remote_test.go b/pkg/ruler/evaluator_remote_test.go index f98b66aa73ce2..c82723c660fd1 100644 --- a/pkg/ruler/evaluator_remote_test.go +++ b/pkg/ruler/evaluator_remote_test.go @@ -19,8 +19,8 @@ import ( "google.golang.org/grpc" "github.com/grafana/loki/v3/pkg/loghttp" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) type mockClient struct { @@ -41,7 +41,7 @@ func TestRemoteEvalQueryTimeout(t *testing.T) { defaultLimits := defaultLimitsTestConfig() defaultLimits.RulerRemoteEvaluationTimeout = timeout - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) cli := mockClient{ @@ -76,7 +76,7 @@ func TestRemoteEvalMaxResponseSize(t *testing.T) { defaultLimits := defaultLimitsTestConfig() defaultLimits.RulerRemoteEvaluationMaxResponseSize = maxSize - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) cli := mockClient{ @@ -109,7 +109,7 @@ func TestRemoteEvalMaxResponseSize(t *testing.T) { func TestRemoteEvalScalar(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) var ( @@ -161,7 +161,7 @@ func TestRemoteEvalScalar(t *testing.T) { // TestRemoteEvalEmptyScalarResponse validates that an empty scalar response is valid and does not cause an error func TestRemoteEvalEmptyScalarResponse(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) cli := mockClient{ @@ -202,7 +202,7 @@ func TestRemoteEvalEmptyScalarResponse(t *testing.T) { // TestRemoteEvalVectorResponse validates that an empty vector response is valid and does not cause an error func TestRemoteEvalVectorResponse(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) now := time.Now() @@ -268,7 +268,7 @@ func TestRemoteEvalVectorResponse(t *testing.T) { // TestRemoteEvalEmptyVectorResponse validates that an empty vector response is valid and does not cause an error func TestRemoteEvalEmptyVectorResponse(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) cli := mockClient{ @@ -307,7 +307,7 @@ func TestRemoteEvalEmptyVectorResponse(t *testing.T) { func TestRemoteEvalErrorResponse(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) var respErr = fmt.Errorf("some error occurred") @@ -332,7 +332,7 @@ func TestRemoteEvalErrorResponse(t *testing.T) { func TestRemoteEvalNon2xxResponse(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) const httpErr = http.StatusInternalServerError @@ -358,7 +358,7 @@ func TestRemoteEvalNon2xxResponse(t *testing.T) { func TestRemoteEvalNonJSONResponse(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) cli := mockClient{ @@ -383,7 +383,7 @@ func TestRemoteEvalNonJSONResponse(t *testing.T) { func TestRemoteEvalUnsupportedResultResponse(t *testing.T) { defaultLimits := defaultLimitsTestConfig() - limits, err := validation.NewOverrides(defaultLimits, nil) + limits, err := runtime.NewOverrides(defaultLimits, nil) require.NoError(t, err) cli := mockClient{ @@ -421,8 +421,8 @@ func TestRemoteEvalUnsupportedResultResponse(t *testing.T) { require.ErrorContains(t, err, fmt.Sprintf("unsupported result type: %q", loghttp.ResultTypeStream)) } -func defaultLimitsTestConfig() validation.Limits { - limits := validation.Limits{} +func defaultLimitsTestConfig() runtime.Limits { + limits := runtime.Limits{} flagext.DefaultValues(&limits) return limits } diff --git a/pkg/ruler/registry_test.go b/pkg/ruler/registry_test.go index 7ab12d8962ae6..7a80cb3f72c1a 100644 --- a/pkg/ruler/registry_test.go +++ b/pkg/ruler/registry_test.go @@ -21,8 +21,8 @@ import ( "github.com/grafana/loki/v3/pkg/ruler/storage/instance" "github.com/grafana/loki/v3/pkg/ruler/util" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util/test" - "github.com/grafana/loki/v3/pkg/validation" ) const enabledRWTenant = "enabled" @@ -164,7 +164,7 @@ var cfg = Config{ func newFakeLimitsBackwardCompat() fakeLimits { return fakeLimits{ - limits: map[string]*validation.Limits{ + limits: map[string]*runtime.Limits{ enabledRWTenant: { RulerRemoteWriteQueueCapacity: 987, }, @@ -172,7 +172,7 @@ func newFakeLimitsBackwardCompat() fakeLimits { RulerRemoteWriteDisabled: true, }, additionalHeadersRWTenant: { - RulerRemoteWriteHeaders: validation.NewOverwriteMarshalingStringMap(map[string]string{ + RulerRemoteWriteHeaders: runtime.NewOverwriteMarshalingStringMap(map[string]string{ user.OrgIDHeaderName: "overridden", fmt.Sprintf(" %s ", user.OrgIDHeaderName): "overridden", strings.ToLower(user.OrgIDHeaderName): "overridden-lower", @@ -181,7 +181,7 @@ func newFakeLimitsBackwardCompat() fakeLimits { }), }, noHeadersRWTenant: { - RulerRemoteWriteHeaders: validation.NewOverwriteMarshalingStringMap(map[string]string{}), + RulerRemoteWriteHeaders: runtime.NewOverwriteMarshalingStringMap(map[string]string{}), }, customRelabelsTenant: { RulerRemoteWriteRelabelConfigs: []*util.RelabelConfig{ @@ -223,7 +223,7 @@ func newFakeLimits() fakeLimits { regex, _ := relabel.NewRegexp(".+:.+") regexCluster, _ := relabel.NewRegexp("__cluster__") return fakeLimits{ - limits: map[string]*validation.Limits{ + limits: map[string]*runtime.Limits{ enabledRWTenant: { RulerRemoteWriteConfig: map[string]config.RemoteWriteConfig{ remote1: { @@ -320,7 +320,7 @@ func setupRegistry(t *testing.T, cfg Config, limits fakeLimits) *walRegistry { Dir: walDir, } - overrides, err := validation.NewOverrides(validation.Limits{}, limits) + overrides, err := runtime.NewOverrides(runtime.Limits{}, limits) require.NoError(t, err) reg := newWALRegistry(log.NewNopLogger(), nil, cfg, overrides) @@ -897,7 +897,7 @@ func TestRelabelConfigOverridesWithErrors(t *testing.T) { } func TestWALRegistryCreation(t *testing.T) { - overrides, err := validation.NewOverrides(validation.Limits{}, nil) + overrides, err := runtime.NewOverrides(runtime.Limits{}, nil) require.NoError(t, err) regEnabled := newWALRegistry(log.NewNopLogger(), nil, Config{ @@ -957,18 +957,18 @@ func TestStorageSetupWithRemoteWriteDisabled(t *testing.T) { } type fakeLimits struct { - limits map[string]*validation.Limits + limits map[string]*runtime.Limits } -func (f fakeLimits) TenantLimits(userID string) *validation.Limits { +func (f fakeLimits) TenantLimits(userID string) *runtime.Limits { limits, ok := f.limits[userID] if !ok { - return &validation.Limits{} + return &runtime.Limits{} } return limits } -func (f fakeLimits) AllByUserID() map[string]*validation.Limits { +func (f fakeLimits) AllByUserID() map[string]*runtime.Limits { return f.limits } diff --git a/pkg/runtime/config.go b/pkg/runtime/config.go deleted file mode 100644 index 1655789dae71d..0000000000000 --- a/pkg/runtime/config.go +++ /dev/null @@ -1,111 +0,0 @@ -package runtime - -import ( - "flag" -) - -type Config struct { - LogStreamCreation bool `yaml:"log_stream_creation"` - LogPushRequest bool `yaml:"log_push_request"` - LogPushRequestStreams bool `yaml:"log_push_request_streams"` - LogDuplicateMetrics bool `yaml:"log_duplicate_metrics"` - LogDuplicateStreamInfo bool `yaml:"log_duplicate_stream_info"` - - // LimitedLogPushErrors is to be implemented and will allow logging push failures at a controlled pace. - LimitedLogPushErrors bool `yaml:"limited_log_push_errors"` -} - -// RegisterFlags adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.BoolVar(&cfg.LogStreamCreation, "operation-config.log-stream-creation", false, "Log every new stream created by a push request (very verbose, recommend to enable via runtime config only).") - f.BoolVar(&cfg.LogPushRequest, "operation-config.log-push-request", false, "Log every push request (very verbose, recommend to enable via runtime config only).") - f.BoolVar(&cfg.LogPushRequestStreams, "operation-config.log-push-request-streams", false, "Log every stream in a push request (very verbose, recommend to enable via runtime config only).") - f.BoolVar(&cfg.LogDuplicateMetrics, "operation-config.log-duplicate-metrics", false, "Log metrics for duplicate lines received.") - f.BoolVar(&cfg.LogDuplicateStreamInfo, "operation-config.log-duplicate-stream-info", false, "Log stream info for duplicate lines received") - f.BoolVar(&cfg.LimitedLogPushErrors, "operation-config.limited-log-push-errors", true, "Log push errors with a rate limited logger, will show client push errors without overly spamming logs.") -} - -// When we load YAML from disk, we want the various per-customer limits -// to default to any values specified on the command line, not default -// command line values. This global contains those values. I (Tom) cannot -// find a nicer way I'm afraid. -var defaultConfig *Config - -// SetDefaultLimitsForYAMLUnmarshalling sets global default limits, used when loading -// Limits from YAML files. This is used to ensure per-tenant limits are defaulted to -// those values. -func SetDefaultLimitsForYAMLUnmarshalling(defaults Config) { - defaultConfig = &defaults -} - -// TenantConfigProvider serves a tenant or default config. -type TenantConfigProvider interface { - TenantConfig(userID string) *Config -} - -// TenantConfigs periodically fetch a set of per-user configs, and provides convenience -// functions for fetching the correct value. -type TenantConfigs struct { - TenantConfigProvider -} - -// DefaultTenantConfigs creates and returns a new TenantConfigs with the defaults populated. -// Only useful for testing, the provider will ignore any tenants passed in. -func DefaultTenantConfigs() *TenantConfigs { - return &TenantConfigs{ - TenantConfigProvider: &defaultsOnlyConfigProvider{}, - } -} - -type defaultsOnlyConfigProvider struct { -} - -// TenantConfig implementation for defaultsOnlyConfigProvider, ignores the tenant input and only returns a default config -func (t *defaultsOnlyConfigProvider) TenantConfig(_ string) *Config { - if defaultConfig == nil { - defaultConfig = &Config{} - defaultConfig.RegisterFlags(flag.NewFlagSet("", flag.PanicOnError)) - } - return defaultConfig -} - -// NewTenantConfigs makes a new TenantConfigs -func NewTenantConfigs(configProvider TenantConfigProvider) (*TenantConfigs, error) { - return &TenantConfigs{ - TenantConfigProvider: configProvider, - }, nil -} - -func (o *TenantConfigs) getOverridesForUser(userID string) *Config { - if o.TenantConfigProvider != nil { - l := o.TenantConfigProvider.TenantConfig(userID) - if l != nil { - return l - } - } - return defaultConfig -} - -func (o *TenantConfigs) LogStreamCreation(userID string) bool { - return o.getOverridesForUser(userID).LogStreamCreation -} - -func (o *TenantConfigs) LogPushRequest(userID string) bool { - return o.getOverridesForUser(userID).LogPushRequest -} - -func (o *TenantConfigs) LogPushRequestStreams(userID string) bool { - return o.getOverridesForUser(userID).LogPushRequestStreams -} - -func (o *TenantConfigs) LogDuplicateMetrics(userID string) bool { - return o.getOverridesForUser(userID).LogDuplicateMetrics -} - -func (o *TenantConfigs) LogDuplicateStreamInfo(userID string) bool { - return o.getOverridesForUser(userID).LogDuplicateStreamInfo -} - -func (o *TenantConfigs) LimitedLogPushErrors(userID string) bool { - return o.getOverridesForUser(userID).LimitedLogPushErrors -} diff --git a/pkg/runtime/limits.go b/pkg/runtime/limits.go new file mode 100644 index 0000000000000..fe5fc0e27d21a --- /dev/null +++ b/pkg/runtime/limits.go @@ -0,0 +1,1281 @@ +package runtime + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "strconv" + "time" + + "github.com/go-kit/log/level" + dskit_flagext "github.com/grafana/dskit/flagext" + + "github.com/pkg/errors" + "github.com/prometheus/common/model" + "github.com/prometheus/common/sigv4" + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/model/labels" + "golang.org/x/time/rate" + "gopkg.in/yaml.v2" + + "github.com/grafana/loki/v3/pkg/compactor/deletionmode" + "github.com/grafana/loki/v3/pkg/compression" + "github.com/grafana/loki/v3/pkg/distributor/shardstreams" + "github.com/grafana/loki/v3/pkg/loghttp/push" + "github.com/grafana/loki/v3/pkg/logql" + "github.com/grafana/loki/v3/pkg/logql/syntax" + ruler_config "github.com/grafana/loki/v3/pkg/ruler/config" + "github.com/grafana/loki/v3/pkg/ruler/util" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding" + "github.com/grafana/loki/v3/pkg/util/flagext" + util_log "github.com/grafana/loki/v3/pkg/util/log" + "github.com/grafana/loki/v3/pkg/util/validation" +) + +const ( + // LocalRateLimitStrat represents a ingestion rate limiting strategy that enforces the limit + // on a per distributor basis. + // + // The actual effective rate limit will be N times higher, where N is the number of distributor replicas. + LocalIngestionRateStrategy = "local" + + // GlobalRateLimitStrat represents a ingestion rate limiting strategy that enforces the rate + // limiting globally, configuring a per-distributor local rate limiter as "ingestion_rate / N", + // where N is the number of distributor replicas (it's automatically adjusted if the + // number of replicas change). + // + // The global strategy requires the distributors to form their own ring, which + // is used to keep track of the current number of healthy distributor replicas. + GlobalIngestionRateStrategy = "global" + + bytesInMB = 1048576 + + defaultPerStreamRateLimit = 3 << 20 // 3MB + DefaultTSDBMaxBytesPerShard = sharding.DefaultTSDBMaxBytesPerShard + defaultPerStreamBurstLimit = 5 * defaultPerStreamRateLimit + + DefaultPerTenantQueryTimeout = "1m" + + defaultMaxStructuredMetadataSize = "64kb" + defaultMaxStructuredMetadataCount = 128 + defaultBloomBuildMaxBlockSize = "200MB" + defaultBloomBuildMaxBloomSize = "128MB" + defaultBloomTaskTargetChunkSize = "20GB" + + defaultBlockedIngestionStatusCode = 260 // 260 is a custom status code to indicate blocked ingestion +) + +// Limits describe all the limits for users; can be used to describe global default +// limits via flags, or per-user limits via yaml config. +// NOTE: we use custom `model.Duration` instead of standard `time.Duration` because, +// to support user-friendly duration format (e.g: "1h30m45s") in JSON value. +type Limits struct { + // Distributor enforced limits. + IngestionRateStrategy string `yaml:"ingestion_rate_strategy" json:"ingestion_rate_strategy"` + IngestionRateMB float64 `yaml:"ingestion_rate_mb" json:"ingestion_rate_mb"` + IngestionBurstSizeMB float64 `yaml:"ingestion_burst_size_mb" json:"ingestion_burst_size_mb"` + MaxLabelNameLength int `yaml:"max_label_name_length" json:"max_label_name_length"` + MaxLabelValueLength int `yaml:"max_label_value_length" json:"max_label_value_length"` + MaxLabelNamesPerSeries int `yaml:"max_label_names_per_series" json:"max_label_names_per_series"` + RejectOldSamples bool `yaml:"reject_old_samples" json:"reject_old_samples"` + RejectOldSamplesMaxAge model.Duration `yaml:"reject_old_samples_max_age" json:"reject_old_samples_max_age"` + CreationGracePeriod model.Duration `yaml:"creation_grace_period" json:"creation_grace_period"` + MaxLineSize flagext.ByteSize `yaml:"max_line_size" json:"max_line_size"` + MaxLineSizeTruncate bool `yaml:"max_line_size_truncate" json:"max_line_size_truncate"` + IncrementDuplicateTimestamp bool `yaml:"increment_duplicate_timestamp" json:"increment_duplicate_timestamp"` + + // Metadata field extraction + DiscoverGenericFields FieldDetectorConfig `yaml:"discover_generic_fields" json:"discover_generic_fields" doc:"description=Experimental: Detect fields from stream labels, structured metadata, or json/logfmt formatted log line and put them into structured metadata of the log entry."` + DiscoverServiceName []string `yaml:"discover_service_name" json:"discover_service_name"` + DiscoverLogLevels bool `yaml:"discover_log_levels" json:"discover_log_levels"` + LogLevelFields []string `yaml:"log_level_fields" json:"log_level_fields"` + + // Ingester enforced limits. + UseOwnedStreamCount bool `yaml:"use_owned_stream_count" json:"use_owned_stream_count"` + MaxLocalStreamsPerUser int `yaml:"max_streams_per_user" json:"max_streams_per_user"` + MaxGlobalStreamsPerUser int `yaml:"max_global_streams_per_user" json:"max_global_streams_per_user"` + UnorderedWrites bool `yaml:"unordered_writes" json:"unordered_writes"` + PerStreamRateLimit flagext.ByteSize `yaml:"per_stream_rate_limit" json:"per_stream_rate_limit"` + PerStreamRateLimitBurst flagext.ByteSize `yaml:"per_stream_rate_limit_burst" json:"per_stream_rate_limit_burst"` + + // Querier enforced limits. + MaxChunksPerQuery int `yaml:"max_chunks_per_query" json:"max_chunks_per_query"` + MaxQuerySeries int `yaml:"max_query_series" json:"max_query_series"` + MaxQueryLookback model.Duration `yaml:"max_query_lookback" json:"max_query_lookback"` + MaxQueryLength model.Duration `yaml:"max_query_length" json:"max_query_length"` + MaxQueryRange model.Duration `yaml:"max_query_range" json:"max_query_range"` + MaxQueryParallelism int `yaml:"max_query_parallelism" json:"max_query_parallelism"` + TSDBMaxQueryParallelism int `yaml:"tsdb_max_query_parallelism" json:"tsdb_max_query_parallelism"` + TSDBMaxBytesPerShard flagext.ByteSize `yaml:"tsdb_max_bytes_per_shard" json:"tsdb_max_bytes_per_shard"` + TSDBShardingStrategy string `yaml:"tsdb_sharding_strategy" json:"tsdb_sharding_strategy"` + TSDBPrecomputeChunks bool `yaml:"tsdb_precompute_chunks" json:"tsdb_precompute_chunks"` + CardinalityLimit int `yaml:"cardinality_limit" json:"cardinality_limit"` + MaxStreamsMatchersPerQuery int `yaml:"max_streams_matchers_per_query" json:"max_streams_matchers_per_query"` + MaxConcurrentTailRequests int `yaml:"max_concurrent_tail_requests" json:"max_concurrent_tail_requests"` + MaxEntriesLimitPerQuery int `yaml:"max_entries_limit_per_query" json:"max_entries_limit_per_query"` + MaxCacheFreshness model.Duration `yaml:"max_cache_freshness_per_query" json:"max_cache_freshness_per_query"` + MaxMetadataCacheFreshness model.Duration `yaml:"max_metadata_cache_freshness" json:"max_metadata_cache_freshness"` + MaxStatsCacheFreshness model.Duration `yaml:"max_stats_cache_freshness" json:"max_stats_cache_freshness"` + MaxQueriersPerTenant uint `yaml:"max_queriers_per_tenant" json:"max_queriers_per_tenant"` + MaxQueryCapacity float64 `yaml:"max_query_capacity" json:"max_query_capacity"` + QueryReadyIndexNumDays int `yaml:"query_ready_index_num_days" json:"query_ready_index_num_days"` + QueryTimeout model.Duration `yaml:"query_timeout" json:"query_timeout"` + + // Query frontend enforced limits. The default is actually parameterized by the queryrange config. + QuerySplitDuration model.Duration `yaml:"split_queries_by_interval" json:"split_queries_by_interval"` + MetadataQuerySplitDuration model.Duration `yaml:"split_metadata_queries_by_interval" json:"split_metadata_queries_by_interval"` + RecentMetadataQuerySplitDuration model.Duration `yaml:"split_recent_metadata_queries_by_interval" json:"split_recent_metadata_queries_by_interval"` + RecentMetadataQueryWindow model.Duration `yaml:"recent_metadata_query_window" json:"recent_metadata_query_window"` + InstantMetricQuerySplitDuration model.Duration `yaml:"split_instant_metric_queries_by_interval" json:"split_instant_metric_queries_by_interval"` + IngesterQuerySplitDuration model.Duration `yaml:"split_ingester_queries_by_interval" json:"split_ingester_queries_by_interval"` + MinShardingLookback model.Duration `yaml:"min_sharding_lookback" json:"min_sharding_lookback"` + MaxQueryBytesRead flagext.ByteSize `yaml:"max_query_bytes_read" json:"max_query_bytes_read"` + MaxQuerierBytesRead flagext.ByteSize `yaml:"max_querier_bytes_read" json:"max_querier_bytes_read"` + VolumeEnabled bool `yaml:"volume_enabled" json:"volume_enabled" doc:"description=Enable log-volume endpoints."` + VolumeMaxSeries int `yaml:"volume_max_series" json:"volume_max_series" doc:"description=The maximum number of aggregated series in a log-volume response"` + + // Ruler defaults and limits. + RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"` + RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"` + RulerAlertManagerConfig *ruler_config.AlertManagerConfig `yaml:"ruler_alertmanager_config" json:"ruler_alertmanager_config" doc:"hidden"` + RulerTenantShardSize int `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"` + + // TODO(dannyk): add HTTP client overrides (basic auth / tls config, etc) + // Ruler remote-write limits. + + // this field is the inversion of the general remote_write.enabled because the zero value of a boolean is false, + // and if it were ruler_remote_write_enabled, it would be impossible to know if the value was explicitly set or default + RulerRemoteWriteDisabled bool `yaml:"ruler_remote_write_disabled" json:"ruler_remote_write_disabled" doc:"description=Disable recording rules remote-write."` + + // deprecated use RulerRemoteWriteConfig instead. + RulerRemoteWriteURL string `yaml:"ruler_remote_write_url" json:"ruler_remote_write_url" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. The URL of the endpoint to send samples to."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteTimeout time.Duration `yaml:"ruler_remote_write_timeout" json:"ruler_remote_write_timeout" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Timeout for requests to the remote write endpoint."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteHeaders OverwriteMarshalingStringMap `yaml:"ruler_remote_write_headers" json:"ruler_remote_write_headers" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Loki itself can't be overwritten."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteRelabelConfigs []*util.RelabelConfig `yaml:"ruler_remote_write_relabel_configs,omitempty" json:"ruler_remote_write_relabel_configs,omitempty" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. List of remote write relabel configurations."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteQueueCapacity int `yaml:"ruler_remote_write_queue_capacity" json:"ruler_remote_write_queue_capacity" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Number of samples to buffer per shard before we block reading of more samples from the WAL. It is recommended to have enough capacity in each shard to buffer several requests to keep throughput up while processing occasional slow remote requests."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteQueueMinShards int `yaml:"ruler_remote_write_queue_min_shards" json:"ruler_remote_write_queue_min_shards" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Minimum number of shards, i.e. amount of concurrency."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteQueueMaxShards int `yaml:"ruler_remote_write_queue_max_shards" json:"ruler_remote_write_queue_max_shards" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum number of shards, i.e. amount of concurrency."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteQueueMaxSamplesPerSend int `yaml:"ruler_remote_write_queue_max_samples_per_send" json:"ruler_remote_write_queue_max_samples_per_send" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum number of samples per send."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteQueueBatchSendDeadline time.Duration `yaml:"ruler_remote_write_queue_batch_send_deadline" json:"ruler_remote_write_queue_batch_send_deadline" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum time a sample will wait in buffer."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteQueueMinBackoff time.Duration `yaml:"ruler_remote_write_queue_min_backoff" json:"ruler_remote_write_queue_min_backoff" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Initial retry delay. Gets doubled for every retry."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteQueueMaxBackoff time.Duration `yaml:"ruler_remote_write_queue_max_backoff" json:"ruler_remote_write_queue_max_backoff" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum retry delay."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteQueueRetryOnRateLimit bool `yaml:"ruler_remote_write_queue_retry_on_ratelimit" json:"ruler_remote_write_queue_retry_on_ratelimit" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Retry upon receiving a 429 status code from the remote-write storage. This is experimental and might change in the future."` + // deprecated use RulerRemoteWriteConfig instead + RulerRemoteWriteSigV4Config *sigv4.SigV4Config `yaml:"ruler_remote_write_sigv4_config" json:"ruler_remote_write_sigv4_config" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Configures AWS's Signature Verification 4 signing process to sign every remote write request."` + + RulerRemoteWriteConfig map[string]config.RemoteWriteConfig `yaml:"ruler_remote_write_config,omitempty" json:"ruler_remote_write_config,omitempty" doc:"description=Configures global and per-tenant limits for remote write clients. A map with remote client id as key."` + + // TODO(dannyk): possible enhancement is to align this with rule group interval + RulerRemoteEvaluationTimeout time.Duration `yaml:"ruler_remote_evaluation_timeout" json:"ruler_remote_evaluation_timeout" doc:"description=Timeout for a remote rule evaluation. Defaults to the value of 'querier.query-timeout'."` + RulerRemoteEvaluationMaxResponseSize int64 `yaml:"ruler_remote_evaluation_max_response_size" json:"ruler_remote_evaluation_max_response_size" doc:"description=Maximum size (in bytes) of the allowable response size from a remote rule evaluation. Set to 0 to allow any response size (default)."` + + // Global and per tenant deletion mode + DeletionMode string `yaml:"deletion_mode" json:"deletion_mode"` + + // Global and per tenant retention + RetentionPeriod model.Duration `yaml:"retention_period" json:"retention_period"` + StreamRetention []StreamRetention `yaml:"retention_stream,omitempty" json:"retention_stream,omitempty" doc:"description=Per-stream retention to apply, if the retention is enable on the compactor side.\nExample:\n retention_stream:\n - selector: '{namespace=\"dev\"}'\n priority: 1\n period: 24h\n- selector: '{container=\"nginx\"}'\n priority: 1\n period: 744h\nSelector is a Prometheus labels matchers that will apply the 'period' retention only if the stream is matching. In case multiple stream are matching, the highest priority will be picked. If no rule is matched the 'retention_period' is used."` + + // Config for overrides, convenient if it goes here. + PerTenantOverrideConfig string `yaml:"per_tenant_override_config" json:"per_tenant_override_config"` + PerTenantOverridePeriod model.Duration `yaml:"per_tenant_override_period" json:"per_tenant_override_period"` + + // Deprecated + CompactorDeletionEnabled bool `yaml:"allow_deletes" json:"allow_deletes" doc:"deprecated|description=Use deletion_mode per tenant configuration instead."` + + ShardStreams shardstreams.Config `yaml:"shard_streams" json:"shard_streams" doc:"description=Define streams sharding behavior."` + + BlockedQueries []*validation.BlockedQuery `yaml:"blocked_queries,omitempty" json:"blocked_queries,omitempty"` + + RequiredLabels []string `yaml:"required_labels,omitempty" json:"required_labels,omitempty" doc:"description=Define a list of required selector labels."` + RequiredNumberLabels int `yaml:"minimum_labels_number,omitempty" json:"minimum_labels_number,omitempty" doc:"description=Minimum number of label matchers a query should contain."` + + IndexGatewayShardSize int `yaml:"index_gateway_shard_size" json:"index_gateway_shard_size"` + + BloomGatewayShardSize int `yaml:"bloom_gateway_shard_size" json:"bloom_gateway_shard_size" category:"experimental"` + BloomGatewayEnabled bool `yaml:"bloom_gateway_enable_filtering" json:"bloom_gateway_enable_filtering" category:"experimental"` + BloomGatewayCacheKeyInterval time.Duration `yaml:"bloom_gateway_cache_key_interval" json:"bloom_gateway_cache_key_interval" category:"experimental"` + + BloomBuildMaxBuilders int `yaml:"bloom_build_max_builders" json:"bloom_build_max_builders" category:"experimental"` + BloomBuildTaskMaxRetries int `yaml:"bloom_build_task_max_retries" json:"bloom_build_task_max_retries" category:"experimental"` + BloomBuilderResponseTimeout time.Duration `yaml:"bloom_build_builder_response_timeout" json:"bloom_build_builder_response_timeout" category:"experimental"` + + BloomCreationEnabled bool `yaml:"bloom_creation_enabled" json:"bloom_creation_enabled" category:"experimental"` + BloomPlanningStrategy string `yaml:"bloom_planning_strategy" json:"bloom_planning_strategy" category:"experimental"` + BloomSplitSeriesKeyspaceBy int `yaml:"bloom_split_series_keyspace_by" json:"bloom_split_series_keyspace_by" category:"experimental"` + BloomTaskTargetSeriesChunkSize flagext.ByteSize `yaml:"bloom_task_target_series_chunk_size" json:"bloom_task_target_series_chunk_size" category:"experimental"` + BloomBlockEncoding string `yaml:"bloom_block_encoding" json:"bloom_block_encoding" category:"experimental"` + BloomPrefetchBlocks bool `yaml:"bloom_prefetch_blocks" json:"bloom_prefetch_blocks" category:"experimental"` + + BloomMaxBlockSize flagext.ByteSize `yaml:"bloom_max_block_size" json:"bloom_max_block_size" category:"experimental"` + BloomMaxBloomSize flagext.ByteSize `yaml:"bloom_max_bloom_size" json:"bloom_max_bloom_size" category:"experimental"` + + AllowStructuredMetadata bool `yaml:"allow_structured_metadata,omitempty" json:"allow_structured_metadata,omitempty" doc:"description=Allow user to send structured metadata in push payload."` + MaxStructuredMetadataSize flagext.ByteSize `yaml:"max_structured_metadata_size" json:"max_structured_metadata_size" doc:"description=Maximum size accepted for structured metadata per log line."` + MaxStructuredMetadataEntriesCount int `yaml:"max_structured_metadata_entries_count" json:"max_structured_metadata_entries_count" doc:"description=Maximum number of structured metadata entries per log line."` + OTLPConfig push.OTLPConfig `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"` + GlobalOTLPConfig push.GlobalOTLPConfig `yaml:"-" json:"-"` + + BlockIngestionUntil dskit_flagext.Time `yaml:"block_ingestion_until" json:"block_ingestion_until"` + BlockIngestionStatusCode int `yaml:"block_ingestion_status_code" json:"block_ingestion_status_code"` + EnforcedLabels []string `yaml:"enforced_labels" json:"enforced_labels" category:"experimental"` + + IngestionPartitionsTenantShardSize int `yaml:"ingestion_partitions_tenant_shard_size" json:"ingestion_partitions_tenant_shard_size" category:"experimental"` + + ShardAggregations []string `yaml:"shard_aggregations,omitempty" json:"shard_aggregations,omitempty" doc:"description=List of LogQL vector and range aggregations that should be sharded."` + + PatternIngesterTokenizableJSONFieldsDefault dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_default" json:"pattern_ingester_tokenizable_json_fields_default" doc:"hidden"` + PatternIngesterTokenizableJSONFieldsAppend dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_append" json:"pattern_ingester_tokenizable_json_fields_append" doc:"hidden"` + PatternIngesterTokenizableJSONFieldsDelete dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_delete" json:"pattern_ingester_tokenizable_json_fields_delete" doc:"hidden"` + MetricAggregationEnabled bool `yaml:"metric_aggregation_enabled" json:"metric_aggregation_enabled"` + + // This config doesn't have a CLI flag registered here because they're registered in + // their own original config struct. + S3SSEType string `yaml:"s3_sse_type" json:"s3_sse_type" doc:"nocli|description=S3 server-side encryption type. Required to enable server-side encryption overrides for a specific tenant. If not set, the default S3 client settings are used."` + S3SSEKMSKeyID string `yaml:"s3_sse_kms_key_id" json:"s3_sse_kms_key_id" doc:"nocli|description=S3 server-side encryption KMS Key ID. Ignored if the SSE type override is not set."` + S3SSEKMSEncryptionContext string `yaml:"s3_sse_kms_encryption_context" json:"s3_sse_kms_encryption_context" doc:"nocli|description=S3 server-side encryption KMS encryption context. If unset and the key ID override is set, the encryption context will not be provided to S3. Ignored if the SSE type override is not set."` + + // Per-tenant overrides that moved from operations_config to limits_config + LogStreamCreation bool `yaml:"log_stream_creation" json:"log_stream_creation"` + LogPushRequest bool `yaml:"log_push_request" json:"log_stream_request"` + LogPushRequestStreams bool `yaml:"log_push_request_streams" json:"log_push_request_streams"` + LogDuplicateMetrics bool `yaml:"log_duplicate_metrics" json:"log_duplicate_metrics"` + LogDuplicateStreamInfo bool `yaml:"log_duplicate_stream_info" json:"log_duplicate_stream_info"` + LimitedLogPushErrors bool `yaml:"limited_log_push_errors" json:"limited_log_push_errors"` +} + +type FieldDetectorConfig struct { + Fields map[string][]string `yaml:"fields,omitempty" json:"fields,omitempty"` +} + +type StreamRetention struct { + Period model.Duration `yaml:"period" json:"period" doc:"description:Retention period applied to the log lines matching the selector."` + Priority int `yaml:"priority" json:"priority" doc:"description:The larger the value, the higher the priority."` + Selector string `yaml:"selector" json:"selector" doc:"description:Stream selector expression."` + Matchers []*labels.Matcher `yaml:"-" json:"-"` // populated during validation. +} + +// LimitError are errors that do not comply with the limits specified. +type LimitError string + +func (e LimitError) Error() string { + return string(e) +} + +// RegisterFlags adds the flags required to config this to the given FlagSet +func (l *Limits) RegisterFlags(f *flag.FlagSet) { + f.StringVar(&l.IngestionRateStrategy, "distributor.ingestion-rate-limit-strategy", "global", "Whether the ingestion rate limit should be applied individually to each distributor instance (local), or evenly shared across the cluster (global). The ingestion rate strategy cannot be overridden on a per-tenant basis.\n- local: enforces the limit on a per distributor basis. The actual effective rate limit will be N times higher, where N is the number of distributor replicas.\n- global: enforces the limit globally, configuring a per-distributor local rate limiter as 'ingestion_rate / N', where N is the number of distributor replicas (it's automatically adjusted if the number of replicas change). The global strategy requires the distributors to form their own ring, which is used to keep track of the current number of healthy distributor replicas.") + f.Float64Var(&l.IngestionRateMB, "distributor.ingestion-rate-limit-mb", 4, "Per-user ingestion rate limit in sample size per second. Sample size includes size of the logs line and the size of structured metadata labels. Units in MB.") + f.Float64Var(&l.IngestionBurstSizeMB, "distributor.ingestion-burst-size-mb", 6, "Per-user allowed ingestion burst size (in sample size). Units in MB. The burst size refers to the per-distributor local rate limiter even in the case of the 'global' strategy, and should be set at least to the maximum logs size expected in a single push request.") + + _ = l.MaxLineSize.Set("256KB") + f.Var(&l.MaxLineSize, "distributor.max-line-size", "Maximum line size on ingestion path. Example: 256kb. Any log line exceeding this limit will be discarded unless `distributor.max-line-size-truncate` is set which in case it is truncated instead of discarding it completely. There is no limit when unset or set to 0.") + f.BoolVar(&l.MaxLineSizeTruncate, "distributor.max-line-size-truncate", false, "Whether to truncate lines that exceed max_line_size.") + f.IntVar(&l.MaxLabelNameLength, "validation.max-length-label-name", 1024, "Maximum length accepted for label names.") + f.IntVar(&l.MaxLabelValueLength, "validation.max-length-label-value", 2048, "Maximum length accepted for label value. This setting also applies to the metric name.") + f.IntVar(&l.MaxLabelNamesPerSeries, "validation.max-label-names-per-series", 15, "Maximum number of label names per series.") + f.BoolVar(&l.RejectOldSamples, "validation.reject-old-samples", true, "Whether or not old samples will be rejected.") + f.BoolVar(&l.IncrementDuplicateTimestamp, "validation.increment-duplicate-timestamps", false, "Alter the log line timestamp during ingestion when the timestamp is the same as the previous entry for the same stream. When enabled, if a log line in a push request has the same timestamp as the previous line for the same stream, one nanosecond is added to the log line. This will preserve the received order of log lines with the exact same timestamp when they are queried, by slightly altering their stored timestamp. NOTE: This is imperfect, because Loki accepts out of order writes, and another push request for the same stream could contain duplicate timestamps to existing entries and they will not be incremented.") + l.DiscoverServiceName = []string{ + "service", + "app", + "application", + "app_name", + "name", + "app_kubernetes_io_name", + "container", + "container_name", + "k8s_container_name", + "component", + "workload", + "job", + "k8s_job_name", + } + f.Var((*dskit_flagext.StringSlice)(&l.DiscoverServiceName), "validation.discover-service-name", "If no service_name label exists, Loki maps a single label from the configured list to service_name. If none of the configured labels exist in the stream, label is set to unknown_service. Empty list disables setting the label.") + f.BoolVar(&l.DiscoverLogLevels, "validation.discover-log-levels", true, "Discover and add log levels during ingestion, if not present already. Levels would be added to Structured Metadata with name level/LEVEL/Level/Severity/severity/SEVERITY/lvl/LVL/Lvl (case-sensitive) and one of the values from 'trace', 'debug', 'info', 'warn', 'error', 'critical', 'fatal' (case insensitive).") + l.LogLevelFields = []string{"level", "LEVEL", "Level", "Severity", "severity", "SEVERITY", "lvl", "LVL", "Lvl"} + f.Var((*dskit_flagext.StringSlice)(&l.LogLevelFields), "validation.log-level-fields", "Field name to use for log levels. If not set, log level would be detected based on pre-defined labels as mentioned above.") + + _ = l.RejectOldSamplesMaxAge.Set("7d") + f.Var(&l.RejectOldSamplesMaxAge, "validation.reject-old-samples.max-age", "Maximum accepted sample age before rejecting.") + _ = l.CreationGracePeriod.Set("10m") + f.Var(&l.CreationGracePeriod, "validation.create-grace-period", "Duration which table will be created/deleted before/after it's needed; we won't accept sample from before this time.") + f.IntVar(&l.MaxEntriesLimitPerQuery, "validation.max-entries-limit", 5000, "Maximum number of log entries that will be returned for a query.") + + f.BoolVar(&l.UseOwnedStreamCount, "ingester.use-owned-stream-count", false, "When true an ingester takes into account only the streams that it owns according to the ring while applying the stream limit.") + f.IntVar(&l.MaxLocalStreamsPerUser, "ingester.max-streams-per-user", 0, "Maximum number of active streams per user, per ingester. 0 to disable.") + f.IntVar(&l.MaxGlobalStreamsPerUser, "ingester.max-global-streams-per-user", 5000, "Maximum number of active streams per user, across the cluster. 0 to disable. When the global limit is enabled, each ingester is configured with a dynamic local limit based on the replication factor and the current number of healthy ingesters, and is kept updated whenever the number of ingesters change.") + + // TODO(ashwanth) Deprecated. This will be removed with the next major release and out-of-order writes would be accepted by default. + f.BoolVar(&l.UnorderedWrites, "ingester.unordered-writes", true, "Deprecated. When true, out-of-order writes are accepted.") + + _ = l.PerStreamRateLimit.Set(strconv.Itoa(defaultPerStreamRateLimit)) + f.Var(&l.PerStreamRateLimit, "ingester.per-stream-rate-limit", "Maximum byte rate per second per stream, also expressible in human readable forms (1MB, 256KB, etc).") + _ = l.PerStreamRateLimitBurst.Set(strconv.Itoa(defaultPerStreamBurstLimit)) + f.Var(&l.PerStreamRateLimitBurst, "ingester.per-stream-rate-limit-burst", "Maximum burst bytes per stream, also expressible in human readable forms (1MB, 256KB, etc). This is how far above the rate limit a stream can 'burst' before the stream is limited.") + + f.IntVar(&l.MaxChunksPerQuery, "store.query-chunk-limit", 2e6, "Maximum number of chunks that can be fetched in a single query.") + + _ = l.MaxQueryLength.Set("721h") + f.Var(&l.MaxQueryLength, "store.max-query-length", "The limit to length of chunk store queries. 0 to disable.") + f.IntVar(&l.MaxQuerySeries, "querier.max-query-series", 500, "Limit the maximum of unique series that is returned by a metric query. When the limit is reached an error is returned.") + _ = l.MaxQueryRange.Set("0s") + f.Var(&l.MaxQueryRange, "querier.max-query-range", "Limit the length of the [range] inside a range query. Default is 0 or unlimited") + _ = l.QueryTimeout.Set(DefaultPerTenantQueryTimeout) + f.Var(&l.QueryTimeout, "querier.query-timeout", "Timeout when querying backends (ingesters or storage) during the execution of a query request. When a specific per-tenant timeout is used, the global timeout is ignored.") + + _ = l.MaxQueryLookback.Set("0s") + f.Var(&l.MaxQueryLookback, "querier.max-query-lookback", "Limit how far back in time series data and metadata can be queried, up until lookback duration ago. This limit is enforced in the query frontend, the querier and the ruler. If the requested time range is outside the allowed range, the request will not fail, but will be modified to only query data within the allowed time range. The default value of 0 does not set a limit.") + f.IntVar(&l.MaxQueryParallelism, "querier.max-query-parallelism", 32, "Maximum number of queries that will be scheduled in parallel by the frontend.") + f.IntVar(&l.TSDBMaxQueryParallelism, "querier.tsdb-max-query-parallelism", 128, "Maximum number of queries will be scheduled in parallel by the frontend for TSDB schemas.") + _ = l.TSDBMaxBytesPerShard.Set(strconv.Itoa(DefaultTSDBMaxBytesPerShard)) + f.Var(&l.TSDBMaxBytesPerShard, "querier.tsdb-max-bytes-per-shard", "Target maximum number of bytes assigned to a single sharded query. Also expressible in human readable forms (1GB, etc). Note: This is a _target_ and not an absolute limit. The actual limit can be higher, but the query planner will try to build shards up to this limit.") + f.StringVar( + &l.TSDBShardingStrategy, + "limits.tsdb-sharding-strategy", + logql.PowerOfTwoVersion.String(), + fmt.Sprintf( + "sharding strategy to use in query planning. Suggested to use %s once all nodes can recognize it.", + logql.BoundedVersion.String(), + ), + ) + f.BoolVar(&l.TSDBPrecomputeChunks, "querier.tsdb-precompute-chunks", false, "Precompute chunks for TSDB queries. This can improve query performance at the cost of increased memory usage by computing chunks once during planning, reducing index calls.") + f.IntVar(&l.CardinalityLimit, "store.cardinality-limit", 1e5, "Cardinality limit for index queries.") + f.IntVar(&l.MaxStreamsMatchersPerQuery, "querier.max-streams-matcher-per-query", 1000, "Maximum number of stream matchers per query.") + f.IntVar(&l.MaxConcurrentTailRequests, "querier.max-concurrent-tail-requests", 10, "Maximum number of concurrent tail requests.") + + _ = l.MinShardingLookback.Set("0s") + f.Var(&l.MinShardingLookback, "frontend.min-sharding-lookback", "Limit queries that can be sharded. Queries within the time range of now and now minus this sharding lookback are not sharded. The default value of 0s disables the lookback, causing sharding of all queries at all times.") + + f.Var(&l.MaxQueryBytesRead, "frontend.max-query-bytes-read", "Max number of bytes a query can fetch. Enforced in log and metric queries only when TSDB is used. This limit is not enforced on log queries without filters. The default value of 0 disables this limit.") + + _ = l.MaxQuerierBytesRead.Set("150GB") + f.Var(&l.MaxQuerierBytesRead, "frontend.max-querier-bytes-read", "Max number of bytes a query can fetch after splitting and sharding. Enforced in log and metric queries only when TSDB is used. This limit is not enforced on log queries without filters. The default value of 0 disables this limit.") + + _ = l.MaxCacheFreshness.Set("10m") + f.Var(&l.MaxCacheFreshness, "frontend.max-cache-freshness", "Most recent allowed cacheable result per-tenant, to prevent caching very recent results that might still be in flux.") + + _ = l.MaxMetadataCacheFreshness.Set("24h") + f.Var(&l.MaxMetadataCacheFreshness, "frontend.max-metadata-cache-freshness", "Do not cache metadata request if the end time is within the frontend.max-metadata-cache-freshness window. Set this to 0 to apply no such limits. Defaults to 24h.") + + _ = l.MaxStatsCacheFreshness.Set("10m") + f.Var(&l.MaxStatsCacheFreshness, "frontend.max-stats-cache-freshness", "Do not cache requests with an end time that falls within Now minus this duration. 0 disables this feature (default).") + + f.UintVar(&l.MaxQueriersPerTenant, "frontend.max-queriers-per-tenant", 0, "Maximum number of queriers that can handle requests for a single tenant. If set to 0 or value higher than number of available queriers, *all* queriers will handle requests for the tenant. Each frontend (or query-scheduler, if used) will select the same set of queriers for the same tenant (given that all queriers are connected to all frontends / query-schedulers). This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL.") + f.Float64Var(&l.MaxQueryCapacity, "frontend.max-query-capacity", 0, "How much of the available query capacity (\"querier\" components in distributed mode, \"read\" components in SSD mode) can be used by a single tenant. Allowed values are 0.0 to 1.0. For example, setting this to 0.5 would allow a tenant to use half of the available queriers for processing the query workload. If set to 0, query capacity is determined by frontend.max-queriers-per-tenant. When both frontend.max-queriers-per-tenant and frontend.max-query-capacity are configured, smaller value of the resulting querier replica count is considered: min(frontend.max-queriers-per-tenant, ceil(querier_replicas * frontend.max-query-capacity)). *All* queriers will handle requests for the tenant if neither limits are applied. This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL. Use this feature in a multi-tenant setup where you need to limit query capacity for certain tenants.") + f.IntVar(&l.QueryReadyIndexNumDays, "store.query-ready-index-num-days", 0, "Number of days of index to be kept always downloaded for queries. Applies only to per user index in boltdb-shipper index store. 0 to disable.") + + f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 0, "Maximum number of rules per rule group per-tenant. 0 to disable.") + f.IntVar(&l.RulerMaxRuleGroupsPerTenant, "ruler.max-rule-groups-per-tenant", 0, "Maximum number of rule groups per-tenant. 0 to disable.") + f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when shuffle-sharding is enabled in the ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") + + f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "Feature renamed to 'runtime configuration', flag deprecated in favor of -runtime-config.file (runtime_config.file in YAML).") + _ = l.RetentionPeriod.Set("0s") + f.Var(&l.RetentionPeriod, "store.retention", "Retention period to apply to stored data, only applies if retention_enabled is true in the compactor config. As of version 2.8.0, a zero value of 0 or 0s disables retention. In previous releases, Loki did not properly honor a zero value to disable retention and a really large value should be used instead.") + + _ = l.PerTenantOverridePeriod.Set("10s") + f.Var(&l.PerTenantOverridePeriod, "limits.per-user-override-period", "Feature renamed to 'runtime configuration'; flag deprecated in favor of -runtime-config.reload-period (runtime_config.period in YAML).") + + _ = l.QuerySplitDuration.Set("1h") + f.Var(&l.QuerySplitDuration, "querier.split-queries-by-interval", "Split queries by a time interval and execute in parallel. The value 0 disables splitting by time. This also determines how cache keys are chosen when result caching is enabled.") + _ = l.InstantMetricQuerySplitDuration.Set("1h") + f.Var(&l.InstantMetricQuerySplitDuration, "querier.split-instant-metric-queries-by-interval", "Split instant metric queries by a time interval and execute in parallel. The value 0 disables splitting instant metric queries by time. This also determines how cache keys are chosen when instant metric query result caching is enabled.") + + _ = l.MetadataQuerySplitDuration.Set("24h") + f.Var(&l.MetadataQuerySplitDuration, "querier.split-metadata-queries-by-interval", "Split metadata queries by a time interval and execute in parallel. The value 0 disables splitting metadata queries by time. This also determines how cache keys are chosen when label/series result caching is enabled.") + + _ = l.RecentMetadataQuerySplitDuration.Set("1h") + f.Var(&l.RecentMetadataQuerySplitDuration, "experimental.querier.split-recent-metadata-queries-by-interval", "Experimental. Split interval to use for the portion of metadata request that falls within `recent_metadata_query_window`. Rest of the request which is outside the window still uses `split_metadata_queries_by_interval`. If set to 0, the entire request defaults to using a split interval of `split_metadata_queries_by_interval.`.") + + f.Var(&l.RecentMetadataQueryWindow, "experimental.querier.recent-metadata-query-window", "Experimental. Metadata query window inside which `split_recent_metadata_queries_by_interval` gets applied, portion of the metadata request that falls in this window is split using `split_recent_metadata_queries_by_interval`. The value 0 disables using a different split interval for recent metadata queries.\n\nThis is added to improve cacheability of recent metadata queries. Query split interval also determines the interval used in cache key. The default split interval of 24h is useful for caching long queries, each cache key holding 1 day's results. But metadata queries are often shorter than 24h, to cache them effectively we need a smaller split interval. `recent_metadata_query_window` along with `split_recent_metadata_queries_by_interval` help configure a shorter split interval for recent metadata queries.") + + _ = l.IngesterQuerySplitDuration.Set("0s") + f.Var(&l.IngesterQuerySplitDuration, "querier.split-ingester-queries-by-interval", "Interval to use for time-based splitting when a request is within the `query_ingesters_within` window; defaults to `split-queries-by-interval` by setting to 0.") + + f.StringVar(&l.DeletionMode, "compactor.deletion-mode", "filter-and-delete", "Deletion mode. Can be one of 'disabled', 'filter-only', or 'filter-and-delete'. When set to 'filter-only' or 'filter-and-delete', and if retention_enabled is true, then the log entry deletion API endpoints are available.") + + // Deprecated + dskit_flagext.DeprecatedFlag(f, "compactor.allow-deletes", "Deprecated. Instead, see compactor.deletion-mode which is another per tenant configuration", util_log.Logger) + + f.IntVar(&l.IndexGatewayShardSize, "index-gateway.shard-size", 0, "The shard size defines how many index gateways should be used by a tenant for querying. If the global shard factor is 0, the global shard factor is set to the deprecated -replication-factor for backwards compatibility reasons.") + + f.IntVar(&l.BloomGatewayShardSize, "bloom-gateway.shard-size", 0, "Experimental. The shard size defines how many bloom gateways should be used by a tenant for querying.") + f.BoolVar(&l.BloomGatewayEnabled, "bloom-gateway.enable-filtering", false, "Experimental. Whether to use the bloom gateway component in the read path to filter chunks.") + f.DurationVar(&l.BloomGatewayCacheKeyInterval, "bloom-gateway.cache-key-interval", 15*time.Minute, "Experimental. Interval for computing the cache key in the Bloom Gateway.") + + f.StringVar(&l.BloomBlockEncoding, "bloom-build.block-encoding", "none", "Experimental. Compression algorithm for bloom block pages.") + f.BoolVar(&l.BloomPrefetchBlocks, "bloom-build.prefetch-blocks", false, "Experimental. Prefetch blocks on bloom gateways as soon as they are built.") + + _ = l.BloomMaxBlockSize.Set(defaultBloomBuildMaxBlockSize) + f.Var(&l.BloomMaxBlockSize, "bloom-build.max-block-size", + fmt.Sprintf( + "Experimental. The maximum bloom block size. A value of 0 sets an unlimited size. Default is %s. The actual block size might exceed this limit since blooms will be added to blocks until the block exceeds the maximum block size.", + defaultBloomBuildMaxBlockSize, + ), + ) + + f.BoolVar(&l.BloomCreationEnabled, "bloom-build.enable", false, "Experimental. Whether to create blooms for the tenant.") + f.StringVar(&l.BloomPlanningStrategy, "bloom-build.planning-strategy", "split_keyspace_by_factor", "Experimental. Bloom planning strategy to use in bloom creation. Can be one of: 'split_keyspace_by_factor', 'split_by_series_chunks_size'") + f.IntVar(&l.BloomSplitSeriesKeyspaceBy, "bloom-build.split-keyspace-by", 256, "Experimental. Only if `bloom-build.planning-strategy` is 'split'. Number of splits to create for the series keyspace when building blooms. The series keyspace is split into this many parts to parallelize bloom creation.") + _ = l.BloomTaskTargetSeriesChunkSize.Set(defaultBloomTaskTargetChunkSize) + f.Var(&l.BloomTaskTargetSeriesChunkSize, "bloom-build.split-target-series-chunk-size", fmt.Sprintf("Experimental. Target chunk size in bytes for bloom tasks. Default is %s.", defaultBloomTaskTargetChunkSize)) + f.IntVar(&l.BloomBuildMaxBuilders, "bloom-build.max-builders", 0, "Experimental. Maximum number of builders to use when building blooms. 0 allows unlimited builders.") + f.DurationVar(&l.BloomBuilderResponseTimeout, "bloom-build.builder-response-timeout", 0, "Experimental. Timeout for a builder to finish a task. If a builder does not respond within this time, it is considered failed and the task will be requeued. 0 disables the timeout.") + f.IntVar(&l.BloomBuildTaskMaxRetries, "bloom-build.task-max-retries", 3, "Experimental. Maximum number of retries for a failed task. If a task fails more than this number of times, it is considered failed and will not be retried. A value of 0 disables this limit.") + + _ = l.BloomMaxBloomSize.Set(defaultBloomBuildMaxBloomSize) + f.Var(&l.BloomMaxBloomSize, "bloom-build.max-bloom-size", + fmt.Sprintf( + "Experimental. The maximum bloom size per log stream. A log stream whose generated bloom filter exceeds this size will be discarded. A value of 0 sets an unlimited size. Default is %s.", + defaultBloomBuildMaxBloomSize, + ), + ) + + l.ShardStreams.RegisterFlagsWithPrefix("shard-streams", f) + f.IntVar(&l.VolumeMaxSeries, "limits.volume-max-series", 1000, "The default number of aggregated series or labels that can be returned from a log-volume endpoint") + + f.BoolVar(&l.AllowStructuredMetadata, "validation.allow-structured-metadata", true, "Allow user to send structured metadata (non-indexed labels) in push payload.") + _ = l.MaxStructuredMetadataSize.Set(defaultMaxStructuredMetadataSize) + f.Var(&l.MaxStructuredMetadataSize, "limits.max-structured-metadata-size", "Maximum size accepted for structured metadata per entry. Default: 64 kb. Any log line exceeding this limit will be discarded. There is no limit when unset or set to 0.") + f.IntVar(&l.MaxStructuredMetadataEntriesCount, "limits.max-structured-metadata-entries-count", defaultMaxStructuredMetadataCount, "Maximum number of structured metadata entries per log line. Default: 128. Any log line exceeding this limit will be discarded. There is no limit when unset or set to 0.") + f.BoolVar(&l.VolumeEnabled, "limits.volume-enabled", true, "Enable log volume endpoint.") + + f.Var(&l.BlockIngestionUntil, "limits.block-ingestion-until", "Block ingestion until the configured date. The time should be in RFC3339 format.") + f.IntVar(&l.BlockIngestionStatusCode, "limits.block-ingestion-status-code", defaultBlockedIngestionStatusCode, "HTTP status code to return when ingestion is blocked. If 200, the ingestion will be blocked without returning an error to the client. By Default, a custom status code (260) is returned to the client along with an error message.") + f.Var((*dskit_flagext.StringSlice)(&l.EnforcedLabels), "validation.enforced-labels", "List of labels that must be present in the stream. If any of the labels are missing, the stream will be discarded. This flag configures it globally for all tenants. Experimental.") + + f.IntVar(&l.IngestionPartitionsTenantShardSize, "limits.ingestion-partition-tenant-shard-size", 0, "The number of partitions a tenant's data should be sharded to when using kafka ingestion. Tenants are sharded across partitions using shuffle-sharding. 0 disables shuffle sharding and tenant is sharded across all partitions.") + + _ = l.PatternIngesterTokenizableJSONFieldsDefault.Set("log,message,msg,msg_,_msg,content") + f.Var(&l.PatternIngesterTokenizableJSONFieldsDefault, "limits.pattern-ingester-tokenizable-json-fields", "List of JSON fields that should be tokenized in the pattern ingester.") + f.Var(&l.PatternIngesterTokenizableJSONFieldsAppend, "limits.pattern-ingester-tokenizable-json-fields-append", "List of JSON fields that should be appended to the default list of tokenizable fields in the pattern ingester.") + f.Var(&l.PatternIngesterTokenizableJSONFieldsDelete, "limits.pattern-ingester-tokenizable-json-fields-delete", "List of JSON fields that should be deleted from the (default U append) list of tokenizable fields in the pattern ingester.") + + f.BoolVar( + &l.MetricAggregationEnabled, + "limits.metric-aggregation-enabled", + false, + "Enable metric aggregation. When enabled, pushed streams will be sampled for bytes and count, and these metric will be written back into Loki as a special __aggregated_metric__ stream, which can be queried for faster histogram queries.", + ) + + // moved from runtime_config to limits_config + f.BoolVar(&l.LogStreamCreation, "operation-config.log-stream-creation", false, "Log every new stream created by a push request (very verbose, recommend to enable via runtime config only).") + f.BoolVar(&l.LogPushRequest, "operation-config.log-push-request", false, "Log every push request (very verbose, recommend to enable via runtime config only).") + f.BoolVar(&l.LogPushRequestStreams, "operation-config.log-push-request-streams", false, "Log every stream in a push request (very verbose, recommend to enable via runtime config only).") + f.BoolVar(&l.LogDuplicateMetrics, "operation-config.log-duplicate-metrics", false, "Log metrics for duplicate lines received.") + f.BoolVar(&l.LogDuplicateStreamInfo, "operation-config.log-duplicate-stream-info", false, "Log stream info for duplicate lines received") + f.BoolVar(&l.LimitedLogPushErrors, "operation-config.limited-log-push-errors", true, "Log push errors with a rate limited logger, will show client push errors without overly spamming logs.") +} + +// SetGlobalOTLPConfig set GlobalOTLPConfig which is used while unmarshaling per-tenant otlp config to use the default list of resource attributes picked as index labels. +func (l *Limits) SetGlobalOTLPConfig(cfg push.GlobalOTLPConfig) { + l.GlobalOTLPConfig = cfg + l.OTLPConfig.ApplyGlobalOTLPConfig(cfg) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (l *Limits) UnmarshalYAML(unmarshal func(interface{}) error) error { + // We want to set c to the defaults and then overwrite it with the input. + // To make unmarshal fill the plain data struct rather than calling UnmarshalYAML + // again, we have to hide it using a type indirection. See prometheus/config. + type plain Limits + + // During startup we wont have a default value so we don't want to overwrite them + if defaultLimits != nil { + b, err := yaml.Marshal(defaultLimits) + if err != nil { + return errors.Wrap(err, "cloning limits (marshaling)") + } + if err := yaml.Unmarshal(b, (*plain)(l)); err != nil { + return errors.Wrap(err, "cloning limits (unmarshaling)") + } + } + if err := unmarshal((*plain)(l)); err != nil { + return err + } + + if defaultLimits != nil { + // apply relevant bits from global otlp config + l.OTLPConfig.ApplyGlobalOTLPConfig(defaultLimits.GlobalOTLPConfig) + } + return nil +} + +// Validate validates that this limits config is valid. +func (l *Limits) Validate() error { + if l.StreamRetention != nil { + for i, rule := range l.StreamRetention { + matchers, err := syntax.ParseMatchers(rule.Selector, true) + if err != nil { + return fmt.Errorf("invalid labels matchers: %w", err) + } + if time.Duration(rule.Period) < 24*time.Hour { + return fmt.Errorf("retention period must be >= 24h was %s", rule.Period) + } + // populate matchers during validation + l.StreamRetention[i].Matchers = matchers + } + } + + if _, err := deletionmode.ParseMode(l.DeletionMode); err != nil { + return err + } + + if l.CompactorDeletionEnabled { + level.Warn(util_log.Logger).Log("msg", "The compactor.allow-deletes configuration option has been deprecated and will be ignored. Instead, use deletion_mode in the limits_configs to adjust deletion functionality") + } + + if l.MaxQueryCapacity < 0 { + level.Warn(util_log.Logger).Log("msg", "setting frontend.max-query-capacity to 0 as it is configured to a value less than 0") + l.MaxQueryCapacity = 0 + } + + if l.MaxQueryCapacity > 1 { + level.Warn(util_log.Logger).Log("msg", "setting frontend.max-query-capacity to 1 as it is configured to a value greater than 1") + l.MaxQueryCapacity = 1 + } + + if err := l.OTLPConfig.Validate(); err != nil { + return err + } + + if _, err := logql.ParseShardVersion(l.TSDBShardingStrategy); err != nil { + return errors.Wrap(err, "invalid tsdb sharding strategy") + } + + if _, err := compression.ParseCodec(l.BloomBlockEncoding); err != nil { + return err + } + + if l.TSDBMaxBytesPerShard <= 0 { + return errors.New("querier.tsdb-max-bytes-per-shard must be greater than 0") + } + + return nil +} + +// When we load YAML from disk, we want the various per-customer limits +// to default to any values specified on the command line, not default +// command line values. This global contains those values. I (Tom) cannot +// find a nicer way I'm afraid. +var defaultLimits *Limits + +// SetDefaultLimitsForYAMLUnmarshalling sets global default limits, used when loading +// Limits from YAML files. This is used to ensure per-tenant limits are defaulted to +// those values. +func SetDefaultLimitsForYAMLUnmarshalling(defaults Limits) { + defaultLimits = &defaults +} + +type ExportedLimits interface { + // DefaultLimits returns the default values of runtime settings + DefaultLimits() *Limits + // AllByUserID gets a mapping of all tenant IDs and limits for that user + AllByUserID() map[string]*Limits +} + +type TenantLimits interface { + // TenantLimits is a function that returns limits for given tenant, or + // nil, if there are no tenant-specific limits. + TenantLimits(userID string) *Limits + // AllByUserID gets a mapping of all tenant IDs and limits for that user + AllByUserID() map[string]*Limits +} + +// Overrides periodically fetch a set of per-user overrides, and provides convenience +// functions for fetching the correct value. +type Overrides struct { + defaultLimits *Limits + tenantLimits TenantLimits +} + +// NewOverrides makes a new Overrides. +func NewOverrides(defaults Limits, tenantLimits TenantLimits) (*Overrides, error) { + return &Overrides{ + tenantLimits: tenantLimits, + defaultLimits: &defaults, + }, nil +} + +func (o *Overrides) AllByUserID() map[string]*Limits { + if o.tenantLimits != nil { + return o.tenantLimits.AllByUserID() + } + return nil +} + +// IngestionRateStrategy returns whether the ingestion rate limit should be individually applied +// to each distributor instance (local) or evenly shared across the cluster (global). +func (o *Overrides) IngestionRateStrategy() string { + // The ingestion rate strategy can't be overridden on a per-tenant basis, + // so here we just pick the value for a not-existing user ID (empty string). + return o.getOverridesForUser("").IngestionRateStrategy +} + +// IngestionRateBytes returns the limit on ingester rate (MBs per second). +func (o *Overrides) IngestionRateBytes(userID string) float64 { + return o.getOverridesForUser(userID).IngestionRateMB * bytesInMB +} + +// IngestionBurstSizeBytes returns the burst size for ingestion rate. +func (o *Overrides) IngestionBurstSizeBytes(userID string) int { + return int(o.getOverridesForUser(userID).IngestionBurstSizeMB * bytesInMB) +} + +// MaxLabelNameLength returns maximum length a label name can be. +func (o *Overrides) MaxLabelNameLength(userID string) int { + return o.getOverridesForUser(userID).MaxLabelNameLength +} + +// MaxLabelValueLength returns maximum length a label value can be. This also is +// the maximum length of a metric name. +func (o *Overrides) MaxLabelValueLength(userID string) int { + return o.getOverridesForUser(userID).MaxLabelValueLength +} + +// MaxLabelNamesPerSeries returns maximum number of label/value pairs timeseries. +func (o *Overrides) MaxLabelNamesPerSeries(userID string) int { + return o.getOverridesForUser(userID).MaxLabelNamesPerSeries +} + +// RejectOldSamples returns true when we should reject samples older than certain +// age. +func (o *Overrides) RejectOldSamples(userID string) bool { + return o.getOverridesForUser(userID).RejectOldSamples +} + +// RejectOldSamplesMaxAge returns the age at which samples should be rejected. +func (o *Overrides) RejectOldSamplesMaxAge(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).RejectOldSamplesMaxAge) +} + +// CreationGracePeriod is misnamed, and actually returns how far into the future +// we should accept samples. +func (o *Overrides) CreationGracePeriod(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).CreationGracePeriod) +} + +func (o *Overrides) UseOwnedStreamCount(userID string) bool { + return o.getOverridesForUser(userID).UseOwnedStreamCount +} + +// MaxLocalStreamsPerUser returns the maximum number of streams a user is allowed to store +// in a single ingester. +func (o *Overrides) MaxLocalStreamsPerUser(userID string) int { + return o.getOverridesForUser(userID).MaxLocalStreamsPerUser +} + +// MaxGlobalStreamsPerUser returns the maximum number of streams a user is allowed to store +// across the cluster. +func (o *Overrides) MaxGlobalStreamsPerUser(userID string) int { + return o.getOverridesForUser(userID).MaxGlobalStreamsPerUser +} + +// MaxChunksPerQuery returns the maximum number of chunks allowed per query. +func (o *Overrides) MaxChunksPerQuery(userID string) int { + return o.getOverridesForUser(userID).MaxChunksPerQuery +} + +// MaxQueryLength returns the limit of the length (in time) of a query. +func (o *Overrides) MaxQueryLength(_ context.Context, userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).MaxQueryLength) +} + +// Compatibility with Cortex interface, this method is set to be removed in 1.12, +// so nooping in Loki until then. +func (o *Overrides) MaxChunksPerQueryFromStore(_ string) int { return 0 } + +// MaxQuerySeries returns the limit of the series of metric queries. +func (o *Overrides) MaxQuerySeries(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).MaxQuerySeries +} + +// MaxQueryRange returns the limit for the max [range] value that can be in a range query +func (o *Overrides) MaxQueryRange(_ context.Context, userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).MaxQueryRange) +} + +// MaxQueriersPerUser returns the maximum number of queriers that can handle requests for this user. +func (o *Overrides) MaxQueriersPerUser(userID string) uint { + return o.getOverridesForUser(userID).MaxQueriersPerTenant +} + +// MaxQueryCapacity returns how much of the available query capacity can be used by this user.. +func (o *Overrides) MaxQueryCapacity(userID string) float64 { + return o.getOverridesForUser(userID).MaxQueryCapacity +} + +// QueryReadyIndexNumDays returns the number of days for which we have to be query ready for a user. +func (o *Overrides) QueryReadyIndexNumDays(userID string) int { + return o.getOverridesForUser(userID).QueryReadyIndexNumDays +} + +// TSDBMaxQueryParallelism returns the limit to the number of sub-queries the +// frontend will process in parallel for TSDB schemas. +func (o *Overrides) TSDBMaxQueryParallelism(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).TSDBMaxQueryParallelism +} + +// TSDBMaxBytesPerShard returns the maximum number of bytes assigned to a specific shard in a tsdb query +func (o *Overrides) TSDBMaxBytesPerShard(userID string) int { + return o.getOverridesForUser(userID).TSDBMaxBytesPerShard.Val() +} + +// TSDBShardingStrategy returns the sharding strategy to use in query planning. +func (o *Overrides) TSDBShardingStrategy(userID string) string { + return o.getOverridesForUser(userID).TSDBShardingStrategy +} + +func (o *Overrides) TSDBPrecomputeChunks(userID string) bool { + return o.getOverridesForUser(userID).TSDBPrecomputeChunks +} + +// MaxQueryParallelism returns the limit to the number of sub-queries the +// frontend will process in parallel. +func (o *Overrides) MaxQueryParallelism(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).MaxQueryParallelism +} + +// CardinalityLimit whether to enforce the presence of a metric name. +func (o *Overrides) CardinalityLimit(userID string) int { + return o.getOverridesForUser(userID).CardinalityLimit +} + +// MaxStreamsMatchersPerQuery returns the limit to number of streams matchers per query. +func (o *Overrides) MaxStreamsMatchersPerQuery(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).MaxStreamsMatchersPerQuery +} + +// MinShardingLookback returns the tenant specific min sharding lookback (e.g from when we should start sharding). +func (o *Overrides) MinShardingLookback(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).MinShardingLookback) +} + +// QuerySplitDuration returns the tenant specific splitby interval applied in the query frontend. +func (o *Overrides) QuerySplitDuration(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).QuerySplitDuration) +} + +// InstantMetricQuerySplitDuration returns the tenant specific instant metric queries splitby interval applied in the query frontend. +func (o *Overrides) InstantMetricQuerySplitDuration(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).InstantMetricQuerySplitDuration) +} + +// MetadataQuerySplitDuration returns the tenant specific metadata splitby interval applied in the query frontend. +func (o *Overrides) MetadataQuerySplitDuration(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).MetadataQuerySplitDuration) +} + +// RecentMetadataQuerySplitDuration returns the tenant specific splitby interval for recent metadata queries. +func (o *Overrides) RecentMetadataQuerySplitDuration(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).RecentMetadataQuerySplitDuration) +} + +// RecentMetadataQueryWindow returns the tenant specific time window used to determine recent metadata queries. +func (o *Overrides) RecentMetadataQueryWindow(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).RecentMetadataQueryWindow) +} + +// IngesterQuerySplitDuration returns the tenant specific splitby interval applied in the query frontend when querying +// during the `query_ingesters_within` window. +func (o *Overrides) IngesterQuerySplitDuration(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).IngesterQuerySplitDuration) +} + +// MaxQueryBytesRead returns the maximum bytes a query can read. +func (o *Overrides) MaxQueryBytesRead(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).MaxQueryBytesRead.Val() +} + +// MaxQuerierBytesRead returns the maximum bytes a sub query can read after splitting and sharding. +func (o *Overrides) MaxQuerierBytesRead(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).MaxQuerierBytesRead.Val() +} + +// MaxConcurrentTailRequests returns the limit to number of concurrent tail requests. +func (o *Overrides) MaxConcurrentTailRequests(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).MaxConcurrentTailRequests +} + +// MaxLineSize returns the maximum size in bytes the distributor should allow. +func (o *Overrides) MaxLineSize(userID string) int { + return o.getOverridesForUser(userID).MaxLineSize.Val() +} + +// MaxLineSizeTruncate returns whether lines longer than max should be truncated. +func (o *Overrides) MaxLineSizeTruncate(userID string) bool { + return o.getOverridesForUser(userID).MaxLineSizeTruncate +} + +// MaxEntriesLimitPerQuery returns the limit to number of entries the querier should return per query. +func (o *Overrides) MaxEntriesLimitPerQuery(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).MaxEntriesLimitPerQuery +} + +func (o *Overrides) QueryTimeout(_ context.Context, userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).QueryTimeout) +} + +func (o *Overrides) MaxCacheFreshness(_ context.Context, userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).MaxCacheFreshness) +} + +func (o *Overrides) MaxMetadataCacheFreshness(_ context.Context, userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).MaxMetadataCacheFreshness) +} + +func (o *Overrides) MaxStatsCacheFreshness(_ context.Context, userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).MaxStatsCacheFreshness) +} + +// MaxQueryLookback returns the max lookback period of queries. +func (o *Overrides) MaxQueryLookback(_ context.Context, userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).MaxQueryLookback) +} + +// RulerTenantShardSize returns shard size (number of rulers) used by this tenant when using shuffle-sharding strategy. +func (o *Overrides) RulerTenantShardSize(userID string) int { + return o.getOverridesForUser(userID).RulerTenantShardSize +} + +func (o *Overrides) IngestionPartitionsTenantShardSize(userID string) int { + return o.getOverridesForUser(userID).IngestionPartitionsTenantShardSize +} + +// RulerMaxRulesPerRuleGroup returns the maximum number of rules per rule group for a given user. +func (o *Overrides) RulerMaxRulesPerRuleGroup(userID string) int { + return o.getOverridesForUser(userID).RulerMaxRulesPerRuleGroup +} + +// RulerMaxRuleGroupsPerTenant returns the maximum number of rule groups for a given user. +func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int { + return o.getOverridesForUser(userID).RulerMaxRuleGroupsPerTenant +} + +// RulerAlertManagerConfig returns the alertmanager configurations to use for a given user. +func (o *Overrides) RulerAlertManagerConfig(userID string) *ruler_config.AlertManagerConfig { + return o.getOverridesForUser(userID).RulerAlertManagerConfig +} + +// RulerRemoteWriteDisabled returns whether remote-write is disabled for a given user or not. +func (o *Overrides) RulerRemoteWriteDisabled(userID string) bool { + return o.getOverridesForUser(userID).RulerRemoteWriteDisabled +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteURL returns the remote-write URL to use for a given user. +func (o *Overrides) RulerRemoteWriteURL(userID string) string { + return o.getOverridesForUser(userID).RulerRemoteWriteURL +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteTimeout returns the duration after which to timeout a remote-write request for a given user. +func (o *Overrides) RulerRemoteWriteTimeout(userID string) time.Duration { + return o.getOverridesForUser(userID).RulerRemoteWriteTimeout +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteHeaders returns the headers to use in a remote-write for a given user. +func (o *Overrides) RulerRemoteWriteHeaders(userID string) map[string]string { + return o.getOverridesForUser(userID).RulerRemoteWriteHeaders.Map() +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteRelabelConfigs returns the write relabel configs to use in a remote-write for a given user. +func (o *Overrides) RulerRemoteWriteRelabelConfigs(userID string) []*util.RelabelConfig { + return o.getOverridesForUser(userID).RulerRemoteWriteRelabelConfigs +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteQueueCapacity returns the queue capacity to use in a remote-write for a given user. +func (o *Overrides) RulerRemoteWriteQueueCapacity(userID string) int { + return o.getOverridesForUser(userID).RulerRemoteWriteQueueCapacity +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteQueueMinShards returns the minimum shards to use in a remote-write for a given user. +func (o *Overrides) RulerRemoteWriteQueueMinShards(userID string) int { + return o.getOverridesForUser(userID).RulerRemoteWriteQueueMinShards +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteQueueMaxShards returns the maximum shards to use in a remote-write for a given user. +func (o *Overrides) RulerRemoteWriteQueueMaxShards(userID string) int { + return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxShards +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteQueueMaxSamplesPerSend returns the max samples to send in a remote-write for a given user. +func (o *Overrides) RulerRemoteWriteQueueMaxSamplesPerSend(userID string) int { + return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxSamplesPerSend +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteQueueBatchSendDeadline returns the maximum time a sample will be buffered before being discarded for a given user. +func (o *Overrides) RulerRemoteWriteQueueBatchSendDeadline(userID string) time.Duration { + return o.getOverridesForUser(userID).RulerRemoteWriteQueueBatchSendDeadline +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteQueueMinBackoff returns the minimum time for an exponential backoff for a given user. +func (o *Overrides) RulerRemoteWriteQueueMinBackoff(userID string) time.Duration { + return o.getOverridesForUser(userID).RulerRemoteWriteQueueMinBackoff +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteQueueMaxBackoff returns the maximum time for an exponential backoff for a given user. +func (o *Overrides) RulerRemoteWriteQueueMaxBackoff(userID string) time.Duration { + return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxBackoff +} + +// Deprecated: use RulerRemoteWriteConfig instead +// RulerRemoteWriteQueueRetryOnRateLimit returns whether to retry failed remote-write requests (429 response) for a given user. +func (o *Overrides) RulerRemoteWriteQueueRetryOnRateLimit(userID string) bool { + return o.getOverridesForUser(userID).RulerRemoteWriteQueueRetryOnRateLimit +} + +// Deprecated: use RulerRemoteWriteConfig instead +func (o *Overrides) RulerRemoteWriteSigV4Config(userID string) *sigv4.SigV4Config { + return o.getOverridesForUser(userID).RulerRemoteWriteSigV4Config +} + +// RulerRemoteWriteConfig returns the remote-write configurations to use for a given user and a given remote client. +func (o *Overrides) RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig { + if c, ok := o.getOverridesForUser(userID).RulerRemoteWriteConfig[id]; ok { + return &c + } + + return nil +} + +// RulerRemoteEvaluationTimeout returns the duration after which to timeout a remote rule evaluation request for a given user. +func (o *Overrides) RulerRemoteEvaluationTimeout(userID string) time.Duration { + // if not defined, use the base query timeout + timeout := o.getOverridesForUser(userID).RulerRemoteEvaluationTimeout + if timeout <= 0 { + return time.Duration(o.getOverridesForUser(userID).QueryTimeout) + } + + return timeout +} + +// RulerRemoteEvaluationMaxResponseSize returns the maximum allowable response size from a remote rule evaluation for a given user. +func (o *Overrides) RulerRemoteEvaluationMaxResponseSize(userID string) int64 { + return o.getOverridesForUser(userID).RulerRemoteEvaluationMaxResponseSize +} + +// RetentionPeriod returns the retention period for a given user. +func (o *Overrides) RetentionPeriod(userID string) time.Duration { + return time.Duration(o.getOverridesForUser(userID).RetentionPeriod) +} + +// StreamRetention returns the retention period for a given user. +func (o *Overrides) StreamRetention(userID string) []StreamRetention { + return o.getOverridesForUser(userID).StreamRetention +} + +func (o *Overrides) UnorderedWrites(userID string) bool { + return o.getOverridesForUser(userID).UnorderedWrites +} + +func (o *Overrides) DeletionMode(userID string) string { + return o.getOverridesForUser(userID).DeletionMode +} + +func (o *Overrides) ShardStreams(userID string) shardstreams.Config { + return o.getOverridesForUser(userID).ShardStreams +} + +func (o *Overrides) BlockedQueries(_ context.Context, userID string) []*validation.BlockedQuery { + return o.getOverridesForUser(userID).BlockedQueries +} + +func (o *Overrides) RequiredLabels(_ context.Context, userID string) []string { + return o.getOverridesForUser(userID).RequiredLabels +} + +func (o *Overrides) RequiredNumberLabels(_ context.Context, userID string) int { + return o.getOverridesForUser(userID).RequiredNumberLabels +} + +func (o *Overrides) DefaultLimits() *Limits { + return o.defaultLimits +} + +func (o *Overrides) PerStreamRateLimit(userID string) RateLimit { + user := o.getOverridesForUser(userID) + + return RateLimit{ + Limit: rate.Limit(float64(user.PerStreamRateLimit.Val())), + Burst: user.PerStreamRateLimitBurst.Val(), + } +} + +func (o *Overrides) IncrementDuplicateTimestamps(userID string) bool { + return o.getOverridesForUser(userID).IncrementDuplicateTimestamp +} + +func (o *Overrides) DiscoverGenericFields(userID string) map[string][]string { + return o.getOverridesForUser(userID).DiscoverGenericFields.Fields +} + +func (o *Overrides) DiscoverServiceName(userID string) []string { + return o.getOverridesForUser(userID).DiscoverServiceName +} + +func (o *Overrides) DiscoverLogLevels(userID string) bool { + return o.getOverridesForUser(userID).DiscoverLogLevels +} + +func (o *Overrides) LogLevelFields(userID string) []string { + return o.getOverridesForUser(userID).LogLevelFields +} + +// VolumeEnabled returns whether volume endpoints are enabled for a user. +func (o *Overrides) VolumeEnabled(userID string) bool { + return o.getOverridesForUser(userID).VolumeEnabled +} + +func (o *Overrides) VolumeMaxSeries(userID string) int { + return o.getOverridesForUser(userID).VolumeMaxSeries +} + +func (o *Overrides) IndexGatewayShardSize(userID string) int { + return o.getOverridesForUser(userID).IndexGatewayShardSize +} + +func (o *Overrides) BloomGatewayShardSize(userID string) int { + return o.getOverridesForUser(userID).BloomGatewayShardSize +} + +func (o *Overrides) BloomGatewayCacheKeyInterval(userID string) time.Duration { + return o.getOverridesForUser(userID).BloomGatewayCacheKeyInterval +} + +func (o *Overrides) BloomGatewayEnabled(userID string) bool { + return o.getOverridesForUser(userID).BloomGatewayEnabled +} + +func (o *Overrides) BloomCreationEnabled(userID string) bool { + return o.getOverridesForUser(userID).BloomCreationEnabled +} + +func (o *Overrides) BloomPlanningStrategy(userID string) string { + return o.getOverridesForUser(userID).BloomPlanningStrategy +} + +func (o *Overrides) BloomSplitSeriesKeyspaceBy(userID string) int { + return o.getOverridesForUser(userID).BloomSplitSeriesKeyspaceBy +} + +func (o *Overrides) BloomTaskTargetSeriesChunksSizeBytes(userID string) uint64 { + return uint64(o.getOverridesForUser(userID).BloomTaskTargetSeriesChunkSize) +} + +func (o *Overrides) BloomBuildMaxBuilders(userID string) int { + return o.getOverridesForUser(userID).BloomBuildMaxBuilders +} + +func (o *Overrides) BuilderResponseTimeout(userID string) time.Duration { + return o.getOverridesForUser(userID).BloomBuilderResponseTimeout +} + +func (o *Overrides) PrefetchBloomBlocks(userID string) bool { + return o.getOverridesForUser(userID).BloomPrefetchBlocks +} + +func (o *Overrides) BloomTaskMaxRetries(userID string) int { + return o.getOverridesForUser(userID).BloomBuildTaskMaxRetries +} + +func (o *Overrides) BloomMaxBlockSize(userID string) int { + return o.getOverridesForUser(userID).BloomMaxBlockSize.Val() +} + +func (o *Overrides) BloomMaxBloomSize(userID string) int { + return o.getOverridesForUser(userID).BloomMaxBloomSize.Val() +} + +func (o *Overrides) BloomBlockEncoding(userID string) string { + return o.getOverridesForUser(userID).BloomBlockEncoding +} + +func (o *Overrides) AllowStructuredMetadata(userID string) bool { + return o.getOverridesForUser(userID).AllowStructuredMetadata +} + +func (o *Overrides) MaxStructuredMetadataSize(userID string) int { + return o.getOverridesForUser(userID).MaxStructuredMetadataSize.Val() +} + +func (o *Overrides) MaxStructuredMetadataCount(userID string) int { + return o.getOverridesForUser(userID).MaxStructuredMetadataEntriesCount +} + +func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig { + return o.getOverridesForUser(userID).OTLPConfig +} + +func (o *Overrides) BlockIngestionUntil(userID string) time.Time { + return time.Time(o.getOverridesForUser(userID).BlockIngestionUntil) +} + +func (o *Overrides) BlockIngestionStatusCode(userID string) int { + return o.getOverridesForUser(userID).BlockIngestionStatusCode +} + +func (o *Overrides) EnforcedLabels(userID string) []string { + return o.getOverridesForUser(userID).EnforcedLabels +} + +func (o *Overrides) ShardAggregations(userID string) []string { + return o.getOverridesForUser(userID).ShardAggregations +} + +func (o *Overrides) PatternIngesterTokenizableJSONFields(userID string) []string { + defaultFields := o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsDefault + appendFields := o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsAppend + deleteFields := o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsDelete + + outputMap := make(map[string]struct{}, len(defaultFields)+len(appendFields)) + + for _, field := range defaultFields { + outputMap[field] = struct{}{} + } + + for _, field := range appendFields { + outputMap[field] = struct{}{} + } + + for _, field := range deleteFields { + delete(outputMap, field) + } + + output := make([]string, 0, len(outputMap)) + for field := range outputMap { + output = append(output, field) + } + + return output +} + +func (o *Overrides) PatternIngesterTokenizableJSONFieldsAppend(userID string) []string { + return o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsAppend +} + +func (o *Overrides) PatternIngesterTokenizableJSONFieldsDelete(userID string) []string { + return o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsDelete +} + +func (o *Overrides) MetricAggregationEnabled(userID string) bool { + return o.getOverridesForUser(userID).MetricAggregationEnabled +} + +// S3SSEType returns the per-tenant S3 SSE type. +func (o *Overrides) S3SSEType(user string) string { + return o.getOverridesForUser(user).S3SSEType +} + +// S3SSEKMSKeyID returns the per-tenant S3 KMS-SSE key id. +func (o *Overrides) S3SSEKMSKeyID(user string) string { + return o.getOverridesForUser(user).S3SSEKMSKeyID +} + +// S3SSEKMSEncryptionContext returns the per-tenant S3 KMS-SSE encryption context. +func (o *Overrides) S3SSEKMSEncryptionContext(user string) string { + return o.getOverridesForUser(user).S3SSEKMSEncryptionContext +} + +func (o *Overrides) LogStreamCreation(userID string) bool { + return o.getOverridesForUser(userID).LogStreamCreation +} + +func (o *Overrides) LogPushRequest(userID string) bool { + return o.getOverridesForUser(userID).LogPushRequest +} + +func (o *Overrides) LogPushRequestStreams(userID string) bool { + return o.getOverridesForUser(userID).LogPushRequestStreams +} + +func (o *Overrides) LogDuplicateMetrics(userID string) bool { + return o.getOverridesForUser(userID).LogDuplicateMetrics +} + +func (o *Overrides) LogDuplicateStreamInfo(userID string) bool { + return o.getOverridesForUser(userID).LogDuplicateStreamInfo +} + +func (o *Overrides) LimitedLogPushErrors(userID string) bool { + return o.getOverridesForUser(userID).LimitedLogPushErrors +} + +func (o *Overrides) getOverridesForUser(userID string) *Limits { + if o.tenantLimits != nil { + l := o.tenantLimits.TenantLimits(userID) + if l != nil { + return l + } + } + return o.defaultLimits +} + +// OverwriteMarshalingStringMap will overwrite the src map when unmarshaling +// as opposed to merging. +type OverwriteMarshalingStringMap struct { + m map[string]string +} + +func NewOverwriteMarshalingStringMap(m map[string]string) OverwriteMarshalingStringMap { + return OverwriteMarshalingStringMap{m: m} +} + +func (sm *OverwriteMarshalingStringMap) Map() map[string]string { + return sm.m +} + +// MarshalJSON explicitly uses the the type receiver and not pointer receiver +// or it won't be called +func (sm OverwriteMarshalingStringMap) MarshalJSON() ([]byte, error) { + return json.Marshal(sm.m) +} + +func (sm *OverwriteMarshalingStringMap) UnmarshalJSON(val []byte) error { + var def map[string]string + if err := json.Unmarshal(val, &def); err != nil { + return err + } + sm.m = def + + return nil +} + +// MarshalYAML explicitly uses the the type receiver and not pointer receiver +// or it won't be called +func (sm OverwriteMarshalingStringMap) MarshalYAML() (interface{}, error) { + return sm.m, nil +} + +func (sm *OverwriteMarshalingStringMap) UnmarshalYAML(unmarshal func(interface{}) error) error { + var def map[string]string + + err := unmarshal(&def) + if err != nil { + return err + } + sm.m = def + + return nil +} diff --git a/pkg/validation/limits_test.go b/pkg/runtime/limits_test.go similarity index 99% rename from pkg/validation/limits_test.go rename to pkg/runtime/limits_test.go index 845ec805c5b37..474aa664086f9 100644 --- a/pkg/validation/limits_test.go +++ b/pkg/runtime/limits_test.go @@ -1,4 +1,4 @@ -package validation +package runtime import ( "encoding/json" diff --git a/pkg/validation/rate.go b/pkg/runtime/rate.go similarity index 95% rename from pkg/validation/rate.go rename to pkg/runtime/rate.go index f8a485126b69f..af52804316186 100644 --- a/pkg/validation/rate.go +++ b/pkg/runtime/rate.go @@ -1,4 +1,4 @@ -package validation +package runtime import "golang.org/x/time/rate" diff --git a/pkg/storage/chunk/tests/caching_fixtures_test.go b/pkg/storage/chunk/tests/caching_fixtures_test.go index fbec1ac4a4fc8..0e6a4a0c186c2 100644 --- a/pkg/storage/chunk/tests/caching_fixtures_test.go +++ b/pkg/storage/chunk/tests/caching_fixtures_test.go @@ -9,13 +9,13 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/grafana/loki/v3/pkg/logqlmodel/stats" + runtime "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/cache" "github.com/grafana/loki/v3/pkg/storage/chunk/client" "github.com/grafana/loki/v3/pkg/storage/chunk/client/gcp" "github.com/grafana/loki/v3/pkg/storage/chunk/client/testutils" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/series/index" - "github.com/grafana/loki/v3/pkg/validation" ) type fixture struct { @@ -43,9 +43,9 @@ var Fixtures = []testutils.Fixture{ fixture{gcp.Fixtures[0]}, } -func defaultLimits() (*validation.Overrides, error) { - var defaults validation.Limits +func defaultLimits() (*runtime.Overrides, error) { + var defaults runtime.Limits flagext.DefaultValues(&defaults) defaults.CardinalityLimit = 5 - return validation.NewOverrides(defaults, nil) + return runtime.NewOverrides(defaults, nil) } diff --git a/pkg/storage/factory_test.go b/pkg/storage/factory_test.go index 52cd6ba749a86..7a1c8cd8cfae9 100644 --- a/pkg/storage/factory_test.go +++ b/pkg/storage/factory_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/client" "github.com/grafana/loki/v3/pkg/storage/chunk/client/cassandra" "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" @@ -21,7 +22,6 @@ import ( "github.com/grafana/loki/v3/pkg/storage/types" "github.com/grafana/loki/v3/pkg/util/constants" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) func TestFactoryStop(t *testing.T) { @@ -29,7 +29,7 @@ func TestFactoryStop(t *testing.T) { cfg Config storeConfig config.ChunkStoreConfig schemaConfig config.SchemaConfig - defaults validation.Limits + defaults runtime.Limits ) flagext.DefaultValues(&cfg, &storeConfig, &schemaConfig, &defaults) schemaConfig.Configs = []config.PeriodConfig{ @@ -46,7 +46,7 @@ func TestFactoryStop(t *testing.T) { }, } - limits, err := validation.NewOverrides(defaults, nil) + limits, err := runtime.NewOverrides(defaults, nil) require.NoError(t, err) store, err := NewStore(cfg, storeConfig, schemaConfig, limits, cm, nil, log.NewNopLogger(), constants.Loki) require.NoError(t, err) @@ -79,12 +79,12 @@ func TestCassandraInMultipleSchemas(t *testing.T) { var ( cfg Config storeConfig config.ChunkStoreConfig - defaults validation.Limits + defaults runtime.Limits ) flagext.DefaultValues(&cfg, &storeConfig, &defaults) cfg.CassandraStorageConfig = cassandraCfg - limits, err := validation.NewOverrides(defaults, nil) + limits, err := runtime.NewOverrides(defaults, nil) require.NoError(t, err) store, err := NewStore(cfg, storeConfig, schemaCfg, limits, cm, nil, log.NewNopLogger(), constants.Loki) @@ -132,7 +132,7 @@ func TestNamedStores(t *testing.T) { }, } - limits, err := validation.NewOverrides(validation.Limits{}, nil) + limits, err := runtime.NewOverrides(runtime.Limits{}, nil) require.NoError(t, err) t.Run("period config referring to configured named store", func(t *testing.T) { diff --git a/pkg/storage/hack/main.go b/pkg/storage/hack/main.go index a9e57affa9940..4b5c51f3ea255 100644 --- a/pkg/storage/hack/main.go +++ b/pkg/storage/hack/main.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/grafana/dskit/flagext" "github.com/grafana/dskit/user" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -19,12 +20,12 @@ import ( "github.com/grafana/loki/v3/pkg/ingester/client" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql/syntax" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" "github.com/grafana/loki/v3/pkg/storage/config" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) var ( @@ -66,7 +67,11 @@ func getStore(cm storage.ClientMetrics) (storage.Store, *config.SchemaConfig, er }, } - store, err := storage.NewStore(storeConfig, config.ChunkStoreConfig{}, schemaCfg, &validation.Overrides{}, cm, prometheus.DefaultRegisterer, util_log.Logger, "cortex") + limits := runtime.Limits{} + flagext.DefaultValues(runtime.Limits{}) + overrides, _ := runtime.NewOverrides(limits, nil) + + store, err := storage.NewStore(storeConfig, config.ChunkStoreConfig{}, schemaCfg, overrides, cm, prometheus.DefaultRegisterer, util_log.Logger, "cortex") return store, &schemaCfg, err } diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index cc17cd0a92ef3..8fc2b2ce714a0 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -9,7 +9,7 @@ import ( "path" "path/filepath" "regexp" - "runtime" + rt "runtime" "testing" "time" @@ -37,6 +37,7 @@ import ( "github.com/grafana/loki/v3/pkg/logqlmodel/stats" "github.com/grafana/loki/v3/pkg/querier/astmapper" "github.com/grafana/loki/v3/pkg/querier/plan" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" "github.com/grafana/loki/v3/pkg/storage/config" @@ -44,12 +45,11 @@ import ( "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/boltdb" "github.com/grafana/loki/v3/pkg/util/constants" "github.com/grafana/loki/v3/pkg/util/marshal" - "github.com/grafana/loki/v3/pkg/validation" ) var ( start = model.Time(1523750400000) - m runtime.MemStats + m rt.MemStats ctx = user.InjectOrgID(context.Background(), "fake") cm = NewClientMetrics() chunkStore = getLocalStore("/tmp/benchmark/", cm) @@ -185,7 +185,7 @@ func benchmarkStoreQuery(b *testing.B, query *logproto.QueryRequest) { var maxHeapInuse uint64 func printHeap(b *testing.B, show bool) { - runtime.ReadMemStats(&m) + rt.ReadMemStats(&m) if m.HeapInuse > maxHeapInuse { maxHeapInuse = m.HeapInuse } @@ -196,7 +196,7 @@ func printHeap(b *testing.B, show bool) { } func getLocalStore(path string, cm ClientMetrics) Store { - limits, err := validation.NewOverrides(validation.Limits{ + limits, err := runtime.NewOverrides(runtime.Limits{ MaxQueryLength: model.Duration(6000 * time.Hour), }, nil) if err != nil { @@ -1318,7 +1318,7 @@ func TestStore_indexPrefixChange(t *testing.T) { }, } - limits, err := validation.NewOverrides(validation.Limits{}, nil) + limits, err := runtime.NewOverrides(runtime.Limits{}, nil) require.NoError(t, err) store, err := NewStore(cfg, config.ChunkStoreConfig{}, schemaConfig, limits, cm, nil, log.NewNopLogger(), constants.Loki) @@ -1435,7 +1435,7 @@ func TestStore_indexPrefixChange(t *testing.T) { } func TestStore_MultiPeriod(t *testing.T) { - limits, err := validation.NewOverrides(validation.Limits{}, nil) + limits, err := runtime.NewOverrides(runtime.Limits{}, nil) require.NoError(t, err) firstStoreDate := parseDate("2019-01-01") @@ -1792,7 +1792,7 @@ func TestStore_BoltdbTsdbSameIndexPrefix(t *testing.T) { tempDir := t.TempDir() ingesterName := "ingester-1" - limits, err := validation.NewOverrides(validation.Limits{}, nil) + limits, err := runtime.NewOverrides(runtime.Limits{}, nil) require.NoError(t, err) // config for BoltDB Shipper @@ -1942,7 +1942,7 @@ func TestStore_SyncStopInteraction(t *testing.T) { tempDir := t.TempDir() ingesterName := "ingester-1" - limits, err := validation.NewOverrides(validation.Limits{}, nil) + limits, err := runtime.NewOverrides(runtime.Limits{}, nil) require.NoError(t, err) // config for BoltDB Shipper diff --git a/pkg/storage/stores/series/index/caching_index_client_test.go b/pkg/storage/stores/series/index/caching_index_client_test.go index 99a9264a0a0aa..90cc122dfb389 100644 --- a/pkg/storage/stores/series/index/caching_index_client_test.go +++ b/pkg/storage/stores/series/index/caching_index_client_test.go @@ -14,9 +14,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/cache" "github.com/grafana/loki/v3/pkg/storage/stores/series/index" - "github.com/grafana/loki/v3/pkg/validation" ) var ctx = user.InjectOrgID(context.Background(), "1") @@ -473,9 +473,9 @@ func yoloString(buf []byte) string { return *((*string)(unsafe.Pointer(&buf))) } -func defaultLimits() (*validation.Overrides, error) { - var defaults validation.Limits +func defaultLimits() (*runtime.Overrides, error) { + var defaults runtime.Limits flagext.DefaultValues(&defaults) defaults.CardinalityLimit = 5 - return validation.NewOverrides(defaults, nil) + return runtime.NewOverrides(defaults, nil) } diff --git a/pkg/storage/stores/series/series_store_test.go b/pkg/storage/stores/series/series_store_test.go index d64fd70b25b59..6d1eb3c95efd6 100644 --- a/pkg/storage/stores/series/series_store_test.go +++ b/pkg/storage/stores/series/series_store_test.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/loki/v3/pkg/ingester/client" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logqlmodel/stats" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/chunk/cache" @@ -29,7 +30,6 @@ import ( "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/series/index" "github.com/grafana/loki/v3/pkg/util/constants" - "github.com/grafana/loki/v3/pkg/validation" ) type configFactory func() config.ChunkStoreConfig @@ -86,7 +86,7 @@ func newTestChunkStoreConfigWithMockStorage(t require.TestingT, schemaCfg config require.NoError(t, err) flagext.DefaultValues(&tbmConfig) - limits, err := validation.NewOverrides(validation.Limits{ + limits, err := runtime.NewOverrides(runtime.Limits{ MaxQueryLength: model.Duration(30 * 24 * time.Hour), }, nil) require.NoError(t, err) diff --git a/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util_test.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util_test.go index c4596f66cb4b6..452dd606d841c 100644 --- a/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util_test.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "go.etcd.io/bbolt" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/chunk/client" @@ -26,7 +27,6 @@ import ( shipper_util "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/util" "github.com/grafana/loki/v3/pkg/util/constants" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) func dayFromTime(t model.Time) config.DayTime { @@ -219,7 +219,7 @@ func newTestStore(t testing.TB, clientMetrics storage.ClientMetrics) *testStore defer func() { }() - limits, err := validation.NewOverrides(validation.Limits{}, nil) + limits, err := runtime.NewOverrides(runtime.Limits{}, nil) require.NoError(t, err) require.NoError(t, schemaCfg.Validate()) diff --git a/pkg/storage/stores/shipper/indexshipper/downloads/table_manager.go b/pkg/storage/stores/shipper/indexshipper/downloads/table_manager.go index 8e258fb7aeafa..97a54bd29c205 100644 --- a/pkg/storage/stores/shipper/indexshipper/downloads/table_manager.go +++ b/pkg/storage/stores/shipper/indexshipper/downloads/table_manager.go @@ -18,11 +18,11 @@ import ( "github.com/prometheus/common/model" "github.com/grafana/loki/v3/pkg/compactor/deletion" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/client/util" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/index" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/storage" - "github.com/grafana/loki/v3/pkg/validation" ) const ( @@ -31,8 +31,7 @@ const ( ) type Limits interface { - AllByUserID() map[string]*validation.Limits - DefaultLimits() *validation.Limits + runtime.ExportedLimits VolumeMaxSeries(userID string) int } diff --git a/pkg/storage/stores/shipper/indexshipper/downloads/table_manager_test.go b/pkg/storage/stores/shipper/indexshipper/downloads/table_manager_test.go index ea360dbd3cf4d..876f711cb8e75 100644 --- a/pkg/storage/stores/shipper/indexshipper/downloads/table_manager_test.go +++ b/pkg/storage/stores/shipper/indexshipper/downloads/table_manager_test.go @@ -11,11 +11,11 @@ import ( "github.com/go-kit/log" "github.com/stretchr/testify/require" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/index" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/storage" - "github.com/grafana/loki/v3/pkg/validation" ) const ( @@ -412,10 +412,10 @@ type mockLimits struct { volumeMaxSeries int } -func (m *mockLimits) AllByUserID() map[string]*validation.Limits { - allByUserID := map[string]*validation.Limits{} +func (m *mockLimits) AllByUserID() map[string]*runtime.Limits { + allByUserID := map[string]*runtime.Limits{} for userID := range m.queryReadyIndexNumDaysByUser { - allByUserID[userID] = &validation.Limits{ + allByUserID[userID] = &runtime.Limits{ QueryReadyIndexNumDays: m.queryReadyIndexNumDaysByUser[userID], } } @@ -423,8 +423,8 @@ func (m *mockLimits) AllByUserID() map[string]*validation.Limits { return allByUserID } -func (m *mockLimits) DefaultLimits() *validation.Limits { - return &validation.Limits{ +func (m *mockLimits) DefaultLimits() *runtime.Limits { + return &runtime.Limits{ QueryReadyIndexNumDays: m.queryReadyIndexNumDaysDefault, } } diff --git a/pkg/storage/stores/shipper/indexshipper/tsdb/head_manager_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/head_manager_test.go index 400e827ec3f76..ba1da67b9b904 100644 --- a/pkg/storage/stores/shipper/indexshipper/tsdb/head_manager_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/head_manager_test.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/dskit/flagext" "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" "github.com/grafana/loki/v3/pkg/storage/chunk/client/util" @@ -26,7 +27,6 @@ import ( "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/v3/pkg/storage/types" - "github.com/grafana/loki/v3/pkg/validation" ) type noopTSDBManager struct { @@ -55,7 +55,7 @@ func (m noopTSDBManager) Start() error { return nil } type zeroValueLimits struct { } -func (m *zeroValueLimits) AllByUserID() map[string]*validation.Limits { +func (m *zeroValueLimits) AllByUserID() map[string]*runtime.Limits { return nil } @@ -63,8 +63,8 @@ func (m *zeroValueLimits) VolumeMaxSeries(_ string) int { return 0 } -func (m *zeroValueLimits) DefaultLimits() *validation.Limits { - return &validation.Limits{ +func (m *zeroValueLimits) DefaultLimits() *runtime.Limits { + return &runtime.Limits{ QueryReadyIndexNumDays: 0, } } diff --git a/pkg/util/limiter/combined_limits.go b/pkg/util/limiter/combined_limits.go index 5c98c6bf9383d..faf22847c9e00 100644 --- a/pkg/util/limiter/combined_limits.go +++ b/pkg/util/limiter/combined_limits.go @@ -6,18 +6,21 @@ import ( "github.com/grafana/loki/v3/pkg/bloomgateway" "github.com/grafana/loki/v3/pkg/compactor" "github.com/grafana/loki/v3/pkg/distributor" + "github.com/grafana/loki/v3/pkg/distributor/writefailures" "github.com/grafana/loki/v3/pkg/indexgateway" "github.com/grafana/loki/v3/pkg/ingester" "github.com/grafana/loki/v3/pkg/pattern" querier_limits "github.com/grafana/loki/v3/pkg/querier/limits" queryrange_limits "github.com/grafana/loki/v3/pkg/querier/queryrange/limits" "github.com/grafana/loki/v3/pkg/ruler" + "github.com/grafana/loki/v3/pkg/runtime" scheduler_limits "github.com/grafana/loki/v3/pkg/scheduler/limits" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/storage/bucket" ) type CombinedLimits interface { + runtime.ExportedLimits compactor.Limits distributor.Limits ingester.Limits @@ -32,4 +35,5 @@ type CombinedLimits interface { bloombuilder.Limits pattern.Limits bucket.SSEConfigProvider + writefailures.Limits } diff --git a/pkg/util/querylimits/limiter_test.go b/pkg/util/querylimits/limiter_test.go index 549972d32a2e9..19e7dd1870568 100644 --- a/pkg/util/querylimits/limiter_test.go +++ b/pkg/util/querylimits/limiter_test.go @@ -9,31 +9,31 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/runtime" ) type mockTenantLimits struct { - limits map[string]*validation.Limits + limits map[string]*runtime.Limits } -func newMockTenantLimits(limits map[string]*validation.Limits) *mockTenantLimits { +func newMockTenantLimits(limits map[string]*runtime.Limits) *mockTenantLimits { return &mockTenantLimits{ limits: limits, } } -func (l *mockTenantLimits) TenantLimits(userID string) *validation.Limits { +func (l *mockTenantLimits) TenantLimits(userID string) *runtime.Limits { return l.limits[userID] } -func (l *mockTenantLimits) AllByUserID() map[string]*validation.Limits { return l.limits } +func (l *mockTenantLimits) AllByUserID() map[string]*runtime.Limits { return l.limits } // end copy pasta func TestLimiter_Defaults(t *testing.T) { // some fake tenant - tLimits := make(map[string]*validation.Limits) - tLimits["fake"] = &validation.Limits{ + tLimits := make(map[string]*runtime.Limits) + tLimits["fake"] = &runtime.Limits{ QueryTimeout: model.Duration(30 * time.Second), MaxQueryLookback: model.Duration(30 * time.Second), MaxQueryLength: model.Duration(30 * time.Second), @@ -45,7 +45,7 @@ func TestLimiter_Defaults(t *testing.T) { MaxQuerierBytesRead: 10, } - overrides, _ := validation.NewOverrides(validation.Limits{}, newMockTenantLimits(tLimits)) + overrides, _ := runtime.NewOverrides(runtime.Limits{}, newMockTenantLimits(tLimits)) l := NewLimiter(log.NewNopLogger(), overrides) expectedLimits := QueryLimits{ @@ -110,8 +110,8 @@ func TestLimiter_Defaults(t *testing.T) { func TestLimiter_RejectHighLimits(t *testing.T) { // some fake tenant - tLimits := make(map[string]*validation.Limits) - tLimits["fake"] = &validation.Limits{ + tLimits := make(map[string]*runtime.Limits) + tLimits["fake"] = &runtime.Limits{ MaxQueryLookback: model.Duration(30 * time.Second), MaxQueryLength: model.Duration(30 * time.Second), MaxQueryRange: model.Duration(30 * time.Second), @@ -122,7 +122,7 @@ func TestLimiter_RejectHighLimits(t *testing.T) { MaxQuerierBytesRead: 10, } - overrides, _ := validation.NewOverrides(validation.Limits{}, newMockTenantLimits(tLimits)) + overrides, _ := runtime.NewOverrides(runtime.Limits{}, newMockTenantLimits(tLimits)) l := NewLimiter(log.NewNopLogger(), overrides) limits := QueryLimits{ MaxQueryLength: model.Duration(2 * 24 * time.Hour), @@ -154,8 +154,8 @@ func TestLimiter_RejectHighLimits(t *testing.T) { func TestLimiter_AcceptLowerLimits(t *testing.T) { // some fake tenant - tLimits := make(map[string]*validation.Limits) - tLimits["fake"] = &validation.Limits{ + tLimits := make(map[string]*runtime.Limits) + tLimits["fake"] = &runtime.Limits{ MaxQueryLookback: model.Duration(30 * time.Second), MaxQueryLength: model.Duration(30 * time.Second), MaxQueryRange: model.Duration(2 * 24 * time.Hour), @@ -166,7 +166,7 @@ func TestLimiter_AcceptLowerLimits(t *testing.T) { MaxQuerierBytesRead: 10, } - overrides, _ := validation.NewOverrides(validation.Limits{}, newMockTenantLimits(tLimits)) + overrides, _ := runtime.NewOverrides(runtime.Limits{}, newMockTenantLimits(tLimits)) l := NewLimiter(log.NewNopLogger(), overrides) limits := QueryLimits{ MaxQueryLength: model.Duration(29 * time.Second), @@ -192,12 +192,12 @@ func TestLimiter_AcceptLowerLimits(t *testing.T) { func TestLimiter_MergeLimits(t *testing.T) { // some fake tenant - tLimits := make(map[string]*validation.Limits) - tLimits["fake"] = &validation.Limits{ + tLimits := make(map[string]*runtime.Limits) + tLimits["fake"] = &runtime.Limits{ RequiredLabels: []string{"one", "two"}, } - overrides, _ := validation.NewOverrides(validation.Limits{}, newMockTenantLimits(tLimits)) + overrides, _ := runtime.NewOverrides(runtime.Limits{}, newMockTenantLimits(tLimits)) l := NewLimiter(log.NewNopLogger(), overrides) limits := QueryLimits{ RequiredLabels: []string{"one", "three"}, diff --git a/pkg/validation/exporter.go b/pkg/validation/exporter.go index 0a5ffc863d5ad..7501b8d6c66d9 100644 --- a/pkg/validation/exporter.go +++ b/pkg/validation/exporter.go @@ -7,22 +7,18 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/util/flagext" ) -type ExportedLimits interface { - AllByUserID() map[string]*Limits - DefaultLimits() *Limits -} - type OverridesExporter struct { - overrides ExportedLimits + overrides runtime.ExportedLimits tenantDesc *prometheus.Desc defaultsDesc *prometheus.Desc } // TODO(jordanrushing): break out overrides from defaults? -func NewOverridesExporter(overrides ExportedLimits) *OverridesExporter { +func NewOverridesExporter(overrides runtime.ExportedLimits) *OverridesExporter { return &OverridesExporter{ overrides: overrides, tenantDesc: prometheus.NewDesc( diff --git a/pkg/validation/exporter_test.go b/pkg/validation/exporter_test.go index 45484bbc13b08..d6bf9927e5be8 100644 --- a/pkg/validation/exporter_test.go +++ b/pkg/validation/exporter_test.go @@ -6,26 +6,28 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/grafana/loki/v3/pkg/runtime" ) type mockTenantLimits struct { - limits map[string]*Limits + limits map[string]*runtime.Limits } -func newMockTenantLimits(limits map[string]*Limits) *mockTenantLimits { +func newMockTenantLimits(limits map[string]*runtime.Limits) *mockTenantLimits { return &mockTenantLimits{ limits: limits, } } -func (l *mockTenantLimits) TenantLimits(userID string) *Limits { +func (l *mockTenantLimits) TenantLimits(userID string) *runtime.Limits { return l.limits[userID] } -func (l *mockTenantLimits) AllByUserID() map[string]*Limits { return l.limits } +func (l *mockTenantLimits) AllByUserID() map[string]*runtime.Limits { return l.limits } func TestOverridesExporter_noConfig(t *testing.T) { - overrides, _ := NewOverrides(Limits{}, newMockTenantLimits(nil)) + overrides, _ := runtime.NewOverrides(runtime.Limits{}, newMockTenantLimits(nil)) exporter := NewOverridesExporter(overrides) count := testutil.CollectAndCount(exporter, "loki_overrides") assert.Equal(t, 0, count) @@ -33,13 +35,13 @@ func TestOverridesExporter_noConfig(t *testing.T) { } func TestOverridesExporter_withConfig(t *testing.T) { - tenantLimits := map[string]*Limits{ + tenantLimits := map[string]*runtime.Limits{ "tenant-a": { MaxQueriersPerTenant: 5, BloomCreationEnabled: true, }, } - overrides, _ := NewOverrides(Limits{}, newMockTenantLimits(tenantLimits)) + overrides, _ := runtime.NewOverrides(runtime.Limits{}, newMockTenantLimits(tenantLimits)) exporter := NewOverridesExporter(overrides) count := testutil.CollectAndCount(exporter, "loki_overrides") assert.Equal(t, 2, count) diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 4ecc566e36389..958ae1a622678 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -1,1234 +1 @@ package validation - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "strconv" - "time" - - "github.com/go-kit/log/level" - dskit_flagext "github.com/grafana/dskit/flagext" - - "github.com/pkg/errors" - "github.com/prometheus/common/model" - "github.com/prometheus/common/sigv4" - "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/model/labels" - "golang.org/x/time/rate" - "gopkg.in/yaml.v2" - - "github.com/grafana/loki/v3/pkg/compactor/deletionmode" - "github.com/grafana/loki/v3/pkg/compression" - "github.com/grafana/loki/v3/pkg/distributor/shardstreams" - "github.com/grafana/loki/v3/pkg/loghttp/push" - "github.com/grafana/loki/v3/pkg/logql" - "github.com/grafana/loki/v3/pkg/logql/syntax" - ruler_config "github.com/grafana/loki/v3/pkg/ruler/config" - "github.com/grafana/loki/v3/pkg/ruler/util" - "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding" - "github.com/grafana/loki/v3/pkg/util/flagext" - util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/util/validation" -) - -const ( - // LocalRateLimitStrat represents a ingestion rate limiting strategy that enforces the limit - // on a per distributor basis. - // - // The actual effective rate limit will be N times higher, where N is the number of distributor replicas. - LocalIngestionRateStrategy = "local" - - // GlobalRateLimitStrat represents a ingestion rate limiting strategy that enforces the rate - // limiting globally, configuring a per-distributor local rate limiter as "ingestion_rate / N", - // where N is the number of distributor replicas (it's automatically adjusted if the - // number of replicas change). - // - // The global strategy requires the distributors to form their own ring, which - // is used to keep track of the current number of healthy distributor replicas. - GlobalIngestionRateStrategy = "global" - - bytesInMB = 1048576 - - defaultPerStreamRateLimit = 3 << 20 // 3MB - DefaultTSDBMaxBytesPerShard = sharding.DefaultTSDBMaxBytesPerShard - defaultPerStreamBurstLimit = 5 * defaultPerStreamRateLimit - - DefaultPerTenantQueryTimeout = "1m" - - defaultMaxStructuredMetadataSize = "64kb" - defaultMaxStructuredMetadataCount = 128 - defaultBloomBuildMaxBlockSize = "200MB" - defaultBloomBuildMaxBloomSize = "128MB" - defaultBloomTaskTargetChunkSize = "20GB" - - defaultBlockedIngestionStatusCode = 260 // 260 is a custom status code to indicate blocked ingestion -) - -// Limits describe all the limits for users; can be used to describe global default -// limits via flags, or per-user limits via yaml config. -// NOTE: we use custom `model.Duration` instead of standard `time.Duration` because, -// to support user-friendly duration format (e.g: "1h30m45s") in JSON value. -type Limits struct { - // Distributor enforced limits. - IngestionRateStrategy string `yaml:"ingestion_rate_strategy" json:"ingestion_rate_strategy"` - IngestionRateMB float64 `yaml:"ingestion_rate_mb" json:"ingestion_rate_mb"` - IngestionBurstSizeMB float64 `yaml:"ingestion_burst_size_mb" json:"ingestion_burst_size_mb"` - MaxLabelNameLength int `yaml:"max_label_name_length" json:"max_label_name_length"` - MaxLabelValueLength int `yaml:"max_label_value_length" json:"max_label_value_length"` - MaxLabelNamesPerSeries int `yaml:"max_label_names_per_series" json:"max_label_names_per_series"` - RejectOldSamples bool `yaml:"reject_old_samples" json:"reject_old_samples"` - RejectOldSamplesMaxAge model.Duration `yaml:"reject_old_samples_max_age" json:"reject_old_samples_max_age"` - CreationGracePeriod model.Duration `yaml:"creation_grace_period" json:"creation_grace_period"` - MaxLineSize flagext.ByteSize `yaml:"max_line_size" json:"max_line_size"` - MaxLineSizeTruncate bool `yaml:"max_line_size_truncate" json:"max_line_size_truncate"` - IncrementDuplicateTimestamp bool `yaml:"increment_duplicate_timestamp" json:"increment_duplicate_timestamp"` - - // Metadata field extraction - DiscoverGenericFields FieldDetectorConfig `yaml:"discover_generic_fields" json:"discover_generic_fields" doc:"description=Experimental: Detect fields from stream labels, structured metadata, or json/logfmt formatted log line and put them into structured metadata of the log entry."` - DiscoverServiceName []string `yaml:"discover_service_name" json:"discover_service_name"` - DiscoverLogLevels bool `yaml:"discover_log_levels" json:"discover_log_levels"` - LogLevelFields []string `yaml:"log_level_fields" json:"log_level_fields"` - - // Ingester enforced limits. - UseOwnedStreamCount bool `yaml:"use_owned_stream_count" json:"use_owned_stream_count"` - MaxLocalStreamsPerUser int `yaml:"max_streams_per_user" json:"max_streams_per_user"` - MaxGlobalStreamsPerUser int `yaml:"max_global_streams_per_user" json:"max_global_streams_per_user"` - UnorderedWrites bool `yaml:"unordered_writes" json:"unordered_writes"` - PerStreamRateLimit flagext.ByteSize `yaml:"per_stream_rate_limit" json:"per_stream_rate_limit"` - PerStreamRateLimitBurst flagext.ByteSize `yaml:"per_stream_rate_limit_burst" json:"per_stream_rate_limit_burst"` - - // Querier enforced limits. - MaxChunksPerQuery int `yaml:"max_chunks_per_query" json:"max_chunks_per_query"` - MaxQuerySeries int `yaml:"max_query_series" json:"max_query_series"` - MaxQueryLookback model.Duration `yaml:"max_query_lookback" json:"max_query_lookback"` - MaxQueryLength model.Duration `yaml:"max_query_length" json:"max_query_length"` - MaxQueryRange model.Duration `yaml:"max_query_range" json:"max_query_range"` - MaxQueryParallelism int `yaml:"max_query_parallelism" json:"max_query_parallelism"` - TSDBMaxQueryParallelism int `yaml:"tsdb_max_query_parallelism" json:"tsdb_max_query_parallelism"` - TSDBMaxBytesPerShard flagext.ByteSize `yaml:"tsdb_max_bytes_per_shard" json:"tsdb_max_bytes_per_shard"` - TSDBShardingStrategy string `yaml:"tsdb_sharding_strategy" json:"tsdb_sharding_strategy"` - TSDBPrecomputeChunks bool `yaml:"tsdb_precompute_chunks" json:"tsdb_precompute_chunks"` - CardinalityLimit int `yaml:"cardinality_limit" json:"cardinality_limit"` - MaxStreamsMatchersPerQuery int `yaml:"max_streams_matchers_per_query" json:"max_streams_matchers_per_query"` - MaxConcurrentTailRequests int `yaml:"max_concurrent_tail_requests" json:"max_concurrent_tail_requests"` - MaxEntriesLimitPerQuery int `yaml:"max_entries_limit_per_query" json:"max_entries_limit_per_query"` - MaxCacheFreshness model.Duration `yaml:"max_cache_freshness_per_query" json:"max_cache_freshness_per_query"` - MaxMetadataCacheFreshness model.Duration `yaml:"max_metadata_cache_freshness" json:"max_metadata_cache_freshness"` - MaxStatsCacheFreshness model.Duration `yaml:"max_stats_cache_freshness" json:"max_stats_cache_freshness"` - MaxQueriersPerTenant uint `yaml:"max_queriers_per_tenant" json:"max_queriers_per_tenant"` - MaxQueryCapacity float64 `yaml:"max_query_capacity" json:"max_query_capacity"` - QueryReadyIndexNumDays int `yaml:"query_ready_index_num_days" json:"query_ready_index_num_days"` - QueryTimeout model.Duration `yaml:"query_timeout" json:"query_timeout"` - - // Query frontend enforced limits. The default is actually parameterized by the queryrange config. - QuerySplitDuration model.Duration `yaml:"split_queries_by_interval" json:"split_queries_by_interval"` - MetadataQuerySplitDuration model.Duration `yaml:"split_metadata_queries_by_interval" json:"split_metadata_queries_by_interval"` - RecentMetadataQuerySplitDuration model.Duration `yaml:"split_recent_metadata_queries_by_interval" json:"split_recent_metadata_queries_by_interval"` - RecentMetadataQueryWindow model.Duration `yaml:"recent_metadata_query_window" json:"recent_metadata_query_window"` - InstantMetricQuerySplitDuration model.Duration `yaml:"split_instant_metric_queries_by_interval" json:"split_instant_metric_queries_by_interval"` - IngesterQuerySplitDuration model.Duration `yaml:"split_ingester_queries_by_interval" json:"split_ingester_queries_by_interval"` - MinShardingLookback model.Duration `yaml:"min_sharding_lookback" json:"min_sharding_lookback"` - MaxQueryBytesRead flagext.ByteSize `yaml:"max_query_bytes_read" json:"max_query_bytes_read"` - MaxQuerierBytesRead flagext.ByteSize `yaml:"max_querier_bytes_read" json:"max_querier_bytes_read"` - VolumeEnabled bool `yaml:"volume_enabled" json:"volume_enabled" doc:"description=Enable log-volume endpoints."` - VolumeMaxSeries int `yaml:"volume_max_series" json:"volume_max_series" doc:"description=The maximum number of aggregated series in a log-volume response"` - - // Ruler defaults and limits. - RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"` - RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"` - RulerAlertManagerConfig *ruler_config.AlertManagerConfig `yaml:"ruler_alertmanager_config" json:"ruler_alertmanager_config" doc:"hidden"` - RulerTenantShardSize int `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"` - - // TODO(dannyk): add HTTP client overrides (basic auth / tls config, etc) - // Ruler remote-write limits. - - // this field is the inversion of the general remote_write.enabled because the zero value of a boolean is false, - // and if it were ruler_remote_write_enabled, it would be impossible to know if the value was explicitly set or default - RulerRemoteWriteDisabled bool `yaml:"ruler_remote_write_disabled" json:"ruler_remote_write_disabled" doc:"description=Disable recording rules remote-write."` - - // deprecated use RulerRemoteWriteConfig instead. - RulerRemoteWriteURL string `yaml:"ruler_remote_write_url" json:"ruler_remote_write_url" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. The URL of the endpoint to send samples to."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteTimeout time.Duration `yaml:"ruler_remote_write_timeout" json:"ruler_remote_write_timeout" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Timeout for requests to the remote write endpoint."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteHeaders OverwriteMarshalingStringMap `yaml:"ruler_remote_write_headers" json:"ruler_remote_write_headers" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Loki itself can't be overwritten."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteRelabelConfigs []*util.RelabelConfig `yaml:"ruler_remote_write_relabel_configs,omitempty" json:"ruler_remote_write_relabel_configs,omitempty" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. List of remote write relabel configurations."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueCapacity int `yaml:"ruler_remote_write_queue_capacity" json:"ruler_remote_write_queue_capacity" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Number of samples to buffer per shard before we block reading of more samples from the WAL. It is recommended to have enough capacity in each shard to buffer several requests to keep throughput up while processing occasional slow remote requests."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMinShards int `yaml:"ruler_remote_write_queue_min_shards" json:"ruler_remote_write_queue_min_shards" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Minimum number of shards, i.e. amount of concurrency."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMaxShards int `yaml:"ruler_remote_write_queue_max_shards" json:"ruler_remote_write_queue_max_shards" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum number of shards, i.e. amount of concurrency."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMaxSamplesPerSend int `yaml:"ruler_remote_write_queue_max_samples_per_send" json:"ruler_remote_write_queue_max_samples_per_send" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum number of samples per send."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueBatchSendDeadline time.Duration `yaml:"ruler_remote_write_queue_batch_send_deadline" json:"ruler_remote_write_queue_batch_send_deadline" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum time a sample will wait in buffer."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMinBackoff time.Duration `yaml:"ruler_remote_write_queue_min_backoff" json:"ruler_remote_write_queue_min_backoff" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Initial retry delay. Gets doubled for every retry."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMaxBackoff time.Duration `yaml:"ruler_remote_write_queue_max_backoff" json:"ruler_remote_write_queue_max_backoff" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum retry delay."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueRetryOnRateLimit bool `yaml:"ruler_remote_write_queue_retry_on_ratelimit" json:"ruler_remote_write_queue_retry_on_ratelimit" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Retry upon receiving a 429 status code from the remote-write storage. This is experimental and might change in the future."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteSigV4Config *sigv4.SigV4Config `yaml:"ruler_remote_write_sigv4_config" json:"ruler_remote_write_sigv4_config" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Configures AWS's Signature Verification 4 signing process to sign every remote write request."` - - RulerRemoteWriteConfig map[string]config.RemoteWriteConfig `yaml:"ruler_remote_write_config,omitempty" json:"ruler_remote_write_config,omitempty" doc:"description=Configures global and per-tenant limits for remote write clients. A map with remote client id as key."` - - // TODO(dannyk): possible enhancement is to align this with rule group interval - RulerRemoteEvaluationTimeout time.Duration `yaml:"ruler_remote_evaluation_timeout" json:"ruler_remote_evaluation_timeout" doc:"description=Timeout for a remote rule evaluation. Defaults to the value of 'querier.query-timeout'."` - RulerRemoteEvaluationMaxResponseSize int64 `yaml:"ruler_remote_evaluation_max_response_size" json:"ruler_remote_evaluation_max_response_size" doc:"description=Maximum size (in bytes) of the allowable response size from a remote rule evaluation. Set to 0 to allow any response size (default)."` - - // Global and per tenant deletion mode - DeletionMode string `yaml:"deletion_mode" json:"deletion_mode"` - - // Global and per tenant retention - RetentionPeriod model.Duration `yaml:"retention_period" json:"retention_period"` - StreamRetention []StreamRetention `yaml:"retention_stream,omitempty" json:"retention_stream,omitempty" doc:"description=Per-stream retention to apply, if the retention is enable on the compactor side.\nExample:\n retention_stream:\n - selector: '{namespace=\"dev\"}'\n priority: 1\n period: 24h\n- selector: '{container=\"nginx\"}'\n priority: 1\n period: 744h\nSelector is a Prometheus labels matchers that will apply the 'period' retention only if the stream is matching. In case multiple stream are matching, the highest priority will be picked. If no rule is matched the 'retention_period' is used."` - - // Config for overrides, convenient if it goes here. - PerTenantOverrideConfig string `yaml:"per_tenant_override_config" json:"per_tenant_override_config"` - PerTenantOverridePeriod model.Duration `yaml:"per_tenant_override_period" json:"per_tenant_override_period"` - - // Deprecated - CompactorDeletionEnabled bool `yaml:"allow_deletes" json:"allow_deletes" doc:"deprecated|description=Use deletion_mode per tenant configuration instead."` - - ShardStreams shardstreams.Config `yaml:"shard_streams" json:"shard_streams" doc:"description=Define streams sharding behavior."` - - BlockedQueries []*validation.BlockedQuery `yaml:"blocked_queries,omitempty" json:"blocked_queries,omitempty"` - - RequiredLabels []string `yaml:"required_labels,omitempty" json:"required_labels,omitempty" doc:"description=Define a list of required selector labels."` - RequiredNumberLabels int `yaml:"minimum_labels_number,omitempty" json:"minimum_labels_number,omitempty" doc:"description=Minimum number of label matchers a query should contain."` - - IndexGatewayShardSize int `yaml:"index_gateway_shard_size" json:"index_gateway_shard_size"` - - BloomGatewayShardSize int `yaml:"bloom_gateway_shard_size" json:"bloom_gateway_shard_size" category:"experimental"` - BloomGatewayEnabled bool `yaml:"bloom_gateway_enable_filtering" json:"bloom_gateway_enable_filtering" category:"experimental"` - BloomGatewayCacheKeyInterval time.Duration `yaml:"bloom_gateway_cache_key_interval" json:"bloom_gateway_cache_key_interval" category:"experimental"` - - BloomBuildMaxBuilders int `yaml:"bloom_build_max_builders" json:"bloom_build_max_builders" category:"experimental"` - BloomBuildTaskMaxRetries int `yaml:"bloom_build_task_max_retries" json:"bloom_build_task_max_retries" category:"experimental"` - BloomBuilderResponseTimeout time.Duration `yaml:"bloom_build_builder_response_timeout" json:"bloom_build_builder_response_timeout" category:"experimental"` - - BloomCreationEnabled bool `yaml:"bloom_creation_enabled" json:"bloom_creation_enabled" category:"experimental"` - BloomPlanningStrategy string `yaml:"bloom_planning_strategy" json:"bloom_planning_strategy" category:"experimental"` - BloomSplitSeriesKeyspaceBy int `yaml:"bloom_split_series_keyspace_by" json:"bloom_split_series_keyspace_by" category:"experimental"` - BloomTaskTargetSeriesChunkSize flagext.ByteSize `yaml:"bloom_task_target_series_chunk_size" json:"bloom_task_target_series_chunk_size" category:"experimental"` - BloomBlockEncoding string `yaml:"bloom_block_encoding" json:"bloom_block_encoding" category:"experimental"` - BloomPrefetchBlocks bool `yaml:"bloom_prefetch_blocks" json:"bloom_prefetch_blocks" category:"experimental"` - - BloomMaxBlockSize flagext.ByteSize `yaml:"bloom_max_block_size" json:"bloom_max_block_size" category:"experimental"` - BloomMaxBloomSize flagext.ByteSize `yaml:"bloom_max_bloom_size" json:"bloom_max_bloom_size" category:"experimental"` - - AllowStructuredMetadata bool `yaml:"allow_structured_metadata,omitempty" json:"allow_structured_metadata,omitempty" doc:"description=Allow user to send structured metadata in push payload."` - MaxStructuredMetadataSize flagext.ByteSize `yaml:"max_structured_metadata_size" json:"max_structured_metadata_size" doc:"description=Maximum size accepted for structured metadata per log line."` - MaxStructuredMetadataEntriesCount int `yaml:"max_structured_metadata_entries_count" json:"max_structured_metadata_entries_count" doc:"description=Maximum number of structured metadata entries per log line."` - OTLPConfig push.OTLPConfig `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"` - GlobalOTLPConfig push.GlobalOTLPConfig `yaml:"-" json:"-"` - - BlockIngestionUntil dskit_flagext.Time `yaml:"block_ingestion_until" json:"block_ingestion_until"` - BlockIngestionStatusCode int `yaml:"block_ingestion_status_code" json:"block_ingestion_status_code"` - EnforcedLabels []string `yaml:"enforced_labels" json:"enforced_labels" category:"experimental"` - - IngestionPartitionsTenantShardSize int `yaml:"ingestion_partitions_tenant_shard_size" json:"ingestion_partitions_tenant_shard_size" category:"experimental"` - - ShardAggregations []string `yaml:"shard_aggregations,omitempty" json:"shard_aggregations,omitempty" doc:"description=List of LogQL vector and range aggregations that should be sharded."` - - PatternIngesterTokenizableJSONFieldsDefault dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_default" json:"pattern_ingester_tokenizable_json_fields_default" doc:"hidden"` - PatternIngesterTokenizableJSONFieldsAppend dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_append" json:"pattern_ingester_tokenizable_json_fields_append" doc:"hidden"` - PatternIngesterTokenizableJSONFieldsDelete dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_delete" json:"pattern_ingester_tokenizable_json_fields_delete" doc:"hidden"` - MetricAggregationEnabled bool `yaml:"metric_aggregation_enabled" json:"metric_aggregation_enabled"` - - // This config doesn't have a CLI flag registered here because they're registered in - // their own original config struct. - S3SSEType string `yaml:"s3_sse_type" json:"s3_sse_type" doc:"nocli|description=S3 server-side encryption type. Required to enable server-side encryption overrides for a specific tenant. If not set, the default S3 client settings are used."` - S3SSEKMSKeyID string `yaml:"s3_sse_kms_key_id" json:"s3_sse_kms_key_id" doc:"nocli|description=S3 server-side encryption KMS Key ID. Ignored if the SSE type override is not set."` - S3SSEKMSEncryptionContext string `yaml:"s3_sse_kms_encryption_context" json:"s3_sse_kms_encryption_context" doc:"nocli|description=S3 server-side encryption KMS encryption context. If unset and the key ID override is set, the encryption context will not be provided to S3. Ignored if the SSE type override is not set."` -} - -type FieldDetectorConfig struct { - Fields map[string][]string `yaml:"fields,omitempty" json:"fields,omitempty"` -} - -type StreamRetention struct { - Period model.Duration `yaml:"period" json:"period" doc:"description:Retention period applied to the log lines matching the selector."` - Priority int `yaml:"priority" json:"priority" doc:"description:The larger the value, the higher the priority."` - Selector string `yaml:"selector" json:"selector" doc:"description:Stream selector expression."` - Matchers []*labels.Matcher `yaml:"-" json:"-"` // populated during validation. -} - -// LimitError are errors that do not comply with the limits specified. -type LimitError string - -func (e LimitError) Error() string { - return string(e) -} - -// RegisterFlags adds the flags required to config this to the given FlagSet -func (l *Limits) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&l.IngestionRateStrategy, "distributor.ingestion-rate-limit-strategy", "global", "Whether the ingestion rate limit should be applied individually to each distributor instance (local), or evenly shared across the cluster (global). The ingestion rate strategy cannot be overridden on a per-tenant basis.\n- local: enforces the limit on a per distributor basis. The actual effective rate limit will be N times higher, where N is the number of distributor replicas.\n- global: enforces the limit globally, configuring a per-distributor local rate limiter as 'ingestion_rate / N', where N is the number of distributor replicas (it's automatically adjusted if the number of replicas change). The global strategy requires the distributors to form their own ring, which is used to keep track of the current number of healthy distributor replicas.") - f.Float64Var(&l.IngestionRateMB, "distributor.ingestion-rate-limit-mb", 4, "Per-user ingestion rate limit in sample size per second. Sample size includes size of the logs line and the size of structured metadata labels. Units in MB.") - f.Float64Var(&l.IngestionBurstSizeMB, "distributor.ingestion-burst-size-mb", 6, "Per-user allowed ingestion burst size (in sample size). Units in MB. The burst size refers to the per-distributor local rate limiter even in the case of the 'global' strategy, and should be set at least to the maximum logs size expected in a single push request.") - - _ = l.MaxLineSize.Set("256KB") - f.Var(&l.MaxLineSize, "distributor.max-line-size", "Maximum line size on ingestion path. Example: 256kb. Any log line exceeding this limit will be discarded unless `distributor.max-line-size-truncate` is set which in case it is truncated instead of discarding it completely. There is no limit when unset or set to 0.") - f.BoolVar(&l.MaxLineSizeTruncate, "distributor.max-line-size-truncate", false, "Whether to truncate lines that exceed max_line_size.") - f.IntVar(&l.MaxLabelNameLength, "validation.max-length-label-name", 1024, "Maximum length accepted for label names.") - f.IntVar(&l.MaxLabelValueLength, "validation.max-length-label-value", 2048, "Maximum length accepted for label value. This setting also applies to the metric name.") - f.IntVar(&l.MaxLabelNamesPerSeries, "validation.max-label-names-per-series", 15, "Maximum number of label names per series.") - f.BoolVar(&l.RejectOldSamples, "validation.reject-old-samples", true, "Whether or not old samples will be rejected.") - f.BoolVar(&l.IncrementDuplicateTimestamp, "validation.increment-duplicate-timestamps", false, "Alter the log line timestamp during ingestion when the timestamp is the same as the previous entry for the same stream. When enabled, if a log line in a push request has the same timestamp as the previous line for the same stream, one nanosecond is added to the log line. This will preserve the received order of log lines with the exact same timestamp when they are queried, by slightly altering their stored timestamp. NOTE: This is imperfect, because Loki accepts out of order writes, and another push request for the same stream could contain duplicate timestamps to existing entries and they will not be incremented.") - l.DiscoverServiceName = []string{ - "service", - "app", - "application", - "app_name", - "name", - "app_kubernetes_io_name", - "container", - "container_name", - "k8s_container_name", - "component", - "workload", - "job", - "k8s_job_name", - } - f.Var((*dskit_flagext.StringSlice)(&l.DiscoverServiceName), "validation.discover-service-name", "If no service_name label exists, Loki maps a single label from the configured list to service_name. If none of the configured labels exist in the stream, label is set to unknown_service. Empty list disables setting the label.") - f.BoolVar(&l.DiscoverLogLevels, "validation.discover-log-levels", true, "Discover and add log levels during ingestion, if not present already. Levels would be added to Structured Metadata with name level/LEVEL/Level/Severity/severity/SEVERITY/lvl/LVL/Lvl (case-sensitive) and one of the values from 'trace', 'debug', 'info', 'warn', 'error', 'critical', 'fatal' (case insensitive).") - l.LogLevelFields = []string{"level", "LEVEL", "Level", "Severity", "severity", "SEVERITY", "lvl", "LVL", "Lvl"} - f.Var((*dskit_flagext.StringSlice)(&l.LogLevelFields), "validation.log-level-fields", "Field name to use for log levels. If not set, log level would be detected based on pre-defined labels as mentioned above.") - - _ = l.RejectOldSamplesMaxAge.Set("7d") - f.Var(&l.RejectOldSamplesMaxAge, "validation.reject-old-samples.max-age", "Maximum accepted sample age before rejecting.") - _ = l.CreationGracePeriod.Set("10m") - f.Var(&l.CreationGracePeriod, "validation.create-grace-period", "Duration which table will be created/deleted before/after it's needed; we won't accept sample from before this time.") - f.IntVar(&l.MaxEntriesLimitPerQuery, "validation.max-entries-limit", 5000, "Maximum number of log entries that will be returned for a query.") - - f.BoolVar(&l.UseOwnedStreamCount, "ingester.use-owned-stream-count", false, "When true an ingester takes into account only the streams that it owns according to the ring while applying the stream limit.") - f.IntVar(&l.MaxLocalStreamsPerUser, "ingester.max-streams-per-user", 0, "Maximum number of active streams per user, per ingester. 0 to disable.") - f.IntVar(&l.MaxGlobalStreamsPerUser, "ingester.max-global-streams-per-user", 5000, "Maximum number of active streams per user, across the cluster. 0 to disable. When the global limit is enabled, each ingester is configured with a dynamic local limit based on the replication factor and the current number of healthy ingesters, and is kept updated whenever the number of ingesters change.") - - // TODO(ashwanth) Deprecated. This will be removed with the next major release and out-of-order writes would be accepted by default. - f.BoolVar(&l.UnorderedWrites, "ingester.unordered-writes", true, "Deprecated. When true, out-of-order writes are accepted.") - - _ = l.PerStreamRateLimit.Set(strconv.Itoa(defaultPerStreamRateLimit)) - f.Var(&l.PerStreamRateLimit, "ingester.per-stream-rate-limit", "Maximum byte rate per second per stream, also expressible in human readable forms (1MB, 256KB, etc).") - _ = l.PerStreamRateLimitBurst.Set(strconv.Itoa(defaultPerStreamBurstLimit)) - f.Var(&l.PerStreamRateLimitBurst, "ingester.per-stream-rate-limit-burst", "Maximum burst bytes per stream, also expressible in human readable forms (1MB, 256KB, etc). This is how far above the rate limit a stream can 'burst' before the stream is limited.") - - f.IntVar(&l.MaxChunksPerQuery, "store.query-chunk-limit", 2e6, "Maximum number of chunks that can be fetched in a single query.") - - _ = l.MaxQueryLength.Set("721h") - f.Var(&l.MaxQueryLength, "store.max-query-length", "The limit to length of chunk store queries. 0 to disable.") - f.IntVar(&l.MaxQuerySeries, "querier.max-query-series", 500, "Limit the maximum of unique series that is returned by a metric query. When the limit is reached an error is returned.") - _ = l.MaxQueryRange.Set("0s") - f.Var(&l.MaxQueryRange, "querier.max-query-range", "Limit the length of the [range] inside a range query. Default is 0 or unlimited") - _ = l.QueryTimeout.Set(DefaultPerTenantQueryTimeout) - f.Var(&l.QueryTimeout, "querier.query-timeout", "Timeout when querying backends (ingesters or storage) during the execution of a query request. When a specific per-tenant timeout is used, the global timeout is ignored.") - - _ = l.MaxQueryLookback.Set("0s") - f.Var(&l.MaxQueryLookback, "querier.max-query-lookback", "Limit how far back in time series data and metadata can be queried, up until lookback duration ago. This limit is enforced in the query frontend, the querier and the ruler. If the requested time range is outside the allowed range, the request will not fail, but will be modified to only query data within the allowed time range. The default value of 0 does not set a limit.") - f.IntVar(&l.MaxQueryParallelism, "querier.max-query-parallelism", 32, "Maximum number of queries that will be scheduled in parallel by the frontend.") - f.IntVar(&l.TSDBMaxQueryParallelism, "querier.tsdb-max-query-parallelism", 128, "Maximum number of queries will be scheduled in parallel by the frontend for TSDB schemas.") - _ = l.TSDBMaxBytesPerShard.Set(strconv.Itoa(DefaultTSDBMaxBytesPerShard)) - f.Var(&l.TSDBMaxBytesPerShard, "querier.tsdb-max-bytes-per-shard", "Target maximum number of bytes assigned to a single sharded query. Also expressible in human readable forms (1GB, etc). Note: This is a _target_ and not an absolute limit. The actual limit can be higher, but the query planner will try to build shards up to this limit.") - f.StringVar( - &l.TSDBShardingStrategy, - "limits.tsdb-sharding-strategy", - logql.PowerOfTwoVersion.String(), - fmt.Sprintf( - "sharding strategy to use in query planning. Suggested to use %s once all nodes can recognize it.", - logql.BoundedVersion.String(), - ), - ) - f.BoolVar(&l.TSDBPrecomputeChunks, "querier.tsdb-precompute-chunks", false, "Precompute chunks for TSDB queries. This can improve query performance at the cost of increased memory usage by computing chunks once during planning, reducing index calls.") - f.IntVar(&l.CardinalityLimit, "store.cardinality-limit", 1e5, "Cardinality limit for index queries.") - f.IntVar(&l.MaxStreamsMatchersPerQuery, "querier.max-streams-matcher-per-query", 1000, "Maximum number of stream matchers per query.") - f.IntVar(&l.MaxConcurrentTailRequests, "querier.max-concurrent-tail-requests", 10, "Maximum number of concurrent tail requests.") - - _ = l.MinShardingLookback.Set("0s") - f.Var(&l.MinShardingLookback, "frontend.min-sharding-lookback", "Limit queries that can be sharded. Queries within the time range of now and now minus this sharding lookback are not sharded. The default value of 0s disables the lookback, causing sharding of all queries at all times.") - - f.Var(&l.MaxQueryBytesRead, "frontend.max-query-bytes-read", "Max number of bytes a query can fetch. Enforced in log and metric queries only when TSDB is used. This limit is not enforced on log queries without filters. The default value of 0 disables this limit.") - - _ = l.MaxQuerierBytesRead.Set("150GB") - f.Var(&l.MaxQuerierBytesRead, "frontend.max-querier-bytes-read", "Max number of bytes a query can fetch after splitting and sharding. Enforced in log and metric queries only when TSDB is used. This limit is not enforced on log queries without filters. The default value of 0 disables this limit.") - - _ = l.MaxCacheFreshness.Set("10m") - f.Var(&l.MaxCacheFreshness, "frontend.max-cache-freshness", "Most recent allowed cacheable result per-tenant, to prevent caching very recent results that might still be in flux.") - - _ = l.MaxMetadataCacheFreshness.Set("24h") - f.Var(&l.MaxMetadataCacheFreshness, "frontend.max-metadata-cache-freshness", "Do not cache metadata request if the end time is within the frontend.max-metadata-cache-freshness window. Set this to 0 to apply no such limits. Defaults to 24h.") - - _ = l.MaxStatsCacheFreshness.Set("10m") - f.Var(&l.MaxStatsCacheFreshness, "frontend.max-stats-cache-freshness", "Do not cache requests with an end time that falls within Now minus this duration. 0 disables this feature (default).") - - f.UintVar(&l.MaxQueriersPerTenant, "frontend.max-queriers-per-tenant", 0, "Maximum number of queriers that can handle requests for a single tenant. If set to 0 or value higher than number of available queriers, *all* queriers will handle requests for the tenant. Each frontend (or query-scheduler, if used) will select the same set of queriers for the same tenant (given that all queriers are connected to all frontends / query-schedulers). This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL.") - f.Float64Var(&l.MaxQueryCapacity, "frontend.max-query-capacity", 0, "How much of the available query capacity (\"querier\" components in distributed mode, \"read\" components in SSD mode) can be used by a single tenant. Allowed values are 0.0 to 1.0. For example, setting this to 0.5 would allow a tenant to use half of the available queriers for processing the query workload. If set to 0, query capacity is determined by frontend.max-queriers-per-tenant. When both frontend.max-queriers-per-tenant and frontend.max-query-capacity are configured, smaller value of the resulting querier replica count is considered: min(frontend.max-queriers-per-tenant, ceil(querier_replicas * frontend.max-query-capacity)). *All* queriers will handle requests for the tenant if neither limits are applied. This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL. Use this feature in a multi-tenant setup where you need to limit query capacity for certain tenants.") - f.IntVar(&l.QueryReadyIndexNumDays, "store.query-ready-index-num-days", 0, "Number of days of index to be kept always downloaded for queries. Applies only to per user index in boltdb-shipper index store. 0 to disable.") - - f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 0, "Maximum number of rules per rule group per-tenant. 0 to disable.") - f.IntVar(&l.RulerMaxRuleGroupsPerTenant, "ruler.max-rule-groups-per-tenant", 0, "Maximum number of rule groups per-tenant. 0 to disable.") - f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when shuffle-sharding is enabled in the ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") - - f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "Feature renamed to 'runtime configuration', flag deprecated in favor of -runtime-config.file (runtime_config.file in YAML).") - _ = l.RetentionPeriod.Set("0s") - f.Var(&l.RetentionPeriod, "store.retention", "Retention period to apply to stored data, only applies if retention_enabled is true in the compactor config. As of version 2.8.0, a zero value of 0 or 0s disables retention. In previous releases, Loki did not properly honor a zero value to disable retention and a really large value should be used instead.") - - _ = l.PerTenantOverridePeriod.Set("10s") - f.Var(&l.PerTenantOverridePeriod, "limits.per-user-override-period", "Feature renamed to 'runtime configuration'; flag deprecated in favor of -runtime-config.reload-period (runtime_config.period in YAML).") - - _ = l.QuerySplitDuration.Set("1h") - f.Var(&l.QuerySplitDuration, "querier.split-queries-by-interval", "Split queries by a time interval and execute in parallel. The value 0 disables splitting by time. This also determines how cache keys are chosen when result caching is enabled.") - _ = l.InstantMetricQuerySplitDuration.Set("1h") - f.Var(&l.InstantMetricQuerySplitDuration, "querier.split-instant-metric-queries-by-interval", "Split instant metric queries by a time interval and execute in parallel. The value 0 disables splitting instant metric queries by time. This also determines how cache keys are chosen when instant metric query result caching is enabled.") - - _ = l.MetadataQuerySplitDuration.Set("24h") - f.Var(&l.MetadataQuerySplitDuration, "querier.split-metadata-queries-by-interval", "Split metadata queries by a time interval and execute in parallel. The value 0 disables splitting metadata queries by time. This also determines how cache keys are chosen when label/series result caching is enabled.") - - _ = l.RecentMetadataQuerySplitDuration.Set("1h") - f.Var(&l.RecentMetadataQuerySplitDuration, "experimental.querier.split-recent-metadata-queries-by-interval", "Experimental. Split interval to use for the portion of metadata request that falls within `recent_metadata_query_window`. Rest of the request which is outside the window still uses `split_metadata_queries_by_interval`. If set to 0, the entire request defaults to using a split interval of `split_metadata_queries_by_interval.`.") - - f.Var(&l.RecentMetadataQueryWindow, "experimental.querier.recent-metadata-query-window", "Experimental. Metadata query window inside which `split_recent_metadata_queries_by_interval` gets applied, portion of the metadata request that falls in this window is split using `split_recent_metadata_queries_by_interval`. The value 0 disables using a different split interval for recent metadata queries.\n\nThis is added to improve cacheability of recent metadata queries. Query split interval also determines the interval used in cache key. The default split interval of 24h is useful for caching long queries, each cache key holding 1 day's results. But metadata queries are often shorter than 24h, to cache them effectively we need a smaller split interval. `recent_metadata_query_window` along with `split_recent_metadata_queries_by_interval` help configure a shorter split interval for recent metadata queries.") - - _ = l.IngesterQuerySplitDuration.Set("0s") - f.Var(&l.IngesterQuerySplitDuration, "querier.split-ingester-queries-by-interval", "Interval to use for time-based splitting when a request is within the `query_ingesters_within` window; defaults to `split-queries-by-interval` by setting to 0.") - - f.StringVar(&l.DeletionMode, "compactor.deletion-mode", "filter-and-delete", "Deletion mode. Can be one of 'disabled', 'filter-only', or 'filter-and-delete'. When set to 'filter-only' or 'filter-and-delete', and if retention_enabled is true, then the log entry deletion API endpoints are available.") - - // Deprecated - dskit_flagext.DeprecatedFlag(f, "compactor.allow-deletes", "Deprecated. Instead, see compactor.deletion-mode which is another per tenant configuration", util_log.Logger) - - f.IntVar(&l.IndexGatewayShardSize, "index-gateway.shard-size", 0, "The shard size defines how many index gateways should be used by a tenant for querying. If the global shard factor is 0, the global shard factor is set to the deprecated -replication-factor for backwards compatibility reasons.") - - f.IntVar(&l.BloomGatewayShardSize, "bloom-gateway.shard-size", 0, "Experimental. The shard size defines how many bloom gateways should be used by a tenant for querying.") - f.BoolVar(&l.BloomGatewayEnabled, "bloom-gateway.enable-filtering", false, "Experimental. Whether to use the bloom gateway component in the read path to filter chunks.") - f.DurationVar(&l.BloomGatewayCacheKeyInterval, "bloom-gateway.cache-key-interval", 15*time.Minute, "Experimental. Interval for computing the cache key in the Bloom Gateway.") - - f.StringVar(&l.BloomBlockEncoding, "bloom-build.block-encoding", "none", "Experimental. Compression algorithm for bloom block pages.") - f.BoolVar(&l.BloomPrefetchBlocks, "bloom-build.prefetch-blocks", false, "Experimental. Prefetch blocks on bloom gateways as soon as they are built.") - - _ = l.BloomMaxBlockSize.Set(defaultBloomBuildMaxBlockSize) - f.Var(&l.BloomMaxBlockSize, "bloom-build.max-block-size", - fmt.Sprintf( - "Experimental. The maximum bloom block size. A value of 0 sets an unlimited size. Default is %s. The actual block size might exceed this limit since blooms will be added to blocks until the block exceeds the maximum block size.", - defaultBloomBuildMaxBlockSize, - ), - ) - - f.BoolVar(&l.BloomCreationEnabled, "bloom-build.enable", false, "Experimental. Whether to create blooms for the tenant.") - f.StringVar(&l.BloomPlanningStrategy, "bloom-build.planning-strategy", "split_keyspace_by_factor", "Experimental. Bloom planning strategy to use in bloom creation. Can be one of: 'split_keyspace_by_factor', 'split_by_series_chunks_size'") - f.IntVar(&l.BloomSplitSeriesKeyspaceBy, "bloom-build.split-keyspace-by", 256, "Experimental. Only if `bloom-build.planning-strategy` is 'split'. Number of splits to create for the series keyspace when building blooms. The series keyspace is split into this many parts to parallelize bloom creation.") - _ = l.BloomTaskTargetSeriesChunkSize.Set(defaultBloomTaskTargetChunkSize) - f.Var(&l.BloomTaskTargetSeriesChunkSize, "bloom-build.split-target-series-chunk-size", fmt.Sprintf("Experimental. Target chunk size in bytes for bloom tasks. Default is %s.", defaultBloomTaskTargetChunkSize)) - f.IntVar(&l.BloomBuildMaxBuilders, "bloom-build.max-builders", 0, "Experimental. Maximum number of builders to use when building blooms. 0 allows unlimited builders.") - f.DurationVar(&l.BloomBuilderResponseTimeout, "bloom-build.builder-response-timeout", 0, "Experimental. Timeout for a builder to finish a task. If a builder does not respond within this time, it is considered failed and the task will be requeued. 0 disables the timeout.") - f.IntVar(&l.BloomBuildTaskMaxRetries, "bloom-build.task-max-retries", 3, "Experimental. Maximum number of retries for a failed task. If a task fails more than this number of times, it is considered failed and will not be retried. A value of 0 disables this limit.") - - _ = l.BloomMaxBloomSize.Set(defaultBloomBuildMaxBloomSize) - f.Var(&l.BloomMaxBloomSize, "bloom-build.max-bloom-size", - fmt.Sprintf( - "Experimental. The maximum bloom size per log stream. A log stream whose generated bloom filter exceeds this size will be discarded. A value of 0 sets an unlimited size. Default is %s.", - defaultBloomBuildMaxBloomSize, - ), - ) - - l.ShardStreams.RegisterFlagsWithPrefix("shard-streams", f) - f.IntVar(&l.VolumeMaxSeries, "limits.volume-max-series", 1000, "The default number of aggregated series or labels that can be returned from a log-volume endpoint") - - f.BoolVar(&l.AllowStructuredMetadata, "validation.allow-structured-metadata", true, "Allow user to send structured metadata (non-indexed labels) in push payload.") - _ = l.MaxStructuredMetadataSize.Set(defaultMaxStructuredMetadataSize) - f.Var(&l.MaxStructuredMetadataSize, "limits.max-structured-metadata-size", "Maximum size accepted for structured metadata per entry. Default: 64 kb. Any log line exceeding this limit will be discarded. There is no limit when unset or set to 0.") - f.IntVar(&l.MaxStructuredMetadataEntriesCount, "limits.max-structured-metadata-entries-count", defaultMaxStructuredMetadataCount, "Maximum number of structured metadata entries per log line. Default: 128. Any log line exceeding this limit will be discarded. There is no limit when unset or set to 0.") - f.BoolVar(&l.VolumeEnabled, "limits.volume-enabled", true, "Enable log volume endpoint.") - - f.Var(&l.BlockIngestionUntil, "limits.block-ingestion-until", "Block ingestion until the configured date. The time should be in RFC3339 format.") - f.IntVar(&l.BlockIngestionStatusCode, "limits.block-ingestion-status-code", defaultBlockedIngestionStatusCode, "HTTP status code to return when ingestion is blocked. If 200, the ingestion will be blocked without returning an error to the client. By Default, a custom status code (260) is returned to the client along with an error message.") - f.Var((*dskit_flagext.StringSlice)(&l.EnforcedLabels), "validation.enforced-labels", "List of labels that must be present in the stream. If any of the labels are missing, the stream will be discarded. This flag configures it globally for all tenants. Experimental.") - - f.IntVar(&l.IngestionPartitionsTenantShardSize, "limits.ingestion-partition-tenant-shard-size", 0, "The number of partitions a tenant's data should be sharded to when using kafka ingestion. Tenants are sharded across partitions using shuffle-sharding. 0 disables shuffle sharding and tenant is sharded across all partitions.") - - _ = l.PatternIngesterTokenizableJSONFieldsDefault.Set("log,message,msg,msg_,_msg,content") - f.Var(&l.PatternIngesterTokenizableJSONFieldsDefault, "limits.pattern-ingester-tokenizable-json-fields", "List of JSON fields that should be tokenized in the pattern ingester.") - f.Var(&l.PatternIngesterTokenizableJSONFieldsAppend, "limits.pattern-ingester-tokenizable-json-fields-append", "List of JSON fields that should be appended to the default list of tokenizable fields in the pattern ingester.") - f.Var(&l.PatternIngesterTokenizableJSONFieldsDelete, "limits.pattern-ingester-tokenizable-json-fields-delete", "List of JSON fields that should be deleted from the (default U append) list of tokenizable fields in the pattern ingester.") - - f.BoolVar( - &l.MetricAggregationEnabled, - "limits.metric-aggregation-enabled", - false, - "Enable metric aggregation. When enabled, pushed streams will be sampled for bytes and count, and these metric will be written back into Loki as a special __aggregated_metric__ stream, which can be queried for faster histogram queries.", - ) -} - -// SetGlobalOTLPConfig set GlobalOTLPConfig which is used while unmarshaling per-tenant otlp config to use the default list of resource attributes picked as index labels. -func (l *Limits) SetGlobalOTLPConfig(cfg push.GlobalOTLPConfig) { - l.GlobalOTLPConfig = cfg - l.OTLPConfig.ApplyGlobalOTLPConfig(cfg) -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (l *Limits) UnmarshalYAML(unmarshal func(interface{}) error) error { - // We want to set c to the defaults and then overwrite it with the input. - // To make unmarshal fill the plain data struct rather than calling UnmarshalYAML - // again, we have to hide it using a type indirection. See prometheus/config. - type plain Limits - - // During startup we wont have a default value so we don't want to overwrite them - if defaultLimits != nil { - b, err := yaml.Marshal(defaultLimits) - if err != nil { - return errors.Wrap(err, "cloning limits (marshaling)") - } - if err := yaml.Unmarshal(b, (*plain)(l)); err != nil { - return errors.Wrap(err, "cloning limits (unmarshaling)") - } - } - if err := unmarshal((*plain)(l)); err != nil { - return err - } - - if defaultLimits != nil { - // apply relevant bits from global otlp config - l.OTLPConfig.ApplyGlobalOTLPConfig(defaultLimits.GlobalOTLPConfig) - } - return nil -} - -// Validate validates that this limits config is valid. -func (l *Limits) Validate() error { - if l.StreamRetention != nil { - for i, rule := range l.StreamRetention { - matchers, err := syntax.ParseMatchers(rule.Selector, true) - if err != nil { - return fmt.Errorf("invalid labels matchers: %w", err) - } - if time.Duration(rule.Period) < 24*time.Hour { - return fmt.Errorf("retention period must be >= 24h was %s", rule.Period) - } - // populate matchers during validation - l.StreamRetention[i].Matchers = matchers - } - } - - if _, err := deletionmode.ParseMode(l.DeletionMode); err != nil { - return err - } - - if l.CompactorDeletionEnabled { - level.Warn(util_log.Logger).Log("msg", "The compactor.allow-deletes configuration option has been deprecated and will be ignored. Instead, use deletion_mode in the limits_configs to adjust deletion functionality") - } - - if l.MaxQueryCapacity < 0 { - level.Warn(util_log.Logger).Log("msg", "setting frontend.max-query-capacity to 0 as it is configured to a value less than 0") - l.MaxQueryCapacity = 0 - } - - if l.MaxQueryCapacity > 1 { - level.Warn(util_log.Logger).Log("msg", "setting frontend.max-query-capacity to 1 as it is configured to a value greater than 1") - l.MaxQueryCapacity = 1 - } - - if err := l.OTLPConfig.Validate(); err != nil { - return err - } - - if _, err := logql.ParseShardVersion(l.TSDBShardingStrategy); err != nil { - return errors.Wrap(err, "invalid tsdb sharding strategy") - } - - if _, err := compression.ParseCodec(l.BloomBlockEncoding); err != nil { - return err - } - - if l.TSDBMaxBytesPerShard <= 0 { - return errors.New("querier.tsdb-max-bytes-per-shard must be greater than 0") - } - - return nil -} - -// When we load YAML from disk, we want the various per-customer limits -// to default to any values specified on the command line, not default -// command line values. This global contains those values. I (Tom) cannot -// find a nicer way I'm afraid. -var defaultLimits *Limits - -// SetDefaultLimitsForYAMLUnmarshalling sets global default limits, used when loading -// Limits from YAML files. This is used to ensure per-tenant limits are defaulted to -// those values. -func SetDefaultLimitsForYAMLUnmarshalling(defaults Limits) { - defaultLimits = &defaults -} - -type TenantLimits interface { - // TenantLimits is a function that returns limits for given tenant, or - // nil, if there are no tenant-specific limits. - TenantLimits(userID string) *Limits - // AllByUserID gets a mapping of all tenant IDs and limits for that user - AllByUserID() map[string]*Limits -} - -// Overrides periodically fetch a set of per-user overrides, and provides convenience -// functions for fetching the correct value. -type Overrides struct { - defaultLimits *Limits - tenantLimits TenantLimits -} - -// NewOverrides makes a new Overrides. -func NewOverrides(defaults Limits, tenantLimits TenantLimits) (*Overrides, error) { - return &Overrides{ - tenantLimits: tenantLimits, - defaultLimits: &defaults, - }, nil -} - -func (o *Overrides) AllByUserID() map[string]*Limits { - if o.tenantLimits != nil { - return o.tenantLimits.AllByUserID() - } - return nil -} - -// IngestionRateStrategy returns whether the ingestion rate limit should be individually applied -// to each distributor instance (local) or evenly shared across the cluster (global). -func (o *Overrides) IngestionRateStrategy() string { - // The ingestion rate strategy can't be overridden on a per-tenant basis, - // so here we just pick the value for a not-existing user ID (empty string). - return o.getOverridesForUser("").IngestionRateStrategy -} - -// IngestionRateBytes returns the limit on ingester rate (MBs per second). -func (o *Overrides) IngestionRateBytes(userID string) float64 { - return o.getOverridesForUser(userID).IngestionRateMB * bytesInMB -} - -// IngestionBurstSizeBytes returns the burst size for ingestion rate. -func (o *Overrides) IngestionBurstSizeBytes(userID string) int { - return int(o.getOverridesForUser(userID).IngestionBurstSizeMB * bytesInMB) -} - -// MaxLabelNameLength returns maximum length a label name can be. -func (o *Overrides) MaxLabelNameLength(userID string) int { - return o.getOverridesForUser(userID).MaxLabelNameLength -} - -// MaxLabelValueLength returns maximum length a label value can be. This also is -// the maximum length of a metric name. -func (o *Overrides) MaxLabelValueLength(userID string) int { - return o.getOverridesForUser(userID).MaxLabelValueLength -} - -// MaxLabelNamesPerSeries returns maximum number of label/value pairs timeseries. -func (o *Overrides) MaxLabelNamesPerSeries(userID string) int { - return o.getOverridesForUser(userID).MaxLabelNamesPerSeries -} - -// RejectOldSamples returns true when we should reject samples older than certain -// age. -func (o *Overrides) RejectOldSamples(userID string) bool { - return o.getOverridesForUser(userID).RejectOldSamples -} - -// RejectOldSamplesMaxAge returns the age at which samples should be rejected. -func (o *Overrides) RejectOldSamplesMaxAge(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).RejectOldSamplesMaxAge) -} - -// CreationGracePeriod is misnamed, and actually returns how far into the future -// we should accept samples. -func (o *Overrides) CreationGracePeriod(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).CreationGracePeriod) -} - -func (o *Overrides) UseOwnedStreamCount(userID string) bool { - return o.getOverridesForUser(userID).UseOwnedStreamCount -} - -// MaxLocalStreamsPerUser returns the maximum number of streams a user is allowed to store -// in a single ingester. -func (o *Overrides) MaxLocalStreamsPerUser(userID string) int { - return o.getOverridesForUser(userID).MaxLocalStreamsPerUser -} - -// MaxGlobalStreamsPerUser returns the maximum number of streams a user is allowed to store -// across the cluster. -func (o *Overrides) MaxGlobalStreamsPerUser(userID string) int { - return o.getOverridesForUser(userID).MaxGlobalStreamsPerUser -} - -// MaxChunksPerQuery returns the maximum number of chunks allowed per query. -func (o *Overrides) MaxChunksPerQuery(userID string) int { - return o.getOverridesForUser(userID).MaxChunksPerQuery -} - -// MaxQueryLength returns the limit of the length (in time) of a query. -func (o *Overrides) MaxQueryLength(_ context.Context, userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).MaxQueryLength) -} - -// Compatibility with Cortex interface, this method is set to be removed in 1.12, -// so nooping in Loki until then. -func (o *Overrides) MaxChunksPerQueryFromStore(_ string) int { return 0 } - -// MaxQuerySeries returns the limit of the series of metric queries. -func (o *Overrides) MaxQuerySeries(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).MaxQuerySeries -} - -// MaxQueryRange returns the limit for the max [range] value that can be in a range query -func (o *Overrides) MaxQueryRange(_ context.Context, userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).MaxQueryRange) -} - -// MaxQueriersPerUser returns the maximum number of queriers that can handle requests for this user. -func (o *Overrides) MaxQueriersPerUser(userID string) uint { - return o.getOverridesForUser(userID).MaxQueriersPerTenant -} - -// MaxQueryCapacity returns how much of the available query capacity can be used by this user.. -func (o *Overrides) MaxQueryCapacity(userID string) float64 { - return o.getOverridesForUser(userID).MaxQueryCapacity -} - -// QueryReadyIndexNumDays returns the number of days for which we have to be query ready for a user. -func (o *Overrides) QueryReadyIndexNumDays(userID string) int { - return o.getOverridesForUser(userID).QueryReadyIndexNumDays -} - -// TSDBMaxQueryParallelism returns the limit to the number of sub-queries the -// frontend will process in parallel for TSDB schemas. -func (o *Overrides) TSDBMaxQueryParallelism(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).TSDBMaxQueryParallelism -} - -// TSDBMaxBytesPerShard returns the maximum number of bytes assigned to a specific shard in a tsdb query -func (o *Overrides) TSDBMaxBytesPerShard(userID string) int { - return o.getOverridesForUser(userID).TSDBMaxBytesPerShard.Val() -} - -// TSDBShardingStrategy returns the sharding strategy to use in query planning. -func (o *Overrides) TSDBShardingStrategy(userID string) string { - return o.getOverridesForUser(userID).TSDBShardingStrategy -} - -func (o *Overrides) TSDBPrecomputeChunks(userID string) bool { - return o.getOverridesForUser(userID).TSDBPrecomputeChunks -} - -// MaxQueryParallelism returns the limit to the number of sub-queries the -// frontend will process in parallel. -func (o *Overrides) MaxQueryParallelism(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).MaxQueryParallelism -} - -// CardinalityLimit whether to enforce the presence of a metric name. -func (o *Overrides) CardinalityLimit(userID string) int { - return o.getOverridesForUser(userID).CardinalityLimit -} - -// MaxStreamsMatchersPerQuery returns the limit to number of streams matchers per query. -func (o *Overrides) MaxStreamsMatchersPerQuery(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).MaxStreamsMatchersPerQuery -} - -// MinShardingLookback returns the tenant specific min sharding lookback (e.g from when we should start sharding). -func (o *Overrides) MinShardingLookback(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).MinShardingLookback) -} - -// QuerySplitDuration returns the tenant specific splitby interval applied in the query frontend. -func (o *Overrides) QuerySplitDuration(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).QuerySplitDuration) -} - -// InstantMetricQuerySplitDuration returns the tenant specific instant metric queries splitby interval applied in the query frontend. -func (o *Overrides) InstantMetricQuerySplitDuration(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).InstantMetricQuerySplitDuration) -} - -// MetadataQuerySplitDuration returns the tenant specific metadata splitby interval applied in the query frontend. -func (o *Overrides) MetadataQuerySplitDuration(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).MetadataQuerySplitDuration) -} - -// RecentMetadataQuerySplitDuration returns the tenant specific splitby interval for recent metadata queries. -func (o *Overrides) RecentMetadataQuerySplitDuration(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).RecentMetadataQuerySplitDuration) -} - -// RecentMetadataQueryWindow returns the tenant specific time window used to determine recent metadata queries. -func (o *Overrides) RecentMetadataQueryWindow(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).RecentMetadataQueryWindow) -} - -// IngesterQuerySplitDuration returns the tenant specific splitby interval applied in the query frontend when querying -// during the `query_ingesters_within` window. -func (o *Overrides) IngesterQuerySplitDuration(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).IngesterQuerySplitDuration) -} - -// MaxQueryBytesRead returns the maximum bytes a query can read. -func (o *Overrides) MaxQueryBytesRead(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).MaxQueryBytesRead.Val() -} - -// MaxQuerierBytesRead returns the maximum bytes a sub query can read after splitting and sharding. -func (o *Overrides) MaxQuerierBytesRead(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).MaxQuerierBytesRead.Val() -} - -// MaxConcurrentTailRequests returns the limit to number of concurrent tail requests. -func (o *Overrides) MaxConcurrentTailRequests(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).MaxConcurrentTailRequests -} - -// MaxLineSize returns the maximum size in bytes the distributor should allow. -func (o *Overrides) MaxLineSize(userID string) int { - return o.getOverridesForUser(userID).MaxLineSize.Val() -} - -// MaxLineSizeTruncate returns whether lines longer than max should be truncated. -func (o *Overrides) MaxLineSizeTruncate(userID string) bool { - return o.getOverridesForUser(userID).MaxLineSizeTruncate -} - -// MaxEntriesLimitPerQuery returns the limit to number of entries the querier should return per query. -func (o *Overrides) MaxEntriesLimitPerQuery(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).MaxEntriesLimitPerQuery -} - -func (o *Overrides) QueryTimeout(_ context.Context, userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).QueryTimeout) -} - -func (o *Overrides) MaxCacheFreshness(_ context.Context, userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).MaxCacheFreshness) -} - -func (o *Overrides) MaxMetadataCacheFreshness(_ context.Context, userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).MaxMetadataCacheFreshness) -} - -func (o *Overrides) MaxStatsCacheFreshness(_ context.Context, userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).MaxStatsCacheFreshness) -} - -// MaxQueryLookback returns the max lookback period of queries. -func (o *Overrides) MaxQueryLookback(_ context.Context, userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).MaxQueryLookback) -} - -// RulerTenantShardSize returns shard size (number of rulers) used by this tenant when using shuffle-sharding strategy. -func (o *Overrides) RulerTenantShardSize(userID string) int { - return o.getOverridesForUser(userID).RulerTenantShardSize -} - -func (o *Overrides) IngestionPartitionsTenantShardSize(userID string) int { - return o.getOverridesForUser(userID).IngestionPartitionsTenantShardSize -} - -// RulerMaxRulesPerRuleGroup returns the maximum number of rules per rule group for a given user. -func (o *Overrides) RulerMaxRulesPerRuleGroup(userID string) int { - return o.getOverridesForUser(userID).RulerMaxRulesPerRuleGroup -} - -// RulerMaxRuleGroupsPerTenant returns the maximum number of rule groups for a given user. -func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int { - return o.getOverridesForUser(userID).RulerMaxRuleGroupsPerTenant -} - -// RulerAlertManagerConfig returns the alertmanager configurations to use for a given user. -func (o *Overrides) RulerAlertManagerConfig(userID string) *ruler_config.AlertManagerConfig { - return o.getOverridesForUser(userID).RulerAlertManagerConfig -} - -// RulerRemoteWriteDisabled returns whether remote-write is disabled for a given user or not. -func (o *Overrides) RulerRemoteWriteDisabled(userID string) bool { - return o.getOverridesForUser(userID).RulerRemoteWriteDisabled -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteURL returns the remote-write URL to use for a given user. -func (o *Overrides) RulerRemoteWriteURL(userID string) string { - return o.getOverridesForUser(userID).RulerRemoteWriteURL -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteTimeout returns the duration after which to timeout a remote-write request for a given user. -func (o *Overrides) RulerRemoteWriteTimeout(userID string) time.Duration { - return o.getOverridesForUser(userID).RulerRemoteWriteTimeout -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteHeaders returns the headers to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteHeaders(userID string) map[string]string { - return o.getOverridesForUser(userID).RulerRemoteWriteHeaders.Map() -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteRelabelConfigs returns the write relabel configs to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteRelabelConfigs(userID string) []*util.RelabelConfig { - return o.getOverridesForUser(userID).RulerRemoteWriteRelabelConfigs -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueCapacity returns the queue capacity to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteQueueCapacity(userID string) int { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueCapacity -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMinShards returns the minimum shards to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteQueueMinShards(userID string) int { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMinShards -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMaxShards returns the maximum shards to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteQueueMaxShards(userID string) int { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxShards -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMaxSamplesPerSend returns the max samples to send in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteQueueMaxSamplesPerSend(userID string) int { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxSamplesPerSend -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueBatchSendDeadline returns the maximum time a sample will be buffered before being discarded for a given user. -func (o *Overrides) RulerRemoteWriteQueueBatchSendDeadline(userID string) time.Duration { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueBatchSendDeadline -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMinBackoff returns the minimum time for an exponential backoff for a given user. -func (o *Overrides) RulerRemoteWriteQueueMinBackoff(userID string) time.Duration { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMinBackoff -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMaxBackoff returns the maximum time for an exponential backoff for a given user. -func (o *Overrides) RulerRemoteWriteQueueMaxBackoff(userID string) time.Duration { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxBackoff -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueRetryOnRateLimit returns whether to retry failed remote-write requests (429 response) for a given user. -func (o *Overrides) RulerRemoteWriteQueueRetryOnRateLimit(userID string) bool { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueRetryOnRateLimit -} - -// Deprecated: use RulerRemoteWriteConfig instead -func (o *Overrides) RulerRemoteWriteSigV4Config(userID string) *sigv4.SigV4Config { - return o.getOverridesForUser(userID).RulerRemoteWriteSigV4Config -} - -// RulerRemoteWriteConfig returns the remote-write configurations to use for a given user and a given remote client. -func (o *Overrides) RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig { - if c, ok := o.getOverridesForUser(userID).RulerRemoteWriteConfig[id]; ok { - return &c - } - - return nil -} - -// RulerRemoteEvaluationTimeout returns the duration after which to timeout a remote rule evaluation request for a given user. -func (o *Overrides) RulerRemoteEvaluationTimeout(userID string) time.Duration { - // if not defined, use the base query timeout - timeout := o.getOverridesForUser(userID).RulerRemoteEvaluationTimeout - if timeout <= 0 { - return time.Duration(o.getOverridesForUser(userID).QueryTimeout) - } - - return timeout -} - -// RulerRemoteEvaluationMaxResponseSize returns the maximum allowable response size from a remote rule evaluation for a given user. -func (o *Overrides) RulerRemoteEvaluationMaxResponseSize(userID string) int64 { - return o.getOverridesForUser(userID).RulerRemoteEvaluationMaxResponseSize -} - -// RetentionPeriod returns the retention period for a given user. -func (o *Overrides) RetentionPeriod(userID string) time.Duration { - return time.Duration(o.getOverridesForUser(userID).RetentionPeriod) -} - -// StreamRetention returns the retention period for a given user. -func (o *Overrides) StreamRetention(userID string) []StreamRetention { - return o.getOverridesForUser(userID).StreamRetention -} - -func (o *Overrides) UnorderedWrites(userID string) bool { - return o.getOverridesForUser(userID).UnorderedWrites -} - -func (o *Overrides) DeletionMode(userID string) string { - return o.getOverridesForUser(userID).DeletionMode -} - -func (o *Overrides) ShardStreams(userID string) shardstreams.Config { - return o.getOverridesForUser(userID).ShardStreams -} - -func (o *Overrides) BlockedQueries(_ context.Context, userID string) []*validation.BlockedQuery { - return o.getOverridesForUser(userID).BlockedQueries -} - -func (o *Overrides) RequiredLabels(_ context.Context, userID string) []string { - return o.getOverridesForUser(userID).RequiredLabels -} - -func (o *Overrides) RequiredNumberLabels(_ context.Context, userID string) int { - return o.getOverridesForUser(userID).RequiredNumberLabels -} - -func (o *Overrides) DefaultLimits() *Limits { - return o.defaultLimits -} - -func (o *Overrides) PerStreamRateLimit(userID string) RateLimit { - user := o.getOverridesForUser(userID) - - return RateLimit{ - Limit: rate.Limit(float64(user.PerStreamRateLimit.Val())), - Burst: user.PerStreamRateLimitBurst.Val(), - } -} - -func (o *Overrides) IncrementDuplicateTimestamps(userID string) bool { - return o.getOverridesForUser(userID).IncrementDuplicateTimestamp -} - -func (o *Overrides) DiscoverGenericFields(userID string) map[string][]string { - return o.getOverridesForUser(userID).DiscoverGenericFields.Fields -} - -func (o *Overrides) DiscoverServiceName(userID string) []string { - return o.getOverridesForUser(userID).DiscoverServiceName -} - -func (o *Overrides) DiscoverLogLevels(userID string) bool { - return o.getOverridesForUser(userID).DiscoverLogLevels -} - -func (o *Overrides) LogLevelFields(userID string) []string { - return o.getOverridesForUser(userID).LogLevelFields -} - -// VolumeEnabled returns whether volume endpoints are enabled for a user. -func (o *Overrides) VolumeEnabled(userID string) bool { - return o.getOverridesForUser(userID).VolumeEnabled -} - -func (o *Overrides) VolumeMaxSeries(userID string) int { - return o.getOverridesForUser(userID).VolumeMaxSeries -} - -func (o *Overrides) IndexGatewayShardSize(userID string) int { - return o.getOverridesForUser(userID).IndexGatewayShardSize -} - -func (o *Overrides) BloomGatewayShardSize(userID string) int { - return o.getOverridesForUser(userID).BloomGatewayShardSize -} - -func (o *Overrides) BloomGatewayCacheKeyInterval(userID string) time.Duration { - return o.getOverridesForUser(userID).BloomGatewayCacheKeyInterval -} - -func (o *Overrides) BloomGatewayEnabled(userID string) bool { - return o.getOverridesForUser(userID).BloomGatewayEnabled -} - -func (o *Overrides) BloomCreationEnabled(userID string) bool { - return o.getOverridesForUser(userID).BloomCreationEnabled -} - -func (o *Overrides) BloomPlanningStrategy(userID string) string { - return o.getOverridesForUser(userID).BloomPlanningStrategy -} - -func (o *Overrides) BloomSplitSeriesKeyspaceBy(userID string) int { - return o.getOverridesForUser(userID).BloomSplitSeriesKeyspaceBy -} - -func (o *Overrides) BloomTaskTargetSeriesChunksSizeBytes(userID string) uint64 { - return uint64(o.getOverridesForUser(userID).BloomTaskTargetSeriesChunkSize) -} - -func (o *Overrides) BloomBuildMaxBuilders(userID string) int { - return o.getOverridesForUser(userID).BloomBuildMaxBuilders -} - -func (o *Overrides) BuilderResponseTimeout(userID string) time.Duration { - return o.getOverridesForUser(userID).BloomBuilderResponseTimeout -} - -func (o *Overrides) PrefetchBloomBlocks(userID string) bool { - return o.getOverridesForUser(userID).BloomPrefetchBlocks -} - -func (o *Overrides) BloomTaskMaxRetries(userID string) int { - return o.getOverridesForUser(userID).BloomBuildTaskMaxRetries -} - -func (o *Overrides) BloomMaxBlockSize(userID string) int { - return o.getOverridesForUser(userID).BloomMaxBlockSize.Val() -} - -func (o *Overrides) BloomMaxBloomSize(userID string) int { - return o.getOverridesForUser(userID).BloomMaxBloomSize.Val() -} - -func (o *Overrides) BloomBlockEncoding(userID string) string { - return o.getOverridesForUser(userID).BloomBlockEncoding -} - -func (o *Overrides) AllowStructuredMetadata(userID string) bool { - return o.getOverridesForUser(userID).AllowStructuredMetadata -} - -func (o *Overrides) MaxStructuredMetadataSize(userID string) int { - return o.getOverridesForUser(userID).MaxStructuredMetadataSize.Val() -} - -func (o *Overrides) MaxStructuredMetadataCount(userID string) int { - return o.getOverridesForUser(userID).MaxStructuredMetadataEntriesCount -} - -func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig { - return o.getOverridesForUser(userID).OTLPConfig -} - -func (o *Overrides) BlockIngestionUntil(userID string) time.Time { - return time.Time(o.getOverridesForUser(userID).BlockIngestionUntil) -} - -func (o *Overrides) BlockIngestionStatusCode(userID string) int { - return o.getOverridesForUser(userID).BlockIngestionStatusCode -} - -func (o *Overrides) EnforcedLabels(userID string) []string { - return o.getOverridesForUser(userID).EnforcedLabels -} - -func (o *Overrides) ShardAggregations(userID string) []string { - return o.getOverridesForUser(userID).ShardAggregations -} - -func (o *Overrides) PatternIngesterTokenizableJSONFields(userID string) []string { - defaultFields := o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsDefault - appendFields := o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsAppend - deleteFields := o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsDelete - - outputMap := make(map[string]struct{}, len(defaultFields)+len(appendFields)) - - for _, field := range defaultFields { - outputMap[field] = struct{}{} - } - - for _, field := range appendFields { - outputMap[field] = struct{}{} - } - - for _, field := range deleteFields { - delete(outputMap, field) - } - - output := make([]string, 0, len(outputMap)) - for field := range outputMap { - output = append(output, field) - } - - return output -} - -func (o *Overrides) PatternIngesterTokenizableJSONFieldsAppend(userID string) []string { - return o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsAppend -} - -func (o *Overrides) PatternIngesterTokenizableJSONFieldsDelete(userID string) []string { - return o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsDelete -} - -func (o *Overrides) MetricAggregationEnabled(userID string) bool { - return o.getOverridesForUser(userID).MetricAggregationEnabled -} - -// S3SSEType returns the per-tenant S3 SSE type. -func (o *Overrides) S3SSEType(user string) string { - return o.getOverridesForUser(user).S3SSEType -} - -// S3SSEKMSKeyID returns the per-tenant S3 KMS-SSE key id. -func (o *Overrides) S3SSEKMSKeyID(user string) string { - return o.getOverridesForUser(user).S3SSEKMSKeyID -} - -// S3SSEKMSEncryptionContext returns the per-tenant S3 KMS-SSE encryption context. -func (o *Overrides) S3SSEKMSEncryptionContext(user string) string { - return o.getOverridesForUser(user).S3SSEKMSEncryptionContext -} - -func (o *Overrides) getOverridesForUser(userID string) *Limits { - if o.tenantLimits != nil { - l := o.tenantLimits.TenantLimits(userID) - if l != nil { - return l - } - } - return o.defaultLimits -} - -// OverwriteMarshalingStringMap will overwrite the src map when unmarshaling -// as opposed to merging. -type OverwriteMarshalingStringMap struct { - m map[string]string -} - -func NewOverwriteMarshalingStringMap(m map[string]string) OverwriteMarshalingStringMap { - return OverwriteMarshalingStringMap{m: m} -} - -func (sm *OverwriteMarshalingStringMap) Map() map[string]string { - return sm.m -} - -// MarshalJSON explicitly uses the the type receiver and not pointer receiver -// or it won't be called -func (sm OverwriteMarshalingStringMap) MarshalJSON() ([]byte, error) { - return json.Marshal(sm.m) -} - -func (sm *OverwriteMarshalingStringMap) UnmarshalJSON(val []byte) error { - var def map[string]string - if err := json.Unmarshal(val, &def); err != nil { - return err - } - sm.m = def - - return nil -} - -// MarshalYAML explicitly uses the the type receiver and not pointer receiver -// or it won't be called -func (sm OverwriteMarshalingStringMap) MarshalYAML() (interface{}, error) { - return sm.m, nil -} - -func (sm *OverwriteMarshalingStringMap) UnmarshalYAML(unmarshal func(interface{}) error) error { - var def map[string]string - - err := unmarshal(&def) - if err != nil { - return err - } - sm.m = def - - return nil -} diff --git a/tools/doc-generator/parse/parser.go b/tools/doc-generator/parse/parser.go index f565bf2dc9c90..b1f57ac9ed5b0 100644 --- a/tools/doc-generator/parse/parser.go +++ b/tools/doc-generator/parse/parser.go @@ -23,9 +23,9 @@ import ( "github.com/prometheus/prometheus/model/relabel" "github.com/grafana/loki/v3/pkg/ruler/util" + "github.com/grafana/loki/v3/pkg/runtime" storage_config "github.com/grafana/loki/v3/pkg/storage/config" - util_validation "github.com/grafana/loki/v3/pkg/util/validation" - "github.com/grafana/loki/v3/pkg/validation" + "github.com/grafana/loki/v3/pkg/util/validation" ) var ( @@ -365,13 +365,13 @@ func getFieldCustomType(t reflect.Type) (string, bool) { return fieldRelabelConfig, true case reflect.TypeOf([]*relabel.Config{}).String(): return fieldRelabelConfig, true - case reflect.TypeOf([]*util_validation.BlockedQuery{}).String(): + case reflect.TypeOf([]*validation.BlockedQuery{}).String(): return "blocked_query...", true case reflect.TypeOf([]*prometheus_config.RemoteWriteConfig{}).String(): return "remote_write_config...", true case reflect.TypeOf(storage_config.PeriodConfig{}).String(): return "period_config", true - case reflect.TypeOf(validation.OverwriteMarshalingStringMap{}).String(): + case reflect.TypeOf(runtime.OverwriteMarshalingStringMap{}).String(): return "headers", true default: return "", false @@ -460,7 +460,7 @@ func getCustomFieldType(t reflect.Type) (string, bool) { return fieldRelabelConfig, true case reflect.TypeOf(&prometheus_config.RemoteWriteConfig{}).String(): return "remote_write_config...", true - case reflect.TypeOf(validation.OverwriteMarshalingStringMap{}).String(): + case reflect.TypeOf(runtime.OverwriteMarshalingStringMap{}).String(): return "headers", true case reflect.TypeOf(relabel.Regexp{}).String(): return fieldString, true diff --git a/tools/doc-generator/parse/root_blocks.go b/tools/doc-generator/parse/root_blocks.go index 12d8b44d2a7b6..614f45164f919 100644 --- a/tools/doc-generator/parse/root_blocks.go +++ b/tools/doc-generator/parse/root_blocks.go @@ -47,7 +47,6 @@ import ( storage_config "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/series/index" "github.com/grafana/loki/v3/pkg/tracing" - "github.com/grafana/loki/v3/pkg/validation" ) var ( @@ -137,7 +136,7 @@ var ( }, { Name: "limits_config", - StructType: []reflect.Type{reflect.TypeOf(validation.Limits{})}, + StructType: []reflect.Type{reflect.TypeOf(runtime.Limits{})}, Desc: "The limits_config block configures global and per-tenant limits in Loki. The values here can be overridden in the `overrides` section of the runtime_config file", }, { @@ -156,11 +155,6 @@ var ( StructType: []reflect.Type{reflect.TypeOf(runtimeconfig.Config{})}, Desc: "Configuration for 'runtime config' module, responsible for reloading runtime configuration file.", }, - { - Name: "operational_config", - StructType: []reflect.Type{reflect.TypeOf(runtime.Config{})}, - Desc: "These are values which allow you to control aspects of Loki's operation, most commonly used for controlling types of higher verbosity logging, the values here can be overridden in the `configs` section of the `runtime_config` file.", - }, { Name: "tracing", StructType: []reflect.Type{reflect.TypeOf(tracing.Config{})}, diff --git a/tools/tsdb/helpers/setup.go b/tools/tsdb/helpers/setup.go index 807adb4dd9897..965e928066523 100644 --- a/tools/tsdb/helpers/setup.go +++ b/tools/tsdb/helpers/setup.go @@ -13,13 +13,13 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors/version" "github.com/grafana/loki/v3/pkg/loki" + "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/storage" "github.com/grafana/loki/v3/pkg/storage/chunk/client/util" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper" "github.com/grafana/loki/v3/pkg/util/cfg" util_log "github.com/grafana/loki/v3/pkg/util/log" - "github.com/grafana/loki/v3/pkg/validation" ) func Setup() (loki.Config, services.Service, string, error) { @@ -81,15 +81,15 @@ func moduleManager(cfg *server.Config) (services.Service, error) { return s, nil } -func DefaultConfigs() (config.ChunkStoreConfig, *validation.Overrides, storage.ClientMetrics) { +func DefaultConfigs() (config.ChunkStoreConfig, *runtime.Overrides, storage.ClientMetrics) { var ( chunkStoreConfig config.ChunkStoreConfig - limits validation.Limits + limits runtime.Limits clientMetrics storage.ClientMetrics ) chunkStoreConfig.RegisterFlags(flag.NewFlagSet("chunk-store", flag.PanicOnError)) limits.RegisterFlags(flag.NewFlagSet("limits", flag.PanicOnError)) - overrides, _ := validation.NewOverrides(limits, nil) + overrides, _ := runtime.NewOverrides(limits, nil) return chunkStoreConfig, overrides, clientMetrics }