From 4f66785645a6b3cf0885344226508b140f301c0c Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Fri, 6 Jun 2025 06:49:24 -0700 Subject: [PATCH 1/9] Add active series limit for nativeHistograms samples Signed-off-by: Paurush Garg --- docs/configuration/config-file-reference.md | 11 +++ pkg/ingester/ingester.go | 5 ++ pkg/ingester/limiter.go | 36 ++++++++-- pkg/ingester/limiter_test.go | 76 +++++++++++++++++++++ pkg/util/validation/limits.go | 33 +++++++-- 5 files changed, 151 insertions(+), 10 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 169be330687..8c0248e10d3 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3606,6 +3606,11 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -ingester.max-series-per-metric [max_series_per_metric: | default = 50000] +# The maximum number of active nativeHistograms series per user, per ingester. 0 +# to disable. +# CLI flag: -ingester.max-native-histograms-series-per-user +[max_native_histograms_series_per_user: | default = 5000000] + # The maximum number of active series per user, across the cluster before # replication. 0 to disable. Supported only if -distributor.shard-by-all-labels # is true. @@ -3617,6 +3622,12 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -ingester.max-global-series-per-metric [max_global_series_per_metric: | default = 0] +# The maximum number of active nativeHistograms series per user, across the +# cluster before replication. 0 to disable. Supported only if +# -distributor.shard-by-all-labels is true. +# CLI flag: -ingester.max-global-native-histograms-series-per-user +[max_global_native_histograms_series_per_user: | default = 0] + # [Experimental] Enable limits per LabelSet. Supported limits per labelSet: # [max_series] [limits_per_label_set: | default = []] diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index c2fe4e45bd2..6e1435e9be2 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -448,6 +448,11 @@ func (u *userTSDB) PreCreation(metric labels.Labels) error { } } + // Total nativeHistograms series limit. + if err := u.limiter.AssertMaxNativeHistogramsSeriesPerUser(u.userID, u.activeSeries.ActiveNativeHistogram()); err != nil { + return err + } + // Total series limit. if err := u.limiter.AssertMaxSeriesPerUser(u.userID, int(u.Head().NumSeries())); err != nil { return err diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index 94dd409b3bc..b7a96dbf342 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -12,10 +12,11 @@ import ( ) var ( - errMaxSeriesPerMetricLimitExceeded = errors.New("per-metric series limit exceeded") - errMaxMetadataPerMetricLimitExceeded = errors.New("per-metric metadata limit exceeded") - errMaxSeriesPerUserLimitExceeded = errors.New("per-user series limit exceeded") - errMaxMetadataPerUserLimitExceeded = errors.New("per-user metric metadata limit exceeded") + errMaxSeriesPerMetricLimitExceeded = errors.New("per-metric series limit exceeded") + errMaxMetadataPerMetricLimitExceeded = errors.New("per-metric metadata limit exceeded") + errMaxSeriesPerUserLimitExceeded = errors.New("per-user series limit exceeded") + errMaxNativeHistogramsSeriesPerUserLimitExceeded = errors.New("per-user nativeHistograms series limit exceeded") + errMaxMetadataPerUserLimitExceeded = errors.New("per-user metric metadata limit exceeded") ) type errMaxSeriesPerLabelSetLimitExceeded struct { @@ -95,6 +96,16 @@ func (l *Limiter) AssertMaxSeriesPerUser(userID string, series int) error { return errMaxSeriesPerUserLimitExceeded } +// AssertMaxNativeHistogramsSeriesPerUser limit has not been reached compared to the current +// number of nativeHistograms series in input and returns an error if so. +func (l *Limiter) AssertMaxNativeHistogramsSeriesPerUser(userID string, series int) error { + if actualLimit := l.maxNativeHistogramsSeriesPerUser(userID); series < actualLimit { + return nil + } + + return errMaxNativeHistogramsSeriesPerUserLimitExceeded +} + // AssertMaxMetricsWithMetadataPerUser limit has not been reached compared to the current // number of metrics with metadata in input and returns an error if so. func (l *Limiter) AssertMaxMetricsWithMetadataPerUser(userID string, metrics int) error { @@ -158,6 +169,15 @@ func (l *Limiter) formatMaxSeriesPerUserError(userID string) error { minNonZero(localLimit, globalLimit), l.AdminLimitMessage, localLimit, globalLimit, actualLimit) } +func (l *Limiter) formatMaxNativeHistogramsSeriesPerUserError(userID string) error { + actualLimit := l.maxNativeHistogramsSeriesPerUser(userID) + localLimit := l.limits.MaxLocalNativeHistogramsSeriesPerUser(userID) + globalLimit := l.limits.MaxGlobalNativeHistogramsSeriesPerUser(userID) + + return fmt.Errorf("per-user nativeHistograms series limit of %d exceeded, %s (local limit: %d global limit: %d actual local limit: %d)", + minNonZero(localLimit, globalLimit), l.AdminLimitMessage, localLimit, globalLimit, actualLimit) +} + func (l *Limiter) formatMaxSeriesPerMetricError(userID string, metric string) error { actualLimit := l.maxSeriesPerMetric(userID) localLimit := l.limits.MaxLocalSeriesPerMetric(userID) @@ -248,6 +268,14 @@ func (l *Limiter) maxSeriesPerUser(userID string) int { ) } +func (l *Limiter) maxNativeHistogramsSeriesPerUser(userID string) int { + return l.maxByLocalAndGlobal( + userID, + l.limits.MaxLocalNativeHistogramsSeriesPerUser, + l.limits.MaxGlobalNativeHistogramsSeriesPerUser, + ) +} + func (l *Limiter) maxMetadataPerUser(userID string) int { return l.maxByLocalAndGlobal( userID, diff --git a/pkg/ingester/limiter_test.go b/pkg/ingester/limiter_test.go index a1043b053e5..d1cbe48c32b 100644 --- a/pkg/ingester/limiter_test.go +++ b/pkg/ingester/limiter_test.go @@ -54,6 +54,19 @@ func TestLimiter_maxSeriesPerUser(t *testing.T) { runLimiterMaxFunctionTest(t, applyLimits, runMaxFn, false) } +func TestLimiter_maxNativeHistogramsSeriesPerUser(t *testing.T) { + applyLimits := func(limits *validation.Limits, localLimit, globalLimit int) { + limits.MaxLocalNativeHistogramsSeriesPerUser = localLimit + limits.MaxGlobalNativeHistogramsSeriesPerUser = globalLimit + } + + runMaxFn := func(limiter *Limiter) int { + return limiter.maxNativeHistogramsSeriesPerUser("test") + } + + runLimiterMaxFunctionTest(t, applyLimits, runMaxFn, false) +} + func TestLimiter_maxMetadataPerUser(t *testing.T) { applyLimits := func(limits *validation.Limits, localLimit, globalLimit int) { limits.MaxLocalMetricsWithMetadataPerUser = localLimit @@ -425,6 +438,69 @@ func TestLimiter_AssertMaxSeriesPerUser(t *testing.T) { } } +func TestLimiter_AssertMaxNativeHistogramsSeriesPerUser(t *testing.T) { + tests := map[string]struct { + maxLocalNativeHistogramsSeriesPerUser int + maxGlobalNativeHistogramsSeriesPerUser int + ringReplicationFactor int + ringIngesterCount int + shardByAllLabels bool + series int + expected error + }{ + "both local and global limit are disabled": { + maxLocalNativeHistogramsSeriesPerUser: 0, + maxGlobalNativeHistogramsSeriesPerUser: 0, + ringReplicationFactor: 1, + ringIngesterCount: 1, + shardByAllLabels: false, + series: 100, + expected: nil, + }, + "current number of series is below the limit": { + maxLocalNativeHistogramsSeriesPerUser: 0, + maxGlobalNativeHistogramsSeriesPerUser: 1000, + ringReplicationFactor: 3, + ringIngesterCount: 10, + shardByAllLabels: true, + series: 299, + expected: nil, + }, + "current number of series is above the limit": { + maxLocalNativeHistogramsSeriesPerUser: 0, + maxGlobalNativeHistogramsSeriesPerUser: 1000, + ringReplicationFactor: 3, + ringIngesterCount: 10, + shardByAllLabels: true, + series: 300, + expected: errMaxNativeHistogramsSeriesPerUserLimitExceeded, + }, + } + + for testName, testData := range tests { + testData := testData + + t.Run(testName, func(t *testing.T) { + // Mock the ring + ring := &ringCountMock{} + ring.On("HealthyInstancesCount").Return(testData.ringIngesterCount) + ring.On("ZonesCount").Return(1) + + // Mock limits + limits, err := validation.NewOverrides(validation.Limits{ + MaxLocalNativeHistogramsSeriesPerUser: testData.maxLocalNativeHistogramsSeriesPerUser, + MaxGlobalNativeHistogramsSeriesPerUser: testData.maxGlobalNativeHistogramsSeriesPerUser, + }, nil) + require.NoError(t, err) + + limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false, "") + actual := limiter.AssertMaxNativeHistogramsSeriesPerUser("test", testData.series) + + assert.Equal(t, testData.expected, actual) + }) + } +} + func TestLimiter_AssertMaxSeriesPerLabelSet(t *testing.T) { tests := map[string]struct { diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 6288956e496..f2770ab6795 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -24,6 +24,7 @@ import ( ) var errMaxGlobalSeriesPerUserValidation = errors.New("the ingester.max-global-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") +var errMaxGlobalNativeHistogramsSeriesPerUserValidation = errors.New("the ingester.max-global-native-histograms-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") var errDuplicateQueryPriorities = errors.New("duplicate entry of priorities found. Make sure they are all unique, including the default priority") var errCompilingQueryPriorityRegex = errors.New("error compiling query priority regex") var errDuplicatePerLabelSetLimit = errors.New("duplicate per labelSet limits found. Make sure they are all unique") @@ -152,12 +153,14 @@ type Limits struct { // Ingester enforced limits. // Series - MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"` - MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"` - MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` - MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` - LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` - EnableNativeHistograms bool `yaml:"enable_native_histograms" json:"enable_native_histograms"` + MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"` + MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"` + MaxLocalNativeHistogramsSeriesPerUser int `yaml:"max_native_histograms_series_per_user" json:"max_native_histograms_series_per_user"` + MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` + MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` + MaxGlobalNativeHistogramsSeriesPerUser int `yaml:"max_global_native_histograms_series_per_user" json:"max_global_native_histograms_series_per_user"` + LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` + EnableNativeHistograms bool `yaml:"enable_native_histograms" json:"enable_native_histograms"` // Metadata MaxLocalMetricsWithMetadataPerUser int `yaml:"max_metadata_per_user" json:"max_metadata_per_user"` @@ -273,6 +276,8 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLocalSeriesPerMetric, "ingester.max-series-per-metric", 50000, "The maximum number of active series per metric name, per ingester. 0 to disable.") f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") + f.IntVar(&l.MaxLocalNativeHistogramsSeriesPerUser, "ingester.max-native-histograms-series-per-user", 5000000, "The maximum number of active nativeHistograms series per user, per ingester. 0 to disable.") + f.IntVar(&l.MaxGlobalNativeHistogramsSeriesPerUser, "ingester.max-global-native-histograms-series-per-user", 0, "The maximum number of active nativeHistograms series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.BoolVar(&l.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") @@ -347,6 +352,12 @@ func (l *Limits) Validate(shardByAllLabels bool) error { return errMaxGlobalSeriesPerUserValidation } + // The ingester.max-global-native-histograms-series-per-user metric is not supported + // if shard-by-all-labels is disabled + if l.MaxGlobalNativeHistogramsSeriesPerUser > 0 && !shardByAllLabels { + return errMaxGlobalNativeHistogramsSeriesPerUserValidation + } + if err := l.RulerExternalLabels.Validate(func(l labels.Label) error { if !model.LabelName(l.Name).IsValid() { return fmt.Errorf("%w: %q", errInvalidLabelName, l.Name) @@ -679,6 +690,11 @@ func (o *Overrides) MaxLocalSeriesPerUser(userID string) int { return o.GetOverridesForUser(userID).MaxLocalSeriesPerUser } +// MaxLocalNativeHistogramsSeriesPerUser returns the maximum number of nativeHistograms series a user is allowed to store in a single ingester. +func (o *Overrides) MaxLocalNativeHistogramsSeriesPerUser(userID string) int { + return o.GetOverridesForUser(userID).MaxLocalNativeHistogramsSeriesPerUser +} + // MaxLocalSeriesPerMetric returns the maximum number of series allowed per metric in a single ingester. func (o *Overrides) MaxLocalSeriesPerMetric(userID string) int { return o.GetOverridesForUser(userID).MaxLocalSeriesPerMetric @@ -689,6 +705,11 @@ func (o *Overrides) MaxGlobalSeriesPerUser(userID string) int { return o.GetOverridesForUser(userID).MaxGlobalSeriesPerUser } +// MaxGlobalNativeHistogramsSeriesPerUser returns the maximum number of nativeHistograms series a user is allowed to store across the cluster. +func (o *Overrides) MaxGlobalNativeHistogramsSeriesPerUser(userID string) int { + return o.GetOverridesForUser(userID).MaxGlobalNativeHistogramsSeriesPerUser +} + // EnableNativeHistograms returns whether the Ingester should accept NativeHistograms samples from this user. func (o *Overrides) EnableNativeHistograms(userID string) bool { return o.GetOverridesForUser(userID).EnableNativeHistograms From 710de245d71729e8ef7fc4ec26c57cc54cc64ed1 Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Fri, 6 Jun 2025 06:58:32 -0700 Subject: [PATCH 2/9] Adding Changelog Signed-off-by: Paurush Garg --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dce8a4f2a40..6e7cafa5c51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +* [ENHANCEMENT] Ingester: Add activeSeries limit specifically for NativeHistograms. #6796 * [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new per-tenant limit at ingestion. #6718 * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603 * [CHANGE] Validate a tenantID when to use a single tenant resolver. #6727 From 20bbde4cf64e4c3d7cce401886002748f51314c0 Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Fri, 6 Jun 2025 07:07:34 -0700 Subject: [PATCH 3/9] Resolving Errror in limiter Signed-off-by: Paurush Garg --- pkg/ingester/limiter.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index b7a96dbf342..7eb305e9aa3 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -145,6 +145,8 @@ func (l *Limiter) FormatError(userID string, err error, lbls labels.Labels) erro switch { case errors.Is(err, errMaxSeriesPerUserLimitExceeded): return l.formatMaxSeriesPerUserError(userID) + case errors.Is(err, errMaxNativeHistogramsSeriesPerUserLimitExceeded): + return l.formatMaxNativeHistogramsSeriesPerUserError(userID) case errors.Is(err, errMaxSeriesPerMetricLimitExceeded): return l.formatMaxSeriesPerMetricError(userID, lbls.Get(labels.MetricName)) case errors.Is(err, errMaxMetadataPerUserLimitExceeded): From 24e65737c5a80b9e49b543967b311e1f20e80c9c Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Fri, 6 Jun 2025 13:48:00 -0700 Subject: [PATCH 4/9] Adding testing Signed-off-by: Paurush Garg --- pkg/ingester/ingester.go | 40 ++++++++++------ pkg/ingester/ingester_test.go | 87 +++++++++++++++++++++++++++++++++++ pkg/ingester/limiter_test.go | 12 +++-- pkg/ingester/user_state.go | 7 +-- 4 files changed, 124 insertions(+), 22 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 6e1435e9be2..3240466272e 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1224,21 +1224,22 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte // Keep track of some stats which are tracked only if the samples will be // successfully committed var ( - succeededSamplesCount = 0 - failedSamplesCount = 0 - succeededHistogramsCount = 0 - failedHistogramsCount = 0 - succeededExemplarsCount = 0 - failedExemplarsCount = 0 - startAppend = time.Now() - sampleOutOfBoundsCount = 0 - sampleOutOfOrderCount = 0 - sampleTooOldCount = 0 - newValueForTimestampCount = 0 - perUserSeriesLimitCount = 0 - perLabelSetSeriesLimitCount = 0 - perMetricSeriesLimitCount = 0 - discardedNativeHistogramCount = 0 + succeededSamplesCount = 0 + failedSamplesCount = 0 + succeededHistogramsCount = 0 + failedHistogramsCount = 0 + succeededExemplarsCount = 0 + failedExemplarsCount = 0 + startAppend = time.Now() + sampleOutOfBoundsCount = 0 + sampleOutOfOrderCount = 0 + sampleTooOldCount = 0 + newValueForTimestampCount = 0 + perUserSeriesLimitCount = 0 + perUserNativeHistogramsSeriesLimitCount = 0 + perLabelSetSeriesLimitCount = 0 + perMetricSeriesLimitCount = 0 + discardedNativeHistogramCount = 0 updateFirstPartial = func(errFn func() error) { if firstPartialErr == nil { @@ -1274,6 +1275,12 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte return makeLimitError(perUserSeriesLimit, i.limiter.FormatError(userID, cause, copiedLabels)) }) + case errors.Is(cause, errMaxNativeHistogramsSeriesPerUserLimitExceeded): + perUserNativeHistogramsSeriesLimitCount++ + updateFirstPartial(func() error { + return makeLimitError(perUserSeriesLimit, i.limiter.FormatError(userID, cause, copiedLabels)) + }) + case errors.Is(cause, errMaxSeriesPerMetricLimitExceeded): perMetricSeriesLimitCount++ updateFirstPartial(func() error { @@ -1517,6 +1524,9 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte if perUserSeriesLimitCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(perUserSeriesLimit, userID).Add(float64(perUserSeriesLimitCount)) } + if perUserNativeHistogramsSeriesLimitCount > 0 { + i.validateMetrics.DiscardedSamples.WithLabelValues(perUserNativeHistogramsSeriesLimit, userID).Add(float64(perUserNativeHistogramsSeriesLimitCount)) + } if perMetricSeriesLimitCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(perMetricSeriesLimit, userID).Add(float64(perMetricSeriesLimitCount)) } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index c90a31df92b..14b3b4a0b0b 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -868,6 +868,93 @@ func TestIngesterUserLimitExceeded(t *testing.T) { } +func TestIngesterUserLimitExceededForNativeHistograms(t *testing.T) { + limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true + limits.MaxLocalNativeHistogramsSeriesPerUser = 1 + limits.MaxLocalSeriesPerUser = 1 + limits.MaxLocalMetricsWithMetadataPerUser = 1 + + userID := "1" + // Series + labels1 := labels.Labels{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "bar"}} + labels3 := labels.Labels{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "biz"}} + sampleNativeHistogram1 := cortexpb.HistogramToHistogramProto(0, tsdbutil.GenerateTestHistogram(1)) + sampleNativeHistogram2 := cortexpb.HistogramToHistogramProto(1, tsdbutil.GenerateTestHistogram(2)) + sampleNativeHistogram3 := cortexpb.HistogramToHistogramProto(0, tsdbutil.GenerateTestHistogram(3)) + + // Metadata + metadata1 := &cortexpb.MetricMetadata{MetricFamilyName: "testmetric", Help: "a help for testmetric", Type: cortexpb.COUNTER} + metadata2 := &cortexpb.MetricMetadata{MetricFamilyName: "testmetric2", Help: "a help for testmetric2", Type: cortexpb.COUNTER} + + dir := t.TempDir() + + chunksDir := filepath.Join(dir, "chunks") + blocksDir := filepath.Join(dir, "blocks") + require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) + require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) + + blocksIngesterGenerator := func(reg prometheus.Registerer) *Ingester { + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) + // Wait until it's ACTIVE + test.Poll(t, time.Second, ring.ACTIVE, func() interface{} { + return ing.lifecycler.GetState() + }) + + return ing + } + + tests := []string{"blocks"} + for i, ingGenerator := range []func(reg prometheus.Registerer) *Ingester{blocksIngesterGenerator} { + t.Run(tests[i], func(t *testing.T) { + reg := prometheus.NewRegistry() + ing := ingGenerator(reg) + + // Append only one series and one metadata first, expect no error. + ctx := user.InjectOrgID(context.Background(), userID) + _, err := ing.Push(ctx, cortexpb.ToWriteRequest([]labels.Labels{labels1}, nil, []*cortexpb.MetricMetadata{metadata1}, []cortexpb.Histogram{sampleNativeHistogram1}, cortexpb.API)) + require.NoError(t, err) + + testLimits := func(reg prometheus.Gatherer) { + // Append to two series, expect series-exceeded error. + _, err = ing.Push(ctx, cortexpb.ToWriteRequest([]labels.Labels{labels1, labels3}, nil, nil, []cortexpb.Histogram{sampleNativeHistogram2, sampleNativeHistogram3}, cortexpb.API)) + httpResp, ok := httpgrpc.HTTPResponseFromError(err) + require.True(t, ok, "returned error is not an httpgrpc response") + assert.Equal(t, http.StatusBadRequest, int(httpResp.Code)) + assert.Equal(t, wrapWithUser(makeLimitError(perUserNativeHistogramsSeriesLimit, ing.limiter.FormatError(userID, errMaxNativeHistogramsSeriesPerUserLimitExceeded, labels1)), userID).Error(), string(httpResp.Body)) + + // Append two metadata, expect no error since metadata is a best effort approach. + _, err = ing.Push(ctx, cortexpb.ToWriteRequest(nil, nil, []*cortexpb.MetricMetadata{metadata1, metadata2}, nil, cortexpb.API)) + require.NoError(t, err) + + // Read samples back via ingester queries. + res, _, err := runTestQuery(ctx, t, ing, labels.MatchEqual, model.MetricNameLabel, "testmetric") + require.NoError(t, err) + require.NotNil(t, res) + + // Verify metadata + m, err := ing.MetricsMetadata(ctx, &client.MetricsMetadataRequest{Limit: -1, LimitPerMetric: -1, Metric: ""}) + require.NoError(t, err) + assert.Equal(t, []*cortexpb.MetricMetadata{metadata1}, m.Metadata) + } + + testLimits(reg) + + // Limits should hold after restart. + services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + // Use new registry to prevent metrics registration panic. + reg = prometheus.NewRegistry() + ing = ingGenerator(reg) + defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + + testLimits(reg) + }) + } + +} + func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []cortexpb.Sample) { for j := 0; j < nSeries; j++ { labels := chunk.BenchmarkLabels.Copy() diff --git a/pkg/ingester/limiter_test.go b/pkg/ingester/limiter_test.go index d1cbe48c32b..239b84ce85e 100644 --- a/pkg/ingester/limiter_test.go +++ b/pkg/ingester/limiter_test.go @@ -656,10 +656,11 @@ func TestLimiter_FormatError(t *testing.T) { // Mock limits limits, err := validation.NewOverrides(validation.Limits{ - MaxGlobalSeriesPerUser: 100, - MaxGlobalSeriesPerMetric: 20, - MaxGlobalMetricsWithMetadataPerUser: 10, - MaxGlobalMetadataPerMetric: 3, + MaxGlobalSeriesPerUser: 100, + MaxGlobalNativeHistogramsSeriesPerUser: 100, + MaxGlobalSeriesPerMetric: 20, + MaxGlobalMetricsWithMetadataPerUser: 10, + MaxGlobalMetadataPerMetric: 3, }, nil) require.NoError(t, err) @@ -669,6 +670,9 @@ func TestLimiter_FormatError(t *testing.T) { actual := limiter.FormatError("user-1", errMaxSeriesPerUserLimitExceeded, lbls) assert.EqualError(t, actual, "per-user series limit of 100 exceeded, please contact administrator to raise it (local limit: 0 global limit: 100 actual local limit: 100)") + actual = limiter.FormatError("user-1", errMaxNativeHistogramsSeriesPerUserLimitExceeded, lbls) + assert.EqualError(t, actual, "per-user nativeHistograms series limit of 100 exceeded, please contact administrator to raise it (local limit: 0 global limit: 100 actual local limit: 100)") + actual = limiter.FormatError("user-1", errMaxSeriesPerMetricLimitExceeded, lbls) assert.EqualError(t, actual, "per-metric series limit of 20 exceeded for metric testMetric, please contact administrator to raise it (local limit: 0 global limit: 20 actual local limit: 20)") diff --git a/pkg/ingester/user_state.go b/pkg/ingester/user_state.go index 9ef89d48a92..5d04cee8a7d 100644 --- a/pkg/ingester/user_state.go +++ b/pkg/ingester/user_state.go @@ -16,9 +16,10 @@ import ( // DiscardedSamples metric labels const ( - perUserSeriesLimit = "per_user_series_limit" - perMetricSeriesLimit = "per_metric_series_limit" - perLabelsetSeriesLimit = "per_labelset_series_limit" + perUserSeriesLimit = "per_user_series_limit" + perUserNativeHistogramsSeriesLimit = "per_user_native_histograms_series_limit" + perMetricSeriesLimit = "per_metric_series_limit" + perLabelsetSeriesLimit = "per_labelset_series_limit" ) const numMetricCounterShards = 128 From 580b7a160df31935e441beaad2550f5963275bd6 Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Tue, 24 Jun 2025 09:53:21 -0700 Subject: [PATCH 5/9] Resolving comments Signed-off-by: Paurush Garg --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 12 +++--- pkg/ingester/ingester.go | 42 ++++++++++----------- pkg/ingester/ingester_test.go | 6 +-- pkg/ingester/limiter.go | 34 ++++++++--------- pkg/ingester/limiter_test.go | 26 ++++++------- pkg/ingester/user_state.go | 8 ++-- pkg/util/validation/limits.go | 38 +++++++++---------- 8 files changed, 84 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e7cafa5c51..444ad8108bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ * [ENHANCEMENT] Metadata Cache: Support inmemory and multi level cache backend. #6829 * [ENHANCEMENT] Store Gateway: Allow to ignore syncing blocks older than certain time using `ignore_blocks_before`. #6830 * [ENHANCEMENT] Distributor: Add native histograms max sample size bytes limit validation. #6834 +* [ENHANCEMENT] Ingester: Add activeSeries limit specifically for Native Histogram. #6796 * [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517 * [BUGFIX] Ingester: Fix labelset data race condition. #6573 * [BUGFIX] Compactor: Cleaner should not put deletion marker for blocks with no-compact marker. #6576 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 8c0248e10d3..aaf7e17a71f 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3606,10 +3606,10 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -ingester.max-series-per-metric [max_series_per_metric: | default = 50000] -# The maximum number of active nativeHistograms series per user, per ingester. 0 +# The maximum number of active native histogram series per user, per ingester. 0 # to disable. -# CLI flag: -ingester.max-native-histograms-series-per-user -[max_native_histograms_series_per_user: | default = 5000000] +# CLI flag: -ingester.max-native-histogram-series-per-user +[max_native_histogram_series_per_user: | default = 5000000] # The maximum number of active series per user, across the cluster before # replication. 0 to disable. Supported only if -distributor.shard-by-all-labels @@ -3622,11 +3622,11 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -ingester.max-global-series-per-metric [max_global_series_per_metric: | default = 0] -# The maximum number of active nativeHistograms series per user, across the +# The maximum number of active native histogram series per user, across the # cluster before replication. 0 to disable. Supported only if # -distributor.shard-by-all-labels is true. -# CLI flag: -ingester.max-global-native-histograms-series-per-user -[max_global_native_histograms_series_per_user: | default = 0] +# CLI flag: -ingester.max-global-native-histogram-series-per-user +[max_global_native_histogram_series_per_user: | default = 0] # [Experimental] Enable limits per LabelSet. Supported limits per labelSet: # [max_series] diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 3240466272e..90e69bd3c38 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -449,7 +449,7 @@ func (u *userTSDB) PreCreation(metric labels.Labels) error { } // Total nativeHistograms series limit. - if err := u.limiter.AssertMaxNativeHistogramsSeriesPerUser(u.userID, u.activeSeries.ActiveNativeHistogram()); err != nil { + if err := u.limiter.AssertMaxNativeHistogramSeriesPerUser(u.userID, u.activeSeries.ActiveNativeHistogram()); err != nil { return err } @@ -1224,22 +1224,22 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte // Keep track of some stats which are tracked only if the samples will be // successfully committed var ( - succeededSamplesCount = 0 - failedSamplesCount = 0 - succeededHistogramsCount = 0 - failedHistogramsCount = 0 - succeededExemplarsCount = 0 - failedExemplarsCount = 0 - startAppend = time.Now() - sampleOutOfBoundsCount = 0 - sampleOutOfOrderCount = 0 - sampleTooOldCount = 0 - newValueForTimestampCount = 0 - perUserSeriesLimitCount = 0 - perUserNativeHistogramsSeriesLimitCount = 0 - perLabelSetSeriesLimitCount = 0 - perMetricSeriesLimitCount = 0 - discardedNativeHistogramCount = 0 + succeededSamplesCount = 0 + failedSamplesCount = 0 + succeededHistogramsCount = 0 + failedHistogramsCount = 0 + succeededExemplarsCount = 0 + failedExemplarsCount = 0 + startAppend = time.Now() + sampleOutOfBoundsCount = 0 + sampleOutOfOrderCount = 0 + sampleTooOldCount = 0 + newValueForTimestampCount = 0 + perUserSeriesLimitCount = 0 + perUserNativeHistogramSeriesLimitCount = 0 + perLabelSetSeriesLimitCount = 0 + perMetricSeriesLimitCount = 0 + discardedNativeHistogramCount = 0 updateFirstPartial = func(errFn func() error) { if firstPartialErr == nil { @@ -1275,8 +1275,8 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte return makeLimitError(perUserSeriesLimit, i.limiter.FormatError(userID, cause, copiedLabels)) }) - case errors.Is(cause, errMaxNativeHistogramsSeriesPerUserLimitExceeded): - perUserNativeHistogramsSeriesLimitCount++ + case errors.Is(cause, errMaxNativeHistogramSeriesPerUserLimitExceeded): + perUserNativeHistogramSeriesLimitCount++ updateFirstPartial(func() error { return makeLimitError(perUserSeriesLimit, i.limiter.FormatError(userID, cause, copiedLabels)) }) @@ -1524,8 +1524,8 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte if perUserSeriesLimitCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(perUserSeriesLimit, userID).Add(float64(perUserSeriesLimitCount)) } - if perUserNativeHistogramsSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perUserNativeHistogramsSeriesLimit, userID).Add(float64(perUserNativeHistogramsSeriesLimitCount)) + if perUserNativeHistogramSeriesLimitCount > 0 { + i.validateMetrics.DiscardedSamples.WithLabelValues(perUserNativeHistogramSeriesLimit, userID).Add(float64(perUserNativeHistogramSeriesLimitCount)) } if perMetricSeriesLimitCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(perMetricSeriesLimit, userID).Add(float64(perMetricSeriesLimitCount)) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 14b3b4a0b0b..0715639c58f 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -868,10 +868,10 @@ func TestIngesterUserLimitExceeded(t *testing.T) { } -func TestIngesterUserLimitExceededForNativeHistograms(t *testing.T) { +func TestIngesterUserLimitExceededForNativeHistogram(t *testing.T) { limits := defaultLimitsTestConfig() limits.EnableNativeHistograms = true - limits.MaxLocalNativeHistogramsSeriesPerUser = 1 + limits.MaxLocalNativeHistogramSeriesPerUser = 1 limits.MaxLocalSeriesPerUser = 1 limits.MaxLocalMetricsWithMetadataPerUser = 1 @@ -923,7 +923,7 @@ func TestIngesterUserLimitExceededForNativeHistograms(t *testing.T) { httpResp, ok := httpgrpc.HTTPResponseFromError(err) require.True(t, ok, "returned error is not an httpgrpc response") assert.Equal(t, http.StatusBadRequest, int(httpResp.Code)) - assert.Equal(t, wrapWithUser(makeLimitError(perUserNativeHistogramsSeriesLimit, ing.limiter.FormatError(userID, errMaxNativeHistogramsSeriesPerUserLimitExceeded, labels1)), userID).Error(), string(httpResp.Body)) + assert.Equal(t, wrapWithUser(makeLimitError(perUserNativeHistogramSeriesLimit, ing.limiter.FormatError(userID, errMaxNativeHistogramSeriesPerUserLimitExceeded, labels1)), userID).Error(), string(httpResp.Body)) // Append two metadata, expect no error since metadata is a best effort approach. _, err = ing.Push(ctx, cortexpb.ToWriteRequest(nil, nil, []*cortexpb.MetricMetadata{metadata1, metadata2}, nil, cortexpb.API)) diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index 7eb305e9aa3..db0cd6e48d7 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -12,11 +12,11 @@ import ( ) var ( - errMaxSeriesPerMetricLimitExceeded = errors.New("per-metric series limit exceeded") - errMaxMetadataPerMetricLimitExceeded = errors.New("per-metric metadata limit exceeded") - errMaxSeriesPerUserLimitExceeded = errors.New("per-user series limit exceeded") - errMaxNativeHistogramsSeriesPerUserLimitExceeded = errors.New("per-user nativeHistograms series limit exceeded") - errMaxMetadataPerUserLimitExceeded = errors.New("per-user metric metadata limit exceeded") + errMaxSeriesPerMetricLimitExceeded = errors.New("per-metric series limit exceeded") + errMaxMetadataPerMetricLimitExceeded = errors.New("per-metric metadata limit exceeded") + errMaxSeriesPerUserLimitExceeded = errors.New("per-user series limit exceeded") + errMaxNativeHistogramSeriesPerUserLimitExceeded = errors.New("per-user native histogram series limit exceeded") + errMaxMetadataPerUserLimitExceeded = errors.New("per-user metric metadata limit exceeded") ) type errMaxSeriesPerLabelSetLimitExceeded struct { @@ -96,14 +96,14 @@ func (l *Limiter) AssertMaxSeriesPerUser(userID string, series int) error { return errMaxSeriesPerUserLimitExceeded } -// AssertMaxNativeHistogramsSeriesPerUser limit has not been reached compared to the current -// number of nativeHistograms series in input and returns an error if so. -func (l *Limiter) AssertMaxNativeHistogramsSeriesPerUser(userID string, series int) error { - if actualLimit := l.maxNativeHistogramsSeriesPerUser(userID); series < actualLimit { +// AssertMaxNativeHistogramSeriesPerUser limit has not been reached compared to the current +// number of native histogram series in input and returns an error if so. +func (l *Limiter) AssertMaxNativeHistogramSeriesPerUser(userID string, series int) error { + if actualLimit := l.maxNativeHistogramSeriesPerUser(userID); series < actualLimit { return nil } - return errMaxNativeHistogramsSeriesPerUserLimitExceeded + return errMaxNativeHistogramSeriesPerUserLimitExceeded } // AssertMaxMetricsWithMetadataPerUser limit has not been reached compared to the current @@ -145,7 +145,7 @@ func (l *Limiter) FormatError(userID string, err error, lbls labels.Labels) erro switch { case errors.Is(err, errMaxSeriesPerUserLimitExceeded): return l.formatMaxSeriesPerUserError(userID) - case errors.Is(err, errMaxNativeHistogramsSeriesPerUserLimitExceeded): + case errors.Is(err, errMaxNativeHistogramSeriesPerUserLimitExceeded): return l.formatMaxNativeHistogramsSeriesPerUserError(userID) case errors.Is(err, errMaxSeriesPerMetricLimitExceeded): return l.formatMaxSeriesPerMetricError(userID, lbls.Get(labels.MetricName)) @@ -172,9 +172,9 @@ func (l *Limiter) formatMaxSeriesPerUserError(userID string) error { } func (l *Limiter) formatMaxNativeHistogramsSeriesPerUserError(userID string) error { - actualLimit := l.maxNativeHistogramsSeriesPerUser(userID) - localLimit := l.limits.MaxLocalNativeHistogramsSeriesPerUser(userID) - globalLimit := l.limits.MaxGlobalNativeHistogramsSeriesPerUser(userID) + actualLimit := l.maxNativeHistogramSeriesPerUser(userID) + localLimit := l.limits.MaxLocalNativeHistogramSeriesPerUser(userID) + globalLimit := l.limits.MaxGlobalNativeHistogramSeriesPerUser(userID) return fmt.Errorf("per-user nativeHistograms series limit of %d exceeded, %s (local limit: %d global limit: %d actual local limit: %d)", minNonZero(localLimit, globalLimit), l.AdminLimitMessage, localLimit, globalLimit, actualLimit) @@ -270,11 +270,11 @@ func (l *Limiter) maxSeriesPerUser(userID string) int { ) } -func (l *Limiter) maxNativeHistogramsSeriesPerUser(userID string) int { +func (l *Limiter) maxNativeHistogramSeriesPerUser(userID string) int { return l.maxByLocalAndGlobal( userID, - l.limits.MaxLocalNativeHistogramsSeriesPerUser, - l.limits.MaxGlobalNativeHistogramsSeriesPerUser, + l.limits.MaxLocalNativeHistogramSeriesPerUser, + l.limits.MaxGlobalNativeHistogramSeriesPerUser, ) } diff --git a/pkg/ingester/limiter_test.go b/pkg/ingester/limiter_test.go index 239b84ce85e..95a1cf47ec3 100644 --- a/pkg/ingester/limiter_test.go +++ b/pkg/ingester/limiter_test.go @@ -56,12 +56,12 @@ func TestLimiter_maxSeriesPerUser(t *testing.T) { func TestLimiter_maxNativeHistogramsSeriesPerUser(t *testing.T) { applyLimits := func(limits *validation.Limits, localLimit, globalLimit int) { - limits.MaxLocalNativeHistogramsSeriesPerUser = localLimit - limits.MaxGlobalNativeHistogramsSeriesPerUser = globalLimit + limits.MaxLocalNativeHistogramSeriesPerUser = localLimit + limits.MaxGlobalNativeHistogramSeriesPerUser = globalLimit } runMaxFn := func(limiter *Limiter) int { - return limiter.maxNativeHistogramsSeriesPerUser("test") + return limiter.maxNativeHistogramSeriesPerUser("test") } runLimiterMaxFunctionTest(t, applyLimits, runMaxFn, false) @@ -473,7 +473,7 @@ func TestLimiter_AssertMaxNativeHistogramsSeriesPerUser(t *testing.T) { ringIngesterCount: 10, shardByAllLabels: true, series: 300, - expected: errMaxNativeHistogramsSeriesPerUserLimitExceeded, + expected: errMaxNativeHistogramSeriesPerUserLimitExceeded, }, } @@ -488,13 +488,13 @@ func TestLimiter_AssertMaxNativeHistogramsSeriesPerUser(t *testing.T) { // Mock limits limits, err := validation.NewOverrides(validation.Limits{ - MaxLocalNativeHistogramsSeriesPerUser: testData.maxLocalNativeHistogramsSeriesPerUser, - MaxGlobalNativeHistogramsSeriesPerUser: testData.maxGlobalNativeHistogramsSeriesPerUser, + MaxLocalNativeHistogramSeriesPerUser: testData.maxLocalNativeHistogramsSeriesPerUser, + MaxGlobalNativeHistogramSeriesPerUser: testData.maxGlobalNativeHistogramsSeriesPerUser, }, nil) require.NoError(t, err) limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false, "") - actual := limiter.AssertMaxNativeHistogramsSeriesPerUser("test", testData.series) + actual := limiter.AssertMaxNativeHistogramSeriesPerUser("test", testData.series) assert.Equal(t, testData.expected, actual) }) @@ -656,11 +656,11 @@ func TestLimiter_FormatError(t *testing.T) { // Mock limits limits, err := validation.NewOverrides(validation.Limits{ - MaxGlobalSeriesPerUser: 100, - MaxGlobalNativeHistogramsSeriesPerUser: 100, - MaxGlobalSeriesPerMetric: 20, - MaxGlobalMetricsWithMetadataPerUser: 10, - MaxGlobalMetadataPerMetric: 3, + MaxGlobalSeriesPerUser: 100, + MaxGlobalNativeHistogramSeriesPerUser: 100, + MaxGlobalSeriesPerMetric: 20, + MaxGlobalMetricsWithMetadataPerUser: 10, + MaxGlobalMetadataPerMetric: 3, }, nil) require.NoError(t, err) @@ -670,7 +670,7 @@ func TestLimiter_FormatError(t *testing.T) { actual := limiter.FormatError("user-1", errMaxSeriesPerUserLimitExceeded, lbls) assert.EqualError(t, actual, "per-user series limit of 100 exceeded, please contact administrator to raise it (local limit: 0 global limit: 100 actual local limit: 100)") - actual = limiter.FormatError("user-1", errMaxNativeHistogramsSeriesPerUserLimitExceeded, lbls) + actual = limiter.FormatError("user-1", errMaxNativeHistogramSeriesPerUserLimitExceeded, lbls) assert.EqualError(t, actual, "per-user nativeHistograms series limit of 100 exceeded, please contact administrator to raise it (local limit: 0 global limit: 100 actual local limit: 100)") actual = limiter.FormatError("user-1", errMaxSeriesPerMetricLimitExceeded, lbls) diff --git a/pkg/ingester/user_state.go b/pkg/ingester/user_state.go index 5d04cee8a7d..062f4d5e1bd 100644 --- a/pkg/ingester/user_state.go +++ b/pkg/ingester/user_state.go @@ -16,10 +16,10 @@ import ( // DiscardedSamples metric labels const ( - perUserSeriesLimit = "per_user_series_limit" - perUserNativeHistogramsSeriesLimit = "per_user_native_histograms_series_limit" - perMetricSeriesLimit = "per_metric_series_limit" - perLabelsetSeriesLimit = "per_labelset_series_limit" + perUserSeriesLimit = "per_user_series_limit" + perUserNativeHistogramSeriesLimit = "per_user_native_histogram_series_limit" + perMetricSeriesLimit = "per_metric_series_limit" + perLabelsetSeriesLimit = "per_labelset_series_limit" ) const numMetricCounterShards = 128 diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index f2770ab6795..30fd70ff2bd 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -24,7 +24,7 @@ import ( ) var errMaxGlobalSeriesPerUserValidation = errors.New("the ingester.max-global-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") -var errMaxGlobalNativeHistogramsSeriesPerUserValidation = errors.New("the ingester.max-global-native-histograms-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") +var errMaxGlobalNativeHistogramSeriesPerUserValidation = errors.New("the ingester.max-global-native-histogram-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") var errDuplicateQueryPriorities = errors.New("duplicate entry of priorities found. Make sure they are all unique, including the default priority") var errCompilingQueryPriorityRegex = errors.New("error compiling query priority regex") var errDuplicatePerLabelSetLimit = errors.New("duplicate per labelSet limits found. Make sure they are all unique") @@ -153,14 +153,14 @@ type Limits struct { // Ingester enforced limits. // Series - MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"` - MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"` - MaxLocalNativeHistogramsSeriesPerUser int `yaml:"max_native_histograms_series_per_user" json:"max_native_histograms_series_per_user"` - MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` - MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` - MaxGlobalNativeHistogramsSeriesPerUser int `yaml:"max_global_native_histograms_series_per_user" json:"max_global_native_histograms_series_per_user"` - LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` - EnableNativeHistograms bool `yaml:"enable_native_histograms" json:"enable_native_histograms"` + MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"` + MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"` + MaxLocalNativeHistogramSeriesPerUser int `yaml:"max_native_histogram_series_per_user" json:"max_native_histogram_series_per_user"` + MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` + MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` + MaxGlobalNativeHistogramSeriesPerUser int `yaml:"max_global_native_histogram_series_per_user" json:"max_global_native_histogram_series_per_user"` + LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` + EnableNativeHistograms bool `yaml:"enable_native_histograms" json:"enable_native_histograms"` // Metadata MaxLocalMetricsWithMetadataPerUser int `yaml:"max_metadata_per_user" json:"max_metadata_per_user"` @@ -276,8 +276,8 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLocalSeriesPerMetric, "ingester.max-series-per-metric", 50000, "The maximum number of active series per metric name, per ingester. 0 to disable.") f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") - f.IntVar(&l.MaxLocalNativeHistogramsSeriesPerUser, "ingester.max-native-histograms-series-per-user", 5000000, "The maximum number of active nativeHistograms series per user, per ingester. 0 to disable.") - f.IntVar(&l.MaxGlobalNativeHistogramsSeriesPerUser, "ingester.max-global-native-histograms-series-per-user", 0, "The maximum number of active nativeHistograms series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") + f.IntVar(&l.MaxLocalNativeHistogramSeriesPerUser, "ingester.max-native-histogram-series-per-user", 5000000, "The maximum number of active native histogram series per user, per ingester. 0 to disable.") + f.IntVar(&l.MaxGlobalNativeHistogramSeriesPerUser, "ingester.max-global-native-histogram-series-per-user", 0, "The maximum number of active native histogram series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.BoolVar(&l.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") @@ -354,8 +354,8 @@ func (l *Limits) Validate(shardByAllLabels bool) error { // The ingester.max-global-native-histograms-series-per-user metric is not supported // if shard-by-all-labels is disabled - if l.MaxGlobalNativeHistogramsSeriesPerUser > 0 && !shardByAllLabels { - return errMaxGlobalNativeHistogramsSeriesPerUserValidation + if l.MaxGlobalNativeHistogramSeriesPerUser > 0 && !shardByAllLabels { + return errMaxGlobalNativeHistogramSeriesPerUserValidation } if err := l.RulerExternalLabels.Validate(func(l labels.Label) error { @@ -690,9 +690,9 @@ func (o *Overrides) MaxLocalSeriesPerUser(userID string) int { return o.GetOverridesForUser(userID).MaxLocalSeriesPerUser } -// MaxLocalNativeHistogramsSeriesPerUser returns the maximum number of nativeHistograms series a user is allowed to store in a single ingester. -func (o *Overrides) MaxLocalNativeHistogramsSeriesPerUser(userID string) int { - return o.GetOverridesForUser(userID).MaxLocalNativeHistogramsSeriesPerUser +// MaxLocalNativeHistogramSeriesPerUser returns the maximum number of nativeHistograms series a user is allowed to store in a single ingester. +func (o *Overrides) MaxLocalNativeHistogramSeriesPerUser(userID string) int { + return o.GetOverridesForUser(userID).MaxLocalNativeHistogramSeriesPerUser } // MaxLocalSeriesPerMetric returns the maximum number of series allowed per metric in a single ingester. @@ -705,9 +705,9 @@ func (o *Overrides) MaxGlobalSeriesPerUser(userID string) int { return o.GetOverridesForUser(userID).MaxGlobalSeriesPerUser } -// MaxGlobalNativeHistogramsSeriesPerUser returns the maximum number of nativeHistograms series a user is allowed to store across the cluster. -func (o *Overrides) MaxGlobalNativeHistogramsSeriesPerUser(userID string) int { - return o.GetOverridesForUser(userID).MaxGlobalNativeHistogramsSeriesPerUser +// MaxGlobalNativeHistogramSeriesPerUser returns the maximum number of nativeHistograms series a user is allowed to store across the cluster. +func (o *Overrides) MaxGlobalNativeHistogramSeriesPerUser(userID string) int { + return o.GetOverridesForUser(userID).MaxGlobalNativeHistogramSeriesPerUser } // EnableNativeHistograms returns whether the Ingester should accept NativeHistograms samples from this user. From ab7970d2b5c67b19172c59cc85f00b166a818cbc Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Tue, 24 Jun 2025 12:15:53 -0700 Subject: [PATCH 6/9] Updated ChangeLog Signed-off-by: Paurush Garg --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 444ad8108bf..0f5598d27aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,9 @@ * [ENHANCEMENT] Distributor: Add min/max schema validation for NativeHistograms. #6766 * [ENHANCEMENT] Ingester: Handle runtime errors in query path #6769 * [ENHANCEMENT] Compactor: Support metadata caching bucket for Cleaner. Can be enabled via `-compactor.cleaner-caching-bucket-enabled` flag. #6778 -* [ENHANCEMENT] Distributor: Add ingestion rate limit for Native Histograms. #6794 * [ENHANCEMENT] Compactor, Store Gateway: Introduce user scanner strategy and user index. #6780 +* [ENHANCEMENT] Distributor: Add ingestion rate limit for Native Histograms. #6794 +* [ENHANCEMENT] Ingester: Add activeSeries limit specifically for Native Histogram. #6796 * [ENHANCEMENT] Querier: Support chunks cache for parquet queryable. #6805 * [ENHANCEMENT] Parquet Storage: Add some metrics for parquet blocks and converter. #6809 #6821 * [ENHANCEMENT] Compactor: Optimize cleaner run time. #6815 @@ -52,7 +53,6 @@ * [ENHANCEMENT] Metadata Cache: Support inmemory and multi level cache backend. #6829 * [ENHANCEMENT] Store Gateway: Allow to ignore syncing blocks older than certain time using `ignore_blocks_before`. #6830 * [ENHANCEMENT] Distributor: Add native histograms max sample size bytes limit validation. #6834 -* [ENHANCEMENT] Ingester: Add activeSeries limit specifically for Native Histogram. #6796 * [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517 * [BUGFIX] Ingester: Fix labelset data race condition. #6573 * [BUGFIX] Compactor: Cleaner should not put deletion marker for blocks with no-compact marker. #6576 From 1085c9147abf551465a8e8fa01bd918f82b814f9 Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Wed, 25 Jun 2025 11:23:54 -0700 Subject: [PATCH 7/9] Resolving comments Signed-off-by: Paurush Garg --- docs/configuration/config-file-reference.md | 3 +- pkg/cortex/cortex.go | 2 +- pkg/cortex/runtime_config.go | 2 +- pkg/ingester/ingester.go | 8 ++--- pkg/ingester/ingester_test.go | 4 ++- pkg/util/validation/limits.go | 9 ++--- pkg/util/validation/limits_test.go | 39 ++++++++++++++++++--- 7 files changed, 51 insertions(+), 16 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index aaf7e17a71f..23582c47caf 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3624,7 +3624,8 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # The maximum number of active native histogram series per user, across the # cluster before replication. 0 to disable. Supported only if -# -distributor.shard-by-all-labels is true. +# -distributor.shard-by-all-labels and ingester.active-series-metrics-enabled is +# true. # CLI flag: -ingester.max-global-native-histogram-series-per-user [max_global_native_histogram_series_per_user: | default = 0] diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 9dec8fa61c8..b141adc8127 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -204,7 +204,7 @@ func (c *Config) Validate(log log.Logger) error { if err := c.BlocksStorage.Validate(); err != nil { return errors.Wrap(err, "invalid TSDB config") } - if err := c.LimitsConfig.Validate(c.Distributor.ShardByAllLabels); err != nil { + if err := c.LimitsConfig.Validate(c.Distributor.ShardByAllLabels, c.Ingester.ActiveSeriesMetricsEnabled); err != nil { return errors.Wrap(err, "invalid limits config") } if err := c.ResourceMonitor.Validate(); err != nil { diff --git a/pkg/cortex/runtime_config.go b/pkg/cortex/runtime_config.go index 554fa708056..bcf59cdfd3a 100644 --- a/pkg/cortex/runtime_config.go +++ b/pkg/cortex/runtime_config.go @@ -79,7 +79,7 @@ func (l runtimeConfigLoader) load(r io.Reader) (interface{}, error) { } for _, ul := range overrides.TenantLimits { - if err := ul.Validate(l.cfg.Distributor.ShardByAllLabels); err != nil { + if err := ul.Validate(l.cfg.Distributor.ShardByAllLabels, l.cfg.Ingester.ActiveSeriesMetricsEnabled); err != nil { return nil, err } } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 90e69bd3c38..72ffed9f388 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -448,13 +448,13 @@ func (u *userTSDB) PreCreation(metric labels.Labels) error { } } - // Total nativeHistograms series limit. - if err := u.limiter.AssertMaxNativeHistogramSeriesPerUser(u.userID, u.activeSeries.ActiveNativeHistogram()); err != nil { + // Total series limit. + if err := u.limiter.AssertMaxSeriesPerUser(u.userID, int(u.Head().NumSeries())); err != nil { return err } - // Total series limit. - if err := u.limiter.AssertMaxSeriesPerUser(u.userID, int(u.Head().NumSeries())); err != nil { + // Total nativeHistograms series limit. + if err := u.limiter.AssertMaxNativeHistogramSeriesPerUser(u.userID, u.activeSeries.ActiveNativeHistogram()); err != nil { return err } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 0715639c58f..2a16da839e8 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -895,7 +895,9 @@ func TestIngesterUserLimitExceededForNativeHistogram(t *testing.T) { require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) blocksIngesterGenerator := func(reg prometheus.Registerer) *Ingester { - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg) + cfg := defaultIngesterTestConfig(t) + cfg.ActiveSeriesMetricsEnabled = false + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) // Wait until it's ACTIVE diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 30fd70ff2bd..02d2b14765c 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -24,7 +24,7 @@ import ( ) var errMaxGlobalSeriesPerUserValidation = errors.New("the ingester.max-global-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") -var errMaxGlobalNativeHistogramSeriesPerUserValidation = errors.New("the ingester.max-global-native-histogram-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") +var errMaxGlobalNativeHistogramSeriesPerUserValidation = errors.New("the ingester.max-global-native-histogram-series-per-user limit is unsupported if distributor.shard-by-all-labels or ingester.active-series-metrics-enabled is disabled") var errDuplicateQueryPriorities = errors.New("duplicate entry of priorities found. Make sure they are all unique, including the default priority") var errCompilingQueryPriorityRegex = errors.New("error compiling query priority regex") var errDuplicatePerLabelSetLimit = errors.New("duplicate per labelSet limits found. Make sure they are all unique") @@ -277,7 +277,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") f.IntVar(&l.MaxLocalNativeHistogramSeriesPerUser, "ingester.max-native-histogram-series-per-user", 5000000, "The maximum number of active native histogram series per user, per ingester. 0 to disable.") - f.IntVar(&l.MaxGlobalNativeHistogramSeriesPerUser, "ingester.max-global-native-histogram-series-per-user", 0, "The maximum number of active native histogram series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") + f.IntVar(&l.MaxGlobalNativeHistogramSeriesPerUser, "ingester.max-global-native-histogram-series-per-user", 0, "The maximum number of active native histogram series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels and ingester.active-series-metrics-enabled is true.") f.BoolVar(&l.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") @@ -345,7 +345,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { // Validate the limits config and returns an error if the validation // doesn't pass -func (l *Limits) Validate(shardByAllLabels bool) error { +func (l *Limits) Validate(shardByAllLabels bool, activeSeriesMetricsEnabled bool) error { // The ingester.max-global-series-per-user metric is not supported // if shard-by-all-labels is disabled if l.MaxGlobalSeriesPerUser > 0 && !shardByAllLabels { @@ -354,7 +354,8 @@ func (l *Limits) Validate(shardByAllLabels bool) error { // The ingester.max-global-native-histograms-series-per-user metric is not supported // if shard-by-all-labels is disabled - if l.MaxGlobalNativeHistogramSeriesPerUser > 0 && !shardByAllLabels { + // or if active-series-metrics-enabled is disabled + if l.MaxGlobalNativeHistogramSeriesPerUser > 0 && (!shardByAllLabels || !activeSeriesMetricsEnabled) { return errMaxGlobalNativeHistogramSeriesPerUserValidation } diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index df7760e3829..9fd603d0610 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -45,9 +45,10 @@ func TestLimits_Validate(t *testing.T) { t.Parallel() tests := map[string]struct { - limits Limits - shardByAllLabels bool - expected error + limits Limits + shardByAllLabels bool + activeSeriesMetricsEnabled bool + expected error }{ "max-global-series-per-user disabled and shard-by-all-labels=false": { limits: Limits{MaxGlobalSeriesPerUser: 0}, @@ -64,6 +65,36 @@ func TestLimits_Validate(t *testing.T) { shardByAllLabels: true, expected: nil, }, + "max-global-native-histogram-series-per-user disabled and shard-by-all-labels=false and active-series-metrics-enabled=false": { + limits: Limits{MaxGlobalSeriesPerUser: 0}, + shardByAllLabels: false, + activeSeriesMetricsEnabled: false, + expected: nil, + }, + "max-global-native-histogram-series-per-user disabled and shard-by-all-labels=true and active-series-metrics-enabled=true": { + limits: Limits{MaxGlobalNativeHistogramSeriesPerUser: 0}, + shardByAllLabels: true, + activeSeriesMetricsEnabled: true, + expected: nil, + }, + "max-global-native-histogram-series-per-user enabled and shard-by-all-labels=true and active-series-metrics-enabled=true": { + limits: Limits{MaxGlobalNativeHistogramSeriesPerUser: 1000}, + shardByAllLabels: true, + activeSeriesMetricsEnabled: true, + expected: nil, + }, + "max-global-native-histogram-series-per-user enabled and shard-by-all-labels=false and active-series-metrics-enabled=true": { + limits: Limits{MaxGlobalNativeHistogramSeriesPerUser: 1000}, + shardByAllLabels: false, + activeSeriesMetricsEnabled: true, + expected: errMaxGlobalNativeHistogramSeriesPerUserValidation, + }, + "max-global-native-histogram-series-per-user enabled and shard-by-all-labels=true and active-series-metrics-enabled=false": { + limits: Limits{MaxGlobalNativeHistogramSeriesPerUser: 1000}, + shardByAllLabels: true, + activeSeriesMetricsEnabled: false, + expected: errMaxGlobalNativeHistogramSeriesPerUserValidation, + }, "external-labels invalid label name": { limits: Limits{RulerExternalLabels: labels.Labels{{Name: "123invalid", Value: "good"}}}, expected: errInvalidLabelName, @@ -78,7 +109,7 @@ func TestLimits_Validate(t *testing.T) { testData := testData t.Run(testName, func(t *testing.T) { - assert.ErrorIs(t, testData.limits.Validate(testData.shardByAllLabels), testData.expected) + assert.ErrorIs(t, testData.limits.Validate(testData.shardByAllLabels, testData.activeSeriesMetricsEnabled), testData.expected) }) } } From 555c6c1100a33e095d3ed73e8cf33a0656f14dfa Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Wed, 25 Jun 2025 12:26:59 -0700 Subject: [PATCH 8/9] Resolving comments Signed-off-by: Paurush Garg --- docs/configuration/config-file-reference.md | 4 ++-- pkg/util/validation/limits.go | 7 ++++++- pkg/util/validation/limits_test.go | 20 ++++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 23582c47caf..2963a87348c 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3607,9 +3607,9 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s [max_series_per_metric: | default = 50000] # The maximum number of active native histogram series per user, per ingester. 0 -# to disable. +# to disable. Supported only if ingester.active-series-metrics-enabled is true. # CLI flag: -ingester.max-native-histogram-series-per-user -[max_native_histogram_series_per_user: | default = 5000000] +[max_native_histogram_series_per_user: | default = 0] # The maximum number of active series per user, across the cluster before # replication. 0 to disable. Supported only if -distributor.shard-by-all-labels diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 02d2b14765c..ec791acf7ee 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -25,6 +25,7 @@ import ( var errMaxGlobalSeriesPerUserValidation = errors.New("the ingester.max-global-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") var errMaxGlobalNativeHistogramSeriesPerUserValidation = errors.New("the ingester.max-global-native-histogram-series-per-user limit is unsupported if distributor.shard-by-all-labels or ingester.active-series-metrics-enabled is disabled") +var errMaxLocalNativeHistogramSeriesPerUserValidation = errors.New("the ingester.max-local-native-histogram-series-per-user limit is unsupported if ingester.active-series-metrics-enabled is disabled") var errDuplicateQueryPriorities = errors.New("duplicate entry of priorities found. Make sure they are all unique, including the default priority") var errCompilingQueryPriorityRegex = errors.New("error compiling query priority regex") var errDuplicatePerLabelSetLimit = errors.New("duplicate per labelSet limits found. Make sure they are all unique") @@ -276,7 +277,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLocalSeriesPerMetric, "ingester.max-series-per-metric", 50000, "The maximum number of active series per metric name, per ingester. 0 to disable.") f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") - f.IntVar(&l.MaxLocalNativeHistogramSeriesPerUser, "ingester.max-native-histogram-series-per-user", 5000000, "The maximum number of active native histogram series per user, per ingester. 0 to disable.") + f.IntVar(&l.MaxLocalNativeHistogramSeriesPerUser, "ingester.max-native-histogram-series-per-user", 0, "The maximum number of active native histogram series per user, per ingester. 0 to disable. Supported only if ingester.active-series-metrics-enabled is true.") f.IntVar(&l.MaxGlobalNativeHistogramSeriesPerUser, "ingester.max-global-native-histogram-series-per-user", 0, "The maximum number of active native histogram series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels and ingester.active-series-metrics-enabled is true.") f.BoolVar(&l.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") @@ -359,6 +360,10 @@ func (l *Limits) Validate(shardByAllLabels bool, activeSeriesMetricsEnabled bool return errMaxGlobalNativeHistogramSeriesPerUserValidation } + if l.MaxLocalNativeHistogramSeriesPerUser > 0 && !activeSeriesMetricsEnabled { + return errMaxLocalNativeHistogramSeriesPerUserValidation + } + if err := l.RulerExternalLabels.Validate(func(l labels.Label) error { if !model.LabelName(l.Name).IsValid() { return fmt.Errorf("%w: %q", errInvalidLabelName, l.Name) diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 9fd603d0610..4b08d6aa19f 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -95,6 +95,26 @@ func TestLimits_Validate(t *testing.T) { activeSeriesMetricsEnabled: false, expected: errMaxGlobalNativeHistogramSeriesPerUserValidation, }, + "max-local-native-histogram-series-per-user disabled and shard-by-all-labels=true and active-series-metrics-enabled=false": { + limits: Limits{MaxLocalNativeHistogramSeriesPerUser: 0}, + activeSeriesMetricsEnabled: false, + expected: nil, + }, + "max-local-native-histogram-series-per-user disabled and shard-by-all-labels=true and active-series-metrics-enabled=true": { + limits: Limits{MaxLocalNativeHistogramSeriesPerUser: 0}, + activeSeriesMetricsEnabled: true, + expected: nil, + }, + "max-local-native-histogram-series-per-user enabled and shard-by-all-labels=true and active-series-metrics-enabled=true": { + limits: Limits{MaxLocalNativeHistogramSeriesPerUser: 1000}, + activeSeriesMetricsEnabled: true, + expected: nil, + }, + "max-local-native-histogram-series-per-user enabled and shard-by-all-labels=true and active-series-metrics-enabled=false": { + limits: Limits{MaxLocalNativeHistogramSeriesPerUser: 1000}, + activeSeriesMetricsEnabled: false, + expected: errMaxLocalNativeHistogramSeriesPerUserValidation, + }, "external-labels invalid label name": { limits: Limits{RulerExternalLabels: labels.Labels{{Name: "123invalid", Value: "good"}}}, expected: errInvalidLabelName, From 6a5e2376084fc8b536c0762f402dc5fd0f7e5801 Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Wed, 25 Jun 2025 21:55:13 -0700 Subject: [PATCH 9/9] Fixing Tests Signed-off-by: Paurush Garg --- pkg/ingester/ingester_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 2a16da839e8..1e898a1ff55 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -754,6 +754,7 @@ func TestIngesterUserLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() limits.EnableNativeHistograms = true limits.MaxLocalSeriesPerUser = 1 + limits.MaxLocalNativeHistogramSeriesPerUser = 1 limits.MaxLocalMetricsWithMetadataPerUser = 1 userID := "1" @@ -872,7 +873,7 @@ func TestIngesterUserLimitExceededForNativeHistogram(t *testing.T) { limits := defaultLimitsTestConfig() limits.EnableNativeHistograms = true limits.MaxLocalNativeHistogramSeriesPerUser = 1 - limits.MaxLocalSeriesPerUser = 1 + limits.MaxLocalSeriesPerUser = 2 limits.MaxLocalMetricsWithMetadataPerUser = 1 userID := "1" @@ -895,9 +896,7 @@ func TestIngesterUserLimitExceededForNativeHistogram(t *testing.T) { require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) blocksIngesterGenerator := func(reg prometheus.Registerer) *Ingester { - cfg := defaultIngesterTestConfig(t) - cfg.ActiveSeriesMetricsEnabled = false - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, reg) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) // Wait until it's ACTIVE @@ -975,6 +974,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() limits.EnableNativeHistograms = true limits.MaxLocalSeriesPerMetric = 1 + limits.MaxLocalNativeHistogramSeriesPerUser = 1 limits.MaxLocalMetadataPerMetric = 1 userID := "1"