Simplify how experimental PromQL functions are enabled

CHANGELOG.md

@@ -15,6 +15,7 @@
 * [CHANGE] Ingester: Set `-ingester.ooo-native-histograms-ingestion-enabled` to true by default. #10483
 * [CHANGE] Ruler: Add `user` and `reason` labels to `cortex_ruler_write_requests_failed_total` and `cortex_ruler_queries_failed_total`; add `user` to
     `cortex_ruler_write_requests_total` and `cortex_ruler_queries_total` metrics. #10536
+* [CHANGE] Querier / Query-frontend: Remove experimental `-querier.promql-experimental-functions-enabled` and `-query-frontend.block-promql-experimental-functions` CLI flags and respective YAML configuration options to enable experimental PromQL functions. Instead access to experimental PromQL functions is always blocked. You can enable them using the per-tenant setting `enabled_promql_experimental_functions`. #10660
 * [FEATURE] Distributor: Add experimental Influx handler. #10153
 * [ENHANCEMENT] Compactor: Expose `cortex_bucket_index_last_successful_update_timestamp_seconds` for all tenants assigned to the compactor before starting the block cleanup job. #10569
 * [ENHANCEMENT] Query Frontend: Return server-side `samples_processed` statistics. #10103

cmd/mimir/config-descriptor.json

@@ -1960,17 +1960,6 @@
           "fieldType": "duration",
           "fieldCategory": "advanced"
         },
-        {
-          "kind": "field",
-          "name": "promql_experimental_functions_enabled",
-          "required": false,
-          "desc": "Enable experimental PromQL functions. This config option should be set on query-frontend too when query sharding is enabled.",
-          "fieldValue": null,
-          "fieldDefaultValue": false,
-          "fieldFlag": "querier.promql-experimental-functions-enabled",
-          "fieldType": "boolean",
-          "fieldCategory": "experimental"
-        },
         {
           "kind": "block",
           "name": "mimir_query_engine",
@@ -6911,17 +6900,6 @@
           "fieldType": "boolean",
           "fieldCategory": "experimental"
         },
-        {
-          "kind": "field",
-          "name": "block_promql_experimental_functions",
-          "required": false,
-          "desc": "True to control access to specific PromQL experimental functions per tenant.",
-          "fieldValue": null,
-          "fieldDefaultValue": false,
-          "fieldFlag": "query-frontend.block-promql-experimental-functions",
-          "fieldType": "boolean",
-          "fieldCategory": "experimental"
-        },
         {
           "kind": "field",
           "name": "query_sharding_target_series_per_shard",

cmd/mimir/help-all.txt.tmpl

@@ -2191,8 +2191,6 @@ Usage of ./cmd/mimir/mimir:
     	If true, when querying ingesters, only the minimum required ingesters required to reach quorum will be queried initially, with other ingesters queried only if needed due to failures from the initial set of ingesters. Enabling this option reduces resource consumption for the happy path at the cost of increased latency for the unhappy path. (default true)
   -querier.minimize-ingester-requests-hedging-delay duration
     	Delay before initiating requests to further ingesters when request minimization is enabled and the initially selected set of ingesters have not all responded. Ignored if -querier.minimize-ingester-requests is not enabled. (default 3s)
-  -querier.promql-experimental-functions-enabled
-    	[experimental] Enable experimental PromQL functions. This config option should be set on query-frontend too when query sharding is enabled.
   -querier.query-engine string
     	[experimental] Query engine to use, either 'prometheus' or 'mimir' (default "prometheus")
   -querier.query-ingesters-within duration
@@ -2275,8 +2273,6 @@ Usage of ./cmd/mimir/mimir:
     	[experimental] Timeout for writing active series responses. 0 means the value from `-server.http-write-timeout` is used. (default 5m0s)
   -query-frontend.align-queries-with-step
     	Mutate incoming queries to align their start and end with their step to improve result caching.
-  -query-frontend.block-promql-experimental-functions
-    	[experimental] True to control access to specific PromQL experimental functions per tenant.
   -query-frontend.cache-errors
     	[experimental] Cache non-transient errors from queries.
   -query-frontend.cache-results

docs/sources/mimir/configure/about-versioning.md

@@ -201,7 +201,6 @@ The following features are currently experimental:
   - Limiting queries based on the estimated number of chunks that will be used (`-querier.max-estimated-fetched-chunks-per-query-multiplier`)
   - Max concurrency for tenant federated queries (`-tenant-federation.max-concurrent`)
   - Maximum response size for active series queries (`-querier.active-series-results-max-size-bytes`)
-  - Enable PromQL experimental functions (`-querier.promql-experimental-functions-enabled`)
   - Allow streaming of `/active_series` responses to the frontend (`-querier.response-streaming-enabled`)
   - [Mimir query engine](https://grafana.com/docs/mimir/<MIMIR_VERSION>/references/architecture/mimir-query-engine) (`-querier.query-engine=mimir` and `-querier.enable-query-engine-fallback`, and all flags beginning with `-querier.mimir-query-engine`)
   - Maximum estimated memory consumption per query limit (`-querier.max-estimated-memory-consumption-per-query`)
@@ -216,6 +215,7 @@ The following features are currently experimental:
   - Caching of non-transient error responses (`-query-frontend.cache-errors`, `-query-frontend.results-cache-ttl-for-errors`)
   - Blocking HTTP requests on a per-tenant basis (configured with the `blocked_requests` limit)
   - Spinning off (as actual range queries) subqueries from instant queries (`-query-frontend.spin-off-instant-subqueries-to-url` and the `instant_queries_with_subquery_spin_off` per-tenant limit)
+  - Enable PromQL experimental functions per-tenant (`-query-frontend.enabled-promql-experimental-functions` and the `enabled_promql_experimental_functions` per-tenant limit)
 - Query-scheduler
   - `-query-scheduler.querier-forget-delay`
 - Store-gateway

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -1644,11 +1644,6 @@ store_gateway_client:
 # CLI flag: -querier.lookback-delta
 [lookback_delta: <duration> | default = 5m]
 
-# (experimental) Enable experimental PromQL functions. This config option should
-# be set on query-frontend too when query sharding is enabled.
-# CLI flag: -querier.promql-experimental-functions-enabled
-[promql_experimental_functions_enabled: <boolean> | default = false]
-
 mimir_query_engine:
   # (experimental) Enable support for aggregation operations in the Mimir query
   # engine. Only applies if the MQE is in use.
@@ -1843,11 +1838,6 @@ results_cache:
 # CLI flag: -query-frontend.prune-queries
 [prune_queries: <boolean> | default = false]
 
-# (experimental) True to control access to specific PromQL experimental
-# functions per tenant.
-# CLI flag: -query-frontend.block-promql-experimental-functions
-[block_promql_experimental_functions: <boolean> | default = false]
-
 # (advanced) How many series a single sharded partial query should load at most.
 # This is not a strict requirement guaranteed to be honoured by query sharding,
 # but a hint given to the query sharding when the query execution is initially

pkg/frontend/querymiddleware/roundtrip.go

@@ -57,19 +57,18 @@ var (
 
 // Config for query_range middleware chain.
 type Config struct {
-	SplitQueriesByInterval           time.Duration `yaml:"split_queries_by_interval" category:"advanced"`
-	ResultsCacheConfig               `yaml:"results_cache"`
-	CacheResults                     bool          `yaml:"cache_results"`
-	CacheErrors                      bool          `yaml:"cache_errors" category:"experimental"`
-	MaxRetries                       int           `yaml:"max_retries" category:"advanced"`
-	NotRunningTimeout                time.Duration `yaml:"not_running_timeout" category:"advanced"`
-	ShardedQueries                   bool          `yaml:"parallelize_shardable_queries"`
-	PrunedQueries                    bool          `yaml:"prune_queries" category:"experimental"`
-	BlockPromQLExperimentalFunctions bool          `yaml:"block_promql_experimental_functions" category:"experimental"`
-	TargetSeriesPerShard             uint64        `yaml:"query_sharding_target_series_per_shard" category:"advanced"`
-	ShardActiveSeriesQueries         bool          `yaml:"shard_active_series_queries" category:"experimental"`
-	UseActiveSeriesDecoder           bool          `yaml:"use_active_series_decoder" category:"experimental"`
-	SpinOffInstantSubqueriesToURL    string        `yaml:"spin_off_instant_subqueries_to_url" category:"experimental"`
+	SplitQueriesByInterval        time.Duration `yaml:"split_queries_by_interval" category:"advanced"`
+	ResultsCacheConfig            `yaml:"results_cache"`
+	CacheResults                  bool          `yaml:"cache_results"`
+	CacheErrors                   bool          `yaml:"cache_errors" category:"experimental"`
+	MaxRetries                    int           `yaml:"max_retries" category:"advanced"`
+	NotRunningTimeout             time.Duration `yaml:"not_running_timeout" category:"advanced"`
+	ShardedQueries                bool          `yaml:"parallelize_shardable_queries"`
+	PrunedQueries                 bool          `yaml:"prune_queries" category:"experimental"`
+	TargetSeriesPerShard          uint64        `yaml:"query_sharding_target_series_per_shard" category:"advanced"`
+	ShardActiveSeriesQueries      bool          `yaml:"shard_active_series_queries" category:"experimental"`
+	UseActiveSeriesDecoder        bool          `yaml:"use_active_series_decoder" category:"experimental"`
+	SpinOffInstantSubqueriesToURL string        `yaml:"spin_off_instant_subqueries_to_url" category:"experimental"`
 
 	// CacheKeyGenerator allows to inject a CacheKeyGenerator to use for generating cache keys.
 	// If nil, the querymiddleware package uses a DefaultCacheKeyGenerator with SplitQueriesByInterval.
@@ -97,7 +96,6 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.CacheErrors, "query-frontend.cache-errors", false, "Cache non-transient errors from queries.")
 	f.BoolVar(&cfg.ShardedQueries, "query-frontend.parallelize-shardable-queries", false, "True to enable query sharding.")
 	f.BoolVar(&cfg.PrunedQueries, "query-frontend.prune-queries", false, "True to enable pruning dead code (eg. expressions that cannot produce any results) and simplifying expressions (eg. expressions that can be evaluated immediately) in queries.")
-	f.BoolVar(&cfg.BlockPromQLExperimentalFunctions, "query-frontend.block-promql-experimental-functions", false, "True to control access to specific PromQL experimental functions per tenant.")
 	f.Uint64Var(&cfg.TargetSeriesPerShard, "query-frontend.query-sharding-target-series-per-shard", 0, "How many series a single sharded partial query should load at most. This is not a strict requirement guaranteed to be honoured by query sharding, but a hint given to the query sharding when the query execution is initially planned. 0 to disable cardinality-based hints.")
 	f.StringVar(&cfg.QueryResultResponseFormat, "query-frontend.query-result-response-format", formatProtobuf, fmt.Sprintf("Format to use when retrieving query results from queriers. Supported values: %s", strings.Join(allFormats, ", ")))
 	f.BoolVar(&cfg.ShardActiveSeriesQueries, "query-frontend.shard-active-series-queries", false, "True to enable sharding of active series queries.")
@@ -211,11 +209,10 @@ func NewTripperware(
 	codec Codec,
 	cacheExtractor Extractor,
 	engineOpts promql.EngineOpts,
-	engineExperimentalFunctionsEnabled bool,
 	ingestStorageTopicOffsetsReaders map[string]*ingest.TopicOffsetsReader,
 	registerer prometheus.Registerer,
 ) (Tripperware, error) {
-	queryRangeTripperware, err := newQueryTripperware(cfg, log, limits, codec, cacheExtractor, engineOpts, engineExperimentalFunctionsEnabled, ingestStorageTopicOffsetsReaders, registerer)
+	queryRangeTripperware, err := newQueryTripperware(cfg, log, limits, codec, cacheExtractor, engineOpts, ingestStorageTopicOffsetsReaders, registerer)
 	if err != nil {
 		return nil, err
 	}
@@ -232,16 +229,16 @@ func newQueryTripperware(
 	codec Codec,
 	cacheExtractor Extractor,
 	engineOpts promql.EngineOpts,
-	engineExperimentalFunctionsEnabled bool,
 	ingestStorageTopicOffsetsReaders map[string]*ingest.TopicOffsetsReader,
 	registerer prometheus.Registerer,
 ) (Tripperware, error) {
 	// Disable concurrency limits for sharded queries.
 	engineOpts.ActiveQueryTracker = nil
 	engine := promql.NewEngine(engineOpts)
 
-	// Experimental functions can only be enabled globally, and not on a per-engine basis.
-	parser.EnableExperimentalFunctions = engineExperimentalFunctionsEnabled
+	// Experimental functions are always enabled globally for all engines. Access to them
+	// is controlled by an experimental functions middleware that reads per-tenant settings.
+	parser.EnableExperimentalFunctions = true
 
 	var c cache.Cache
 	if cfg.CacheResults || cfg.cardinalityBasedShardingEnabled() {
@@ -501,23 +498,19 @@ func newQueryMiddlewares(
 		queryInstantMiddleware = append(queryInstantMiddleware, newInstrumentMiddleware("retry", metrics), newRetryMiddleware(log, cfg.MaxRetries, retryMiddlewareMetrics))
 	}
 
-	if parser.EnableExperimentalFunctions && cfg.BlockPromQLExperimentalFunctions {
-		// We only need to check for tenant-specific settings if experimental functions are enabled globally
-		// and if we want to control access to them per tenant.
-		// Does not apply to remote read as those are executed remotely and the enabling of PromQL experimental
-		// functions for those are not controlled here.
-		experimentalFunctionsMiddleware := newExperimentalFunctionsMiddleware(limits, log)
-		queryRangeMiddleware = append(
-			queryRangeMiddleware,
-			newInstrumentMiddleware("experimental_functions", metrics),
-			experimentalFunctionsMiddleware,
-		)
-		queryInstantMiddleware = append(
-			queryInstantMiddleware,
-			newInstrumentMiddleware("experimental_functions", metrics),
-			experimentalFunctionsMiddleware,
-		)
-	}
+	// Does not apply to remote read as those are executed remotely and the enabling of PromQL experimental
+	// functions for those are not controlled here.
+	experimentalFunctionsMiddleware := newExperimentalFunctionsMiddleware(limits, log)
+	queryRangeMiddleware = append(
+		queryRangeMiddleware,
+		newInstrumentMiddleware("experimental_functions", metrics),
+		experimentalFunctionsMiddleware,
+	)
+	queryInstantMiddleware = append(
+		queryInstantMiddleware,
+		newInstrumentMiddleware("experimental_functions", metrics),
+		experimentalFunctionsMiddleware,
+	)
 
 	return
 }

pkg/frontend/querymiddleware/roundtrip_test.go

@@ -89,7 +89,6 @@ func TestTripperware_RangeQuery(t *testing.T) {
 			MaxSamples: 1000,
 			Timeout:    time.Minute,
 		},
-		true,
 		nil,
 		nil,
 	)
@@ -142,7 +141,6 @@ func TestTripperware_InstantQuery(t *testing.T) {
 			MaxSamples: 1000,
 			Timeout:    time.Minute,
 		},
-		true,
 		nil,
 		nil,
 	)
@@ -474,7 +472,6 @@ func TestTripperware_Metrics(t *testing.T) {
 					MaxSamples: 1000,
 					Timeout:    time.Minute,
 				},
-				true,
 				nil,
 				reg,
 			)
@@ -540,7 +537,6 @@ func TestTripperware_BlockedRequests(t *testing.T) {
 			MaxSamples: 1000,
 			Timeout:    time.Minute,
 		},
-		true,
 		nil,
 		nil,
 	)
@@ -593,14 +589,12 @@ func TestMiddlewaresConsistency(t *testing.T) {
 	cfg.CacheResults = true
 	cfg.ShardedQueries = true
 	cfg.PrunedQueries = true
-	cfg.BlockPromQLExperimentalFunctions = true
 	cfg.SpinOffInstantSubqueriesToURL = "http://localhost"
 
 	// Ensure all features are enabled, so that we assert on all middlewares.
 	require.NotZero(t, cfg.CacheResults)
 	require.NotZero(t, cfg.ShardedQueries)
 	require.NotZero(t, cfg.PrunedQueries)
-	require.NotZero(t, cfg.BlockPromQLExperimentalFunctions)
 	require.NotZero(t, cfg.SplitQueriesByInterval)
 	require.NotZero(t, cfg.MaxRetries)
 
@@ -834,7 +828,6 @@ func TestTripperware_RemoteRead(t *testing.T) {
 					MaxSamples: 1000,
 					Timeout:    time.Minute,
 				},
-				true,
 				nil,
 				reg,
 			)
@@ -969,7 +962,6 @@ func TestTripperware_ShouldSupportReadConsistencyOffsetsInjection(t *testing.T)
 			MaxSamples: 1000,
 			Timeout:    time.Minute,
 		},
-		true,
 		map[string]*ingest.TopicOffsetsReader{querierapi.ReadConsistencyOffsetsHeader: offsetsReader},
 		nil,
 	)

pkg/mimir/modules.go

@@ -765,7 +765,7 @@ func (t *Mimir) initQueryFrontendTopicOffsetsReaders() (services.Service, error)
 func (t *Mimir) initQueryFrontendTripperware() (serv services.Service, err error) {
 	promqlEngineRegisterer := prometheus.WrapRegistererWith(prometheus.Labels{"engine": "query-frontend"}, t.Registerer)
 
-	engineOpts, _, engineExperimentalFunctionsEnabled := engine.NewPromQLEngineOptions(t.Cfg.Querier.EngineConfig, t.ActivityTracker, util_log.Logger, promqlEngineRegisterer)
+	engineOpts, _ := engine.NewPromQLEngineOptions(t.Cfg.Querier.EngineConfig, t.ActivityTracker, util_log.Logger, promqlEngineRegisterer)
 
 	tripperware, err := querymiddleware.NewTripperware(
 		t.Cfg.Frontend.QueryMiddleware,
@@ -774,7 +774,6 @@ func (t *Mimir) initQueryFrontendTripperware() (serv services.Service, err error
 		t.QueryFrontendCodec,
 		querymiddleware.PrometheusResponseExtractor{},
 		engineOpts,
-		engineExperimentalFunctionsEnabled,
 		t.QueryFrontendTopicOffsetsReaders,
 		t.Registerer,
 	)

pkg/querier/engine/config.go

@@ -31,8 +31,6 @@ type Config struct {
 	// series is considered stale.
 	LookbackDelta time.Duration `yaml:"lookback_delta" category:"advanced"`
 
-	PromQLExperimentalFunctionsEnabled bool `yaml:"promql_experimental_functions_enabled" category:"experimental"`
-
 	MimirQueryEngine streamingpromql.Features `yaml:"mimir_query_engine" category:"experimental"`
 }
 
@@ -50,14 +48,12 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&cfg.MaxSamples, "querier.max-samples", 50e6, sharedWithQueryFrontend("Maximum number of samples a single query can load into memory."))
 	f.DurationVar(&cfg.DefaultEvaluationInterval, "querier.default-evaluation-interval", time.Minute, sharedWithQueryFrontend("The default evaluation interval or step size for subqueries."))
 	f.DurationVar(&cfg.LookbackDelta, "querier.lookback-delta", 5*time.Minute, sharedWithQueryFrontend("Time since the last sample after which a time series is considered stale and ignored by expression evaluations."))
-	f.BoolVar(&cfg.PromQLExperimentalFunctionsEnabled, "querier.promql-experimental-functions-enabled", false, sharedWithQueryFrontend("Enable experimental PromQL functions."))
 
 	cfg.MimirQueryEngine.RegisterFlags(f)
 }
 
-// NewPromQLEngineOptions returns the PromQL engine options based on the provided config and a boolean
-// to indicate whether the experimental PromQL functions should be enabled.
-func NewPromQLEngineOptions(cfg Config, activityTracker *activitytracker.ActivityTracker, logger log.Logger, reg prometheus.Registerer) (promql.EngineOpts, streamingpromql.EngineOpts, bool) {
+// NewPromQLEngineOptions returns the PromQL engine options based on the provided config.
+func NewPromQLEngineOptions(cfg Config, activityTracker *activitytracker.ActivityTracker, logger log.Logger, reg prometheus.Registerer) (promql.EngineOpts, streamingpromql.EngineOpts) {
 	commonOpts := promql.EngineOpts{
 		Logger:               util_log.SlogFromGoKit(logger),
 		Reg:                  reg,
@@ -77,5 +73,5 @@ func NewPromQLEngineOptions(cfg Config, activityTracker *activitytracker.Activit
 		Features:   cfg.MimirQueryEngine,
 	}
 
-	return commonOpts, mqeOpts, cfg.PromQLExperimentalFunctionsEnabled
+	return commonOpts, mqeOpts
 }

pkg/querier/querier.go

@@ -158,10 +158,11 @@ func New(cfg Config, limits *validation.Overrides, distributor Distributor, quer
 		return lazyquery.NewLazyQuerier(querier), nil
 	})
 
-	opts, mqeOpts, engineExperimentalFunctionsEnabled := engine.NewPromQLEngineOptions(cfg.EngineConfig, tracker, logger, reg)
+	opts, mqeOpts := engine.NewPromQLEngineOptions(cfg.EngineConfig, tracker, logger, reg)
 
-	// Experimental functions can only be enabled globally, and not on a per-engine basis.
-	parser.EnableExperimentalFunctions = engineExperimentalFunctionsEnabled
+	// Experimental functions are always enabled globally for all engines. Access to them
+	// is controlled by an experimental functions middleware that reads per-tenant settings.
+	parser.EnableExperimentalFunctions = true
 
 	var eng promql.QueryEngine