From daff5f1bf2d958647b7d21fef6f0e5f6dd757c8f Mon Sep 17 00:00:00 2001 From: David Reiss Date: Wed, 11 Jan 2023 16:53:05 -0800 Subject: [PATCH] Add separate dynamic config knobs for internal-frontend rate limiting (#3800) --- common/dynamicconfig/constants.go | 9 ++++++++- service/frontend/fx.go | 25 +++++++++++++++++++++---- service/frontend/service.go | 4 ++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 21497ce45fe..b15148ee698 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -165,7 +165,7 @@ const ( FrontendMaxNamespaceRPSPerInstance = "frontend.namespaceRPS" // FrontendMaxNamespaceBurstPerInstance is workflow namespace burst limit FrontendMaxNamespaceBurstPerInstance = "frontend.namespaceBurst" - // FrontendMaxNamespaceCountPerInstance is workflow namespace count limit per second + // FrontendMaxNamespaceCountPerInstance limits concurrent task queue polls per namespace per instance FrontendMaxNamespaceCountPerInstance = "frontend.namespaceCount" // FrontendMaxNamespaceVisibilityRPSPerInstance is namespace rate limit per second for visibility APIs. // This config is EXPERIMENTAL and may be changed or removed in a later release. @@ -177,11 +177,18 @@ const ( // The limit is evenly distributed among available frontend service instances. // If this is set, it overwrites per instance limit "frontend.namespaceRPS". FrontendGlobalNamespaceRPS = "frontend.globalNamespaceRPS" + // InternalFrontendGlobalNamespaceRPS is workflow namespace rate limit per second across + // all internal-frontends. + InternalFrontendGlobalNamespaceRPS = "internal-frontend.globalNamespaceRPS" // FrontendGlobalNamespaceVisibilityRPS is workflow namespace rate limit per second for the whole cluster for visibility API. // The limit is evenly distributed among available frontend service instances. // If this is set, it overwrites per instance limit "frontend.namespaceRPS.visibility". // This config is EXPERIMENTAL and may be changed or removed in a later release. FrontendGlobalNamespaceVisibilityRPS = "frontend.globalNamespaceRPS.visibility" + // InternalFrontendGlobalNamespaceVisibilityRPS is workflow namespace rate limit per second + // across all internal-frontends. + // This config is EXPERIMENTAL and may be changed or removed in a later release. + InternalFrontendGlobalNamespaceVisibilityRPS = "internal-frontend.globalNamespaceRPS.visibility" // FrontendThrottledLogRPS is the rate limit on number of log messages emitted per second for throttled logger FrontendThrottledLogRPS = "frontend.throttledLogRPS" // FrontendShutdownDrainDuration is the duration of traffic drain during shutdown diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 8e6f79a8379..4cade1c10a7 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -270,14 +270,28 @@ func RateLimitInterceptorProvider( } func NamespaceRateLimitInterceptorProvider( + serviceName primitives.ServiceName, serviceConfig *Config, namespaceRegistry namespace.Registry, frontendServiceResolver membership.ServiceResolver, ) *interceptor.NamespaceRateLimitInterceptor { + var globalNamespaceRPS, globalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter + + switch serviceName { + case primitives.FrontendService: + globalNamespaceRPS = serviceConfig.GlobalNamespaceRPS + globalNamespaceVisibilityRPS = serviceConfig.GlobalNamespaceVisibilityRPS + case primitives.InternalFrontendService: + globalNamespaceRPS = serviceConfig.InternalFEGlobalNamespaceRPS + globalNamespaceVisibilityRPS = serviceConfig.InternalFEGlobalNamespaceVisibilityRPS + default: + panic("invalid service name") + } + rateFn := func(namespace string) float64 { return namespaceRPS( serviceConfig.MaxNamespaceRPSPerInstance, - serviceConfig.GlobalNamespaceRPS, + globalNamespaceRPS, frontendServiceResolver, namespace, ) @@ -286,7 +300,7 @@ func NamespaceRateLimitInterceptorProvider( visibilityRateFn := func(namespace string) float64 { return namespaceRPS( serviceConfig.MaxNamespaceVisibilityRPSPerInstance, - serviceConfig.GlobalNamespaceVisibilityRPS, + globalNamespaceVisibilityRPS, frontendServiceResolver, namespace, ) @@ -393,8 +407,11 @@ func FEReplicatorNamespaceReplicationQueueProvider( return replicatorNamespaceReplicationQueue } -func ServiceResolverProvider(membershipMonitor membership.Monitor) (membership.ServiceResolver, error) { - return membershipMonitor.GetResolver(primitives.FrontendService) +func ServiceResolverProvider( + membershipMonitor membership.Monitor, + serviceName primitives.ServiceName, +) (membership.ServiceResolver, error) { + return membershipMonitor.GetResolver(serviceName) } func AdminHandlerProvider( diff --git a/service/frontend/service.go b/service/frontend/service.go index 435dfb6b181..ee3a0b0f915 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -79,7 +79,9 @@ type Config struct { MaxNamespaceVisibilityRPSPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter MaxNamespaceVisibilityBurstPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter GlobalNamespaceRPS dynamicconfig.IntPropertyFnWithNamespaceFilter + InternalFEGlobalNamespaceRPS dynamicconfig.IntPropertyFnWithNamespaceFilter GlobalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter + InternalFEGlobalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter MaxIDLengthLimit dynamicconfig.IntPropertyFn WorkerBuildIdSizeLimit dynamicconfig.IntPropertyFn DisallowQuery dynamicconfig.BoolPropertyFnWithNamespaceFilter @@ -193,7 +195,9 @@ func NewConfig(dc *dynamicconfig.Collection, numHistoryShards int32, esIndexName MaxNamespaceVisibilityRPSPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceVisibilityRPSPerInstance, 10), MaxNamespaceVisibilityBurstPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceVisibilityBurstPerInstance, 10), GlobalNamespaceRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendGlobalNamespaceRPS, 0), + InternalFEGlobalNamespaceRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.InternalFrontendGlobalNamespaceRPS, 0), GlobalNamespaceVisibilityRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendGlobalNamespaceVisibilityRPS, 0), + InternalFEGlobalNamespaceVisibilityRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.InternalFrontendGlobalNamespaceVisibilityRPS, 0), MaxIDLengthLimit: dc.GetIntProperty(dynamicconfig.MaxIDLengthLimit, 1000), WorkerBuildIdSizeLimit: dc.GetIntProperty(dynamicconfig.WorkerBuildIdSizeLimit, 1000), MaxBadBinaries: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxBadBinaries, namespace.MaxBadBinaries),