Use separate metric for resource exhausted error in task processing (#…

…3463)
temporalio · Oct 10, 2022 · 71a1de3 · 71a1de3
1 parent 62cd143
commit 71a1de3
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/service/history/queues/executable.go b/service/history/queues/executable.go
@@ -237,19 +237,20 @@ func (e *executableImpl) HandleErr(err error) (retErr error) {
 				e.taggedMetricsHandler.Histogram(TaskAttempt, metrics.Dimensionless).Record(int64(e.attempt))
 				e.logger.Error("Critical error processing task, retrying.", tag.Error(err), tag.OperationCritical)
 			}
-
-			if common.IsResourceExhausted(retErr) {
-				e.resourceExhaustedCount++
-			} else {
-				e.resourceExhaustedCount = 0
-			}
 		}
 	}()
 
 	if err == nil {
 		return nil
 	}
 
+	if common.IsResourceExhausted(err) {
+		e.resourceExhaustedCount++
+		e.taggedMetricsHandler.Counter(TaskThrottledCounter).Record(1)
+		return err
+	}
+	e.resourceExhaustedCount = 0
+
 	if _, isNotFound := err.(*serviceerror.NotFound); isNotFound {
 		return nil
 	}

diff --git a/service/history/queues/metrics.go b/service/history/queues/metrics.go
@@ -43,6 +43,7 @@ const (
 	TaskStandbyRetryCounter         = "task_errors_standby_retry_counter"
 	TaskWorkflowBusyCounter         = "task_errors_workflow_busy"
 	TaskNotActiveCounter            = "task_errors_not_active_counter"
+	TaskThrottledCounter            = "task_errors_throttled"
 	TaskLoadLatency                 = "task_latency_load"                     // latency from task generation to task loading (persistence scheduleToStart)
 	TaskScheduleLatency             = "task_latency_schedule"                 // latency from task submission to in-memory queue to processing (in-memory scheduleToStart)
 	TaskProcessingLatency           = "task_latency_processing"               // latency for processing task one time
@@ -53,7 +54,6 @@ const (
 	TaskNoUserQueueLatency          = "task_latency_queue_nouserlatency"      // same as TaskQueueLatency, but excludes workflow lock latency
 	TaskUserLatency                 = "task_latency_userlatency"              // workflow lock latency across multiple attempts
 	TaskReschedulerPendingTasks     = "task_rescheduler_pending_tasks"
-	TaskThrottledCounter            = "task_throttled_counter"
 	TasksDependencyTaskNotCompleted = "task_dependency_task_not_completed"
 
 	TaskBatchCompleteCounter    = "task_batch_complete_counter"