From 177cab9f076f581e886b9935c4e70a30ce469dc4 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan <55104035+aiqiaoy@users.noreply.github.com> Date: Fri, 5 Apr 2024 14:09:33 -0400 Subject: [PATCH 1/4] backoff if we retried polling for more than 50 times in less than 30minutes --- src/Runner.Listener/MessageListener.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 403c2bfc720..8ab6c003cd9 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -236,6 +236,7 @@ public async Task GetNextMessageAsync(CancellationToken token) ArgUtil.NotNull(_settings, nameof(_settings)); bool encounteringError = false; int continuousError = 0; + int continuousEmptyMessage = 0; string errorMessage = string.Empty; Stopwatch heartbeat = new(); heartbeat.Restart(); @@ -359,10 +360,21 @@ public async Task GetNextMessageAsync(CancellationToken token) if (message == null) { + continuousEmptyMessage++; + if (continuousEmptyMessage > 50) + { + // retried more than 50 times in less than 30mins and still getting empty message + // something is not right on the service side, backoff for 15-30s before retry + _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval); + Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds); + await HostContext.Delay(_getNextMessageRetryInterval, token); + } + if (heartbeat.Elapsed > TimeSpan.FromMinutes(30)) { Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes."); heartbeat.Restart(); + continuousEmptyMessage = 0; } else { From 3f38bf3bc246dc5d0aeaff65051baee4a308c032 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan <55104035+aiqiaoy@users.noreply.github.com> Date: Fri, 5 Apr 2024 18:34:34 +0000 Subject: [PATCH 2/4] run dotnet format --- src/Runner.Listener/MessageListener.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 8ab6c003cd9..6dd9e1de1ba 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; From 0456a88930e7f3cbad241dcbbc9808ec19a2cd03 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan <55104035+aiqiaoy@users.noreply.github.com> Date: Tue, 9 Apr 2024 13:35:25 -0400 Subject: [PATCH 3/4] move delay to after no message trace --- src/Runner.Listener/MessageListener.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 6dd9e1de1ba..099386b3187 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -361,15 +361,6 @@ public async Task GetNextMessageAsync(CancellationToken token) if (message == null) { continuousEmptyMessage++; - if (continuousEmptyMessage > 50) - { - // retried more than 50 times in less than 30mins and still getting empty message - // something is not right on the service side, backoff for 15-30s before retry - _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval); - Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds); - await HostContext.Delay(_getNextMessageRetryInterval, token); - } - if (heartbeat.Elapsed > TimeSpan.FromMinutes(30)) { Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes."); @@ -380,6 +371,15 @@ public async Task GetNextMessageAsync(CancellationToken token) { Trace.Verbose($"No message retrieved from session '{_session.SessionId}'."); } + + if (continuousEmptyMessage > 50) + { + // retried more than 50 times in less than 30mins and still getting empty message + // something is not right on the service side, backoff for 15-30s before retry + _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval); + Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds); + await HostContext.Delay(_getNextMessageRetryInterval, token); + } continue; } From ab89ec400918352d6d4f3567b5a1fb47bcf35853 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan <55104035+aiqiaoy@users.noreply.github.com> Date: Tue, 9 Apr 2024 17:49:03 +0000 Subject: [PATCH 4/4] run dotnet format --- src/Runner.Listener/MessageListener.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 099386b3187..7b51423e6f5 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -371,7 +371,7 @@ public async Task GetNextMessageAsync(CancellationToken token) { Trace.Verbose($"No message retrieved from session '{_session.SessionId}'."); } - + if (continuousEmptyMessage > 50) { // retried more than 50 times in less than 30mins and still getting empty message