From 77da0f3a0e8bcb714c322436529a0c42e3f46b27 Mon Sep 17 00:00:00 2001
From: Sergey Galuzo <sergal@microsoft.com>
Date: Sat, 27 Jan 2024 14:49:41 -0800
Subject: [PATCH] Add retries to deal with small GP databases.

---
 .../Features/Storage/SqlStoreClient.cs        | 27 +++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs
index afa6ab96a9..38b22c1e6d 100644
--- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs
+++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs
@@ -214,8 +214,31 @@ internal async Task<long> MergeResourcesGetTransactionVisibilityAsync(Cancellati
                 cmd.Parameters.AddWithValue("@HeartbeatDate", heartbeatDate.Value);
             }
 
-            await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken);
-            return ((long)transactionIdParam.Value, (int)sequenceParam.Value);
+            // Code below has retries on execution timeouts.
+            // Reason: GP databases are created with single data file. When database is heavily loaded by writes, single data file leads to long (up to several minutes) IO waits.
+            // These waits cause intermittent execution timeouts even for very short (~10msec) calls.
+            var start = DateTime.UtcNow;
+            var timeoutRetries = 0;
+            while (true)
+            {
+                try
+                {
+                    await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken);
+                    return ((long)transactionIdParam.Value, (int)sequenceParam.Value);
+                }
+                catch (Exception e)
+                {
+                    if (e.IsExecutionTimeout() && timeoutRetries++ < 3)
+                    {
+                        _logger.LogWarning(e, $"Error on {nameof(MergeResourcesBeginTransactionAsync)} timeoutRetries={{TimeoutRetries}}", timeoutRetries);
+                        await TryLogEvent(nameof(MergeResourcesBeginTransactionAsync), "Warn", $"timeout retries={timeoutRetries}", start, cancellationToken);
+                        await Task.Delay(5000, cancellationToken);
+                        continue;
+                    }
+
+                    throw;
+                }
+            }
         }
 
         internal async Task<int> MergeResourcesDeleteInvisibleHistory(long transactionId, CancellationToken cancellationToken)