From df064cf7b4d66ca36d29d441d9c9dc98e94f0db0 Mon Sep 17 00:00:00 2001
From: Stephen Salinas <ssalinas@hubspot.com>
Date: Tue, 28 Apr 2020 07:59:19 -0400
Subject: [PATCH] Uncaught exception can result in failed deploy

---
 .../scheduler/SingularityDeployChecker.java         | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java
index 84ed03d662..f956ec9c5e 100644
--- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java
+++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java
@@ -181,7 +181,7 @@ private void checkDeploy(final SingularityPendingDeploy pendingDeploy, final Lis
     }
 
     SingularityDeployResult deployResult =
-      getDeployResult(request, requestWithState.getState(), cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployMatchingTasks, allOtherMatchingTasks, inactiveDeployMatchingTasks);
+      getDeployResultSafe(request, requestWithState.getState(), cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployMatchingTasks, allOtherMatchingTasks, inactiveDeployMatchingTasks);
 
     LOG.info("Deploy {} had result {} after {}", pendingDeployMarker, deployResult, JavaUtils.durationFromMillis(System.currentTimeMillis() - pendingDeployMarker.getTimestamp()));
 
@@ -632,6 +632,17 @@ private LoadBalancerRequestId getLoadBalancerRequestId(SingularityPendingDeploy
       LoadBalancerRequestType.DEPLOY, Optional.<Integer>empty());
   }
 
+  private SingularityDeployResult getDeployResultSafe(final SingularityRequest request, final RequestState requestState, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy,
+                                                      final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks,
+                                                      final Collection<SingularityTaskId> inactiveDeployMatchingTasks) {
+    try {
+      return getDeployResult(request, requestState, cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployActiveTasks, otherActiveTasks, inactiveDeployMatchingTasks);
+    } catch (Exception e) {
+      LOG.error("Uncaught exception processing deploy {} - {}", pendingDeploy.getDeployMarker().getRequestId(), pendingDeploy.getDeployMarker().getDeployId(), e);
+      return new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, String.format("Uncaught exception: %s", e.getMessage()));
+    }
+  }
+
   private SingularityDeployResult getDeployResult(final SingularityRequest request, final RequestState requestState, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy,
     final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks,
     final Collection<SingularityTaskId> inactiveDeployMatchingTasks) {