YARN-3477 TimelineClientImpl swallows exceptions

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java

@@ -338,7 +338,7 @@ private void addTimelineDelegationToken(
     }
     credentials.addToken(timelineService, timelineDelegationToken);
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Add timline delegation token into credentials: "
+      LOG.debug("Add timeline delegation token to credentials: "
           + timelineDelegationToken);
     }
     DataOutputBuffer dob = new DataOutputBuffer();
@@ -354,8 +354,10 @@ private void addTimelineDelegationToken(
           return timelineClient.getDelegationToken(timelineDTRenewer);
         } catch (Exception e ) {
           if (timelineServiceBestEffort) {
-            LOG.warn("Failed to get delegation token from the timeline server: "
+            LOG.warn("Failed to get delegation token from the timeline server;" +
+                " timeline client no longer publishing data: "
                 + e.getMessage());
+            LOG.debug("Full exception details", e);
             return null;
           }
           throw e;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java

@@ -20,6 +20,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.net.ConnectException;
 import java.net.HttpURLConnection;
@@ -90,6 +91,10 @@ public class TimelineClientImpl extends TimelineClient {
   private static final String RESOURCE_URI_STR = "/ws/v1/timeline/";
   private static final Joiner JOINER = Joiner.on("");
   public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
+  public static final String ERROR_NO_ATS_RESPONSE
+      = "Failed to get the response from the timeline server";
+  public static final String ERROR_RETRIES_EXCEEDED =
+    "Failed to connect to timeline server";
 
   private static Options opts;
   private static final String ENTITY_DATA_TYPE = "entity";
@@ -177,11 +182,13 @@ public Object retryOn(TimelineClientRetryOp op)
       retried = false;
 
       // keep trying
+      Exception lastException;
       while (true) {
         try {
           // try perform the op, if fail, keep retrying
           return op.run();
         } catch (IOException | RuntimeException e) {
+          lastException = e;
           // break if there's no retries left
           if (leftRetries == 0) {
             break;
@@ -201,23 +208,32 @@ public Object retryOn(TimelineClientRetryOp op)
           Thread.sleep(retryInterval);
         } catch (InterruptedException ie) {
           LOG.warn("Client retry sleep interrupted! ");
+          throw (InterruptedIOException)
+              (new InterruptedIOException(ie.toString()).initCause(ie));
         }
       }
-      throw new RuntimeException("Failed to connect to timeline server. "
-          + "Connection retries limit exceeded. "
-          + "The posted timeline event may be missing");
-    };
+      // reached only if the retry count has been exceeded.
+      // therefore, lastException no-null
+      String message = ERROR_RETRIES_EXCEEDED
+          + " Connection retries limit (" + maxRetries + ") exceeded."
+          + " The posted timeline event may be missing : " + lastException;
+      LOG.warn(message, lastException);
+
+      throw new RuntimeException(message, lastException);
+    }
 
     private void logException(Exception e, int leftRetries) {
       if (leftRetries > 0) {
         LOG.info("Exception caught by TimelineClientConnectionRetry,"
               + " will try " + leftRetries + " more time(s).\nMessage: "
-              + e.getMessage());
+              + e);
+        LOG.debug("Failure", e);
       } else {
         // note that maxRetries may be -1 at the very beginning
         LOG.info("ConnectionException caught by TimelineClientConnectionRetry,"
             + " will keep retrying.\nMessage: "
-            + e.getMessage());
+            + e);
+        LOG.debug("Failure", e);
       }
     }
   }
@@ -244,8 +260,8 @@ public boolean shouldRetryOn(Exception e) {
       try {
         return (ClientResponse) connectionRetry.retryOn(jerseyRetryOp);
       } catch (IOException e) {
-        throw new ClientHandlerException("Jersey retry failed!\nMessage: "
-              + e.getMessage());
+        throw new ClientHandlerException("Jersey retry failed against " + resURI
+            + "\nException: " + e, e);
       }
     }
   }
@@ -324,20 +340,29 @@ public ClientResponse run() throws Exception {
         }
       });
     } catch (UndeclaredThrowableException e) {
-        throw new IOException(e.getCause());
+      Throwable cause = e.getCause();
+      if (cause instanceof IOException) {
+        throw (IOException) cause;
+      } else {
+        throw new IOException(cause);
+      }
     } catch (InterruptedException ie) {
-      throw new IOException(ie);
+      throw (InterruptedIOException)
+          (new InterruptedIOException(ie.toString()).initCause(ie));
     }
     if (resp == null ||
         resp.getClientResponseStatus() != ClientResponse.Status.OK) {
-      String msg =
-          "Failed to get the response from the timeline server.";
-      LOG.error(msg);
-      if (LOG.isDebugEnabled() && resp != null) {
-        String output = resp.getEntity(String.class);
-        LOG.debug("HTTP error code: " + resp.getStatus()
-            + " Server response : \n" + output);
+      String msg = ERROR_NO_ATS_RESPONSE +" at " + resURI;
+      if (resp != null) {
+        int status = resp.getStatus();
+        msg += " -status code=" + status;
+        if (LOG.isDebugEnabled()) {
+          String output = resp.getEntity(String.class);
+          LOG.debug("HTTP error code: " + status
+              + " Server response : \n" + output);
+        }
       }
+      LOG.error(msg);
       throw new YarnException(msg);
     }
     return resp;
@@ -452,7 +477,8 @@ public Object run() throws IOException {
         } catch (UndeclaredThrowableException e) {
           throw new IOException(e.getCause());
         } catch (InterruptedException e) {
-          throw new IOException(e);
+          throw (InterruptedIOException)
+              (new InterruptedIOException(e.toString()).initCause(e));
         }
       }