From 409e24dde4a7665a10a2a9510f2bddb18504247b Mon Sep 17 00:00:00 2001 From: Paul Schoenfelder Date: Mon, 7 Mar 2022 15:02:43 -0500 Subject: [PATCH 1/2] Add decom reasons to task cleaning messages --- .../scheduler/SingularityScheduler.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java index da81924838..9b9323c45d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java @@ -24,6 +24,7 @@ import com.hubspot.singularity.SingularityDeployStatisticsBuilder; import com.hubspot.singularity.SingularityKilledTaskIdRecord; import com.hubspot.singularity.SingularityMachineAbstraction; +import com.hubspot.singularity.SingularityMachineStateHistoryUpdate; import com.hubspot.singularity.SingularityMainModule; import com.hubspot.singularity.SingularityManagedThreadPoolFactory; import com.hubspot.singularity.SingularityPendingDeploy; @@ -152,7 +153,8 @@ private void cleanupTaskDueToDecomission( final Map> requestIdsToUserToReschedule, final Set matchingTaskIds, SingularityTask task, - SingularityMachineAbstraction decommissioningObject + SingularityMachineAbstraction decommissioningObject, + Optional decomissionReason ) { requestIdsToUserToReschedule.put( task.getTaskRequest().getRequest().getId(), @@ -175,9 +177,10 @@ private void cleanupTaskDueToDecomission( task.getTaskId(), Optional.of( String.format( - "%s %s is decomissioning", + "%s %s is decomissioning%s", decommissioningObject.getTypeName(), - decommissioningObject.getName() + decommissioningObject.getName(), + decomissionReason.map(r -> String.format(" (message: %s)", r)).orElse("") ) ), Optional.empty(), @@ -211,6 +214,17 @@ public void checkForDecomissions() { for (SingularityAgent agent : agents.keySet()) { boolean foundTask = false; + List history = agentManager.getHistory( + agent.getId() + ); + + // TODO: Is there a guaranteed order these come out of ZK in to prevent sort? + // Probably nbd because this list will be very short, but still + SingularityMachineStateHistoryUpdate stateHistoryUpdate = history + .stream() + .filter(update -> update.getState() == MachineState.STARTING_DECOMMISSION) + .max(Comparator.comparing(SingularityMachineStateHistoryUpdate::getTimestamp)) + .get(); for (SingularityTask activeTask : taskManager.getTasksOnAgent( activeTaskIds, @@ -220,7 +234,8 @@ public void checkForDecomissions() { requestIdsToUserToReschedule, matchingTaskIds, activeTask, - agent + agent, + stateHistoryUpdate.getMessage() ); foundTask = true; } @@ -256,7 +271,8 @@ public void checkForDecomissions() { requestIdsToUserToReschedule, matchingTaskIds, maybeTask.get(), - rack + rack, + Optional.empty() ); } } From 8bb3e952218526883d413bb96272941c1b35cfc7 Mon Sep 17 00:00:00 2001 From: Paul Schoenfelder Date: Tue, 8 Mar 2022 16:52:16 -0500 Subject: [PATCH 2/2] Cleanup --- .../com/hubspot/singularity/scheduler/SingularityScheduler.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java index 9b9323c45d..2436dd452e 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java @@ -218,8 +218,6 @@ public void checkForDecomissions() { agent.getId() ); - // TODO: Is there a guaranteed order these come out of ZK in to prevent sort? - // Probably nbd because this list will be very short, but still SingularityMachineStateHistoryUpdate stateHistoryUpdate = history .stream() .filter(update -> update.getState() == MachineState.STARTING_DECOMMISSION)