Merge pull request #98 from Duke-GCB/podmonitor-value-error-96

Fix error when PodMonitor cleanup is called before remove

calrissian/k8s.py

@@ -263,9 +263,12 @@ def add(self, pod):
         PodMonitor.pod_names.append(pod.metadata.name)
 
     def remove(self, pod):
-        log.info('PodMonitor removing {}'.format(pod.metadata.name))
         # This has to look up the pod by something unique
-        PodMonitor.pod_names.remove(pod.metadata.name)
+        if pod.metadata.name in PodMonitor.pod_names:
+            log.info('PodMonitor removing {}'.format(pod.metadata.name))
+            PodMonitor.pod_names.remove(pod.metadata.name)
+        else:
+            log.warning('PodMonitor {} has already been removed'.format(pod.metadata.name))
 
     @staticmethod
     def cleanup():

tests/test_k8s.py

@@ -365,6 +365,22 @@ def test_delete_pods_calls_podmonitor(self, mock_pod_monitor):
         delete_pods()
         self.assertTrue(mock_pod_monitor.cleanup.called)
 
+    @patch('calrissian.k8s.KubernetesClient')
+    @patch('calrissian.k8s.log')
+    def test_remove_after_cleanup(self, mock_log, mock_client):
+        # Depending upon timing cleanup may get called before we receive a remove pod event
+        pod = self.make_mock_pod('pod-123')
+        with PodMonitor() as monitor:
+            monitor.add(pod)
+        PodMonitor.cleanup()
+        with PodMonitor() as monitor:
+            monitor.remove(pod)
+        mock_log.info.assert_has_calls([
+            call('PodMonitor adding pod-123'),
+            call('PodMonitor deleting pod pod-123'),
+        ])
+        mock_log.warning.assert_called_with('PodMonitor pod-123 has already been removed')
+
 
 class CompletionResultTestCase(TestCase):