diff --git a/fastmorph/__init__.py b/fastmorph/__init__.py
index 41193ab..b9196bc 100644
--- a/fastmorph/__init__.py
+++ b/fastmorph/__init__.py
@@ -271,7 +271,7 @@ def fill_holes(
 
   if return_fill_count:
     for label in removed_set:
-      del fill_counts[label]
+      fill_counts.pop(label, None)
     ret.append(fill_counts)
 
   if return_removed: