make enable_sequential_cpu_offload more generic for third-party devices (#4191)

* make enable_sequential_cpu_offload more generic for third-party devices

* make style

Author: ji-huazhong

src/diffusers/pipelines/pipeline_utils.py

@@ -1127,7 +1127,9 @@ def enable_sequential_cpu_offload(self, gpu_id: int = 0, device: Union[torch.dev
 
         if self.device.type != "cpu":
             self.to("cpu", silence_dtype_warnings=True)
-            torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)
+            device_mod = getattr(torch, self.device.type, None)
+            if hasattr(device_mod, "empty_cache") and device_mod.is_available():
+                device_mod.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)
 
         for name, model in self.components.items():
             if not isinstance(model, torch.nn.Module):