diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt
index 078386c457bd..7204eead5864 100644
--- a/requirements/requirements-dev.txt
+++ b/requirements/requirements-dev.txt
@@ -16,5 +16,5 @@ sphinx
 sphinx-rtd-theme
 tensorboard
 torchvision
-transformers
+transformers>=4.32.1
 wandb
diff --git a/requirements/requirements-inf.txt b/requirements/requirements-inf.txt
index 848a7f7a485d..27371e623f26 100644
--- a/requirements/requirements-inf.txt
+++ b/requirements/requirements-inf.txt
@@ -1,5 +1,5 @@
 google
 lm-eval==0.3.0
 protobuf
-transformers
+transformers>=4.32.1
 transformers[sentencepiece]
diff --git a/tests/unit/inference/quantization/test_intX_quantization.py b/tests/unit/inference/quantization/test_intX_quantization.py
index 56df2b232d15..fd6a8e5ad2e1 100644
--- a/tests/unit/inference/quantization/test_intX_quantization.py
+++ b/tests/unit/inference/quantization/test_intX_quantization.py
@@ -55,7 +55,7 @@ def quantization_test_helper(pre_quant_type: torch.dtype, num_bits: int):
 
 def zero3_post_init_quantization_test_helper(cpu_offload: bool, nvme_offload: bool, bits: int):
     import deepspeed
-    from transformers.deepspeed import HfDeepSpeedConfig
+    from transformers.integrations.deepspeed import HfDeepSpeedConfig
 
     def get_zero3_ds_config(hf_config: OPTConfig, cpu_offload: bool, nvme_offload: bool, bits: int) -> Dict:
         GB = 1 << 30
@@ -172,7 +172,7 @@ def get_zero3_ds_config(hf_config: OPTConfig, cpu_offload: bool, nvme_offload: b
 
 def zero3_quantized_initialization_test_helper(cpu_offload: bool, nvme_offload: bool, bits: int):
     import deepspeed
-    from transformers.deepspeed import HfDeepSpeedConfig
+    from transformers.integrations.deepspeed import HfDeepSpeedConfig
 
     def get_zero3_ds_config(hf_config: OPTConfig, cpu_offload: bool, nvme_offload: bool, bits: int) -> Dict:
         GB = 1 << 30
diff --git a/tests/unit/runtime/zero/test_zero_nesting_init.py b/tests/unit/runtime/zero/test_zero_nesting_init.py
index 143e7e997b13..15d82fd8be00 100644
--- a/tests/unit/runtime/zero/test_zero_nesting_init.py
+++ b/tests/unit/runtime/zero/test_zero_nesting_init.py
@@ -8,7 +8,7 @@
 from unit.common import DistributedTest
 
 from transformers import VisionEncoderDecoderModel
-from transformers.deepspeed import HfDeepSpeedConfig
+from transformers.integrations.deepspeed import HfDeepSpeedConfig
 
 import deepspeed