diff --git a/DependencyInjection/FrameworkExtension.php b/DependencyInjection/FrameworkExtension.php
index a1f23a837..f7b80aeef 100644
--- a/DependencyInjection/FrameworkExtension.php
+++ b/DependencyInjection/FrameworkExtension.php
@@ -1867,6 +1867,10 @@ private function registerSerializerConfiguration(array $config, ContainerBuilder
             $container->removeDefinition('serializer.normalizer.mime_message');
         }
 
+        if ($container->getParameter('kernel.debug')) {
+            $container->removeDefinition('serializer.mapping.cache_class_metadata_factory');
+        }
+
         // compat with Symfony < 6.3
         if (!is_subclass_of(ProblemNormalizer::class, SerializerAwareInterface::class)) {
             $container->getDefinition('serializer.normalizer.problem')
@@ -1875,10 +1879,6 @@ private function registerSerializerConfiguration(array $config, ContainerBuilder
 
         $serializerLoaders = [];
         if (isset($config['enable_annotations']) && $config['enable_annotations']) {
-            if ($container->getParameter('kernel.debug')) {
-                $container->removeDefinition('serializer.mapping.cache_class_metadata_factory');
-            }
-
             $annotationLoader = new Definition(
                 AnnotationLoader::class,
                 [new Reference('annotation_reader', ContainerInterface::NULL_ON_INVALID_REFERENCE)]
diff --git a/Tests/DependencyInjection/FrameworkExtensionTestCase.php b/Tests/DependencyInjection/FrameworkExtensionTestCase.php
index 97831b04e..e86cc3975 100644
--- a/Tests/DependencyInjection/FrameworkExtensionTestCase.php
+++ b/Tests/DependencyInjection/FrameworkExtensionTestCase.php
@@ -1572,6 +1572,12 @@ public function testSerializerCacheActivated()
     public function testSerializerCacheUsedWithoutAnnotationsAndMappingFiles()
     {
         $container = $this->createContainerFromFile('serializer_mapping_without_annotations', ['kernel.debug' => true, 'kernel.container_class' => __CLASS__]);
+        $this->assertFalse($container->hasDefinition('serializer.mapping.cache_class_metadata_factory'));
+    }
+
+    public function testSerializerCacheUsedWithoutAnnotationsAndMappingFilesNoDebug()
+    {
+        $container = $this->createContainerFromFile('serializer_mapping_without_annotations', ['kernel.debug' => false, 'kernel.container_class' => __CLASS__]);
         $this->assertTrue($container->hasDefinition('serializer.mapping.cache_class_metadata_factory'));
     }