diff --git a/src/Microsoft.OpenApi/Interfaces/IOpenApiReferenceHolder.cs b/src/Microsoft.OpenApi/Interfaces/IOpenApiReferenceHolder.cs
index c244263f6..8883a90f5 100644
--- a/src/Microsoft.OpenApi/Interfaces/IOpenApiReferenceHolder.cs
+++ b/src/Microsoft.OpenApi/Interfaces/IOpenApiReferenceHolder.cs
@@ -34,6 +34,6 @@ public interface IOpenApiReferenceHolder : IOpenApiSerializable
         /// <summary>
         /// Reference object.
         /// </summary>
-        OpenApiReference Reference { get; set; }
+        OpenApiReference Reference { get; init; }
     }
 }
diff --git a/src/Microsoft.OpenApi/Models/OpenApiReference.cs b/src/Microsoft.OpenApi/Models/OpenApiReference.cs
index ea614ae0a..bed22a7c3 100644
--- a/src/Microsoft.OpenApi/Models/OpenApiReference.cs
+++ b/src/Microsoft.OpenApi/Models/OpenApiReference.cs
@@ -78,10 +78,11 @@ public class OpenApiReference : IOpenApiSerializable, IOpenApiDescribedElement,
         /// </summary>
         public bool IsFragment { get; init; }
 
+        private OpenApiDocument openApiDocument;        
         /// <summary>
         /// The OpenApiDocument that is hosting the OpenApiReference instance. This is used to enable dereferencing the reference.
         /// </summary>
-        public OpenApiDocument HostDocument { get; init; }
+        public OpenApiDocument HostDocument { get => openApiDocument; init => openApiDocument = value; }
 
         /// <summary>
         /// Gets the full reference string for v3.0.
@@ -291,5 +292,16 @@ private string GetReferenceTypeNameAsV2(ReferenceType type)
                           // to indicate that the reference is not pointing to any object.
             };
         }
+
+        /// <summary>
+        /// Sets the host document after deserialization or before serialization.
+        /// This method is internal on purpose to avoid consumers mutating the host document.
+        /// </summary>
+        /// <param name="currentDocument">Host document to set if none is present</param>
+        internal void EnsureHostDocumentIsSet(OpenApiDocument currentDocument)
+        {
+            Utils.CheckArgumentNull(currentDocument);
+            openApiDocument ??= currentDocument;
+        }
     }
 }
diff --git a/src/Microsoft.OpenApi/Models/References/BaseOpenApiReferenceHolder.cs b/src/Microsoft.OpenApi/Models/References/BaseOpenApiReferenceHolder.cs
index 86b31fa06..4d1eaf4c0 100644
--- a/src/Microsoft.OpenApi/Models/References/BaseOpenApiReferenceHolder.cs
+++ b/src/Microsoft.OpenApi/Models/References/BaseOpenApiReferenceHolder.cs
@@ -71,7 +71,7 @@ protected BaseOpenApiReferenceHolder(string referenceId, OpenApiDocument hostDoc
     /// <inheritdoc/>
     public bool UnresolvedReference { get => Reference is null || Target is null; }
     /// <inheritdoc/>
-    public OpenApiReference Reference { get; set; }
+    public OpenApiReference Reference { get; init; }
     /// <inheritdoc/>
     public abstract V CopyReferenceAsTargetElementWithOverrides(V source);
     /// <inheritdoc/>
diff --git a/src/Microsoft.OpenApi/Services/ReferenceHostDocumentSetter.cs b/src/Microsoft.OpenApi/Services/ReferenceHostDocumentSetter.cs
index 7a9685ba4..c660d21bd 100644
--- a/src/Microsoft.OpenApi/Services/ReferenceHostDocumentSetter.cs
+++ b/src/Microsoft.OpenApi/Services/ReferenceHostDocumentSetter.cs
@@ -21,13 +21,7 @@ public ReferenceHostDocumentSetter(OpenApiDocument currentDocument)
         /// <inheritdoc/>
         public override void Visit(IOpenApiReferenceHolder referenceHolder)
         {
-            if (referenceHolder.Reference != null)
-            {
-                referenceHolder.Reference = new OpenApiReference(referenceHolder.Reference)
-                {
-                    HostDocument = _currentDocument,
-                };
-            }
+            referenceHolder.Reference?.EnsureHostDocumentIsSet(_currentDocument);
         }
     }
 }
diff --git a/test/Microsoft.OpenApi.Readers.Tests/ReferenceService/TryLoadReferenceV2Tests.cs b/test/Microsoft.OpenApi.Readers.Tests/ReferenceService/TryLoadReferenceV2Tests.cs
index 2380c07e3..9208fabd8 100644
--- a/test/Microsoft.OpenApi.Readers.Tests/ReferenceService/TryLoadReferenceV2Tests.cs
+++ b/test/Microsoft.OpenApi.Readers.Tests/ReferenceService/TryLoadReferenceV2Tests.cs
@@ -118,11 +118,7 @@ public async Task LoadResponseAndSchemaReference()
                     }
                 };
 
-            var schemaReference = (OpenApiSchemaReference)expected.Content["application/json"].Schema;
-            schemaReference.Reference = new OpenApiReference(schemaReference.Reference)
-            {
-                HostDocument = result.Document,
-            };
+            ((OpenApiSchemaReference)expected.Content["application/json"].Schema).Reference.EnsureHostDocumentIsSet(result.Document);
             var actual = reference.Target;
 
             // Assert
diff --git a/test/Microsoft.OpenApi.Readers.Tests/V3Tests/OpenApiDocumentTests.cs b/test/Microsoft.OpenApi.Readers.Tests/V3Tests/OpenApiDocumentTests.cs
index 44775f27d..6a5d80f0f 100644
--- a/test/Microsoft.OpenApi.Readers.Tests/V3Tests/OpenApiDocumentTests.cs
+++ b/test/Microsoft.OpenApi.Readers.Tests/V3Tests/OpenApiDocumentTests.cs
@@ -1046,11 +1046,11 @@ public async Task ParseModifiedPetStoreDocumentWithTagAndSecurityShouldSucceed()
                     }
             };
 
-            tagReference1.Reference = new OpenApiReference(tagReference1.Reference) {HostDocument = expected };
-            tagReference2.Reference = new OpenApiReference(tagReference2.Reference) {HostDocument = expected };
-            petSchemaReference.Reference = new OpenApiReference(petSchemaReference.Reference) {HostDocument = expected };
-            newPetSchemaReference.Reference = new OpenApiReference(newPetSchemaReference.Reference) {HostDocument = expected };
-            errorModelSchemaReference.Reference = new OpenApiReference(errorModelSchemaReference.Reference) {HostDocument = expected };
+            tagReference1.Reference.EnsureHostDocumentIsSet(expected);
+            tagReference2.Reference.EnsureHostDocumentIsSet(expected);
+            petSchemaReference.Reference.EnsureHostDocumentIsSet(expected);
+            newPetSchemaReference.Reference.EnsureHostDocumentIsSet(expected);
+            errorModelSchemaReference.Reference.EnsureHostDocumentIsSet(expected);
 
             actual.Document.Should().BeEquivalentTo(expected, options => options
             .IgnoringCyclicReferences()
@@ -1284,7 +1284,7 @@ public async Task ParseDocWithRefsUsingProxyReferencesSucceeds()
             var outputDoc = (await doc.SerializeAsYamlAsync(OpenApiSpecVersion.OpenApi3_0)).MakeLineBreaksEnvironmentNeutral();
             var expectedParam = expected.Paths["/pets"].Operations[OperationType.Get].Parameters[0];
             var expectedParamReference = Assert.IsType<OpenApiParameterReference>(expectedParam);
-            expectedParamReference.Reference = new OpenApiReference(expectedParamReference.Reference) {HostDocument = doc};
+            expectedParamReference.Reference.EnsureHostDocumentIsSet(doc);
 
             var actualParamReference = Assert.IsType<OpenApiParameterReference>(actualParam);
 
diff --git a/test/Microsoft.OpenApi.Tests/PublicApi/PublicApi.approved.txt b/test/Microsoft.OpenApi.Tests/PublicApi/PublicApi.approved.txt
index f8679c32d..5b6c34d8e 100644
--- a/test/Microsoft.OpenApi.Tests/PublicApi/PublicApi.approved.txt
+++ b/test/Microsoft.OpenApi.Tests/PublicApi/PublicApi.approved.txt
@@ -224,8 +224,8 @@ namespace Microsoft.OpenApi.Interfaces
     }
     public interface IOpenApiReferenceHolder : Microsoft.OpenApi.Interfaces.IOpenApiElement, Microsoft.OpenApi.Interfaces.IOpenApiSerializable
     {
-        Microsoft.OpenApi.Models.OpenApiReference Reference { get; set; }
         bool UnresolvedReference { get; }
+        Microsoft.OpenApi.Models.OpenApiReference Reference { get; init; }
     }
     public interface IOpenApiReferenceHolder<out T, V> : Microsoft.OpenApi.Interfaces.IOpenApiElement, Microsoft.OpenApi.Interfaces.IOpenApiReferenceHolder, Microsoft.OpenApi.Interfaces.IOpenApiSerializable
         where out T : Microsoft.OpenApi.Interfaces.IOpenApiReferenceable, V
@@ -1243,9 +1243,9 @@ namespace Microsoft.OpenApi.Models.References
         protected readonly T _target;
         protected BaseOpenApiReferenceHolder(Microsoft.OpenApi.Models.References.BaseOpenApiReferenceHolder<T, V> source) { }
         protected BaseOpenApiReferenceHolder(string referenceId, Microsoft.OpenApi.Models.OpenApiDocument hostDocument, Microsoft.OpenApi.Models.ReferenceType referenceType, string externalResource = null) { }
-        public Microsoft.OpenApi.Models.OpenApiReference Reference { get; set; }
         public virtual T Target { get; }
         public bool UnresolvedReference { get; }
+        public Microsoft.OpenApi.Models.OpenApiReference Reference { get; init; }
         public abstract V CopyReferenceAsTargetElementWithOverrides(V source);
         public virtual void SerializeAsV2(Microsoft.OpenApi.Writers.IOpenApiWriter writer) { }
         public virtual void SerializeAsV3(Microsoft.OpenApi.Writers.IOpenApiWriter writer) { }