Fail at startup when ASP.NET OpenAPI is registered

Author: bkoelman

Diff for: src/JsonApiDotNetCore/Configuration/ApplicationBuilderExtensions.cs

@@ -1,6 +1,7 @@
 using JsonApiDotNetCore.Middleware;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
 
 namespace JsonApiDotNetCore.Configuration;
 
@@ -23,6 +24,7 @@ public static class ApplicationBuilderExtensions
     public static void UseJsonApi(this IApplicationBuilder builder)
     {
         ArgumentNullException.ThrowIfNull(builder);
+        AssertAspNetCoreOpenApiIsNotRegistered(builder.ApplicationServices);
 
         using (IServiceScope scope = builder.ApplicationServices.CreateScope())
         {
@@ -46,4 +48,33 @@ public static void UseJsonApi(this IApplicationBuilder builder)
 
         builder.UseMiddleware<JsonApiMiddleware>();
     }
+
+    private static void AssertAspNetCoreOpenApiIsNotRegistered(IServiceProvider serviceProvider)
+    {
+        Type? optionsType = TryLoadOptionsType();
+
+        if (optionsType != null)
+        {
+            Type configureType = typeof(IConfigureOptions<>).MakeGenericType(optionsType);
+            object? configureInstance = serviceProvider.GetService(configureType);
+
+            if (configureInstance != null)
+            {
+                throw new InvalidOperationException("JsonApiDotNetCore is incompatible with ASP.NET OpenAPI. " +
+                    "Replace 'services.AddOpenApi()' with 'services.AddOpenApiForJsonApi()' from the JsonApiDotNetCore.OpenApi.Swashbuckle NuGet package.");
+            }
+        }
+    }
+
+    private static Type? TryLoadOptionsType()
+    {
+        try
+        {
+            return Type.GetType("Microsoft.AspNetCore.OpenApi.OpenApiOptions, Microsoft.AspNetCore.OpenApi");
+        }
+        catch (FileLoadException)
+        {
+            return null;
+        }
+    }
 }

Diff for: test/DiscoveryTests/AspNetOpenApiTests.cs

@@ -0,0 +1,31 @@
+#if !NET8_0
+using FluentAssertions;
+using JsonApiDotNetCore.Configuration;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.Extensions.DependencyInjection;
+using Xunit;
+
+namespace DiscoveryTests;
+
+public sealed class AspNetOpenApiTests
+{
+    [Fact]
+    public async Task Throws_when_AspNet_OpenApi_is_registered()
+    {
+        // Arrange
+        WebApplicationBuilder builder = WebApplication.CreateEmptyBuilder(new WebApplicationOptions());
+        builder.WebHost.UseTestServer();
+        builder.Services.AddJsonApi();
+        builder.Services.AddOpenApi();
+        await using WebApplication app = builder.Build();
+
+        // Act
+        Action action = app.UseJsonApi;
+
+        // Assert
+        action.Should().ThrowExactly<InvalidOperationException>().WithMessage("JsonApiDotNetCore is incompatible with ASP.NET OpenAPI. " +
+            "Replace 'services.AddOpenApi()' with 'services.AddOpenApiForJsonApi()' from the JsonApiDotNetCore.OpenApi.Swashbuckle NuGet package.");
+    }
+}
+#endif

Diff for: test/DiscoveryTests/DiscoveryTests.csproj

@@ -14,5 +14,6 @@
     <PackageReference Include="coverlet.collector" Version="$(CoverletVersion)" PrivateAssets="All" />
     <PackageReference Include="GitHubActionsTestLogger" Version="$(GitHubActionsTestLoggerVersion)" PrivateAssets="All" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(TestSdkVersion)" />
+    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Condition="'$(TargetFramework)' != 'net8.0'" Version="$(AspNetCoreVersion)" />
   </ItemGroup>
 </Project>