Cohosting semantic tokens tests

src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/LanguageClient/Cohost/CohostSemanticTokensRangeEndpoint.cs

@@ -9,11 +9,13 @@
 using Microsoft.AspNetCore.Razor;
 using Microsoft.AspNetCore.Razor.LanguageServer.Hosting;
 using Microsoft.AspNetCore.Razor.Telemetry;
+using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.ExternalAccess.Razor.Cohost;
 using Microsoft.CodeAnalysis.Razor.Logging;
 using Microsoft.CodeAnalysis.Razor.Remote;
 using Microsoft.CodeAnalysis.Razor.SemanticTokens;
 using Microsoft.CodeAnalysis.Razor.Workspaces;
+using Microsoft.CodeAnalysis.Text;
 using Microsoft.VisualStudio.LanguageServer.Protocol;
 using Microsoft.VisualStudio.Razor.LanguageClient.Extensions;
 using Microsoft.VisualStudio.Razor.Settings;
@@ -68,11 +70,16 @@ internal sealed class CohostSemanticTokensRangeEndpoint(
     protected override RazorTextDocumentIdentifier? GetRazorTextDocumentIdentifier(SemanticTokensRangeParams request)
         => request.TextDocument.ToRazorTextDocumentIdentifier();
 
-    protected override async Task<SemanticTokens?> HandleRequestAsync(SemanticTokensRangeParams request, RazorCohostRequestContext context, CancellationToken cancellationToken)
+    protected override Task<SemanticTokens?> HandleRequestAsync(SemanticTokensRangeParams request, RazorCohostRequestContext context, CancellationToken cancellationToken)
     {
         var razorDocument = context.TextDocument.AssumeNotNull();
         var span = request.Range.ToLinePositionSpan();
 
+        return HandleRequestAsync(razorDocument, span, cancellationToken);
+    }
+
+    private async Task<SemanticTokens?> HandleRequestAsync(TextDocument razorDocument, LinePositionSpan span, CancellationToken cancellationToken)
+    {
         var colorBackground = _clientSettingsManager.GetClientSettings().AdvancedSettings.ColorBackground;
 
         var correlationId = Guid.NewGuid();
@@ -93,4 +100,12 @@ internal sealed class CohostSemanticTokensRangeEndpoint(
 
         return null;
     }
+
+    internal TestAccessor GetTestAccessor() => new(this);
+
+    internal readonly struct TestAccessor(CohostSemanticTokensRangeEndpoint instance)
+    {
+        public Task<SemanticTokens?> HandleRequestAsync(TextDocument razorDocument, LinePositionSpan span, CancellationToken cancellationToken)
+            => instance.HandleRequestAsync(razorDocument, span, cancellationToken);
+    }
 }

src/Razor/test/Microsoft.AspNetCore.Razor.Test.Common.Tooling/TestImportProjectFeature.cs

@@ -26,7 +26,7 @@ public IReadOnlyList<RazorProjectItem> GetImports(RazorProjectItem projectItem)
     private void AddHierarchicalImports(RazorProjectItem projectItem, List<RazorProjectItem> imports)
     {
         // We want items in descending order. FindHierarchicalItems returns items in ascending order.
-        var importProjectItems = ProjectEngine.FileSystem.FindHierarchicalItems(projectItem.FilePath, "_Imports.cshtml").Reverse();
+        var importProjectItems = ProjectEngine.FileSystem.FindHierarchicalItems(projectItem.FilePath, "_ViewImports.cshtml").Reverse();
         imports.AddRange(importProjectItems);
     }
 }

src/Razor/test/Microsoft.AspNetCore.Razor.Test.Common.Tooling/TestProjectData.cs

@@ -25,10 +25,10 @@ static TestProjectData()
         SomeProject = new HostProject(Path.Combine(someProjectPath, "SomeProject.csproj"), someProjectObjPath, RazorConfiguration.Default, "SomeProject");
         SomeProjectFile1 = new HostDocument(Path.Combine(someProjectPath, "File1.cshtml"), "File1.cshtml", FileKinds.Legacy);
         SomeProjectFile2 = new HostDocument(Path.Combine(someProjectPath, "File2.cshtml"), "File2.cshtml", FileKinds.Legacy);
-        SomeProjectImportFile = new HostDocument(Path.Combine(someProjectPath, "_Imports.cshtml"), "_Imports.cshtml", FileKinds.Legacy);
+        SomeProjectImportFile = new HostDocument(Path.Combine(someProjectPath, "_ViewImports.cshtml"), "_ViewImports.cshtml", FileKinds.Legacy);
         SomeProjectNestedFile3 = new HostDocument(Path.Combine(someProjectPath, "Nested", "File3.cshtml"), "Nested\\File3.cshtml", FileKinds.Legacy);
         SomeProjectNestedFile4 = new HostDocument(Path.Combine(someProjectPath, "Nested", "File4.cshtml"), "Nested\\File4.cshtml", FileKinds.Legacy);
-        SomeProjectNestedImportFile = new HostDocument(Path.Combine(someProjectPath, "Nested", "_Imports.cshtml"), "Nested\\_Imports.cshtml", FileKinds.Legacy);
+        SomeProjectNestedImportFile = new HostDocument(Path.Combine(someProjectPath, "Nested", "_ViewImports.cshtml"), "Nested\\_ViewImports.cshtml", FileKinds.Legacy);
         SomeProjectComponentFile1 = new HostDocument(Path.Combine(someProjectPath, "File1.razor"), "File1.razor", FileKinds.Component);
         SomeProjectComponentFile2 = new HostDocument(Path.Combine(someProjectPath, "File2.razor"), "File2.razor", FileKinds.Component);
         SomeProjectComponentImportFile1 = new HostDocument(Path.Combine(someProjectPath, "_Imports.razor"), "_Imports.razor", FileKinds.Component);
@@ -42,10 +42,10 @@ static TestProjectData()
         AnotherProject = new HostProject(Path.Combine(anotherProjectPath, "AnotherProject.csproj"), anotherProjectObjPath, RazorConfiguration.Default, "AnotherProject");
         AnotherProjectFile1 = new HostDocument(Path.Combine(anotherProjectPath, "File1.cshtml"), "File1.cshtml", FileKinds.Legacy);
         AnotherProjectFile2 = new HostDocument(Path.Combine(anotherProjectPath, "File2.cshtml"), "File2.cshtml", FileKinds.Legacy);
-        AnotherProjectImportFile = new HostDocument(Path.Combine(anotherProjectPath, "_Imports.cshtml"), "_Imports.cshtml", FileKinds.Legacy);
+        AnotherProjectImportFile = new HostDocument(Path.Combine(anotherProjectPath, "_ViewImports.cshtml"), "_ViewImports.cshtml", FileKinds.Legacy);
         AnotherProjectNestedFile3 = new HostDocument(Path.Combine(anotherProjectPath, "Nested", "File3.cshtml"), "Nested\\File1.cshtml", FileKinds.Legacy);
         AnotherProjectNestedFile4 = new HostDocument(Path.Combine(anotherProjectPath, "Nested", "File4.cshtml"), "Nested\\File2.cshtml", FileKinds.Legacy);
-        AnotherProjectNestedImportFile = new HostDocument(Path.Combine(anotherProjectPath, "Nested", "_Imports.cshtml"), "Nested\\_Imports.cshtml", FileKinds.Legacy);
+        AnotherProjectNestedImportFile = new HostDocument(Path.Combine(anotherProjectPath, "Nested", "_ViewImports.cshtml"), "Nested\\_ViewImports.cshtml", FileKinds.Legacy);
         AnotherProjectComponentFile1 = new HostDocument(Path.Combine(anotherProjectPath, "File1.razor"), "File1.razor", FileKinds.Component);
         AnotherProjectComponentFile2 = new HostDocument(Path.Combine(anotherProjectPath, "File2.razor"), "File2.razor", FileKinds.Component);
         AnotherProjectNestedComponentFile3 = new HostDocument(Path.Combine(anotherProjectPath, "Nested", "File3.razor"), "Nested\\File1.razor", FileKinds.Component);

src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/Cohost/CohostSemanticTokensRangeEndpointTest.cs

@@ -0,0 +1,202 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System.IO;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Razor.Language;
+using Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
+using Microsoft.AspNetCore.Razor.PooledObjects;
+using Microsoft.AspNetCore.Razor.Telemetry;
+using Microsoft.CodeAnalysis.Razor.Settings;
+using Microsoft.CodeAnalysis.Razor.Workspaces;
+using Microsoft.CodeAnalysis.Remote.Razor.SemanticTokens;
+using Microsoft.CodeAnalysis.Text;
+using Microsoft.VisualStudio.Razor.Settings;
+using Microsoft.VisualStudio.Utilities;
+using Roslyn.Test.Utilities;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.VisualStudio.Razor.LanguageClient.Cohost;
+
+public class CohostSemanticTokensRangeEndpointTest(ITestOutputHelper testOutputHelper) : CohostTestBase(testOutputHelper)
+{
+    [Theory]
+    [CombinatorialData]
+    public async Task Razor(bool colorBackground)
+    {
+        var input = """
+            @page "/"
+            @using System
+
+            <div>This is some HTML</div>
+
+            <InputText Value="someValue" />
+
+            @* hello there *@
+            <!-- how are you? -->
+
+            @if (true)
+            {
+                <text>Html!</text>
+            }
+
+            @code
+            {
+                // I am also good, thanks for asking
+
+                /*
+                    No problem.
+                */
+
+                private string someValue;
+
+                public void M()
+                {
+                    RenderFragment x = @<div>This is some HTML in a render fragment</div>;
+                }
+            }
+            """;
+
+        await VerifySemanticTokensAsync(input, colorBackground);
+    }
+
+    [Theory]
+    [CombinatorialData]
+    public async Task Legacy(bool colorBackground)
+    {
+        var input = """
+            @page "/"
+            @using System
+
+            <div>This is some HTML</div>
+
+            <component type="typeof(Component)" render-mode="ServerPrerendered" />
+
+            @functions
+            {
+                public void M()
+                {
+                }
+            }
+
+            @section MySection {
+                <div>Section content</div>
+            }
+            """;
+
+        await VerifySemanticTokensAsync(input, colorBackground, fileKind: FileKinds.Legacy);
+    }
+
+    private async Task VerifySemanticTokensAsync(string input, bool colorBackground, string? fileKind = null, [CallerMemberName] string? testName = null)
+    {
+        var document = CreateProjectAndRazorDocument(input, fileKind);
+        var sourceText = await document.GetTextAsync(DisposalToken);
+
+        var legend = TestRazorSemanticTokensLegendService.Instance;
+
+        // We need to manually initialize the OOP service so we can get semantic token info later
+        RemoteSemanticTokensLegendService.SetLegend(legend.TokenTypes.All, legend.TokenModifiers.All);
+
+        var clientSettingsManager = new ClientSettingsManager([], null, null);
+        clientSettingsManager.Update(ClientAdvancedSettings.Default with { ColorBackground = colorBackground });
+
+        var endpoint = new CohostSemanticTokensRangeEndpoint(RemoteServiceProvider, clientSettingsManager, legend, NoOpTelemetryReporter.Instance, LoggerFactory);
+
+        var span = new LinePositionSpan(new(0, 0), new(sourceText.Lines.Count, 0));
+
+        var result = await endpoint.GetTestAccessor().HandleRequestAsync(document, span, DisposalToken);
+
+        var actualFileContents = GetTestOutput(sourceText, result?.Data);
+
+        if (colorBackground)
+        {
+            testName += "_with_background";
+        }
+
+        var baselineFileName = $@"TestFiles\SemanticTokens\{testName}.txt";
+        if (GenerateBaselines.ShouldGenerate)
+        {
+            WriteBaselineFile(actualFileContents, baselineFileName);
+        }
+
+        var expectedFileContents = GetBaselineFileContents(baselineFileName);
+        AssertEx.EqualOrDiff(expectedFileContents, actualFileContents);
+    }
+
+    private string GetBaselineFileContents(string baselineFileName)
+    {
+        var semanticFile = TestFile.Create(baselineFileName, GetType().Assembly);
+        if (!semanticFile.Exists())
+        {
+            return string.Empty;
+        }
+
+        return semanticFile.ReadAllText();
+    }
+
+    private static void WriteBaselineFile(string fileContents, string baselineFileName)
+    {
+        var projectPath = TestProject.GetProjectDirectory(typeof(CohostSemanticTokensRangeEndpointTest), layer: TestProject.Layer.Tooling);
+        var baselineFileFullPath = Path.Combine(projectPath, baselineFileName);
+        File.WriteAllText(baselineFileFullPath, fileContents);
+    }
+
+    private static string GetTestOutput(SourceText sourceText, int[]? data)
+    {
+        if (data == null)
+        {
+            return string.Empty;
+        }
+
+        using var _ = StringBuilderPool.GetPooledObject(out var builder);
+        builder.AppendLine("Line Δ, Char Δ, Length, Type, Modifier(s), Text");
+        var tokenTypes = TestRazorSemanticTokensLegendService.Instance.TokenTypes.All;
+        var prevLength = 0;
+        var lineIndex = 0;
+        var lineOffset = 0;
+        for (var i = 0; i < data.Length; i += 5)
+        {
+            var lineDelta = data[i];
+            var charDelta = data[i + 1];
+            var length = data[i + 2];
+
+            Assert.False(i != 0 && lineDelta == 0 && charDelta == 0, "line delta and character delta are both 0, which is invalid as we shouldn't be producing overlapping tokens");
+            Assert.False(i != 0 && lineDelta == 0 && charDelta < prevLength, "Previous length is longer than char offset from previous start, meaning tokens will overlap");
+
+            if (lineDelta != 0)
+            {
+                lineOffset = 0;
+            }
+
+            lineIndex += lineDelta;
+            lineOffset += charDelta;
+
+            var type = tokenTypes[data[i + 3]];
+            var modifier = GetTokenModifierString(data[i + 4]);
+            var text = sourceText.GetSubTextString(new TextSpan(sourceText.Lines[lineIndex].Start + lineOffset, length));
+            builder.AppendLine($"{lineDelta} {charDelta} {length} {type} {modifier} [{text}]");
+
+            prevLength = length;
+        }
+
+        return builder.ToString();
+    }
+
+    private static string GetTokenModifierString(int tokenModifiers)
+    {
+        var modifiers = TestRazorSemanticTokensLegendService.Instance.TokenModifiers.All;
+
+        var modifiersBuilder = ArrayBuilder<string>.GetInstance();
+        for (var i = 0; i < modifiers.Length; i++)
+        {
+            if ((tokenModifiers & (1 << i % 32)) != 0)
+            {
+                modifiersBuilder.Add(modifiers[i]);
+            }
+        }
+
+        return $"[{string.Join(", ", modifiersBuilder.ToArrayAndFree())}]";
+    }
+}

src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/Cohost/CohostTestBase.cs

@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using Basic.Reference.Assemblies;
 using Microsoft.AspNetCore.Razor;
+using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Test.Common;
 using Microsoft.AspNetCore.Razor.Test.Common.Workspaces;
 using Microsoft.CodeAnalysis;
@@ -18,6 +19,7 @@ namespace Microsoft.VisualStudio.Razor.LanguageClient.Cohost;
 
 public abstract class CohostTestBase(ITestOutputHelper testOutputHelper) : WorkspaceTestBase(testOutputHelper)
 {
+    private const string CSharpVirtualDocumentSuffix = ".g.cs";
     private IRemoteServiceProvider? _remoteServiceProvider;
 
     private protected IRemoteServiceProvider RemoteServiceProvider => _remoteServiceProvider.AssumeNotNull();
@@ -28,12 +30,27 @@ protected override async Task InitializeAsync()
 
         var exportProvider = AddDisposable(await RemoteMefComposition.CreateExportProviderAsync());
         _remoteServiceProvider = AddDisposable(new TestRemoteServiceProvider(exportProvider));
+
+        RemoteLanguageServerFeatureOptions.SetOptions(new()
+        {
+            CSharpVirtualDocumentSuffix = CSharpVirtualDocumentSuffix,
+            HtmlVirtualDocumentSuffix = ".g.html",
+            IncludeProjectKeyInGeneratedFilePath = false,
+            UsePreciseSemanticTokenRanges = false,
+            UseRazorCohostServer = true
+        });
     }
 
-    protected TextDocument CreateProjectAndRazorDocument(string contents)
+    protected TextDocument CreateProjectAndRazorDocument(string contents, string? fileKind = null)
     {
+        // Using IsLegacy means null == component, so easier for test authors
+        var isComponent = !FileKinds.IsLegacy(fileKind);
+
+        var documentFilePath = isComponent
+            ? TestProjectData.SomeProjectComponentFile1.FilePath
+            : TestProjectData.SomeProjectFile1.FilePath;
+
         var projectFilePath = TestProjectData.SomeProject.FilePath;
-        var documentFilePath = TestProjectData.SomeProjectComponentFile1.FilePath;
         var projectName = Path.GetFileNameWithoutExtension(projectFilePath);
         var projectId = ProjectId.CreateNewId(debugName: projectName);
         var documentId = DocumentId.CreateNewId(projectId, debugName: documentFilePath);
@@ -56,16 +73,29 @@ protected TextDocument CreateProjectAndRazorDocument(string contents)
                 documentFilePath,
                 SourceText.From(contents),
                 filePath: documentFilePath)
+            .AddDocument(
+                DocumentId.CreateNewId(projectId),
+                name: documentFilePath + CSharpVirtualDocumentSuffix,
+                SourceText.From(""),
+                filePath: documentFilePath + CSharpVirtualDocumentSuffix)
             .AddAdditionalDocument(
                 DocumentId.CreateNewId(projectId),
-                name: "_Imports.razor",
+                name: TestProjectData.SomeProjectComponentImportFile1.FilePath,
                 text: SourceText.From("""
                     @using Microsoft.AspNetCore.Components
                     @using Microsoft.AspNetCore.Components.Authorization
+                    @using Microsoft.AspNetCore.Components.Forms
                     @using Microsoft.AspNetCore.Components.Routing
                     @using Microsoft.AspNetCore.Components.Web
                     """),
-                filePath: TestProjectData.SomeProjectComponentImportFile1.FilePath);
+                filePath: TestProjectData.SomeProjectComponentImportFile1.FilePath)
+            .AddAdditionalDocument(
+                DocumentId.CreateNewId(projectId),
+                name: "_ViewImports.cshtml",
+                text: SourceText.From("""
+                    @addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers
+                    """),
+                filePath: TestProjectData.SomeProjectImportFile.FilePath);
 
         return solution.GetAdditionalDocument(documentId).AssumeNotNull();
     }

src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/Microsoft.VisualStudio.LanguageServices.Razor.Test.csproj

@@ -30,4 +30,8 @@
     <ProjectReference Include="..\Microsoft.AspNetCore.Razor.Test.Common.Tooling\Microsoft.AspNetCore.Razor.Test.Common.Tooling.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <EmbeddedResource Include="TestFiles\**\*" />
+  </ItemGroup>
+
 </Project>