Use Verify library for approval tests (#2330)

* Used Verify library for approval tests

* Removed unnecessary item group

* Approval -> Verify renaming

* Avoid duplication of directory name for VerifyTests

* Disabled diff tool for better console experience

* Renamed files after classes

* Updated documentation

---------

Co-authored-by: Alina Smirnova <BlueOstrich@yandex.ru>

docs/articles/contributing/running-tests.md

@@ -14,8 +14,8 @@ dotnet test -c Release -f net5.0 BenchmarkDotNet.sln
 
 You should be able to run all of tests from your IDE as well.
 
-## Approval Tests
+## Verify Tests
 
-For some unit tests (e.g. for exporter tests) BenchmarkDotNet uses [approval tests'](https://approvaltests.com/) implementation for .NET: [ApprovalTests.Net](https://github.com/approvals/ApprovalTests.Net).
-* The expected value for each test is stored in a `*.approved.txt` file located near the test source file in the repository. ApprovalTests.NET generates approved file's names automatically according test name and its parameters. This files must be added under the source control.
-* It also creates a `*.received` file for each failed test. You can use different reporters for convenient file comparison. By default we use XUnit2Reporter, so you can find test run results on the test runner console as usual. You can add [UseReporter(typeof(KDiffReporter))] on test class and then ApprovalTests will open KDiff for each failed test. This way you can easily understand what's the difference between approved and received values and choose the correct one.
+For some unit tests (e.g. for exporter tests) BenchmarkDotNet uses [Verify](https://github.com/VerifyTests/Verify).
+* The expected value for each test is stored in a `*.verified.txt` file located near the test source file in the repository. Verify generates verified file's names automatically according test name and its parameters. This files must be added under the source control.
+* It also creates a `*.received` file for each failed test. You can use diff tools for convenient file comparison. By default you can find test run results on the test runner console as usual. You can comment out the line ```result.DisableDiff()``` in ```VerifySettingsFactory.Create``` method and then Verify will open KDiff for each failed test. This way you can easily understand what's the difference between verified and received values and choose the correct one.

tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesApprovalTests.cs → tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs

@@ -2,32 +2,27 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Threading;
-using ApprovalTests;
-using ApprovalTests.Namers;
-using ApprovalTests.Reporters;
+using System.Threading.Tasks;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Tests.Mocks;
 using BenchmarkDotNet.Attributes;
-using BenchmarkDotNet.Tests.XUnit;
+using BenchmarkDotNet.Tests.Builders;
 using BenchmarkDotNet.Validators;
 using JetBrains.Annotations;
+using VerifyXunit;
 using Xunit;
 
 namespace BenchmarkDotNet.Tests.Attributes
 {
-    // In case of failed approval tests, use the following reporter:
-    // [UseReporter(typeof(KDiffReporter))]
-    [UseReporter(typeof(PatchedXUnit2Reporter))]
-    [UseApprovalSubdirectory("ApprovedFiles")]
-    [Collection("ApprovalTests")]
-    public class ParamsAllValuesApprovalTests : IDisposable
+    [Collection("VerifyTests")]
+    [UsesVerify]
+    public class ParamsAllValuesVerifyTests : IDisposable
     {
         private readonly CultureInfo initCulture;
 
-        public ParamsAllValuesApprovalTests() => initCulture = Thread.CurrentThread.CurrentCulture;
+        public ParamsAllValuesVerifyTests() => initCulture = Thread.CurrentThread.CurrentCulture;
 
         [UsedImplicitly]
         public static TheoryData<Type> GetBenchmarkTypes()
@@ -40,10 +35,8 @@ public static TheoryData<Type> GetBenchmarkTypes()
 
         [Theory]
         [MemberData(nameof(GetBenchmarkTypes))]
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        public void BenchmarkShouldProduceSummary(Type benchmarkType)
+        public Task BenchmarkShouldProduceSummary(Type benchmarkType)
         {
-            NamerFactory.AdditionalInformation = benchmarkType.Name;
             Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
 
             var logger = new AccumulationLogger();
@@ -60,7 +53,9 @@ public void BenchmarkShouldProduceSummary(Type benchmarkType)
             foreach (var error in errors)
                 logger.WriteLineError("* " + error.Message);
 
-            Approvals.Verify(logger.GetLog());
+            var settings = VerifySettingsFactory.Create();
+            settings.UseTextForParameters(benchmarkType.Name);
+            return Verifier.Verify(logger.GetLog(), settings);
         }
 
         public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture;

tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj

@@ -16,12 +16,12 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
+    <PackageReference Include="Verify.Xunit" Version="20.3.2" />
     <PackageReference Include="xunit" Version="2.4.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="ApprovalTests" Version="3.1.0" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
     <ProjectReference Include="..\..\src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" />

tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs

@@ -0,0 +1,15 @@
+using VerifyTests;
+
+namespace BenchmarkDotNet.Tests.Builders
+{
+    public static class VerifySettingsFactory
+    {
+        public static VerifySettings Create()
+        {
+            var result = new VerifySettings();
+            result.UseDirectory("VerifiedFiles");
+            result.DisableDiff();
+            return result;
+        }
+    }
+}

tests/BenchmarkDotNet.Tests/Exporters/CommonExporterApprovalTests.cs → tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs

@@ -1,34 +1,29 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
-using System.Runtime.CompilerServices;
 using System.Threading;
-using ApprovalTests;
-using ApprovalTests.Namers;
-using ApprovalTests.Reporters;
+using System.Threading.Tasks;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Exporters.Json;
 using BenchmarkDotNet.Exporters.Xml;
 using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Tests.Builders;
 using BenchmarkDotNet.Tests.Mocks;
-using BenchmarkDotNet.Tests.XUnit;
 using JetBrains.Annotations;
+using VerifyXunit;
 using Xunit;
 
 namespace BenchmarkDotNet.Tests.Exporters
 {
-    // In case of failed approval tests, use the following reporter:
-    // [UseReporter(typeof(KDiffReporter))]
-    [Collection("ApprovalTests")]
-    [UseReporter(typeof(PatchedXUnit2Reporter))]
-    [UseApprovalSubdirectory("ApprovedFiles")]
-    public class CommonExporterApprovalTests : IDisposable
+    [Collection("VerifyTests")]
+    [UsesVerify]
+    public class CommonExporterVerifyTests : IDisposable
     {
         private readonly CultureInfo initCulture;
 
-        public CommonExporterApprovalTests()
+        public CommonExporterVerifyTests()
         {
             initCulture = Thread.CurrentThread.CurrentCulture;
         }
@@ -44,12 +39,10 @@ public CommonExporterApprovalTests()
         public static TheoryData<string> CultureInfoNames => TheoryDataHelper.Create(CultureInfos.Keys);
 
         [Theory]
-        [MethodImpl(MethodImplOptions.NoInlining)] // required by the Approval test framework, do NOT remove
         [MemberData(nameof(CultureInfoNames))]
-        public void Exporters(string cultureInfoName)
+        public Task Exporters(string cultureInfoName)
         {
             var cultureInfo = CultureInfos[cultureInfoName];
-            NamerFactory.AdditionalInformation = $"{GetName(cultureInfo)}";
             Thread.CurrentThread.CurrentCulture = cultureInfo;
 
             var logger = new AccumulationLogger();
@@ -61,7 +54,9 @@ public void Exporters(string cultureInfoName)
                 exporter.ExportToLog(MockFactory.CreateSummary(config.WithCultureInfo(cultureInfo)), logger);
             }
 
-            Approvals.Verify(logger.GetLog());
+            var settings = VerifySettingsFactory.Create();
+            settings.UseTextForParameters(GetName(cultureInfo));
+            return Verifier.Verify(logger.GetLog(), settings);
         }
 
         private static void PrintTitle(AccumulationLogger logger, IExporter exporter)

tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs → tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs

@@ -2,33 +2,28 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Threading;
-using ApprovalTests;
-using ApprovalTests.Namers;
-using ApprovalTests.Reporters;
+using System.Threading.Tasks;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Tests.Mocks;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
-using BenchmarkDotNet.Tests.XUnit;
+using BenchmarkDotNet.Tests.Builders;
 using BenchmarkDotNet.Validators;
 using JetBrains.Annotations;
+using VerifyXunit;
 using Xunit;
 
 namespace BenchmarkDotNet.Tests.Exporters
 {
-    // In case of failed approval tests, use the following reporter:
-    // [UseReporter(typeof(KDiffReporter))]
-    [UseReporter(typeof(PatchedXUnit2Reporter))]
-    [UseApprovalSubdirectory("ApprovedFiles")]
-    [Collection("ApprovalTests")]
-    public class MarkdownExporterApprovalTests : IDisposable
+    [Collection("VerifyTests")]
+    [UsesVerify]
+    public class MarkdownExporterVerifyTests : IDisposable
     {
         private readonly CultureInfo initCulture;
 
-        public MarkdownExporterApprovalTests() => initCulture = Thread.CurrentThread.CurrentCulture;
+        public MarkdownExporterVerifyTests() => initCulture = Thread.CurrentThread.CurrentCulture;
 
         [UsedImplicitly]
         public static TheoryData<Type> GetGroupBenchmarkTypes()
@@ -41,10 +36,8 @@ public static TheoryData<Type> GetGroupBenchmarkTypes()
 
         [Theory]
         [MemberData(nameof(GetGroupBenchmarkTypes))]
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        public void GroupExporterTest(Type benchmarkType)
+        public Task GroupExporterTest(Type benchmarkType)
         {
-            NamerFactory.AdditionalInformation = benchmarkType.Name;
             Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
 
             var logger = new AccumulationLogger();
@@ -61,7 +54,9 @@ public void GroupExporterTest(Type benchmarkType)
             foreach (var error in errors)
                 logger.WriteLineError("* " + error.Message);
 
-            Approvals.Verify(logger.GetLog());
+            var settings = VerifySettingsFactory.Create();
+            settings.UseTextForParameters(benchmarkType.Name);
+            return Verifier.Verify(logger.GetLog(), settings);
         }
 
         public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture;

tests/BenchmarkDotNet.Tests/GlobFilterTests.cs

@@ -1,6 +1,4 @@
 using System.Linq;
-using System.Threading;
-using ApprovalUtilities.Utilities;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Running;

tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs

@@ -1,22 +1,18 @@
-using System.Runtime.CompilerServices;
-using System.Text;
-using ApprovalTests;
-using ApprovalTests.Namers;
-using ApprovalTests.Reporters;
+using System.Text;
+using System.Threading.Tasks;
 using BenchmarkDotNet.Portability.Cpu;
-using BenchmarkDotNet.Tests.XUnit;
+using BenchmarkDotNet.Tests.Builders;
+using VerifyXunit;
 using Xunit;
 
 namespace BenchmarkDotNet.Tests.Portability.Cpu
 {
-    [Collection("ApprovalTests")]
-    [UseReporter(typeof(PatchedXUnit2Reporter))]
-    [UseApprovalSubdirectory("ApprovedFiles")]
+    [Collection("VerifyTests")]
+    [UsesVerify]
     public class CpuInfoFormatterTests
     {
         [Fact]
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        public void FormatTest()
+        public Task FormatTest()
         {
             var captions = new StringBuilder();
             foreach (var processorName in new[] { null, "", "Intel" })
@@ -28,7 +24,8 @@ public void FormatTest()
                 captions.AppendLine(CpuInfoFormatter.Format(mockCpuInfo));
             }
 
-            Approvals.Verify(captions.ToString());
+            var settings = VerifySettingsFactory.Create();
+            return Verifier.Verify(captions.ToString(), settings);
         }
     }
 }

tests/BenchmarkDotNet.Tests/Properties/AssemblyInfo.cs

@@ -1,5 +1,3 @@
 using System.Runtime.InteropServices;
-using ApprovalTests.Reporters;
 
 [assembly: Guid("16c47abf-43e0-4db4-8151-36ca7a4082ae")]
-[assembly: IgnoreLineEndings(true)]