Skip to content

Commit

Permalink
feat: allow the processing of multiple trx files
Browse files Browse the repository at this point in the history
  • Loading branch information
ricardofslp authored and joaoopereira committed Sep 9, 2023
1 parent 59f1243 commit 3d82209
Showing 14 changed files with 327 additions and 104 deletions.
6 changes: 3 additions & 3 deletions src/Analyzers/ITestResultsAnalyzer.cs
Original file line number Diff line number Diff line change
@@ -4,11 +4,11 @@ namespace dotnet.test.rerun.Analyzers;

public interface ITestResultsAnalyzer
{
string GetFailedTestsFilter(IFileInfo trxFile);
string GetFailedTestsFilter(IFileInfo[] trxFiles);

IFileInfo? GetTrxFile(IDirectoryInfo resultsDirectory);
IFileInfo[] GetTrxFiles(IDirectoryInfo resultsDirectory, DateTime startSearchTime);

void AddLastTrxFile(IDirectoryInfo resultsDirectory);
void AddLastTrxFiles(IDirectoryInfo resultsDirectory, DateTime startSearchTime);

HashSet<string> GetReportFiles();
}
54 changes: 29 additions & 25 deletions src/Analyzers/TestResultsAnalyzer.cs
Original file line number Diff line number Diff line change
@@ -14,43 +14,47 @@ public TestResultsAnalyzer(ILogger logger)
Log = logger;
reportFiles = new ();
}
public string GetFailedTestsFilter(IFileInfo trxFile)

public string GetFailedTestsFilter(IFileInfo[] trxFiles)
{
const string outcome = "Failed";
var trx = TrxDeserializer.Deserialize(trxFile.FullName);
reportFiles.Add(trxFile.FullName);
var failedTests = new List<string>();
foreach (var trxFile in trxFiles)
failedTests.AddRange(GetFailedTestsFilter(trxFile));

var tests = trx.Results.UnitTestResults
.Where(t => t.Outcome.Equals(outcome, StringComparison.InvariantCultureIgnoreCase))
.Select(t => $"FullyQualifiedName~{(t.TestName.IndexOf("(") > 0 ? t.TestName.Substring(0, t.TestName.IndexOf("(")) : t.TestName).TrimEnd()}")
.Distinct()
.ToList();

if (tests.Count == 0)
{
Log.Warning($"No tests found with the Outcome {outcome}");
return "";
}

var testFilter = string.Join(" | ", tests);
var testFilter = string.Join(" | ", failedTests);
Log.Debug(testFilter);
return testFilter;
}

public IFileInfo? GetTrxFile(IDirectoryInfo resultsDirectory)
public IFileInfo[] GetTrxFiles(IDirectoryInfo resultsDirectory, DateTime startSearchTime)
=> resultsDirectory.Exists ?
resultsDirectory.EnumerateFiles("*.trx").MaxBy(f => f.Name) :
default;
resultsDirectory.EnumerateFiles("*.trx").Where(file => file.CreationTime >= startSearchTime).ToArray() :
Array.Empty<IFileInfo>();

public void AddLastTrxFile(IDirectoryInfo resultsDirectory)
public void AddLastTrxFiles(IDirectoryInfo resultsDirectory, DateTime startSearchTime)
{
var fileInfo = GetTrxFile(resultsDirectory);

if (fileInfo is not null)
foreach (var fileInfo in GetTrxFiles(resultsDirectory, startSearchTime))
reportFiles.Add(fileInfo.FullName);
}

public HashSet<string> GetReportFiles()
=> reportFiles;

private List<string> GetFailedTestsFilter(IFileInfo trxFile)
{
const string outcome = "Failed";
var trx = TrxDeserializer.Deserialize(trxFile.FullName);
reportFiles.Add(trxFile.FullName);

var tests = trx.Results?.UnitTestResults
.Where(t => t.Outcome.Equals(outcome, StringComparison.InvariantCultureIgnoreCase))
.Select(t => $"FullyQualifiedName~{(t.TestName.IndexOf("(") > 0 ? t.TestName.Substring(0, t.TestName.IndexOf("(")) : t.TestName).TrimEnd()}")
.Distinct()
.ToList() ?? new List<string>();

if (tests.Count == 0)
Log.Warning($"No tests found with the Outcome {outcome} in file {trxFile.Name}");

return tests;
}
}
19 changes: 6 additions & 13 deletions src/RerunCommand/RerunCommand.cs
Original file line number Diff line number Diff line change
@@ -45,28 +45,20 @@ public RerunCommand(ILogger logger,
public async Task Run()
{
var startOfProcess = DateTime.Now;
var startOfDotnetRun = DateTime.Now;
IDirectoryInfo resultsDirectory = FileSystem.DirectoryInfo.New(Config.ResultsDirectory);
var oldTrxFile = TestResultsAnalyzer.GetTrxFile(resultsDirectory);
await DotNetTestRunner.Test(Config, resultsDirectory.FullName);
if (DotNetTestRunner.GetErrorCode() == ErrorCode.FailedTests)
{
var attempt = 1;
while (attempt <= Config.RerunMaxAttempts)
{
await Task.Delay(Config.Delay);
var trxFile = TestResultsAnalyzer.GetTrxFile(resultsDirectory);
var trxFiles = TestResultsAnalyzer.GetTrxFiles(resultsDirectory, startOfDotnetRun);

if (trxFile != null)
if (trxFiles.Length > 0)
{
if (oldTrxFile != null &&
trxFile.FullName.Equals(oldTrxFile.FullName, StringComparison.InvariantCultureIgnoreCase))
{
Log.Error("No new trx file was generated");
break;
}

var testsToRerun = TestResultsAnalyzer.GetFailedTestsFilter(trxFile);
oldTrxFile = trxFile;
var testsToRerun = TestResultsAnalyzer.GetFailedTestsFilter(trxFiles);
if (string.IsNullOrEmpty(testsToRerun))
{
Environment.ExitCode = 0;
@@ -77,6 +69,7 @@ public async Task Run()
Log.Information($"Rerun attempt {attempt}/{Config.RerunMaxAttempts}");
Log.Warning($"Found Failed tests. Rerun filter: {testsToRerun}");
Config.Filter = testsToRerun;
startOfDotnetRun = DateTime.Now;
await DotNetTestRunner.Test(Config, resultsDirectory.FullName);
attempt++;
}
@@ -95,7 +88,7 @@ public async Task Run()

if (Config.DeleteReportFiles)
{
TestResultsAnalyzer.AddLastTrxFile(resultsDirectory);
TestResultsAnalyzer.AddLastTrxFiles(resultsDirectory, startOfDotnetRun);
DeleteFiles();
}

17 changes: 17 additions & 0 deletions test/dotnet-test-rerun.Common/Utilities/TestUtilities.cs
Original file line number Diff line number Diff line change
@@ -32,6 +32,23 @@ public static void CopyFixture(string fixtureName, DirectoryInfo target)
CopyAll(source, target);
}

/// <summary>
/// Copies the fixture to a directory
/// </summary>
/// <param name="folder"></param>
/// <param name="fixtureName"></param>
/// <param name="target"></param>
public static void CopyFixtureFile(string folder, string fixtureName, DirectoryInfo target)
{
Directory.CreateDirectory(target.FullName);
var source = new DirectoryInfo(System.IO.Path.GetFullPath(
System.IO.Path.Join(
AppDomain.CurrentDomain.BaseDirectory,
"..", "..", "..", "Fixtures", folder)));
var file = source.GetFiles().FirstOrDefault(file => file.Name == fixtureName);
file!.CopyTo(Path.Combine(target.FullName, file.Name), true);
}

/// <summary>
/// Copies all files from a source directory to a target directory
/// </summary>
24 changes: 23 additions & 1 deletion test/dotnet-test-rerun.IntegrationTests/DotNetTestRerunTests.cs
Original file line number Diff line number Diff line change
@@ -10,7 +10,6 @@
using dotnet.test.rerun.RerunCommand;
using FluentAssertions;
using Xunit.Abstractions;
using Xunit.Sdk;

namespace dotnet_test_rerun.IntegrationTests;

@@ -267,6 +266,29 @@ public async Task DotnetTestRerun_FailingXUnit_WithDeleteFiles()
var files = FileSystem.Directory.EnumerateFiles(testDir, "*trx");
files.Should().HaveCount(0);
}

[Fact]
public async Task DotnetTestRerun_FailingNUnit_PassOnSecond_TwoFailingProjects()
{
// Arrange
Environment.ExitCode = 0;

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("NUnitMultipleFailingProjects");

// Assert
output.Should().Contain("Passed!");
output.Should().Contain("Failed!", Exactly.Times(3));
output.Should().Contain("Rerun filter:",
Exactly.Twice());
output.Should().Contain("Failed: 2, Passed: 1",
Exactly.Once());
output.Should().Contain("Failed: 1, Passed: 1",
Exactly.Twice());
output.Should().Contain("Failed: 0, Passed: 1",
Exactly.Twice());
Environment.ExitCode.Should().Be(0);
}

private async Task<string> RunDotNetTestRerunAndCollectOutputMessage(string proj, string extraArgs = "", string? dir = null)
{
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@

Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NUnitTestPassOnSecondRunFirstExample", "NUnitTestPassOnSecondRunFirstExample\NUnitTestPassOnSecondRunFirstExample.csproj", "{05625348-7376-4E0C-98EE-AEC93C1EA85B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NUnitTestPassOnThirdRunSecondExample", "NUnitTestPassOnThirdRunSecondExample\NUnitTestPassOnThirdRunSecondExample.csproj", "{F65E419F-ED1C-49F1-A32F-CBD95FFCB0B8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{05625348-7376-4E0C-98EE-AEC93C1EA85B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{05625348-7376-4E0C-98EE-AEC93C1EA85B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{05625348-7376-4E0C-98EE-AEC93C1EA85B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{05625348-7376-4E0C-98EE-AEC93C1EA85B}.Release|Any CPU.Build.0 = Release|Any CPU
{F65E419F-ED1C-49F1-A32F-CBD95FFCB0B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F65E419F-ED1C-49F1-A32F-CBD95FFCB0B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F65E419F-ED1C-49F1-A32F-CBD95FFCB0B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F65E419F-ED1C-49F1-A32F-CBD95FFCB0B8}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

<IsPackable>false</IsPackable>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
<PackageReference Include="NUnit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
<PackageReference Include="NUnit.Analyzers" Version="3.3.0" />
<PackageReference Include="coverlet.collector" Version="3.1.2" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
namespace NUnitTestPassOnSecondRunFirstExample;


public class Tests
{
private int number = 2;

[Test, Order(1)]
public void NUnitTestPassOnSecondRunFirstExample_FirstNumberCompare()
{
number = 3;
Assert.AreEqual(3, number);
}

[Test, Order(2)]
public void NUnitTestPassOnSecondRunFirstExample_SecondNumberCompare()
{
Assert.AreEqual(2, number);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
global using NUnit.Framework;
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

<IsPackable>false</IsPackable>

<RootNamespace>NUnitTestPassOnSecondRunSecondExample</RootNamespace>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
<PackageReference Include="NUnit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
<PackageReference Include="NUnit.Analyzers" Version="3.3.0" />
<PackageReference Include="coverlet.collector" Version="3.1.2" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
namespace NUnitTestPassOnSecondRunSecondExample;


public class Tests
{
private int firstNumber = 2;
private int secondNumber = 2;

[Test, Order(1)]
public void NUnitTestPassOnThirdRunSecondExample_FirstNumberCompare()
{
firstNumber = 3;
Assert.AreEqual(3, firstNumber);
}

[Test, Order(2)]
public void NUnitTestPassOnThirdRunSecondExample_SecondSimpleNumberCompare()
{
secondNumber = 4;
Assert.AreEqual(2, firstNumber);
Assert.AreEqual(4, secondNumber);
}

[Test, Order(3)]
public void NUnitTestPassOnThirdRunSecondExample_ThirdSimpleNumberCompare()
{
Assert.AreEqual(2, secondNumber);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
global using NUnit.Framework;
Loading

0 comments on commit 3d82209

Please # to comment.