Skip to content

Add tests for test templates #49102

New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Merged
merged 4 commits into from
May 23, 2025
Merged

Add tests for test templates #49102

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0da5e1b
add 1 test
mariam-abdulla May 22, 2025
49cc12e
Add tests
mariam-abdulla May 22, 2025
fc7d935
Merge branch 'main' into dev/mabdullah/create-tests-for-test-templates
mariam-abdulla May 22, 2025
e7e1b40
Merge branch 'main' into dev/mabdullah/create-tests-for-test-templates
mariam-abdulla May 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
259 changes: 259 additions & 0 deletions test/dotnet-new.IntegrationTests/DotnetNewTestTemplatesTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Immutable;

namespace Microsoft.DotNet.Cli.New.IntegrationTests
{
public class DotnetNewTestTemplatesTests : BaseIntegrationTest
{
private readonly ITestOutputHelper _log;

private static readonly ImmutableArray<string> SupportedTargetFrameworks =
[
"net10.0",
];

private static readonly (string ProjectTemplateName, string ItemTemplateName, string[] Languages, bool SupportsTestingPlatform)[] AvailableItemTemplates =
[
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How will you make sure you add any new ones here that get added in the future? Is this going to be documented somewhere?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From discussions with PMs, there is no support to add support for any other test framework nor any strong push on the support of non-MSTest ones. NUnit and xUnit are already barely maintained (we just bump version with exclusion to major version).

Still, we are happy to add some doc about these tests. Do you have any docs related to templates we could use?

("nunit", "nunit-test", Languages.All, false),
("mstest", "mstest-class", Languages.All, false),
];

private static readonly (string ProjectTemplateName, string[] Languages, bool RunDotnetTest, bool SupportsTestingPlatform)[] AvailableProjectTemplates =
[
("nunit", Languages.All, true, false),
("xunit", Languages.All, true, false),
("nunit-playwright", new[] { Languages.CSharp }, false, false),
];

public DotnetNewTestTemplatesTests(ITestOutputHelper log) : base(log)
{
_log = log;
}

public static class Languages
{
public const string CSharp = "c#";
public const string FSharp = "f#";
public const string VisualBasic = "vb";
public static readonly string[] All =
[CSharp, FSharp, VisualBasic];
}

private class NullTestOutputHelper : ITestOutputHelper
{
public void WriteLine(string message) { }

public void WriteLine(string format, params object[] args) { }
}

static DotnetNewTestTemplatesTests()
{
string templatePackagePath = Path.Combine(
RepoTemplatePackages,
"Microsoft.DotNet.Common.ProjectTemplates.10.0",
"content");

var dummyLog = new NullTestOutputHelper();

new DotnetNewCommand(dummyLog, "uninstall", templatePackagePath)
.WithCustomHive(CreateTemporaryFolder(folderName: "Home"))
.WithWorkingDirectory(CreateTemporaryFolder())
.Execute();

new DotnetNewCommand(dummyLog, "install", templatePackagePath)
.WithCustomHive(CreateTemporaryFolder(folderName: "Home"))
.WithWorkingDirectory(CreateTemporaryFolder())
.Execute()
.Should().ExitWith(0);
}

[Theory]
[MemberData(nameof(GetTemplateItemsToTest))]
public void ItemTemplate_CanBeInstalledAndTestArePassing(string targetFramework, string projectTemplate, string itemTemplate, string language)
{
string testProjectName = GenerateTestProjectName();
string outputDirectory = CreateTemporaryFolder(folderName: "Home");
string workingDirectory = CreateTemporaryFolder();

// Create new test project: dotnet new <projectTemplate> -n <testProjectName> -f <targetFramework> -lang <language>
string args = $"{projectTemplate} -n {testProjectName} -f {targetFramework} -lang {language} -o {outputDirectory}";
new DotnetNewCommand(_log, args)
.WithCustomHive(outputDirectory).WithRawArguments()
.WithWorkingDirectory(workingDirectory)
.Execute()
.Should().ExitWith(0);

var itemName = "test";

// Add test item to test project: dotnet new <itemTemplate> -n <test> -lang <language> -o <testProjectName>
new DotnetNewCommand(_log, $"{itemTemplate} -n {itemName} -lang {language} -o {outputDirectory}")
.WithCustomHive(outputDirectory).WithRawArguments()
.WithWorkingDirectory(workingDirectory)
.Execute()
.Should().ExitWith(0);

if (language == Languages.FSharp)
{
// f# projects don't include all files by default, so the file is created
// but the project ignores it until you manually add it into the project
// in the right order
AddItemToFsproj(itemName, outputDirectory, testProjectName);
}

var result = new DotnetTestCommand(_log, false)
.WithWorkingDirectory(outputDirectory)
.Execute(outputDirectory);

result.Should().ExitWith(0);

result.StdOut.Should().Contain("Passed!");
result.StdOut.Should().MatchRegex(@"Passed:\s*2");

Directory.Delete(outputDirectory, true);
Directory.Delete(workingDirectory, true);
}

[Theory]
[MemberData(nameof(GetTemplateProjectsToTest))]
public void ProjectTemplate_CanBeInstalledAndTestsArePassing(string targetFramework, string projectTemplate, string language, bool runDotnetTest)
{
string testProjectName = GenerateTestProjectName();
string outputDirectory = CreateTemporaryFolder(folderName: "Home");
string workingDirectory = CreateTemporaryFolder();

// Create new test project: dotnet new <projectTemplate> -n <testProjectName> -f <targetFramework> -lang <language>
string args = $"{projectTemplate} -n {testProjectName} -f {targetFramework} -lang {language} -o {outputDirectory}";
new DotnetNewCommand(_log, args)
.WithCustomHive(outputDirectory).WithRawArguments()
.WithWorkingDirectory(workingDirectory)
.Execute()
.Should().ExitWith(0);

if (runDotnetTest)
{
var result = new DotnetTestCommand(_log, false)
.WithWorkingDirectory(outputDirectory)
.Execute(outputDirectory);

result.Should().ExitWith(0);

result.StdOut.Should().Contain("Passed!");
result.StdOut.Should().MatchRegex(@"Passed:\s*1");
}

Directory.Delete(outputDirectory, true);
Directory.Delete(workingDirectory, true);
}

[Theory]
[MemberData(nameof(GetMSTestAndPlaywrightCoverageAndRunnerCombinations))]
public void MSTestAndPlaywrightProjectTemplate_WithCoverageToolAndTestRunner_CanBeInstalledAndTestsArePassing(
string projectTemplate,
string targetFramework,
string language,
string coverageTool,
string testRunner,
bool runDotnetTest)
{
string testProjectName = GenerateTestProjectName();
string outputDirectory = CreateTemporaryFolder(folderName: "Home");
string workingDirectory = CreateTemporaryFolder();

// Create new test project: dotnet new <projectTemplate> -n <testProjectName> -f <targetFramework> -lang <language> --coverage-tool <coverageTool> --test-runner <testRunner>
string args = $"{projectTemplate} -n {testProjectName} -f {targetFramework} -lang {language} -o {outputDirectory} --coverage-tool {coverageTool} --test-runner {testRunner}";
new DotnetNewCommand(_log, args)
.WithCustomHive(outputDirectory).WithRawArguments()
.WithWorkingDirectory(workingDirectory)
.Execute()
.Should().ExitWith(0);

if (runDotnetTest)
{
var result = new DotnetTestCommand(_log, false)
.WithWorkingDirectory(outputDirectory)
.Execute(outputDirectory);

result.Should().ExitWith(0);

result.StdOut.Should().Contain("Passed!");
result.StdOut.Should().MatchRegex(@"Passed:\s*1");
}

Directory.Delete(outputDirectory, true);
Directory.Delete(workingDirectory, true);
}

private void AddItemToFsproj(string itemName, string outputDirectory, string projectName)
{
var fsproj = Path.Combine(outputDirectory, $"{projectName}.fsproj");
var lines = File.ReadAllLines(fsproj).ToList();

lines.Insert(lines.IndexOf(" <ItemGroup>") + 1, $@" <Compile Include=""{itemName}.fs""/>");
File.WriteAllLines(fsproj, lines);
}

private static string GenerateTestProjectName()
{
// Avoiding VB errors because root namespace must not start with number or contain dashes
return "Test_" + Guid.NewGuid().ToString("N");
}

public static IEnumerable<object[]> GetTemplateItemsToTest()
{
foreach (var targetFramework in SupportedTargetFrameworks)
{
foreach (var (projectTemplate, itemTemplate, languages, supportsTestingPlatform) in AvailableItemTemplates)
{
foreach (var language in languages)
{
yield return new object[] { targetFramework, projectTemplate, itemTemplate, language };
}
}
}
}

public static IEnumerable<object[]> GetTemplateProjectsToTest()
{
foreach (var targetFramework in SupportedTargetFrameworks)
{
foreach (var (projectTemplate, languages, runDotnetTest, supportsTestingPlatform) in AvailableProjectTemplates)
{
foreach (var language in languages)
{
yield return new object[] { targetFramework, projectTemplate, language, runDotnetTest };
}
}
}
}

public static IEnumerable<object[]> GetMSTestAndPlaywrightCoverageAndRunnerCombinations()
{
var coverageTools = new[] { "Microsoft.CodeCoverage", "coverlet" };
var testRunners = new[] { "VSTest", "Microsoft.Testing.Platform" };
foreach (var targetFramework in SupportedTargetFrameworks)
{
// mstest: all languages, runDotnetTest = true
foreach (var language in Languages.All)
{
foreach (var coverageTool in coverageTools)
{
foreach (var testRunner in testRunners)
{
yield return new object[] { "mstest", targetFramework, language, coverageTool, testRunner, true };
}
}
}
// mstest-playwright: only c#, runDotnetTest = false
foreach (var coverageTool in coverageTools)
{
foreach (var testRunner in testRunners)
{
yield return new object[] { "mstest-playwright", targetFramework, Languages.CSharp, coverageTool, testRunner, false };
}
}
}
}
}
}