Skip to content

Add tests for test templates #49102

New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Merged
merged 4 commits into from
May 23, 2025
Merged

Add tests for test templates #49102

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0da5e1b
add 1 test
mariam-abdulla May 22, 2025
49cc12e
Add tests
mariam-abdulla May 22, 2025
fc7d935
Merge branch 'main' into dev/mabdullah/create-tests-for-test-templates
mariam-abdulla May 22, 2025
e7e1b40
Merge branch 'main' into dev/mabdullah/create-tests-for-test-templates
mariam-abdulla May 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
259 changes: 259 additions & 0 deletions test/dotnet-new.IntegrationTests/DotnetNewTestTemplatesTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Immutable;

namespace Microsoft.DotNet.Cli.New.IntegrationTests
{
public class DotnetNewTestTemplatesTests : BaseIntegrationTest
{
private readonly ITestOutputHelper _log;

private static readonly ImmutableArray<string> SupportedTargetFrameworks =
[
"net10.0",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you want to target any downlevel versions? In net11, would these tests target net11 and net10 or be updated to just net11? If you're only ever going to target current, you might be able to reuse the values in ToolsetInfo.cs so as to reduce the work needed when upgrading TFMs.

];

private static readonly (string ProjectTemplateName, string ItemTemplateName, string[] Languages, bool SupportsTestingPlatform)[] AvailableItemTemplates =
[
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How will you make sure you add any new ones here that get added in the future? Is this going to be documented somewhere?

("nunit", "nunit-test", Languages.All, false),
("mstest", "mstest-class", Languages.All, false),
];

private static readonly (string ProjectTemplateName, string[] Languages, bool RunDotnetTest, bool SupportsTestingPlatform)[] AvailableProjectTemplates =
[
("nunit", Languages.All, true, false),
("xunit", Languages.All, true, false),
("nunit-playwright", new[] { Languages.CSharp }, false, false),
];

public DotnetNewTestTemplatesTests(ITestOutputHelper log) : base(log)
{
_log = log;
}

public static class Languages
{
public const string CSharp = "c#";
public const string FSharp = "f#";
public const string VisualBasic = "vb";
public static readonly string[] All =
[CSharp, FSharp, VisualBasic];
}

private class NullTestOutputHelper : ITestOutputHelper
{
public void WriteLine(string message) { }

public void WriteLine(string format, params object[] args) { }
}

static DotnetNewTestTemplatesTests()
{
string templatePackagePath = Path.Combine(
RepoTemplatePackages,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You've confirmed that this is the live location in the build, correct? I think it is but just double checking it's not the stage 0 (global.json) one as that version of templates is usually a month behind.

"Microsoft.DotNet.Common.ProjectTemplates.10.0",
"content");

var dummyLog = new NullTestOutputHelper();

new DotnetNewCommand(dummyLog, "uninstall", templatePackagePath)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Help me understand why you uninstall first? Just to ensure the install works if it was already installed?

.WithCustomHive(CreateTemporaryFolder(folderName: "Home"))
.WithWorkingDirectory(CreateTemporaryFolder())
.Execute();

new DotnetNewCommand(dummyLog, "install", templatePackagePath)
.WithCustomHive(CreateTemporaryFolder(folderName: "Home"))
.WithWorkingDirectory(CreateTemporaryFolder())
.Execute()
.Should().ExitWith(0);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Can use .Pass() to be a bit more obvious

}

[Theory]
[MemberData(nameof(GetTemplateItemsToTest))]
public void ItemTemplate_CanBeInstalledAndTestArePassing(string targetFramework, string projectTemplate, string itemTemplate, string language)
{
string testProjectName = GenerateTestProjectName();
string outputDirectory = CreateTemporaryFolder(folderName: "Home");
string workingDirectory = CreateTemporaryFolder();

// Create new test project: dotnet new <projectTemplate> -n <testProjectName> -f <targetFramework> -lang <language>
string args = $"{projectTemplate} -n {testProjectName} -f {targetFramework} -lang {language} -o {outputDirectory}";
new DotnetNewCommand(_log, args)
.WithCustomHive(outputDirectory).WithRawArguments()
.WithWorkingDirectory(workingDirectory)
.Execute()
.Should().ExitWith(0);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same nit on Pass()


var itemName = "test";

// Add test item to test project: dotnet new <itemTemplate> -n <test> -lang <language> -o <testProjectName>
new DotnetNewCommand(_log, $"{itemTemplate} -n {itemName} -lang {language} -o {outputDirectory}")
.WithCustomHive(outputDirectory).WithRawArguments()
.WithWorkingDirectory(workingDirectory)
.Execute()
.Should().ExitWith(0);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same nit


if (language == Languages.FSharp)
{
// f# projects don't include all files by default, so the file is created
// but the project ignores it until you manually add it into the project
// in the right order
AddItemToFsproj(itemName, outputDirectory, testProjectName);
}

var result = new DotnetTestCommand(_log, false)
.WithWorkingDirectory(outputDirectory)
.Execute(outputDirectory);

result.Should().ExitWith(0);

result.StdOut.Should().Contain("Passed!");
result.StdOut.Should().MatchRegex(@"Passed:\s*2");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are two tests that run?


Directory.Delete(outputDirectory, true);
Directory.Delete(workingDirectory, true);
}

[Theory]
[MemberData(nameof(GetTemplateProjectsToTest))]
public void ProjectTemplate_CanBeInstalledAndTestsArePassing(string targetFramework, string projectTemplate, string language, bool runDotnetTest)
{
string testProjectName = GenerateTestProjectName();
string outputDirectory = CreateTemporaryFolder(folderName: "Home");
string workingDirectory = CreateTemporaryFolder();

// Create new test project: dotnet new <projectTemplate> -n <testProjectName> -f <targetFramework> -lang <language>
string args = $"{projectTemplate} -n {testProjectName} -f {targetFramework} -lang {language} -o {outputDirectory}";
new DotnetNewCommand(_log, args)
.WithCustomHive(outputDirectory).WithRawArguments()
.WithWorkingDirectory(workingDirectory)
.Execute()
.Should().ExitWith(0);

if (runDotnetTest)
{
var result = new DotnetTestCommand(_log, false)
.WithWorkingDirectory(outputDirectory)
.Execute(outputDirectory);

result.Should().ExitWith(0);

result.StdOut.Should().Contain("Passed!");
result.StdOut.Should().MatchRegex(@"Passed:\s*1");
}

Directory.Delete(outputDirectory, true);
Directory.Delete(workingDirectory, true);
}

[Theory]
[MemberData(nameof(GetMSTestAndPlaywrightCoverageAndRunnerCombinations))]
public void MSTestAndPlaywrightProjectTemplate_WithCoverageToolAndTestRunner_CanBeInstalledAndTestsArePassing(
string projectTemplate,
string targetFramework,
string language,
string coverageTool,
string testRunner,
bool runDotnetTest)
{
string testProjectName = GenerateTestProjectName();
string outputDirectory = CreateTemporaryFolder(folderName: "Home");
string workingDirectory = CreateTemporaryFolder();

// Create new test project: dotnet new <projectTemplate> -n <testProjectName> -f <targetFramework> -lang <language> --coverage-tool <coverageTool> --test-runner <testRunner>
string args = $"{projectTemplate} -n {testProjectName} -f {targetFramework} -lang {language} -o {outputDirectory} --coverage-tool {coverageTool} --test-runner {testRunner}";
new DotnetNewCommand(_log, args)
.WithCustomHive(outputDirectory).WithRawArguments()
.WithWorkingDirectory(workingDirectory)
.Execute()
.Should().ExitWith(0);

if (runDotnetTest)
{
var result = new DotnetTestCommand(_log, false)
.WithWorkingDirectory(outputDirectory)
.Execute(outputDirectory);

result.Should().ExitWith(0);

result.StdOut.Should().Contain("Passed!");
result.StdOut.Should().MatchRegex(@"Passed:\s*1");
}

Directory.Delete(outputDirectory, true);
Directory.Delete(workingDirectory, true);
}

private void AddItemToFsproj(string itemName, string outputDirectory, string projectName)
{
var fsproj = Path.Combine(outputDirectory, $"{projectName}.fsproj");
var lines = File.ReadAllLines(fsproj).ToList();

lines.Insert(lines.IndexOf(" <ItemGroup>") + 1, $@" <Compile Include=""{itemName}.fs""/>");
File.WriteAllLines(fsproj, lines);
}

private static string GenerateTestProjectName()
{
// Avoiding VB errors because root namespace must not start with number or contain dashes
return "Test_" + Guid.NewGuid().ToString("N");
}

public static IEnumerable<object[]> GetTemplateItemsToTest()
{
foreach (var targetFramework in SupportedTargetFrameworks)
{
foreach (var (projectTemplate, itemTemplate, languages, supportsTestingPlatform) in AvailableItemTemplates)
{
foreach (var language in languages)
{
yield return new object[] { targetFramework, projectTemplate, itemTemplate, language };
}
}
}
}

public static IEnumerable<object[]> GetTemplateProjectsToTest()
{
foreach (var targetFramework in SupportedTargetFrameworks)
{
foreach (var (projectTemplate, languages, runDotnetTest, supportsTestingPlatform) in AvailableProjectTemplates)
{
foreach (var language in languages)
{
yield return new object[] { targetFramework, projectTemplate, language, runDotnetTest };
}
}
}
}

public static IEnumerable<object[]> GetMSTestAndPlaywrightCoverageAndRunnerCombinations()
{
var coverageTools = new[] { "Microsoft.CodeCoverage", "coverlet" };
var testRunners = new[] { "VSTest", "Microsoft.Testing.Platform" };
foreach (var targetFramework in SupportedTargetFrameworks)
{
// mstest: all languages, runDotnetTest = true
foreach (var language in Languages.All)
{
foreach (var coverageTool in coverageTools)
{
foreach (var testRunner in testRunners)
{
yield return new object[] { "mstest", targetFramework, language, coverageTool, testRunner, true };
}
}
}
// mstest-playwright: only c#, runDotnetTest = false
foreach (var coverageTool in coverageTools)
{
foreach (var testRunner in testRunners)
{
yield return new object[] { "mstest-playwright", targetFramework, Languages.CSharp, coverageTool, testRunner, false };
}
}
}
}
}
}