From 1c592d935bf0b23629b158510ef0e28a0ff558d6 Mon Sep 17 00:00:00 2001 From: Arun Mahapatra Date: Mon, 29 Aug 2016 16:29:15 +0530 Subject: [PATCH] Refactor integration tests (#19) Add an integration test framework that runs tests for vstest.console.exe. Create test assets for integration tests. The test framework provides standard way to query for test assets, test runner etc. irrespective of IDE or CI. * Fix TestImpact smoke tests. Use full path in test environment. Sign testimpact listener assembly. * Add /tests and /testCaseFilter arguments for vstest.console. Refactor tests to check for all IArgumentProcessor implementations in command map. * Rename test asset to not conflict with actual unit test projects. * Fix inproc data collector interface implementation. * Add set and reset environment to test script. A few variables like TP_ROOT_DIR, TPT_TargetFramework, TPT_TargetRuntime are set before the test run. Fix the usage of environment variable in integration tests. * Update netci.groovy to run smoke tests. --- TestPlatform.sln | 34 +++ netci.groovy | 2 + scripts/stylecop.json | 11 + scripts/stylecop.test.ruleset | 7 + scripts/test.ps1 | 16 ++ .../EnableStaticLoggersArgumentProcessor.cs | 203 ------------------ .../Utilities/ArgumentProcessorFactory.cs | 48 ++--- .../DiscoveryTests.cs | 20 ++ .../ExecutionTests.cs | 33 +++ .../Microsoft.TestPlatform.SmokeTests.xproj} | 6 +- .../Properties/AssemblyInfo.cs | 4 +- .../TestImpactTests.cs | 63 ++++++ .../project.json | 45 ++++ .../End2EndTests.cs | 138 ------------ .../project.json | 35 --- .../IntegrationTestBase.cs} | 179 ++++++++------- .../IntegrationTestEnvironment.cs | 185 ++++++++++++++++ ...icrosoft.TestPlatform.TestUtilities.xproj} | 0 .../Properties/AssemblyInfo.cs | 0 .../project.json | 33 +++ .../SampleUnitTestProject.csproj | 103 --------- .../SampleUnitTestProject/packages.config | 5 - .../TestImpactListener.Tests.csproj | 67 ------ .../Properties/AssemblyInfo.cs | 0 .../SimpleTestProject/SimpleTestProject.xproj | 21 ++ .../SimpleTestProject}/UnitTest1.cs | 0 .../SimpleTestProject}/project.json | 4 +- .../TestImpactListener.Tests/Class1.cs | 11 +- .../Properties/AssemblyInfo.cs | 0 .../TestImpactListener.Tests/TITestDllKey.snk | Bin .../TestImpactListener.Tests.xproj | 19 ++ .../TestImpactListener.Tests/app.config | 0 .../TestImpactListener.Tests/project.json | 33 +++ .../ExecutionManager.cs | 99 --------- ...ableStaticLoggersArgumentProcessorTests.cs | 44 ---- .../ArgumentProcessorFactoryTests.cs | 74 ++++--- 36 files changed, 701 insertions(+), 841 deletions(-) create mode 100644 scripts/stylecop.json create mode 100644 scripts/stylecop.test.ruleset delete mode 100644 src/vstest.console/Processors/EnableStaticLoggersArgumentProcessor.cs create mode 100644 test/Integration/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs create mode 100644 test/Integration/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs rename test/{Microsoft.TestPlatform.End2EndTests/Microsoft.TestPlatform.End2EndTests.xproj => Integration/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.xproj} (81%) rename test/{Microsoft.TestPlatform.End2EndTests => Integration/Microsoft.TestPlatform.SmokeTests}/Properties/AssemblyInfo.cs (82%) create mode 100644 test/Integration/Microsoft.TestPlatform.SmokeTests/TestImpactTests.cs create mode 100644 test/Integration/Microsoft.TestPlatform.SmokeTests/project.json delete mode 100644 test/Microsoft.TestPlatform.End2EndTests/End2EndTests.cs delete mode 100644 test/Microsoft.TestPlatform.End2EndTests/project.json rename test/{TestPlatform.TestUtilities/VSTestConsoleTestBase.cs => Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs} (60%) create mode 100644 test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs rename test/{TestPlatform.TestUtilities/TestPlatform.TestUtilities.xproj => Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.xproj} (100%) rename test/{TestPlatform.TestUtilities => Microsoft.TestPlatform.TestUtilities}/Properties/AssemblyInfo.cs (100%) create mode 100644 test/Microsoft.TestPlatform.TestUtilities/project.json delete mode 100644 test/Samples/SampleUnitTestProject/SampleUnitTestProject.csproj delete mode 100644 test/Samples/SampleUnitTestProject/packages.config delete mode 100644 test/Samples/TestImpactListener.Tests/TestImpactListener.Tests.csproj rename test/{Samples/SampleUnitTestProject => TestAssets/SimpleTestProject}/Properties/AssemblyInfo.cs (100%) create mode 100644 test/TestAssets/SimpleTestProject/SimpleTestProject.xproj rename test/{Samples/SampleUnitTestProject => TestAssets/SimpleTestProject}/UnitTest1.cs (100%) rename test/{TestPlatform.TestUtilities => TestAssets/SimpleTestProject}/project.json (88%) rename test/{Samples => TestAssets}/TestImpactListener.Tests/Class1.cs (86%) rename test/{Samples => TestAssets}/TestImpactListener.Tests/Properties/AssemblyInfo.cs (100%) rename test/{Samples => TestAssets}/TestImpactListener.Tests/TITestDllKey.snk (100%) create mode 100644 test/TestAssets/TestImpactListener.Tests/TestImpactListener.Tests.xproj rename test/{Samples => TestAssets}/TestImpactListener.Tests/app.config (100%) create mode 100644 test/TestAssets/TestImpactListener.Tests/project.json delete mode 100644 test/TestPlatform.TestUtilities/ExecutionManager.cs delete mode 100644 test/vstest.console.UnitTests/Processors/EnableStaticLoggersArgumentProcessorTests.cs diff --git a/TestPlatform.sln b/TestPlatform.sln index 95cd56cb8e..72c4f295e8 100644 --- a/TestPlatform.sln +++ b/TestPlatform.sln @@ -83,6 +83,18 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "datacollector.x86", "src\da EndProject Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "datacollector.x86.UnitTests", "test\datacollector.x86.UnitTests\datacollector.x86.UnitTests.xproj", "{00AA21F3-31E4-4748-AC0B-C4EADB41CA24}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Integration", "Integration", "{6DA46479-C688-4296-A6E7-F20C20CDA3A9}" +EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.TestPlatform.SmokeTests", "test\Integration\Microsoft.TestPlatform.SmokeTests\Microsoft.TestPlatform.SmokeTests.xproj", "{C1497516-ACB5-49AF-A676-51DB65FF8252}" +EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.TestPlatform.TestUtilities", "test\Microsoft.TestPlatform.TestUtilities\Microsoft.TestPlatform.TestUtilities.xproj", "{F52A4D48-90B3-4004-8C98-D2786CC9B965}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TestAssets", "TestAssets", "{50D7D355-08F6-4DFD-AEAA-9BCE41C94C18}" +EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "TestImpactListener.Tests", "test\TestAssets\TestImpactListener.Tests\TestImpactListener.Tests.xproj", "{D7F7A9F5-5646-44E7-990C-500844E9272E}" +EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "SimpleTestProject", "test\TestAssets\SimpleTestProject\SimpleTestProject.xproj", "{0CC51637-B665-47B0-A093-042D31785928}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -205,6 +217,22 @@ Global {00AA21F3-31E4-4748-AC0B-C4EADB41CA24}.Debug|Any CPU.Build.0 = Debug|Any CPU {00AA21F3-31E4-4748-AC0B-C4EADB41CA24}.Release|Any CPU.ActiveCfg = Release|Any CPU {00AA21F3-31E4-4748-AC0B-C4EADB41CA24}.Release|Any CPU.Build.0 = Release|Any CPU + {C1497516-ACB5-49AF-A676-51DB65FF8252}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C1497516-ACB5-49AF-A676-51DB65FF8252}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C1497516-ACB5-49AF-A676-51DB65FF8252}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C1497516-ACB5-49AF-A676-51DB65FF8252}.Release|Any CPU.Build.0 = Release|Any CPU + {F52A4D48-90B3-4004-8C98-D2786CC9B965}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F52A4D48-90B3-4004-8C98-D2786CC9B965}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F52A4D48-90B3-4004-8C98-D2786CC9B965}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F52A4D48-90B3-4004-8C98-D2786CC9B965}.Release|Any CPU.Build.0 = Release|Any CPU + {D7F7A9F5-5646-44E7-990C-500844E9272E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D7F7A9F5-5646-44E7-990C-500844E9272E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D7F7A9F5-5646-44E7-990C-500844E9272E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D7F7A9F5-5646-44E7-990C-500844E9272E}.Release|Any CPU.Build.0 = Release|Any CPU + {0CC51637-B665-47B0-A093-042D31785928}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0CC51637-B665-47B0-A093-042D31785928}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0CC51637-B665-47B0-A093-042D31785928}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0CC51637-B665-47B0-A093-042D31785928}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -241,5 +269,11 @@ Global {3572E78C-5AA5-4F68-876D-FC5322677263} = {D8EF073C-279A-4279-912D-E9D4B0635E17} {00DFB5C7-3850-4A65-986B-713F200482D4} = {D8EF073C-279A-4279-912D-E9D4B0635E17} {00AA21F3-31E4-4748-AC0B-C4EADB41CA24} = {463031A2-7F16-4E38-9944-1F5161D04933} + {6DA46479-C688-4296-A6E7-F20C20CDA3A9} = {463031A2-7F16-4E38-9944-1F5161D04933} + {C1497516-ACB5-49AF-A676-51DB65FF8252} = {6DA46479-C688-4296-A6E7-F20C20CDA3A9} + {F52A4D48-90B3-4004-8C98-D2786CC9B965} = {463031A2-7F16-4E38-9944-1F5161D04933} + {50D7D355-08F6-4DFD-AEAA-9BCE41C94C18} = {463031A2-7F16-4E38-9944-1F5161D04933} + {D7F7A9F5-5646-44E7-990C-500844E9272E} = {50D7D355-08F6-4DFD-AEAA-9BCE41C94C18} + {0CC51637-B665-47B0-A093-042D31785928} = {50D7D355-08F6-4DFD-AEAA-9BCE41C94C18} EndGlobalSection EndGlobal diff --git a/netci.groovy b/netci.groovy index 19fcae1a91..6c9f4c021e 100644 --- a/netci.groovy +++ b/netci.groovy @@ -16,12 +16,14 @@ def branch = GithubBranchName // Define your build/test strings here def buildString = """call build.cmd -c ${configuration}""" def testString = """call test.cmd -c ${configuration}""" + def smoketestString = """call test.cmd -c ${configuration} -p smoke""" // Create a new job for windows build def newJob = job(newJobName) { steps { batchFile(buildString) batchFile(testString) + batchFile(smoketestString) } } diff --git a/scripts/stylecop.json b/scripts/stylecop.json new file mode 100644 index 0000000000..dabdc2c167 --- /dev/null +++ b/scripts/stylecop.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", + "settings": { + "documentationRules": { + "companyName": "Microsoft Corporation", + "copyrightText": "Copyright (c) {companyName}. All rights reserved.\nLicensed under the MIT license. See LICENSE file in the project root for full license information.", + "xmlHeader": false, + "fileNamingConvention": "metadata" + }, + } +} diff --git a/scripts/stylecop.test.ruleset b/scripts/stylecop.test.ruleset new file mode 100644 index 0000000000..ac7a04afad --- /dev/null +++ b/scripts/stylecop.test.ruleset @@ -0,0 +1,7 @@ + + + + + + + diff --git a/scripts/test.ps1 b/scripts/test.ps1 index 44974cbe59..9a1d697fbb 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -108,9 +108,12 @@ function Invoke-Test } elseif (!($testContainerName -match $Script:TPT_Pattern)) { Write-Log ".. . $testContainerName doesn't match test container pattern '$($Script:TPT_Pattern)'. Skipped from run." } else { + Set-TestEnvironment + Write-Verbose "vstest.console.exe $testContainerPath /platform:$testArchitecture /testAdapterPath:$testAdapterPath" $output = & $vstestConsolePath $testContainerPath /platform:$testArchitecture /testAdapterPath:"$testAdapterPath" + Reset-TestEnvironment #Write-Verbose "$dotnetExe test $_ --configuration $Configuration" #& $dotnetExe test $_ --configuration $Configuration @@ -128,6 +131,7 @@ function Invoke-Test continue } } + } Write-Log ".. Test: Complete." @@ -174,6 +178,18 @@ function Set-ScriptFailed $Script:ScriptFailed = $true } +function Set-TestEnvironment +{ + $env:TPT_TargetFramework = $Script:TPT_TargetFramework + $env:TPT_TargetRuntime = $Script:TPT_TargetRuntime +} + +function Reset-TestEnvironment +{ + $env:TPT_TargetFramework = $null + $env:TPT_TargetRuntime = $null +} + # Execute build $timer = Start-Timer Write-Log "Build started: args = '$args'" diff --git a/src/vstest.console/Processors/EnableStaticLoggersArgumentProcessor.cs b/src/vstest.console/Processors/EnableStaticLoggersArgumentProcessor.cs deleted file mode 100644 index 9f7d99f338..0000000000 --- a/src/vstest.console/Processors/EnableStaticLoggersArgumentProcessor.cs +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors -{ - using Microsoft.VisualStudio.TestPlatform.Common.Logging; - using System; - using System.Collections.Generic; - - /// - /// An argument processor that enables all configured loggers. - /// - internal class EnableStaticLoggersArgumentProcessor : IArgumentProcessor - { - #region Constants - - /// - /// The command name. - /// - public const string CommandName = "/EnableStaticLoggers"; - - #endregion - - private Lazy metadata; - - private Lazy executor; - - /// - /// Gets or sets the executor. - /// - public Lazy Executor - { - get - { - if (this.executor == null) - { - this.executor = new Lazy(() => new EnableStaticLoggersArgumentExecutor()); - } - - return this.executor; - } - - set - { - this.executor = value; - } - } - - /// - /// Gets the metadata. - /// - public Lazy Metadata - { - get - { - if (this.metadata == null) - { - this.metadata = new Lazy(() => new EnableStaticLoggersArgumentProcessorCapabilities()); - } - - return this.metadata; - } - } - } - - /// - /// The argument capabilities. - /// - internal class EnableStaticLoggersArgumentProcessorCapabilities : BaseArgumentProcessorCapabilities - { - /// - /// Gets the command name. - /// - public override string CommandName => EnableStaticLoggersArgumentProcessor.CommandName; - - /// - /// Gets a value indicating whether allow multiple. - /// - public override bool AllowMultiple => false; - - /// - /// Gets a value indicating whether is action. - /// - public override bool IsAction => false; - - /// - /// Gets the priority. - /// - public override ArgumentProcessorPriority Priority => ArgumentProcessorPriority.Logging; - } - - /// - /// The argument e xecutor. - /// - internal class EnableStaticLoggersArgumentExecutor : IArgumentExecutor - { - #region Constructor - - /// - /// Default constructor. - /// - public EnableStaticLoggersArgumentExecutor() - { - } - - #endregion - - #region IArgumentProcessor - - /// - /// Enables the configured loggers. - /// - /// The argument used to initialize the processor. - /// is not used. - public void Initialize(string argument) - { - var logManager = TestLoggerManager.Instance; -#if NET451 - foreach (var logger in TestPlatFormSettings.Loggers.Cast()) - { - string loggerIdentifier = null; - Dictionary parameters = null; - bool parseSucceeded = LoggerUtilities.TryParseLoggerArgument(logger.Uri, out loggerIdentifier, out parameters); - - if (parseSucceeded) - { - Uri loggerUri = null; - try - { - loggerUri = new Uri(loggerIdentifier); - } - catch (UriFormatException) - { - Logger.SendMessage( - TestMessageLevel.Error, - String.Format( - CultureInfo.CurrentUICulture, - Resources.LoggerUriInvalid, - logger.Uri)); - } - - if (loggerUri != null) - { - try - { - logManager.AddLogger(loggerUri, null); - } - catch (InvalidOperationException e) - { - Logger.SendMessage( - TestMessageLevel.Error, - e.Message); - } - } - } - else - { - Logger.SendMessage( - TestMessageLevel.Error, - String.Format( - CultureInfo.CurrentUICulture, - Resources.LoggerUriInvalid, - logger.Uri)); - } - } -#else - //// todo : write code after getting clarity on config file format in dotnet core -#endif - - } - - public ArgumentProcessorResult Execute() - { - // Nothing to do. - return ArgumentProcessorResult.Success; - } - - #endregion - - #region Private Methods - -#if NET451 - /// - /// The settings for the test platform. - /// - private static TestPlatformSection TestPlatFormSettings - { - get - { - var section = ConfigurationManager.GetSection(TestPlatformSection.SectionName) as TestPlatformSection; - - if (section == null) - { - section = new TestPlatformSection(); - } - - return section; - } - } -#endif - #endregion - - } -} diff --git a/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs b/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs index 04e3ef8fb8..7d3d7dce9f 100644 --- a/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs +++ b/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs @@ -28,7 +28,7 @@ internal class ArgumentProcessorFactory /// /// Available argument processors. /// - private IEnumerable argumentProcessors; + private readonly IEnumerable argumentProcessors; private Dictionary commandToProcessorMap; private Dictionary specialCommandToProcessorMap; @@ -45,8 +45,7 @@ internal class ArgumentProcessorFactory /// /// This is not public because the static Create method should be used to access the instance. /// - protected ArgumentProcessorFactory( - IEnumerable argumentProcessors) + protected ArgumentProcessorFactory(IEnumerable argumentProcessors) { Contract.Requires(argumentProcessors != null); this.argumentProcessors = argumentProcessors; @@ -56,22 +55,6 @@ protected ArgumentProcessorFactory( #region Static Methods - public static IEnumerable DefaultArgumentProcessors => new List { - new HelpArgumentProcessor(), - new TestSourceArgumentProcessor(), - new ListTestsArgumentProcessor(), - new RunTestsArgumentProcessor(), - new TestAdapterPathArgumentProcessor(), - new OutputArgumentProcessor(), - new BuildBasePathArgumentProcessor(), - new ConfigurationArgumentProcessor(), - new PortArgumentProcessor(), - new RunSettingsArgumentProcessor(), - new PlatformArgumentProcessor(), - new EnableLoggerArgumentProcessor(), - new ParallelArgumentProcessor() - }; - /// /// Creates ArgumentProcessorFactory. /// @@ -82,15 +65,6 @@ internal static ArgumentProcessorFactory Create() return new ArgumentProcessorFactory(DefaultArgumentProcessors); } - /// - /// Creates ArgumentProcessorFactory with given list of processors - /// - /// ArgumentProcessorFactory. - internal static ArgumentProcessorFactory Create(IEnumerable processorList) - { - return new ArgumentProcessorFactory(processorList); - } - #endregion #region Properties @@ -203,6 +177,24 @@ public IEnumerable GetArgumentProcessorsToAlwaysExecute() #region Private Methods + private static IEnumerable DefaultArgumentProcessors => new List { + new HelpArgumentProcessor(), + new TestSourceArgumentProcessor(), + new ListTestsArgumentProcessor(), + new RunTestsArgumentProcessor(), + new RunSpecificTestsArgumentProcessor(), + new TestAdapterPathArgumentProcessor(), + new TestCaseFilterArgumentProcessor(), + new OutputArgumentProcessor(), + new BuildBasePathArgumentProcessor(), + new ConfigurationArgumentProcessor(), + new PortArgumentProcessor(), + new RunSettingsArgumentProcessor(), + new PlatformArgumentProcessor(), + new EnableLoggerArgumentProcessor(), + new ParallelArgumentProcessor() + }; + /// /// Checks the provided argument to see if it could be a test source and returns the test source /// argument processor if it could be a test source. diff --git a/test/Integration/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs b/test/Integration/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs new file mode 100644 index 0000000000..5bf266b13b --- /dev/null +++ b/test/Integration/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.SmokeTests +{ + using Microsoft.TestPlatform.TestUtilities; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class DiscoveryTests : IntegrationTestBase + { + [TestMethod] + public void DiscoverAllTests() + { + this.InvokeVsTestForDiscovery(this.GetSampleTestAssembly(), this.GetTestAdapterPath()); + var listOfTests = new string[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; + this.ValidateDiscoveredTests(listOfTests); + } + } +} \ No newline at end of file diff --git a/test/Integration/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs b/test/Integration/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs new file mode 100644 index 0000000000..36aa2cbd11 --- /dev/null +++ b/test/Integration/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.SmokeTests +{ + using Microsoft.TestPlatform.TestUtilities; + + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class ExecutionTests : IntegrationTestBase + { + [TestMethod] + public void RunAllTestExecution() + { + this.InvokeVsTestForExecution(this.GetSampleTestAssembly(), this.GetTestAdapterPath()); + this.ValidateSummaryStatus(1, 1, 1); + this.ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); + this.ValidateFailedTests("SampleUnitTestProject.UnitTest1.FailingTest"); + this.ValidateSkippedTests("SampleUnitTestProject.UnitTest1.SkippingTest"); + } + + [TestMethod] + public void RunSelectedTests() + { + var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty); + arguments = string.Concat(arguments, " /Tests:PassingTest"); + this.InvokeVsTest(arguments); + this.ValidateSummaryStatus(1, 0, 0); + this.ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); + } + } +} diff --git a/test/Microsoft.TestPlatform.End2EndTests/Microsoft.TestPlatform.End2EndTests.xproj b/test/Integration/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.xproj similarity index 81% rename from test/Microsoft.TestPlatform.End2EndTests/Microsoft.TestPlatform.End2EndTests.xproj rename to test/Integration/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.xproj index 8a89f794a5..856c216d79 100644 --- a/test/Microsoft.TestPlatform.End2EndTests/Microsoft.TestPlatform.End2EndTests.xproj +++ b/test/Integration/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.xproj @@ -7,9 +7,9 @@ c1497516-acb5-49af-a676-51db65ff8252 - TestingMSTest - ..\..\artifacts\obj\$(MSBuildProjectName) - ..\..\artifacts\ + Microsoft.TestPlatform.SmokeTests + ..\..\..\artifacts\obj\$(MSBuildProjectName) + ..\..\..\artifacts\ v4.5.2 diff --git a/test/Microsoft.TestPlatform.End2EndTests/Properties/AssemblyInfo.cs b/test/Integration/Microsoft.TestPlatform.SmokeTests/Properties/AssemblyInfo.cs similarity index 82% rename from test/Microsoft.TestPlatform.End2EndTests/Properties/AssemblyInfo.cs rename to test/Integration/Microsoft.TestPlatform.SmokeTests/Properties/AssemblyInfo.cs index a91bf84dca..23906b17bb 100644 --- a/test/Microsoft.TestPlatform.End2EndTests/Properties/AssemblyInfo.cs +++ b/test/Integration/Microsoft.TestPlatform.SmokeTests/Properties/AssemblyInfo.cs @@ -1,5 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + using System.Reflection; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following diff --git a/test/Integration/Microsoft.TestPlatform.SmokeTests/TestImpactTests.cs b/test/Integration/Microsoft.TestPlatform.SmokeTests/TestImpactTests.cs new file mode 100644 index 0000000000..d57bbffe78 --- /dev/null +++ b/test/Integration/Microsoft.TestPlatform.SmokeTests/TestImpactTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.SmokeTests +{ + using System.IO; + + using Microsoft.TestPlatform.TestUtilities; + + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class TestImpactTests : IntegrationTestBase + { + [TestMethod] + public void RunAllWithTestImpactSettings() + { + var runSettings = this.GetInProcDataCollectionRunsettingsFile(); + + this.InvokeVsTestForExecution(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), runSettings); + this.ValidateSummaryStatus(1, 1, 1); + this.ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); + this.ValidateFailedTests("SampleUnitTestProject.UnitTest1.FailingTest"); + this.ValidateSkippedTests("SampleUnitTestProject.UnitTest1.SkippingTest"); + + ValidateInProcDataCollectionOutput(); + } + + private static void ValidateInProcDataCollectionOutput() + { + var fileName = Path.Combine(Path.GetTempPath(), "inproctest.txt"); + Assert.IsTrue(File.Exists(fileName), "Datacollector test file doesn't exist: {0}.", fileName); + var actual = File.ReadAllText(fileName); + var expected = @"TestSessionStart : 4312 TestCaseStart : PassingTest TestCaseEnd : PassingTest TestCaseStart : FailingTest TestCaseEnd : FailingTest TestCaseStart : SkippingTest TestCaseEnd : SkippingTest TestSessionEnd"; + actual = actual.Replace(" ", string.Empty).Replace("\r\n", string.Empty); + expected = expected.Replace(" ", string.Empty).Replace("\r\n", string.Empty); + Assert.AreEqual(expected, actual); + } + + private string GetInProcDataCollectionRunsettingsFile() + { + var runSettings = Path.Combine(Path.GetDirectoryName(this.GetSampleTestAssembly()), "runsettingstest.runsettings"); + var testEnvironment = new IntegrationTestEnvironment(); + var inprocasm = testEnvironment.GetTestAsset("TestImpactListener.Tests.dll"); + var fileContents = @" + + + + + 4312 + + + + + "; + + fileContents = string.Format(fileContents, "'" + inprocasm + "'"); + File.WriteAllText(runSettings, fileContents); + + return runSettings; + } + } +} \ No newline at end of file diff --git a/test/Integration/Microsoft.TestPlatform.SmokeTests/project.json b/test/Integration/Microsoft.TestPlatform.SmokeTests/project.json new file mode 100644 index 0000000000..5deec2ea84 --- /dev/null +++ b/test/Integration/Microsoft.TestPlatform.SmokeTests/project.json @@ -0,0 +1,45 @@ +{ + "version": "15.0.0-*", + + "buildOptions": { + "delaySign": true, + "keyFile": "../../../scripts/key.snk", + "warningsAsErrors": true, + "additionalArguments": [ "/ruleset:../../../scripts/stylecop.test.ruleset", "/additionalFile:../../../scripts/stylecop.json" ] + }, + + "dependencies": { + "MSTest.TestFramework": "1.0.0-preview", + "MSTest.TestAdapter": { + "version": "1.0.3-preview", + "exclude": "compile" + }, + "StyleCop.Analyzers": { + "version": "1.0.0", + "type": "build" + }, + "Microsoft.TestPlatform.TestUtilities": "15.0.0-*" + }, + + "frameworks": { + "netcoreapp1.0": { + "imports": [ + "dnxcore50", + "portable-net45+win8" + ], + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0" + }, + "dotnet-test-mstest": { + "version": "1.0.1-preview", + "exclude": "compile" + } + } + }, + "net46": { } + }, + + "testRunner": "mstest" +} \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.End2EndTests/End2EndTests.cs b/test/Microsoft.TestPlatform.End2EndTests/End2EndTests.cs deleted file mode 100644 index 8778002e53..0000000000 --- a/test/Microsoft.TestPlatform.End2EndTests/End2EndTests.cs +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace TestingMSTest -{ - using System.IO; - - using Microsoft.VisualStudio.TestTools.UnitTesting; - - using TestPlatform.TestUtilities; - - [TestClass] - public class End2EndTests:VsTestConsoleTestBase - { -#if DEBUG - private const string TestAssemblyRelativePath = @"Samples\SampleUnitTestProject\bin\Debug\SampleUnitTestProject.dll"; -#else - private const string TestAssemblyRelativePath = @"Samples\SampleUnitTestProject\bin\Release\SampleUnitTestProject.dll"; -#endif - private const string TestAdapterRelativePath = @"Samples\packages\MSTest.TestAdapter.1.0.0-preview\build"; - - private string testAssembly; - - private string testAdapter; - - [TestInitialize] - public void InitializeTests() - { - this.testAssembly = GetSampleTestAssembly(); - this.testAdapter = GetTestAdapterPath(); - } - - - - [TestMethod] - [TestCategory("EndToEnd")] - public void RunAllTestExecution() - { - this.InvokeVsTestForExecution(this.testAssembly, this.testAdapter); - this.ValidateSummaryStatus(1, 1, 1); - this.ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); - this.ValidateFailedTests("SampleUnitTestProject.UnitTest1.FailingTest"); - this.ValidateSkippedTests("SampleUnitTestProject.UnitTest1.SkippingTest"); - } - - [TestMethod] - [TestCategory("EndToEnd")] - public void DiscoverAllTests() - { - this.InvokeVsTestForDiscovery(this.testAssembly, this.testAdapter); - var listOfTests = new string[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; - this.ValidateDiscoveredTests(listOfTests); - } - - [TestMethod] - [TestCategory("EndToEnd")] - public void RunSelectedTests() - { - var arguments = PrepareArguments(this.testAssembly, this.testAdapter, string.Empty); - arguments = string.Concat(arguments, " /Tests:PassingTest"); - this.InvokeVsTest(arguments); - this.ValidateSummaryStatus(1, 0, 0); - this.ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); - } - - [TestMethod] - [TestCategory("EndToEnd")] - public void RunAllWithTestImpactSettings() - { - var runSettings = GetInProcDataCollectionRunsettignsFile(); - - this.InvokeVsTestForExecution(this.testAssembly, this.testAdapter, runSettings); - this.ValidateSummaryStatus(1, 1, 1); - this.ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); - this.ValidateFailedTests("SampleUnitTestProject.UnitTest1.FailingTest"); - this.ValidateSkippedTests("SampleUnitTestProject.UnitTest1.SkippingTest"); - - ValidateInProcDataCollectionOutput(); - } - -#region PrivateMethods - private static string GetSampleTestAssembly() - { - var currentDirectoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory()); - - var testAssembly = Path.Combine(currentDirectoryInfo.Parent?.FullName, TestAssemblyRelativePath); - - return testAssembly; - } - private static string GetTestAdapterPath() - { - var currentDirectoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory()); - - var testAdapterPath = Path.Combine(currentDirectoryInfo.Parent?.FullName, TestAdapterRelativePath); - - return testAdapterPath; - } - - private static string GetInProcDataCollectionRunsettignsFile() - { - var runSettings = Path.Combine(Path.GetDirectoryName(GetSampleTestAssembly()), "runsettingstest.runsettings"); -#if DEBUG - var realtiveInProcPath = @"Samples\TestImpactListener.Tests\bin\Debug\TestImpactListener.Tests.dll"; -#else - var realtiveInProcPath = @"Samples\TestImpactListener.Tests\bin\Release\TestImpactListener.Tests.dll"; -#endif - var currentDirectoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory()); - var inprocasm = Path.Combine(currentDirectoryInfo.Parent?.FullName, realtiveInProcPath); - var fileContents = @" - - - - - 4312 - - - - - "; - - fileContents = string.Format(fileContents, "'" + inprocasm + "'"); - File.WriteAllText(runSettings, fileContents); - - return runSettings; - } - - private static void ValidateInProcDataCollectionOutput() - { - var fileName = Path.Combine(Path.GetTempPath(), "inproctest.txt"); - Assert.IsTrue(File.Exists(fileName)); - var actual = File.ReadAllText(fileName); - var expected = @"TestSessionStart : 4312 TestCaseStart : PassingTest TestCaseEnd : PassingTest TestCaseStart : FailingTest TestCaseEnd : FailingTest TestCaseStart : SkippingTest TestCaseEnd : SkippingTest TestSessionEnd"; - actual = actual.Replace(" ", "").Replace("\r\n", ""); - expected = expected.Replace(" ", "").Replace("\r\n", ""); - Assert.AreEqual(expected, actual); - } -#endregion - } -} diff --git a/test/Microsoft.TestPlatform.End2EndTests/project.json b/test/Microsoft.TestPlatform.End2EndTests/project.json deleted file mode 100644 index 35ebca1030..0000000000 --- a/test/Microsoft.TestPlatform.End2EndTests/project.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "version": "1.0.0-*", - - - - "dependencies": { - "dotnet-test-mstest": { - "version": "1.0.1-preview", - "exclude": "compile" - }, - - "MSTest.TestFramework": "1.0.0-preview", - "TestPlatform.TestUtilities": "1.0.0-*" - - }, - - - "frameworks": { - "netcoreapp1.0": { - "imports": [ - "dnxcore50", - "portable-net45+win8" - ], - "dependencies": { - "Microsoft.NETCore.App": { - "type": "platform", - "version": "1.0.0-rc2-3002702" - } - } - }, - "net461": { } - }, - - "testRunner": "mstest" - } diff --git a/test/TestPlatform.TestUtilities/VSTestConsoleTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs similarity index 60% rename from test/TestPlatform.TestUtilities/VSTestConsoleTestBase.cs rename to test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 2dfa4e82df..1236fc97ea 100644 --- a/test/TestPlatform.TestUtilities/VSTestConsoleTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -1,46 +1,79 @@ // Copyright (c) Microsoft. All rights reserved. -namespace TestPlatform.TestUtilities +namespace Microsoft.TestPlatform.TestUtilities { + using System; + using System.Diagnostics; using System.Linq; using System.Text.RegularExpressions; using Microsoft.VisualStudio.TestTools.UnitTesting; - + /// - /// The VS test console test base. + /// Base class for integration tests. /// - public class VsTestConsoleTestBase + public class IntegrationTestBase { private const string TestSummaryStatusMessageFormat = "Total tests: {0}. Passed: {1}. Failed: {2}. Skipped: {3}"; private string standardTestOutput = string.Empty; - private string standardTestError = string.Empty; + private readonly IntegrationTestEnvironment testEnvironment; + + private const string TestAdapterRelativePath = @"MSTest.TestAdapter\1.0.3-preview\build\_common"; + + public IntegrationTestBase() + { + this.testEnvironment = new IntegrationTestEnvironment(); + } + /// - /// The invoke VS test. + /// Prepare arguments for vstest.console.exe. /// - /// - /// The arguments. - /// + /// Name of the test assembly. + /// Path to test adapter. + /// Text of run settings. + /// Command line arguments string. + public static string PrepareArguments(string testAssembly, string testAdapterPath, string runSettings) + { + string arguments; + if (string.IsNullOrWhiteSpace(runSettings)) + { + arguments = string.Concat("\"", testAssembly, "\"", " /testadapterpath:\"", testAdapterPath, "\""); + } + else + { + arguments = string.Concat( + "\"", + testAssembly, + "\"", + " /testadapterpath:\"", + testAdapterPath, + "\"", + " /settings:\"", + runSettings, + "\""); + } + + return arguments; + } + + /// + /// Invokes vstest.console with specified arguments. + /// + /// Arguments provided to vstest.console.exe public void InvokeVsTest(string arguments) { - ExecutionManager.Execute(arguments, out this.standardTestOutput, out this.standardTestError); + Execute(arguments, out this.standardTestOutput, out this.standardTestError); this.FormatStandardOutCome(); } /// - /// The invoke VS test. + /// Invokes vstest.console to execute tests in a test assembly. /// - /// - /// The test assembly. - /// - /// - /// The test Adapter Path. - /// - /// - /// The run Settings. - /// + /// A test assembly. + /// Path to test adapters. + /// Run settings for execution. public void InvokeVsTestForExecution(string testAssembly, string testAdapterPath, string runSettings = "") { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings); @@ -48,17 +81,11 @@ public void InvokeVsTestForExecution(string testAssembly, string testAdapterPath } /// - /// The invoke VS test for discovery. + /// Invokes vstest.console to discover tests in a test assembly. "/listTests" is appended to the arguments. /// - /// - /// The test assembly. - /// - /// - /// The test adapter path. - /// - /// - /// The run Settings. - /// + /// A test assembly. + /// Path to test adapters. + /// Run settings for execution. public void InvokeVsTestForDiscovery(string testAssembly, string testAdapterPath, string runSettings = "") { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings); @@ -67,11 +94,11 @@ public void InvokeVsTestForDiscovery(string testAssembly, string testAdapterPath } /// - /// Validate if the overall Test count and results are matching. + /// Validate if the overall test count and results are matching. /// - /// passed test count - /// failed test count - /// skipped test count + /// Passed test count + /// Failed test count + /// Skipped test count public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, int skippedTestsCount) { var summaryStatus = string.Format( @@ -87,7 +114,7 @@ public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, in /// /// Validates if the test results have the specified set of passed tests. /// - /// The set of passed tests. + /// Set of passed tests. /// Provide the full test name similar to this format SampleTest.TestCode.TestMethodPass. public void ValidatePassedTests(params string[] passedTests) { @@ -102,7 +129,7 @@ public void ValidatePassedTests(params string[] passedTests) /// /// Validates if the test results have the specified set of failed tests. /// - /// The set of failed tests. + /// Set of failed tests. /// /// Provide the full test name similar to this format SampleTest.TestCode.TestMethodFailed. /// Also validates whether these tests have stack trace info. @@ -136,11 +163,9 @@ public void ValidateSkippedTests(params string[] skippedTests) } /// - /// The validate discovered tests. + /// Validate if the discovered tests list contains provided tests. /// - /// - /// The discovered tests list. - /// + /// List of tests expected to be discovered. public void ValidateDiscoveredTests(params string[] discoveredTestsList) { foreach (var test in discoveredTestsList) @@ -151,50 +176,21 @@ public void ValidateDiscoveredTests(params string[] discoveredTestsList) } } - /// - /// The prepare arguments. - /// - /// - /// The test assembly. - /// - /// - /// The test adapter path. - /// - /// - /// The run settings. - /// - /// - /// The . - /// - public static string PrepareArguments(string testAssembly, string testAdapterPath, string runSettings) + protected string GetSampleTestAssembly() { - string arguments; - if (string.IsNullOrWhiteSpace(runSettings)) - { - arguments = string.Concat("\"", testAssembly, "\"", " /testadapterpath:\"", testAdapterPath, "\""); - } - else - { - arguments = string.Concat( - "\"", - testAssembly, - "\"", - " /testadapterpath:\"", - testAdapterPath, - "\"", - " /settings:\"", - runSettings, - "\""); - } + return this.testEnvironment.GetTestAsset("SimpleTestProject.dll"); + } - return arguments; + protected string GetTestAdapterPath() + { + return this.testEnvironment.GetNugetPackage(TestAdapterRelativePath); } /// /// Gets the test method name from full name. /// - /// test case complete name - /// just the test name + /// Fully qualified name of the test. + /// Simple name of the test. private static string GetTestMethodName(string testFullName) { string testMethodName = string.Empty; @@ -208,10 +204,37 @@ private static string GetTestMethodName(string testFullName) return testMethodName; } + private static void Execute(string args, out string stdOut, out string stdError) + { + var testEnvironment = new IntegrationTestEnvironment(); + + using (Process vstestconsole = new Process()) + { + Console.WriteLine("IntegrationTestBase.Execute: Starting vstest.console.exe"); + vstestconsole.StartInfo.FileName = testEnvironment.GetConsoleRunnerPath(); + vstestconsole.StartInfo.Arguments = args; + vstestconsole.StartInfo.UseShellExecute = false; + //vstestconsole.StartInfo.WorkingDirectory = testEnvironment.PublishDirectory; + vstestconsole.StartInfo.RedirectStandardError = true; + vstestconsole.StartInfo.RedirectStandardOutput = true; + vstestconsole.StartInfo.CreateNoWindow = true; + + Console.WriteLine("IntegrationTestBase.Execute: Path = {0}", vstestconsole.StartInfo.FileName); + Console.WriteLine("IntegrationTestBase.Execute: Arguments = {0}", vstestconsole.StartInfo.Arguments); + + vstestconsole.Start(); + stdError = vstestconsole.StandardError.ReadToEnd(); + stdOut = vstestconsole.StandardOutput.ReadToEnd(); + + vstestconsole.WaitForExit(); + Console.WriteLine("IntegrationTestBase.Execute: Stopped vstest.console.exe. Exit code = {0}", vstestconsole.ExitCode); + } + } + private void FormatStandardOutCome() { this.standardTestError = Regex.Replace(this.standardTestError, @"\s+", " "); this.standardTestOutput = Regex.Replace(this.standardTestOutput, @"\s+", " "); } } -} +} \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs new file mode 100644 index 0000000000..10b2adcdc1 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TestPlatform.TestUtilities +{ + using System; + using System.IO; + + using Microsoft.VisualStudio.TestTools.UnitTesting; + + /// + /// Provider for test environment configuration. + /// Currently reads configuration from environment variables. We may support a + /// different provider later. E.g. run settings. + /// + public class IntegrationTestEnvironment + { + private readonly string testPlatformRootDirectory; + + public IntegrationTestEnvironment() + { + // These environment variables are set in scripts/test.ps1 or scripts/test.sh. + this.testPlatformRootDirectory = Environment.GetEnvironmentVariable("TP_ROOT_DIR"); + this.TargetFramework = Environment.GetEnvironmentVariable("TPT_TargetFramework"); + this.TargetRuntime = Environment.GetEnvironmentVariable("TPT_TargetRuntime"); + + // If the variables are not set, valid defaults are assumed. + if (string.IsNullOrEmpty(this.TargetFramework)) + { + // Run integration tests for net46 by default. + this.TargetFramework = "net46"; + } + + if (string.IsNullOrEmpty(this.TargetRuntime)) + { + this.TargetRuntime = "win7-x64"; + } + + if (string.IsNullOrEmpty(this.testPlatformRootDirectory)) + { + // Running in VS/IDE. Use artifacts directory as root. + this.testPlatformRootDirectory = Path.GetFullPath(@"..\..\.."); + this.TestAssetsPath = Path.Combine(this.testPlatformRootDirectory, @"artifacts\test\TestAssets"); + this.PublishDirectory = Path.Combine( + this.testPlatformRootDirectory, + "artifacts", + @"src\Microsoft.TestPlatform.VSIXCreator\bin", + this.BuildConfiguration, + "net461", + this.TargetRuntime); + } + else + { + // Running in command line/CI + this.TestAssetsPath = Path.Combine(this.testPlatformRootDirectory, @"test\TestAssets"); + this.PublishDirectory = Path.Combine( + this.testPlatformRootDirectory, + "artifacts", + this.BuildConfiguration, + this.TargetFramework, + this.TargetRuntime); + } + + // There is an assumption that integration tests will always run from a source enlistment. + // Need to remove this assumption when we move to a CDP. + this.PackageDirectory = Path.Combine(this.testPlatformRootDirectory, @"packages"); + } + + /// + /// Gets the build configuration for the test run. + /// + public string BuildConfiguration + { + get + { +#if DEBUG + return "Debug"; +#else + return "Release"; +#endif + } + } + + /// + /// Gets the nuget packages directory for enlistment. + /// + public string PackageDirectory + { + get; + private set; + } + + /// + /// Gets the publish directory for vstest.console package. + /// + public string PublishDirectory + { + get; + private set; + } + + /// + /// Gets the target framework. + /// Supported values = net46, netcoreapp1.0. + /// + public string TargetFramework + { + get; + private set; + } + + /// + /// Gets the target runtime. + /// Supported values = win7-x64. + /// + public string TargetRuntime + { + get; + private set; + } + + /// + /// Gets the root directory for test assets. + /// + public string TestAssetsPath + { + get; + } + + /// + /// Gets the full path to a test asset. + /// + /// Name of the asset with extension. E.g. SimpleUnitTest.dll + /// Full path to the test asset. + /// + /// Test assets follow several conventions: + /// (a) They are built for supported frameworks. See . + /// (b) They are built for provided build configuration. + /// (c) Name of the test asset matches the parent directory name. E.g. TestAssets\SimpleUnitTest\SimpleUnitTest.xproj must + /// produce TestAssets\SimpleUnitTest\bin\Debug\SimpleUnitTest.dll + /// + public string GetTestAsset(string assetName) + { + var simpleAssetName = Path.GetFileNameWithoutExtension(assetName); + var assetPath = Path.Combine( + this.TestAssetsPath, + simpleAssetName, + "bin", + this.BuildConfiguration, + this.TargetFramework, + assetName); + + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + + return assetPath; + } + + /// + /// Gets the full path to a nuget package. + /// + /// Suffix for the nuget package. + /// Complete path to a nuget package. + /// GetNugetPackage("foobar") will return a path to packages\foobar. + public string GetNugetPackage(string packageSuffix) + { + var packagePath = Path.Combine(this.PackageDirectory, packageSuffix); + + Assert.IsTrue(Directory.Exists(packagePath), "GetNugetPackage: Directory not found: {0}.", packagePath); + + return packagePath; + } + + /// + /// Gets the path to vstest.console.exe. + /// + /// Full path to vstest.console.exe + public string GetConsoleRunnerPath() + { + var vstestConsolePath = Path.Combine(this.PublishDirectory, "vstest.console.exe"); + + Assert.IsTrue(File.Exists(vstestConsolePath), "GetConsoleRunnerPath: Path not found: {0}", vstestConsolePath); + + return vstestConsolePath; + } + } +} \ No newline at end of file diff --git a/test/TestPlatform.TestUtilities/TestPlatform.TestUtilities.xproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.xproj similarity index 100% rename from test/TestPlatform.TestUtilities/TestPlatform.TestUtilities.xproj rename to test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.xproj diff --git a/test/TestPlatform.TestUtilities/Properties/AssemblyInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/Properties/AssemblyInfo.cs similarity index 100% rename from test/TestPlatform.TestUtilities/Properties/AssemblyInfo.cs rename to test/Microsoft.TestPlatform.TestUtilities/Properties/AssemblyInfo.cs diff --git a/test/Microsoft.TestPlatform.TestUtilities/project.json b/test/Microsoft.TestPlatform.TestUtilities/project.json new file mode 100644 index 0000000000..959f01fc62 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/project.json @@ -0,0 +1,33 @@ +{ + "version": "15.0.0-*", + + "buildOptions": { + "delaySign": true, + "keyFile": "../../scripts/key.snk", + "warningsAsErrors": true + }, + + "dependencies": { + "MSTest.TestFramework": "1.0.0-preview" + }, + + "frameworks": { + "netcoreapp1.0": { + "imports": [ + "dnxcore50", + "portable-net45+win8" + ], + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0" + } + } + }, + "net46": { + "frameworkAssemblies": { + "System.Runtime": "" + } + } + } +} diff --git a/test/Samples/SampleUnitTestProject/SampleUnitTestProject.csproj b/test/Samples/SampleUnitTestProject/SampleUnitTestProject.csproj deleted file mode 100644 index be29ca25d4..0000000000 --- a/test/Samples/SampleUnitTestProject/SampleUnitTestProject.csproj +++ /dev/null @@ -1,103 +0,0 @@ - - - - - - Debug - AnyCPU - {37C76C3D-947E-48A6-AF68-2D7C9EC35F6F} - Library - Properties - SampleUnitTestProject - SampleUnitTestProject - v4.5.2 - 512 - {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} - 10.0 - $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) - $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages - False - UnitTest - - - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\MSTest.TestFramework.1.0.0-preview\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.dll - True - - - ..\packages\MSTest.TestFramework.1.0.0-preview\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll - True - - - - - - - - - - - - - - - - - - Designer - - - - - - - False - - - False - - - False - - - False - - - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - - - \ No newline at end of file diff --git a/test/Samples/SampleUnitTestProject/packages.config b/test/Samples/SampleUnitTestProject/packages.config deleted file mode 100644 index 8b1e1f4e26..0000000000 --- a/test/Samples/SampleUnitTestProject/packages.config +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/test/Samples/TestImpactListener.Tests/TestImpactListener.Tests.csproj b/test/Samples/TestImpactListener.Tests/TestImpactListener.Tests.csproj deleted file mode 100644 index 96836bfb34..0000000000 --- a/test/Samples/TestImpactListener.Tests/TestImpactListener.Tests.csproj +++ /dev/null @@ -1,67 +0,0 @@ - - - - - Debug - AnyCPU - {F060289C-6E1A-4B21-9EB5-7F111A79449C} - Library - Properties - TestImpactListener.Tests - TestImpactListener.Tests - v4.6.1 - 512 - - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - true - - - TITestDllKey.snk - - - - ..\artifacts\src\Microsoft.TestPlatform.ObjectModel\bin\Debug\net461\Microsoft.VisualStudio.TestPlatform.ObjectModel.dll - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/test/Samples/SampleUnitTestProject/Properties/AssemblyInfo.cs b/test/TestAssets/SimpleTestProject/Properties/AssemblyInfo.cs similarity index 100% rename from test/Samples/SampleUnitTestProject/Properties/AssemblyInfo.cs rename to test/TestAssets/SimpleTestProject/Properties/AssemblyInfo.cs diff --git a/test/TestAssets/SimpleTestProject/SimpleTestProject.xproj b/test/TestAssets/SimpleTestProject/SimpleTestProject.xproj new file mode 100644 index 0000000000..2da94a3df4 --- /dev/null +++ b/test/TestAssets/SimpleTestProject/SimpleTestProject.xproj @@ -0,0 +1,21 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + + 0CC51637-B665-47B0-A093-042D31785928 + SampleUnitTestProject + .\obj + ..\..\..\artifacts\ + v4.5.2 + + + + 2.0 + + + diff --git a/test/Samples/SampleUnitTestProject/UnitTest1.cs b/test/TestAssets/SimpleTestProject/UnitTest1.cs similarity index 100% rename from test/Samples/SampleUnitTestProject/UnitTest1.cs rename to test/TestAssets/SimpleTestProject/UnitTest1.cs diff --git a/test/TestPlatform.TestUtilities/project.json b/test/TestAssets/SimpleTestProject/project.json similarity index 88% rename from test/TestPlatform.TestUtilities/project.json rename to test/TestAssets/SimpleTestProject/project.json index 3f6bc39b3e..4e2d3f5085 100644 --- a/test/TestPlatform.TestUtilities/project.json +++ b/test/TestAssets/SimpleTestProject/project.json @@ -14,11 +14,11 @@ "dependencies": { "Microsoft.NETCore.App": { "type": "platform", - "version": "1.0.0-rc2-3002702" + "version": "1.0.0" } } }, - "net461": { + "net46": { "frameworkAssemblies": { "System.Runtime": "" } diff --git a/test/Samples/TestImpactListener.Tests/Class1.cs b/test/TestAssets/TestImpactListener.Tests/Class1.cs similarity index 86% rename from test/Samples/TestImpactListener.Tests/Class1.cs rename to test/TestAssets/TestImpactListener.Tests/Class1.cs index 0c14633f92..09989eb34b 100644 --- a/test/Samples/TestImpactListener.Tests/Class1.cs +++ b/test/TestAssets/TestImpactListener.Tests/Class1.cs @@ -13,7 +13,7 @@ namespace TestImpactListener.Tests /// public class TIListenerTests : InProcDataCollection { - private string fileName; + private readonly string fileName; /// /// Initializes a new instance of the class. @@ -23,6 +23,11 @@ public TIListenerTests() this.fileName = Path.Combine(Path.GetTempPath(), "inproctest.txt"); } + public void Initialize(IDataCollectionSink dataCollectionSink) + { + throw new NotImplementedException(); + } + /// /// The test session start. /// @@ -58,8 +63,8 @@ public void TestCaseStart(TestCaseStartArgs testCaseStartArgs) /// public void TestCaseEnd(TestCaseEndArgs testCaseEndArgs) { - Console.WriteLine("TestCase Name:{0}, TestCase ID:{1}, OutCome:{2}", testCaseEndArgs.TestCase.DisplayName, testCaseEndArgs.TestCase.Id, testCaseEndArgs.TestOutcome); - File.AppendAllText(this.fileName, "TestCaseEnd : " + testCaseEndArgs.TestCase.DisplayName + "\r\n"); + Console.WriteLine("TestCase Name:{0}, TestCase ID:{1}, OutCome:{2}", testCaseEndArgs.DataCollectionContext.TestCase.DisplayName, testCaseEndArgs.DataCollectionContext.TestCase.Id, testCaseEndArgs.TestOutcome); + File.AppendAllText(this.fileName, "TestCaseEnd : " + testCaseEndArgs.DataCollectionContext.TestCase.DisplayName + "\r\n"); } /// diff --git a/test/Samples/TestImpactListener.Tests/Properties/AssemblyInfo.cs b/test/TestAssets/TestImpactListener.Tests/Properties/AssemblyInfo.cs similarity index 100% rename from test/Samples/TestImpactListener.Tests/Properties/AssemblyInfo.cs rename to test/TestAssets/TestImpactListener.Tests/Properties/AssemblyInfo.cs diff --git a/test/Samples/TestImpactListener.Tests/TITestDllKey.snk b/test/TestAssets/TestImpactListener.Tests/TITestDllKey.snk similarity index 100% rename from test/Samples/TestImpactListener.Tests/TITestDllKey.snk rename to test/TestAssets/TestImpactListener.Tests/TITestDllKey.snk diff --git a/test/TestAssets/TestImpactListener.Tests/TestImpactListener.Tests.xproj b/test/TestAssets/TestImpactListener.Tests/TestImpactListener.Tests.xproj new file mode 100644 index 0000000000..1148f8629e --- /dev/null +++ b/test/TestAssets/TestImpactListener.Tests/TestImpactListener.Tests.xproj @@ -0,0 +1,19 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + d7f7a9f5-5646-44e7-990c-500844e9272e + TestImpactListener.Tests + .\obj + ..\..\..\artifacts\ + v4.5.2 + + + 2.0 + + + \ No newline at end of file diff --git a/test/Samples/TestImpactListener.Tests/app.config b/test/TestAssets/TestImpactListener.Tests/app.config similarity index 100% rename from test/Samples/TestImpactListener.Tests/app.config rename to test/TestAssets/TestImpactListener.Tests/app.config diff --git a/test/TestAssets/TestImpactListener.Tests/project.json b/test/TestAssets/TestImpactListener.Tests/project.json new file mode 100644 index 0000000000..1a4797507c --- /dev/null +++ b/test/TestAssets/TestImpactListener.Tests/project.json @@ -0,0 +1,33 @@ +{ + "version": "1.0.0-*", + + "buildOptions": { + "keyFile": "TITestDllKey.snk", + "warningsAsErrors": true + }, + + "dependencies": { + "MSTest.TestFramework": "1.0.0-preview", + "Microsoft.TestPlatform.ObjectModel": "15.0.0-*" + }, + + "frameworks": { + "netcoreapp1.0": { + "imports": [ + "dnxcore50", + "portable-net45+win8" + ], + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0" + } + } + }, + "net46": { + "frameworkAssemblies": { + "System.Runtime": "" + } + } + } +} diff --git a/test/TestPlatform.TestUtilities/ExecutionManager.cs b/test/TestPlatform.TestUtilities/ExecutionManager.cs deleted file mode 100644 index 54203f5444..0000000000 --- a/test/TestPlatform.TestUtilities/ExecutionManager.cs +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace TestPlatform.TestUtilities -{ - using System.Diagnostics; - using System.IO; - - /// - /// The execution manager. - /// - public static class ExecutionManager - { -#if DEBUG - private const string RelativeVsTestPath = @"artifacts\src\Microsoft.TestPlatform.VSIXCreator\bin\Debug\net461\win7-x64"; -#else - private const string RelativeVsTestPath = @"artifacts\src\Microsoft.TestPlatform.VSIXCreator\bin\Release\net461\win7-x64"; -#endif - - /// - /// Gets the executing location. - /// - public static string ExecutingLocation - { - get - { - return Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName); - } - } - - /// - /// The execute. - /// - /// - /// The args. - /// - /// - /// The standard out. - /// - /// - /// The standard error. - /// - public static void Execute(string args, out string stdOut, out string stdError) - { - stdError = string.Empty; - stdOut = string.Empty; - - using (Process vstestconsole = new Process()) - { - vstestconsole.StartInfo.FileName = GetVstestConsolePath(); - vstestconsole.StartInfo.Arguments = args; - vstestconsole.StartInfo.UseShellExecute = false; - vstestconsole.StartInfo.WorkingDirectory = GetBaseDirectory(); - vstestconsole.StartInfo.RedirectStandardError = true; - vstestconsole.StartInfo.RedirectStandardOutput = true; - vstestconsole.StartInfo.CreateNoWindow = true; - vstestconsole.Start(); - - stdError = vstestconsole.StandardError.ReadToEnd(); - stdOut = vstestconsole.StandardOutput.ReadToEnd(); - - vstestconsole.WaitForExit(); - } - } - - /// - /// The get VS test console path. - /// - /// - /// The . - /// - public static string GetVstestConsolePath() - { - var path = Path.Combine(GetBaseDirectory(), "vstest.console.exe"); - return path; - } - - /// - /// The get test adapter path. - /// - /// - /// The . - /// - public static string GetTestAdapterPath() - { - var path = Path.Combine(GetBaseDirectory(), "Adapter"); - return path; - } - - public static string GetBaseDirectory() - { - var baseDirectory = string.Empty; - - var directoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory()); - baseDirectory = Path.Combine(directoryInfo.Parent?.Parent.FullName, RelativeVsTestPath); - - return baseDirectory; - } - } -} diff --git a/test/vstest.console.UnitTests/Processors/EnableStaticLoggersArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/EnableStaticLoggersArgumentProcessorTests.cs deleted file mode 100644 index d0f476a277..0000000000 --- a/test/vstest.console.UnitTests/Processors/EnableStaticLoggersArgumentProcessorTests.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors -{ - using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class EnableStaticLoggerArgumentProcessorTests - { - [TestMethod] - public void GetMetadataShouldReturnEnableLoggerArgumentProcessorCapabilities() - { - EnableStaticLoggersArgumentProcessor processor = new EnableStaticLoggersArgumentProcessor(); - Assert.IsTrue(processor.Metadata.Value is EnableStaticLoggersArgumentProcessorCapabilities); - } - - [TestMethod] - public void GetExecutorShouldReturnEnableStaticLoggersArgumentExecutor() - { - var processor = new EnableStaticLoggersArgumentProcessor(); - Assert.IsTrue(processor.Executor.Value is EnableStaticLoggersArgumentExecutor); - } - - [TestMethod] - public void CapabilitiesShouldAppropriateProperties() - { - EnableStaticLoggersArgumentProcessorCapabilities capabilities = new EnableStaticLoggersArgumentProcessorCapabilities(); - Assert.AreEqual("/EnableStaticLoggers", capabilities.CommandName); - Assert.AreEqual(false, capabilities.AllowMultiple); - Assert.AreEqual(false, capabilities.IsAction); - Assert.AreEqual(ArgumentProcessorPriority.Logging, capabilities.Priority); - } - - public void ExecutorExecuteShouldReturnArgumentProcessorResultSuccess() - { - var executor = new EnableLoggerArgumentExecutor(null); - var result = executor.Execute(); - Assert.AreEqual(ArgumentProcessorResult.Success, result); - } - - // todo : Add test cases for NET451 and dotnet core - } -} diff --git a/test/vstest.console.UnitTests/Processors/Utilities/ArgumentProcessorFactoryTests.cs b/test/vstest.console.UnitTests/Processors/Utilities/ArgumentProcessorFactoryTests.cs index 5bb07d3c90..60a9437c96 100644 --- a/test/vstest.console.UnitTests/Processors/Utilities/ArgumentProcessorFactoryTests.cs +++ b/test/vstest.console.UnitTests/Processors/Utilities/ArgumentProcessorFactoryTests.cs @@ -3,7 +3,12 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors.Utilities { using System; + using System.Collections; using System.Collections.Generic; + using System.Collections.Immutable; + using System.Linq; + using System.Reflection; + using TestPlatform.CommandLine.Processors; using TestTools.UnitTesting; @@ -11,46 +16,51 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors.U public class ArgumentProcessorFactoryTests { [TestMethod] - public void BuildCommadMapsForProcessorWithValidShortCommandNameAddsShortCommandNameToMap() + public void BuildCommadMapsForProcessorWithIsSpecialCommandSetAddsProcessorToSpecialMap() { - var configProcessor = new ConfigurationArgumentProcessor(); - IEnumerable processors = new List() - { - //Adding Processor which has a Short Command Name - configProcessor - }; - ArgumentProcessorFactory factory = ArgumentProcessorFactory.Create(processors); - - Assert.IsTrue(factory.CommandToProcessorMap.ContainsKey("/c")); - Assert.IsTrue(factory.CommandToProcessorMap.ContainsKey("/Configuration")); - Assert.IsTrue(factory.CommandToProcessorMap.ContainsValue(configProcessor)); + var specialCommands = GetArgumentProcessors(specialCommandFilter: true) + .Select(a => a.Metadata.Value.CommandName) + .ToImmutableSortedSet(); + var factory = ArgumentProcessorFactory.Create(); + + CollectionAssert.AreEquivalent( + specialCommands, + factory.SpecialCommandToProcessorMap.Keys.ToImmutableSortedSet()); } [TestMethod] - public void BuildCommadMapsForProcessorWithIsSpecialCommandSetAddsProcessorToSpecialMap() + public void BuildCommadMapsForMultipleProcessorAddsProcessorToAppropriateMaps() { - var sourceProcessor = new TestSourceArgumentProcessor(); - IEnumerable processors = new List(){sourceProcessor}; - ArgumentProcessorFactory factory = ArgumentProcessorFactory.Create(processors); + var commandProcessors = GetArgumentProcessors(specialCommandFilter: false); + var commands = commandProcessors.Select(a => a.Metadata.Value.CommandName); + var shortCommands = commandProcessors.Where(a => !string.IsNullOrEmpty(a.Metadata.Value.ShortCommandName)) + .Select(a => a.Metadata.Value.ShortCommandName); + + ArgumentProcessorFactory factory = ArgumentProcessorFactory.Create(); - Assert.IsTrue(factory.SpecialCommandToProcessorMap.ContainsKey("TestSource")); - Assert.IsTrue(factory.SpecialCommandToProcessorMap.ContainsValue(sourceProcessor)); + // Expect command processors to contain both long and short commands. + CollectionAssert.AreEquivalent( + commands.Concat(shortCommands).ToImmutableSortedSet(), + factory.CommandToProcessorMap.Keys.ToImmutableSortedSet()); } - [TestMethod] - public void BuildCommadMapsForMultipleProcessorAddsProcessorToAppropriateMaps() + private static IEnumerable GetArgumentProcessors(bool specialCommandFilter) { - var configProcessor = new ConfigurationArgumentProcessor(); - var sourceProcessor = new TestSourceArgumentProcessor(); - IEnumerable processors = new List() { sourceProcessor, configProcessor }; - ArgumentProcessorFactory factory = ArgumentProcessorFactory.Create(processors); - - Assert.IsTrue(factory.SpecialCommandToProcessorMap.ContainsKey("TestSource")); - Assert.IsTrue(factory.SpecialCommandToProcessorMap.ContainsValue(sourceProcessor)); - Assert.IsTrue(factory.CommandToProcessorMap.ContainsKey("/c")); - Assert.IsTrue(factory.CommandToProcessorMap.ContainsKey("/Configuration")); - Assert.IsTrue(factory.CommandToProcessorMap.ContainsValue(configProcessor)); + var allProcessors = typeof(ArgumentProcessorFactory).GetTypeInfo() + .Assembly.GetTypes() + .Where(t => !t.Name.Equals("IArgumentProcessor") && typeof(IArgumentProcessor).IsAssignableFrom(t)); + + foreach (var processor in allProcessors) + { + var instance = Activator.CreateInstance(processor) as IArgumentProcessor; + Assert.IsNotNull(instance, "Unable to instantiate processor: {0}", processor); + + var specialProcessor = instance.Metadata.Value.IsSpecialCommand; + if ((specialCommandFilter && specialProcessor) || (!specialCommandFilter && !specialProcessor)) + { + yield return instance; + } + } } } -} - +} \ No newline at end of file