diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs index 2fd1b39333..c44f9af324 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs @@ -161,7 +161,7 @@ internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, D continue; } - var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings.MapInconclusiveToFailed); + var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings); if (unitTestResult.DatarowIndex >= 0) { diff --git a/src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs b/src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs index 4a2b46639e..8c0ebd2a54 100644 --- a/src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs +++ b/src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs @@ -14,9 +14,9 @@ internal static class UnitTestOutcomeHelper /// Converts the parameter unitTestOutcome to testOutcome /// /// The unit Test Outcome. - /// Should map inconclusive to failed. + /// Current MSTest settings /// The Test platforms outcome. - internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, bool mapInconclusiveToFailed) + internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, MSTestSettings currentSettings) { switch (unitTestOutcome) { @@ -29,14 +29,21 @@ internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, bool return TestOutcome.Failed; case UnitTestOutcome.NotRunnable: - return TestOutcome.None; + { + if (currentSettings.MapNotRunnableToFailed) + { + return TestOutcome.Failed; + } + + return TestOutcome.None; + } case UnitTestOutcome.Ignored: return TestOutcome.Skipped; case UnitTestOutcome.Inconclusive: { - if (mapInconclusiveToFailed) + if (currentSettings.MapInconclusiveToFailed) { return TestOutcome.Failed; } diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs index 4643581082..d6e679820a 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs +++ b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs @@ -51,6 +51,7 @@ public MSTestSettings() { this.CaptureDebugTraces = true; this.MapInconclusiveToFailed = false; + this.MapNotRunnableToFailed = false; this.EnableBaseClassTestMethodsFromOtherAssemblies = true; this.ForcedLegacyMode = false; this.TestSettingsFile = null; @@ -121,6 +122,11 @@ private set /// public bool MapInconclusiveToFailed { get; private set; } + /// + /// Gets a value indicating whether a not runnable result be mapped to failed test. + /// + public bool MapNotRunnableToFailed { get; private set; } + /// /// Gets a value indicating whether to enable discovery of test methods from base classes in a different assembly from the inheriting test class. /// @@ -159,6 +165,7 @@ public static void PopulateSettings(MSTestSettings settings) CurrentSettings.ForcedLegacyMode = settings.ForcedLegacyMode; CurrentSettings.TestSettingsFile = settings.TestSettingsFile; CurrentSettings.MapInconclusiveToFailed = settings.MapInconclusiveToFailed; + CurrentSettings.MapNotRunnableToFailed = settings.MapNotRunnableToFailed; CurrentSettings.EnableBaseClassTestMethodsFromOtherAssemblies = settings.EnableBaseClassTestMethodsFromOtherAssemblies; CurrentSettings.ParallelizationWorkers = settings.ParallelizationWorkers; CurrentSettings.ParallelizationScope = settings.ParallelizationScope; @@ -280,6 +287,7 @@ private static MSTestSettings ToSettings(XmlReader reader) // // true // false + // false // false // 5000 // @@ -350,6 +358,16 @@ private static MSTestSettings ToSettings(XmlReader reader) break; } + case "MAPNOTRUNNABLETOFAILED": + { + if (bool.TryParse(reader.ReadInnerXml(), out result)) + { + settings.MapNotRunnableToFailed = result; + } + + break; + } + case "SETTINGSFILE": { string fileName = reader.ReadInnerXml(); diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs index c8aae7fd21..43bee38e66 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs @@ -147,9 +147,9 @@ internal UnitTestResult(UnitTestOutcome outcome, string errorMessage) /// The test Case. /// The start Time. /// The end Time. - /// Indication to map inconclusive tests to failed. + /// Current MSTest settings. /// The . - internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, DateTimeOffset endTime, bool mapInconclusiveToFailed) + internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, DateTimeOffset endTime, MSTestSettings currentSettings) { Debug.Assert(testCase != null, "testCase"); @@ -159,7 +159,7 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da Duration = this.Duration, ErrorMessage = this.ErrorMessage, ErrorStackTrace = this.ErrorStackTrace, - Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, mapInconclusiveToFailed), + Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, currentSettings), StartTime = startTime, EndTime = endTime }; diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs index 8b0039f150..515a8f25f9 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs @@ -10,6 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution using System; using System.Linq; using System.Reflection; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -64,8 +65,16 @@ public void ToTestResultShouldReturnConvertedTestResultWithFieldsSet() var startTime = DateTimeOffset.Now; var endTime = DateTimeOffset.Now; + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + // Act - var testResult = result.ToTestResult(testCase, startTime, endTime, mapInconclusiveToFailed: false); + var testResult = result.ToTestResult(testCase, startTime, endTime, adapterSettings); // Validate Assert.AreEqual(testCase, testResult.TestCase); @@ -87,7 +96,15 @@ public void ToTestResultForUniTestResultWithStandardOutShouldReturnTestResultWit StandardOut = "DummyOutput" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, mapInconclusiveToFailed: false); + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("DummyOutput") && m.Category.Equals("StdOutMsgs"))); } @@ -99,7 +116,14 @@ public void ToTestResultForUniTestResultWithDebugTraceShouldReturnTestResultWith DebugTrace = "DummyDebugTrace" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, mapInconclusiveToFailed: false); + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("\n\nDebug Trace:\nDummyDebugTrace") && m.Category.Equals("StdOutMsgs"))); } } diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs index 91b70b3609..8c7f4c89bd 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs @@ -6,79 +6,92 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Helpers extern alias FrameworkV1; extern alias FrameworkV2; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; - using Assert = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.Assert; using TestClass = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute; + using TestInitialize = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestInitializeAttribute; using TestMethod = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute; - using UTF = FrameworkV2::Microsoft.VisualStudio.TestTools.UnitTesting; - [TestClass] public class UnitTestOutcomeHelperTests { + private MSTestSettings adapterSettings; + + [TestInitialize] + public void TestInit() + { + string runSettingxml = + @" + + + "; + + this.adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + } + [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomePassedShouldReturnTestOutcomePassed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, this.adapterSettings); Assert.AreEqual(TestOutcome.Passed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeFailedShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, this.adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeErrorShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, this.adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeNone() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, this.adapterSettings); Assert.AreEqual(TestOutcome.None, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeTimeoutShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, this.adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeIgnoredShouldReturnTestOutcomeSkipped() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, this.adapterSettings); Assert.AreEqual(TestOutcome.Skipped, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInconclusiveShouldReturnTestOutcomeSkipped() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, this.adapterSettings); Assert.AreEqual(TestOutcome.Skipped, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotFoundShouldReturnTestOutcomeNotFound() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, this.adapterSettings); Assert.AreEqual(TestOutcome.NotFound, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInProgressShouldReturnTestOutcomeNone() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, this.adapterSettings); Assert.AreEqual(TestOutcome.None, resultOutcome); } } diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs index 3704926ba5..36f292f1c2 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs @@ -64,6 +64,20 @@ public void MapInconclusiveToFailedIsByDefaultFalseWhenNotSpecified() Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, false); } + [TestMethod] + public void MapNotRunnableToFailedIsByDefaultFalseWhenNotSpecified() + { + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, false); + } + [TestMethod] public void MapInconclusiveToFailedShouldBeConsumedFromRunSettingsWhenSpecified() { @@ -79,6 +93,21 @@ public void MapInconclusiveToFailedShouldBeConsumedFromRunSettingsWhenSpecified( Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); } + [TestMethod] + public void MapNotRunnableToFailedShouldBeConsumedFromRunSettingsWhenSpecified() + { + string runSettingxml = + @" + + True + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); + } + [TestMethod] public void ForcedLegacyModeIsByDefaultFalseWhenNotSpecified() { @@ -566,6 +595,7 @@ public void GetSettingsShouldBeAbleToReadSettingsAfterThePlatformServiceReadsIts @" True + True True DummyPath\\TestSettings1.testsettings @@ -608,6 +638,7 @@ public void GetSettingsShouldBeAbleToReadSettingsAfterThePlatformServiceReadsIts // Assert. Assert.IsTrue(dummyPlatformSpecificSetting); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual("DummyPath\\\\TestSettings1.testsettings", adapterSettings.TestSettingsFile); } @@ -618,6 +649,7 @@ public void GetSettingsShouldBeAbleToReadSettingsIfThePlatformServiceDoesNotUnde @" True + True foobar true true @@ -663,6 +695,7 @@ public void GetSettingsShouldBeAbleToReadSettingsIfThePlatformServiceDoesNotUnde // Assert. Assert.IsTrue(dummyPlatformSpecificSetting); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.AreEqual("DummyPath\\\\TestSettings1.testsettings", adapterSettings.TestSettingsFile); @@ -724,6 +757,7 @@ public void GetSettingsShouldWorkIfThereAreCommentsInTheXML() True + True True true @@ -770,6 +804,7 @@ public void GetSettingsShouldWorkIfThereAreCommentsInTheXML() // Assert. Assert.IsTrue(dummyPlatformSpecificSetting); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); } @@ -825,6 +860,7 @@ public void PopulateSettingsShouldFillInSettingsFromSettingsObject() False True + True DummyPath\\TestSettings1.testsettings true true @@ -837,6 +873,7 @@ public void PopulateSettingsShouldFillInSettingsFromSettingsObject() Assert.AreEqual(MSTestSettings.CurrentSettings.CaptureDebugTraces, false); Assert.AreEqual(MSTestSettings.CurrentSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(MSTestSettings.CurrentSettings.MapNotRunnableToFailed, true); Assert.AreEqual(MSTestSettings.CurrentSettings.ForcedLegacyMode, true); Assert.AreEqual(MSTestSettings.CurrentSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.IsFalse(string.IsNullOrEmpty(MSTestSettings.CurrentSettings.TestSettingsFile)); @@ -850,6 +887,7 @@ public void PopulateSettingsShouldInitializeDefaultAdapterSettingsWhenDiscoveryC MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; Assert.AreEqual(adapterSettings.CaptureDebugTraces, true); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, false); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, false); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); } @@ -861,6 +899,7 @@ public void PopulateSettingsShouldInitializeDefaultSettingsWhenRunSettingsIsNull MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; Assert.AreEqual(adapterSettings.CaptureDebugTraces, true); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, false); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, false); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); } @@ -873,6 +912,7 @@ public void PopulateSettingsShouldInitializeDefaultSettingsWhenRunSettingsXmlIsE MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; Assert.AreEqual(adapterSettings.CaptureDebugTraces, true); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, false); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, false); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); } @@ -905,6 +945,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestSection() @" True + True DummyPath\\TestSettings1.testsettings true true @@ -920,6 +961,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestSection() Assert.IsNotNull(adapterSettings); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.IsFalse(string.IsNullOrEmpty(adapterSettings.TestSettingsFile)); @@ -932,6 +974,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestV2Section() @" True + True DummyPath\\TestSettings1.testsettings true true @@ -947,6 +990,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestV2Section() Assert.IsNotNull(adapterSettings); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.IsFalse(string.IsNullOrEmpty(adapterSettings.TestSettingsFile)); @@ -959,6 +1003,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestV2OverMSTestV1Sect @" True + True true @@ -977,6 +1022,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestV2OverMSTestV1Sect Assert.IsNotNull(adapterSettings); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, false); Assert.AreEqual(adapterSettings.CaptureDebugTraces, true); diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs index f6ce578efd..ef6db77911 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs @@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.ObjectMode using System.Collections.Generic; using System.Linq; using System.Reflection; - + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -64,8 +64,16 @@ public void ToTestResultShouldReturnConvertedTestResultWithFieldsSet() var startTime = DateTimeOffset.Now; var endTime = DateTimeOffset.Now; + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + // Act - var testResult = result.ToTestResult(testCase, startTime, endTime, false); + var testResult = result.ToTestResult(testCase, startTime, endTime, adapterSettings); // Validate Assert.AreEqual(testCase, testResult.TestCase); @@ -87,7 +95,16 @@ public void ToTestResultForUniTestResultWithStandardOutShouldReturnTestResultWit StandardOut = "DummyOutput" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("DummyOutput") && m.Category.Equals("StdOutMsgs"))); } @@ -99,7 +116,16 @@ public void ToTestResultForUniTestResultWithStandardErrorShouldReturnTestResultW StandardError = "DummyError" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("DummyError") && m.Category.Equals("StdErrMsgs"))); } @@ -111,7 +137,16 @@ public void ToTestResultForUniTestResultWithDebugTraceShouldReturnTestResultWith DebugTrace = "DummyDebugTrace" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("\n\nDebug Trace:\nDummyDebugTrace") && m.Category.Equals("StdOutMsgs"))); } @@ -123,7 +158,16 @@ public void ToTestResultForUniTestResultWithTestContextMessagesShouldReturnTestR TestContextMessages = "KeepMovingForward" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("\n\nTestContext Messages:\nKeepMovingForward") && m.Category.Equals("StdOutMsgs"))); } @@ -135,7 +179,16 @@ public void ToTestResultForUniTestResultWithResultFilesShouldReturnTestResultWit ResultFiles = new List() { "dummy://DummyFile.txt" } }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.AreEqual(testresult.Attachments.Count, 1); Assert.AreEqual(testresult.Attachments[0].Attachments[0].Description, "dummy://DummyFile.txt"); @@ -149,7 +202,16 @@ public void ToTestResultForUniTestResultWithNoResultFilesShouldReturnTestResultW ResultFiles = null }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.AreEqual(testresult.Attachments.Count, 0); } @@ -168,7 +230,16 @@ public void ToTestResultForUniTestResultWithParentInfoShouldReturnTestResultWith InnerResultsCount = innerResultsCount }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.AreEqual(executionId, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.ExecutionIdProperty)); Assert.AreEqual(parentExecId, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.ParentExecIdProperty)); @@ -178,71 +249,167 @@ public void ToTestResultForUniTestResultWithParentInfoShouldReturnTestResultWith [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomePassedShouldReturnTestOutcomePassed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, adapterSettings); Assert.AreEqual(TestOutcome.Passed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeFailedShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeErrorShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] - public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeNone() + public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeNoneWhenNotSpecified() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, adapterSettings); Assert.AreEqual(TestOutcome.None, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeTimeoutShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeIgnoredShouldReturnTestOutcomeSkipped() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, adapterSettings); Assert.AreEqual(TestOutcome.Skipped, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInconclusiveShouldReturnTestOutcomeSkipped() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, adapterSettings); Assert.AreEqual(TestOutcome.Skipped, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInconclusiveShouldReturnTestOutcomeFailedWhenSpecifiedSo() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, true); + string runSettingxml = + @" + + true + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotFoundShouldReturnTestOutcomeNotFound() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, adapterSettings); Assert.AreEqual(TestOutcome.NotFound, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInProgressShouldReturnTestOutcomeNone() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, adapterSettings); Assert.AreEqual(TestOutcome.None, resultOutcome); } + + [TestMethod] + public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeFailedWhenSpecifiedSo() + { + string runSettingxml = + @" + + true + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, adapterSettings); + Assert.AreEqual(TestOutcome.Failed, resultOutcome); + } } }