Refactor integration tests (microsoft#19)

Add an integration test framework that runs tests for vstest.console.exe.
Create test assets for integration tests.
The test framework provides standard way to query for test assets, test runner
etc. irrespective of IDE or CI.

* Fix TestImpact smoke tests.

Use full path in test environment. Sign testimpact listener assembly.

* Add /tests and /testCaseFilter arguments for vstest.console.

Refactor tests to check for all IArgumentProcessor implementations in command
map.

* Rename test asset to not conflict with actual unit test projects.

* Fix inproc data collector interface implementation.

* Add set and reset environment to test script.

A few variables like TP_ROOT_DIR, TPT_TargetFramework, TPT_TargetRuntime are set before the test run.
Fix the usage of environment variable in integration tests.

* Update netci.groovy to run smoke tests.

TestPlatform.sln

@@ -83,6 +83,18 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "datacollector.x86", "src\da
 EndProject
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "datacollector.x86.UnitTests", "test\datacollector.x86.UnitTests\datacollector.x86.UnitTests.xproj", "{00AA21F3-31E4-4748-AC0B-C4EADB41CA24}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Integration", "Integration", "{6DA46479-C688-4296-A6E7-F20C20CDA3A9}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.TestPlatform.SmokeTests", "test\Integration\Microsoft.TestPlatform.SmokeTests\Microsoft.TestPlatform.SmokeTests.xproj", "{C1497516-ACB5-49AF-A676-51DB65FF8252}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.TestPlatform.TestUtilities", "test\Microsoft.TestPlatform.TestUtilities\Microsoft.TestPlatform.TestUtilities.xproj", "{F52A4D48-90B3-4004-8C98-D2786CC9B965}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TestAssets", "TestAssets", "{50D7D355-08F6-4DFD-AEAA-9BCE41C94C18}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "TestImpactListener.Tests", "test\TestAssets\TestImpactListener.Tests\TestImpactListener.Tests.xproj", "{D7F7A9F5-5646-44E7-990C-500844E9272E}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "SimpleTestProject", "test\TestAssets\SimpleTestProject\SimpleTestProject.xproj", "{0CC51637-B665-47B0-A093-042D31785928}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -205,6 +217,22 @@ Global
 		{00AA21F3-31E4-4748-AC0B-C4EADB41CA24}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{00AA21F3-31E4-4748-AC0B-C4EADB41CA24}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{00AA21F3-31E4-4748-AC0B-C4EADB41CA24}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C1497516-ACB5-49AF-A676-51DB65FF8252}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C1497516-ACB5-49AF-A676-51DB65FF8252}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C1497516-ACB5-49AF-A676-51DB65FF8252}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C1497516-ACB5-49AF-A676-51DB65FF8252}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F52A4D48-90B3-4004-8C98-D2786CC9B965}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F52A4D48-90B3-4004-8C98-D2786CC9B965}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F52A4D48-90B3-4004-8C98-D2786CC9B965}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F52A4D48-90B3-4004-8C98-D2786CC9B965}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D7F7A9F5-5646-44E7-990C-500844E9272E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D7F7A9F5-5646-44E7-990C-500844E9272E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D7F7A9F5-5646-44E7-990C-500844E9272E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D7F7A9F5-5646-44E7-990C-500844E9272E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0CC51637-B665-47B0-A093-042D31785928}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0CC51637-B665-47B0-A093-042D31785928}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0CC51637-B665-47B0-A093-042D31785928}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0CC51637-B665-47B0-A093-042D31785928}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -241,5 +269,11 @@ Global
 		{3572E78C-5AA5-4F68-876D-FC5322677263} = {D8EF073C-279A-4279-912D-E9D4B0635E17}
 		{00DFB5C7-3850-4A65-986B-713F200482D4} = {D8EF073C-279A-4279-912D-E9D4B0635E17}
 		{00AA21F3-31E4-4748-AC0B-C4EADB41CA24} = {463031A2-7F16-4E38-9944-1F5161D04933}
+		{6DA46479-C688-4296-A6E7-F20C20CDA3A9} = {463031A2-7F16-4E38-9944-1F5161D04933}
+		{C1497516-ACB5-49AF-A676-51DB65FF8252} = {6DA46479-C688-4296-A6E7-F20C20CDA3A9}
+		{F52A4D48-90B3-4004-8C98-D2786CC9B965} = {463031A2-7F16-4E38-9944-1F5161D04933}
+		{50D7D355-08F6-4DFD-AEAA-9BCE41C94C18} = {463031A2-7F16-4E38-9944-1F5161D04933}
+		{D7F7A9F5-5646-44E7-990C-500844E9272E} = {50D7D355-08F6-4DFD-AEAA-9BCE41C94C18}
+		{0CC51637-B665-47B0-A093-042D31785928} = {50D7D355-08F6-4DFD-AEAA-9BCE41C94C18}
 	EndGlobalSection
 EndGlobal

netci.groovy

@@ -16,12 +16,14 @@ def branch = GithubBranchName
         // Define your build/test strings here
         def buildString = """call build.cmd -c ${configuration}"""
         def testString = """call test.cmd -c ${configuration}"""
+        def smoketestString = """call test.cmd -c ${configuration} -p smoke"""
 
         // Create a new job for windows build
         def newJob = job(newJobName) {
             steps {
                 batchFile(buildString)
                 batchFile(testString)
+                batchFile(smoketestString)
             }
         }
 

scripts/stylecop.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json",
+  "settings": {
+    "documentationRules": {
+      "companyName": "Microsoft Corporation",
+      "copyrightText": "Copyright (c) {companyName}. All rights reserved.\nLicensed under the MIT license. See LICENSE file in the project root for full license information.",
+      "xmlHeader": false,
+      "fileNamingConvention": "metadata"
+    },
+  }
+}

scripts/stylecop.test.ruleset

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RuleSet Name="Rules for test platform tests" Description="Code analysis rules for test projects." ToolsVersion="14.0">
+  <Rules AnalyzerId="StyleCop.Analyzers" RuleNamespace="StyleCop.Analyzers">
+    <!-- XML documentation checks are not required for test projects -->
+    <Rule Id="SA1652" Action="None" />
+  </Rules>
+</RuleSet>

scripts/test.ps1

@@ -108,9 +108,12 @@ function Invoke-Test
                 } elseif (!($testContainerName -match $Script:TPT_Pattern)) {
                     Write-Log ".. . $testContainerName doesn't match test container pattern '$($Script:TPT_Pattern)'. Skipped from run."
                 } else {
+                    Set-TestEnvironment
+
                     Write-Verbose "vstest.console.exe $testContainerPath /platform:$testArchitecture /testAdapterPath:$testAdapterPath"
                     $output = & $vstestConsolePath $testContainerPath /platform:$testArchitecture /testAdapterPath:"$testAdapterPath"
 
+                    Reset-TestEnvironment
                     #Write-Verbose "$dotnetExe test $_ --configuration $Configuration"
                     #& $dotnetExe test $_ --configuration $Configuration
 
@@ -128,6 +131,7 @@ function Invoke-Test
                             continue
                         }
                     }
+
                 }
 
                 Write-Log ".. Test: Complete."
@@ -174,6 +178,18 @@ function Set-ScriptFailed
     $Script:ScriptFailed = $true
 }
 
+function Set-TestEnvironment
+{
+    $env:TPT_TargetFramework = $Script:TPT_TargetFramework
+    $env:TPT_TargetRuntime = $Script:TPT_TargetRuntime
+}
+
+function Reset-TestEnvironment
+{
+    $env:TPT_TargetFramework = $null
+    $env:TPT_TargetRuntime = $null
+}
+
 # Execute build
 $timer = Start-Timer
 Write-Log "Build started: args = '$args'"