From 3bb2cefa3ff51b8fe80b773912df39380be74448 Mon Sep 17 00:00:00 2001 From: Medeni Baykal <433724+Haplois@users.noreply.github.com> Date: Fri, 3 Sep 2021 10:25:34 +0200 Subject: [PATCH] Cherry-picking the changes from 2.2.7 (#958) * Allow opting-out of ITestDataSource test discovery. * Fixed missing strong-name and Authenticode signatures (#956) * Updated signature verification to include DLLs --- scripts/Build.ps1 | 5 +- scripts/build/TestFx.Sign.props | 21 +++ scripts/build/TestFx.Sign.targets | 35 +++++ scripts/build/TestFx.props | 7 +- scripts/build/TestFx.targets | 30 +--- scripts/common.lib.ps1 | 9 ++ scripts/verify-sign.ps1 | 143 +++++++++++++++--- .../Discovery/AssemblyEnumerator.cs | 12 +- .../PlatformServices.NetCore.csproj | 7 +- .../PlatformServices.WinUI.csproj | 7 +- .../Extension.WinUI/Extension.WinUI.csproj | 6 +- .../Attributes/DiscoverInternalsAttribute.cs | 2 +- .../TestDataSourceDiscoveryAttribute.cs | 30 ++++ .../TestDataSourceDiscoveryOption.cs | 23 +++ .../MSTest.Core/MSTest.Core.csproj | 2 + .../Discovery/AssemblyEnumeratorTests.cs | 6 + 16 files changed, 283 insertions(+), 62 deletions(-) create mode 100644 scripts/build/TestFx.Sign.props create mode 100644 scripts/build/TestFx.Sign.targets create mode 100644 src/TestFramework/MSTest.Core/Attributes/TestDataSourceDiscoveryAttribute.cs create mode 100644 src/TestFramework/MSTest.Core/Attributes/TestDataSourceDiscoveryOption.cs diff --git a/scripts/Build.ps1 b/scripts/Build.ps1 index 80a1b4f3b1..774f42c2d8 100644 --- a/scripts/Build.ps1 +++ b/scripts/Build.ps1 @@ -222,7 +222,10 @@ function Invoke-MSBuild([string]$solution, $buildTarget = $Target, $hasVsixExten "-m") Write-Log " $buildTarget`: $solution..." - & "$msbuild" $argument; + & { + $PSNativeCommandArgumentPassing = 'Legacy' + & "$msbuild" $argument; + } if ($lastExitCode -ne 0) { throw "Build failed with an exit code of '$lastExitCode'." diff --git a/scripts/build/TestFx.Sign.props b/scripts/build/TestFx.Sign.props new file mode 100644 index 0000000000..5d296fc0c3 --- /dev/null +++ b/scripts/build/TestFx.Sign.props @@ -0,0 +1,21 @@ + + + + $([MSBuild]::NormalizeDirectory('$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildProjectDirectory), 'LICENSE'))')) + $(RepoRoot)packages + Release + + true + + + + $(MSBuildThisFileDirectory)key.snk + true + Test + true + + + true + $(RepoRoot)artifacts\$(Configuration)\$(MSBuildProjectName)\ + + \ No newline at end of file diff --git a/scripts/build/TestFx.Sign.targets b/scripts/build/TestFx.Sign.targets new file mode 100644 index 0000000000..26bca45947 --- /dev/null +++ b/scripts/build/TestFx.Sign.targets @@ -0,0 +1,35 @@ + + + + true + + + + + + false + + + + + + + Microsoft400 + StrongName + + + + + false + + + + + + + Microsoft400 + StrongName + + + + \ No newline at end of file diff --git a/scripts/build/TestFx.props b/scripts/build/TestFx.props index ddc819bced..aed1b94467 100644 --- a/scripts/build/TestFx.props +++ b/scripts/build/TestFx.props @@ -7,6 +7,7 @@ + Debug @@ -14,12 +15,6 @@ en-US 512 true - $(MSBuildThisFileDirectory)key.snk - true - true - - true - $(RepoRoot)artifacts\$(Configuration)\$(MSBuildProjectName)\ $(RepoRoot)artifacts\$(Configuration)\$(MSBuildProjectName)\obj\ true diff --git a/scripts/build/TestFx.targets b/scripts/build/TestFx.targets index c6155c40fd..760d9e948c 100644 --- a/scripts/build/TestFx.targets +++ b/scripts/build/TestFx.targets @@ -2,13 +2,7 @@ - - - - - false - - + @@ -21,28 +15,6 @@ - - - - Microsoft400 - StrongName - - - - - false - - - - - - - Microsoft400 - StrongName - - - - 0.1 diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1 index 6f7b4714c4..75b9273cc6 100644 --- a/scripts/common.lib.ps1 +++ b/scripts/common.lib.ps1 @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT license. See LICENSE file in the project root for full license information. +Add-Type -AssemblyName System.IO.Compression.FileSystem + # Common utilities for building solution and running tests $TF_ROOT_DIR = (Get-Item (Split-Path $MyInvocation.MyCommand.Path)).Parent.FullName @@ -294,4 +296,11 @@ function Install-DotNetCli { } catch {} Write-Log "Install-DotNetCli: Complete." +} + +function Unzip +{ + param([string]$zipfile, [string]$outpath) + + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath) } \ No newline at end of file diff --git a/scripts/verify-sign.ps1 b/scripts/verify-sign.ps1 index 5f34d67315..61fbed4125 100644 --- a/scripts/verify-sign.ps1 +++ b/scripts/verify-sign.ps1 @@ -1,14 +1,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT license. See LICENSE file in the project root for full license information. -# Build script for Test Platform. - [CmdletBinding()] Param( [Parameter(Mandatory=$false)] [ValidateSet("Debug", "Release")] [Alias("c")] - [System.String] $Configuration = "Debug" + [string] $Configuration = "Debug", + [string] $ArtifactsDirectory = "", + [switch] $Force ) . $PSScriptRoot\common.lib.ps1 @@ -16,41 +16,148 @@ Param( # # Variables # -$rootDirectory = (Get-Item (Split-Path $MyInvocation.MyCommand.Path)).Parent.FullName +if(-not [string]::IsNullOrWhiteSpace($ArtifactsDirectory)) { + $TF_OUT_DIR = $ArtifactsDirectory +} # # Signing configuration # Write-Verbose "Setup build configuration." -$TPB_Configuration = $Configuration -function Verify-NugetPackages +$TF_Configuration = $Configuration +$TF_AssembliesPattern = @("Microsoft.VisualStudio.TestPlatform.*.dll", "Microsoft.TestPlatform.*.dll") +$script:ErrorCount = 0 + +function Test-Assembly ([string] $Path) { - Write-Log "Verify-NugetPackages: Start" + $signature = Get-AuthenticodeSignature -FilePath $Path - $nugetInstallPath = Locate-NuGet + if ($signature.Status -eq "Valid") { + if ($signature.SignerCertificate.Subject -eq "CN=Microsoft Corporation, O=Microsoft Corporation, L=Redmond, S=Washington, C=US") { + Write-Debug "Valid ($($signature.SignerCertificate.Thumbprint)): $Path" + } + elseif ($signature.SignerCertificate.Subject -eq "CN=Microsoft 3rd Party Application Component, O=Microsoft Corporation, L=Redmond, S=Washington, C=US") { + Write-Debug "Valid ($($signature.SignerCertificate.Thumbprint)): $Path [3rd Party]" + } + else { + # For legacy components + # CN=Microsoft Corporation, OU=AOC, O=Microsoft Corporation, L=Redmond, S=Washington, C=US + if ($signature.SignerCertificate.Thumbprint -eq "49D59D86505D82942A076388693F4FB7B21254EE") { + Write-Debug "Valid ($($signature.SignerCertificate.Thumbprint)): $Path [Legacy Prod Signed]" + } + else { + Write-FailLog "Invalid ($($signature.SignerCertificate.Thumbprint)). File: $Path. [$($signature.SignerCertificate.Subject)]" + } + } + } + else { + Write-FailLog "Not signed. File: $Path." + } +} + +function Test-Assemblies ([string] $Path) +{ + foreach ($pattern in $TF_AssembliesPattern) { + Get-ChildItem -Recurse -Include $pattern $Path | Where-Object { (!$_.PSIsContainer) } | ForEach-Object { + Test-Assembly $_.FullName + } + } +} + +function Test-NugetPackage ([string] $Path) { + $packageFolder = [System.IO.Path]::GetDirectoryName($Path) + $fileName = [System.IO.Path]::GetFileNameWithoutExtension($Path) + $out = Join-Path $packageFolder $fileName + + try { + Write-ToCI "Verifing assemblies in $Path" -type "group" + Write-Debug "Extracting..." + if (Test-Path $out) { + if (-not $Force) { + Write-FailLog "Folder already exists: $out" + return + } + + Remove-Item $out -Recurse -Force + } + + Unzip $Path $out + + Test-Assemblies $out + } finally { + if (Test-Path $out) { + Remove-Item $out -Recurse -Force + } + Write-ToCI -type "endgroup" + } +} - Write-Log "Using nuget.exe installed at $nugetInstallPath" +function Test-NugetPackages +{ + Write-Debug "Test-NugetPackages" + + $nugetInstallPath = Locate-NuGet + Write-Debug "Using nuget.exe installed at $nugetInstallPath" - $artifactsDirectory = Join-Path $rootDirectory "artifacts" - $artifactsConfigDirectory = Join-Path $artifactsDirectory $TPB_Configuration + $artifactsConfigDirectory = Join-Path $TF_OUT_DIR $TF_Configuration $packagesDirectory = Join-Path $artifactsConfigDirectory "MSTestPackages" + Get-ChildItem -Filter *.nupkg $packagesDirectory | ForEach-Object { - & $nugetInstallPath verify -signature -CertificateFingerprint "3F9001EA83C560D712C24CF213C3D312CB3BFF51EE89435D3430BD06B5D0EECE;AA12DA22A49BCE7D5C1AE64CC1F3D892F150DA76140F210ABD2CBFFCA2C18A27;" $_.FullName + try { + Write-ToCI "Verifing $($_.FullName)" -type "group" + & $nugetInstallPath verify -signature -CertificateFingerprint "3F9001EA83C560D712C24CF213C3D312CB3BFF51EE89435D3430BD06B5D0EECE;AA12DA22A49BCE7D5C1AE64CC1F3D892F150DA76140F210ABD2CBFFCA2C18A27;" $_.FullName + Test-NugetPackage -path $_.FullName + } finally { + Write-ToCI -type "endgroup" + } } - Write-Log "Verify-NugetPackages: Complete" + Write-Debug "Test-NugetPackages: Complete" } -function Write-Log ([string] $message) +function Write-FailLog ([string] $message) +{ + $script:ErrorCount = $script:ErrorCount + 1 + Write-ToCI -message $message -type "error" +} + +function Write-Debug ([string] $message) +{ + Write-ToCI -message $message -type "debug" +} + +function Write-ToCI ([string] $message, [string]$type, [switch]$vso) { $currentColor = $Host.UI.RawUI.ForegroundColor - $Host.UI.RawUI.ForegroundColor = "Green" - if ($message) + + if($type -eq "error") { + $Host.UI.RawUI.ForegroundColor = "Red" + } + + if ($message -or $vso -or $type) { - Write-Output "... $message" + $prefix = "" + if ($vso) { + $prefix = "vso" + } + + Write-Output "##$prefix[$type]$message" } $Host.UI.RawUI.ForegroundColor = $currentColor } -Verify-NugetPackages +try { + Write-ToCI "Variables used: " -type "group" + Get-ChildItem variable:TF_* + Write-Output "" + Write-Output "" +} finally { + Write-ToCI -type "endgroup" +} + +Test-NugetPackages + +if ($script:ErrorCount -gt 0) { + Write-ToCI -message "Verification failed, $($script:ErrorCount) errors found!" -type "task.logissue" -vso +} \ No newline at end of file diff --git a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs index 722c2e4aec..bf45deed6e 100644 --- a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs @@ -102,6 +102,7 @@ internal ICollection EnumerateAssembly(string assemblyFileName, var types = this.GetTypes(assembly, assemblyFileName, warningMessages); var discoverInternals = assembly.GetCustomAttribute() != null; + var testDataSourceDiscovery = assembly.GetCustomAttribute()?.DiscoveryOption ?? UTF.TestDataSourceDiscoveryOption.DuringDiscovery; foreach (var type in types) { @@ -110,7 +111,7 @@ internal ICollection EnumerateAssembly(string assemblyFileName, continue; } - var testsInType = this.DiscoverTestsInType(assemblyFileName, runSettingsXml, assembly, type, warningMessages, discoverInternals); + var testsInType = this.DiscoverTestsInType(assemblyFileName, runSettingsXml, assembly, type, warningMessages, discoverInternals, testDataSourceDiscovery); tests.AddRange(testsInType); } @@ -206,7 +207,7 @@ internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFile return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator); } - private IEnumerable DiscoverTestsInType(string assemblyFileName, string runSettingsXml, Assembly assembly, Type type, List warningMessages, bool discoverInternals = false) + private IEnumerable DiscoverTestsInType(string assemblyFileName, string runSettingsXml, Assembly assembly, Type type, List warningMessages, bool discoverInternals = false, UTF.TestDataSourceDiscoveryOption discoveryOption = UTF.TestDataSourceDiscoveryOption.DuringExecution) { var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(assemblyFileName); sourceLevelParameters = RunSettingsUtilities.GetTestRunParameters(runSettingsXml)?.ConcatWithOverwrites(sourceLevelParameters) @@ -231,9 +232,12 @@ private IEnumerable DiscoverTestsInType(string assemblyFileName { foreach (var test in unitTestCases) { - if (this.DynamicDataAttached(sourceLevelParameters, assembly, test, tests)) + if (discoveryOption == UTF.TestDataSourceDiscoveryOption.DuringDiscovery) { - continue; + if (this.DynamicDataAttached(sourceLevelParameters, assembly, test, tests)) + { + continue; + } } tests.Add(test); diff --git a/src/Adapter/PlatformServices.NetCore/PlatformServices.NetCore.csproj b/src/Adapter/PlatformServices.NetCore/PlatformServices.NetCore.csproj index 9eb94fd028..46c0f48cea 100644 --- a/src/Adapter/PlatformServices.NetCore/PlatformServices.NetCore.csproj +++ b/src/Adapter/PlatformServices.NetCore/PlatformServices.NetCore.csproj @@ -2,6 +2,7 @@ NetCore + Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices @@ -14,6 +15,11 @@ true + + + + + @@ -77,5 +83,4 @@ Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices - \ No newline at end of file diff --git a/src/Adapter/PlatformServices.WinUI/PlatformServices.WinUI.csproj b/src/Adapter/PlatformServices.WinUI/PlatformServices.WinUI.csproj index 6d1e0feea6..1b32ebd154 100644 --- a/src/Adapter/PlatformServices.WinUI/PlatformServices.WinUI.csproj +++ b/src/Adapter/PlatformServices.WinUI/PlatformServices.WinUI.csproj @@ -2,6 +2,7 @@ NetCore + Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices @@ -18,6 +19,11 @@ true + + + + + @@ -98,5 +104,4 @@ Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices - \ No newline at end of file diff --git a/src/TestFramework/Extension.WinUI/Extension.WinUI.csproj b/src/TestFramework/Extension.WinUI/Extension.WinUI.csproj index 5ba1aef9aa..3d5de357e4 100644 --- a/src/TestFramework/Extension.WinUI/Extension.WinUI.csproj +++ b/src/TestFramework/Extension.WinUI/Extension.WinUI.csproj @@ -20,6 +20,11 @@ false win10-x86;win10-x64;win10-arm64 + + + + + @@ -43,5 +48,4 @@ $(OutputPath)\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.XML Extensions\WinUI - diff --git a/src/TestFramework/MSTest.Core/Attributes/DiscoverInternalsAttribute.cs b/src/TestFramework/MSTest.Core/Attributes/DiscoverInternalsAttribute.cs index 063e82f05d..78b6e62223 100644 --- a/src/TestFramework/MSTest.Core/Attributes/DiscoverInternalsAttribute.cs +++ b/src/TestFramework/MSTest.Core/Attributes/DiscoverInternalsAttribute.cs @@ -11,7 +11,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting /// internal in addition to test classes and test methods which are declared public. When this attribute is not /// present in a test assembly the tests in such classes will not be discovered. /// - [AttributeUsage(AttributeTargets.Assembly)] + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)] public class DiscoverInternalsAttribute : Attribute { } diff --git a/src/TestFramework/MSTest.Core/Attributes/TestDataSourceDiscoveryAttribute.cs b/src/TestFramework/MSTest.Core/Attributes/TestDataSourceDiscoveryAttribute.cs new file mode 100644 index 0000000000..3b9700f099 --- /dev/null +++ b/src/TestFramework/MSTest.Core/Attributes/TestDataSourceDiscoveryAttribute.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting +{ + using System; + + /// + /// Specifies how to discover ITestDataSource tests. + /// + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)] + public class TestDataSourceDiscoveryAttribute : Attribute + { + /// + /// Initializes a new instance of the class. + /// + /// + /// Sets which to use when discovering ITestDataSource tests. + /// + public TestDataSourceDiscoveryAttribute(TestDataSourceDiscoveryOption discoveryOption) + { + this.DiscoveryOption = discoveryOption; + } + + /// + /// Gets specified discovery option. + /// + public TestDataSourceDiscoveryOption DiscoveryOption { get; } + } +} diff --git a/src/TestFramework/MSTest.Core/Attributes/TestDataSourceDiscoveryOption.cs b/src/TestFramework/MSTest.Core/Attributes/TestDataSourceDiscoveryOption.cs new file mode 100644 index 0000000000..c8167cca6a --- /dev/null +++ b/src/TestFramework/MSTest.Core/Attributes/TestDataSourceDiscoveryOption.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting +{ + /// + /// Supported TestDataSource discovery modes + /// + public enum TestDataSourceDiscoveryOption + { + /// + /// Discover tests during execution. + /// This was the default option on version 2.2.3 and before. + /// + DuringExecution = 1, + + /// + /// Discover and expand ITestDataSource based tests. + /// This is the default behavior after version 2.2.3. + /// + DuringDiscovery = 2 + } +} diff --git a/src/TestFramework/MSTest.Core/MSTest.Core.csproj b/src/TestFramework/MSTest.Core/MSTest.Core.csproj index d7cab44955..e899d7b9e2 100644 --- a/src/TestFramework/MSTest.Core/MSTest.Core.csproj +++ b/src/TestFramework/MSTest.Core/MSTest.Core.csproj @@ -45,6 +45,8 @@ + + diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs index b0d99d1e98..4564227b1b 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs @@ -424,6 +424,12 @@ private static Mock CreateMockTestableAssembly() true)) .Returns(new Attribute[0]); + mockAssembly + .Setup(a => a.GetCustomAttributes( + typeof(FrameworkV2::Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceDiscoveryAttribute), + true)) + .Returns(new Attribute[0]); + return mockAssembly; }