[main] Source code updates from dotnet/dotnet

eng/common/build.ps1

@@ -21,6 +21,7 @@ Param(
   [switch] $publish,
   [switch] $clean,
   [switch][Alias('pb')]$productBuild,
+  [switch]$fromVMR,
   [switch][Alias('bl')]$binaryLog,
   [switch][Alias('nobl')]$excludeCIBinarylog,
   [switch] $ci,
@@ -74,6 +75,7 @@ function Print-Usage() {
   Write-Host "  -nativeToolsOnMachine   Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
   Write-Host "  -nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
   Write-Host "  -buildCheck             Sets /check msbuild parameter"
+  Write-Host "  -fromVMR                Set when building from within the VMR"
   Write-Host ""
 
   Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -127,7 +129,8 @@ function Build {
     /p:Deploy=$deploy `
     /p:Test=$test `
     /p:Pack=$pack `
-    /p:DotNetBuildRepo=$productBuild `
+    /p:DotNetBuild=$productBuild `
+    /p:DotNetBuildFromVMR=$fromVMR `
     /p:IntegrationTest=$integrationTest `
     /p:PerformanceTest=$performanceTest `
     /p:Sign=$sign `

eng/common/build.sh

@@ -43,6 +43,7 @@ usage()
   echo "  --nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
   echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   echo "  --buildCheck <value>     Sets /check msbuild parameter"
+  echo "  --fromVMR                Set when building from within the VMR"
   echo ""
   echo "Command line arguments not listed above are passed thru to msbuild."
   echo "Arguments can also be passed in with a single hyphen."
@@ -64,6 +65,7 @@ restore=false
 build=false
 source_build=false
 product_build=false
+from_vmr=false
 rebuild=false
 test=false
 integration_test=false
@@ -89,7 +91,7 @@ verbosity='minimal'
 runtime_source_feed=''
 runtime_source_feed_key=''
 
-properties=''
+properties=()
 while [[ $# > 0 ]]; do
   opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
@@ -129,19 +131,22 @@ while [[ $# > 0 ]]; do
     -pack)
       pack=true
       ;;
-    -sourcebuild|-sb)
+    -sourcebuild|-source-build|-sb)
       build=true
       source_build=true
       product_build=true
       restore=true
       pack=true
       ;;
-    -productbuild|-pb)
+    -productbuild|-product-build|-pb)
       build=true
       product_build=true
       restore=true
       pack=true
       ;;
+    -fromvmr|-from-vmr)
+      from_vmr=true
+      ;;
     -test|-t)
       test=true
       ;;
@@ -187,7 +192,7 @@ while [[ $# > 0 ]]; do
       shift
       ;;
     *)
-      properties="$properties $1"
+      properties+=("$1")
       ;;
   esac
 
@@ -221,7 +226,7 @@ function Build {
   InitializeCustomToolset
 
   if [[ ! -z "$projects" ]]; then
-    properties="$properties /p:Projects=$projects"
+    properties+=("/p:Projects=$projects")
   fi
 
   local bl=""
@@ -241,8 +246,9 @@ function Build {
     /p:RepoRoot="$repo_root" \
     /p:Restore=$restore \
     /p:Build=$build \
-    /p:DotNetBuildRepo=$product_build \
+    /p:DotNetBuild=$product_build \
     /p:DotNetBuildSourceOnly=$source_build \
+    /p:DotNetBuildFromVMR=$from_vmr \
     /p:Rebuild=$rebuild \
     /p:Test=$test \
     /p:Pack=$pack \
@@ -251,7 +257,7 @@ function Build {
     /p:Sign=$sign \
     /p:Publish=$publish \
     /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \
-    $properties
+    "${properties[@]}"
 
   ExitWithExitCode 0
 }

eng/common/core-templates/post-build/post-build.yml

@@ -193,9 +193,6 @@ stages:
             buildId: $(AzDOBuildId)
             artifactName: PackageArtifacts
             checkDownloadedFiles: true
-            itemPattern: |
-              **
-              !**/Microsoft.SourceBuild.Intermediate.*.nupkg
 
         # This is necessary whenever we want to publish/restore to an AzDO private feed
         # Since sdk-task.ps1 tries to restore packages we need to do this authentication here

eng/common/core-templates/steps/install-microbuild.yml

@@ -48,38 +48,3 @@ steps:
             eq(variables['_SignType'], 'real')
           )
         ))
-
-    # Workaround for ESRP CLI on Linux - https://github.com/dotnet/source-build/issues/4964
-    - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
-      - task: UseDotNet@2
-        displayName: Install .NET 9.0 SDK for ESRP CLI Workaround
-        inputs:
-          packageType: sdk
-          version: 9.0.x
-          installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet
-          workingDirectory: ${{ parameters.microBuildOutputFolder }}
-        condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
-
-      - task: PowerShell@2
-        displayName: Workaround for ESRP CLI on Linux
-        inputs:
-          targetType: 'inline'
-          script: |
-            Write-Host "Copying Linux Path"
-            $MBSIGN_APPFOLDER = '$(MBSIGN_APPFOLDER)'
-            $MBSIGN_APPFOLDER = ($MBSIGN_APPFOLDER -replace '/build', '')
-
-            $versionRegex = '\d+\.\d+\.\d+'
-            $package = Get-ChildItem -Path $MBSIGN_APPFOLDER -Directory |
-              Where-Object { $_.Name -match $versionRegex }
-
-            if ($package.Count -ne 1) {
-                Write-Host "There should be exactly one matching subfolder, but found $($package.Count)."
-                exit 1
-            }
-
-            $MBSIGN_APPFOLDER = $package[0].FullName + '/build'
-            $MBSIGN_APPFOLDER | Write-Host
-            $SignConfigPath = $MBSIGN_APPFOLDER + '/signconfig.xml'
-            Copy-Item -Path "$(MBSIGN_APPFOLDER)/signconfig.xml" -Destination $SignConfigPath -Force
-        condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))

eng/common/sdl/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Guardian.Cli" version="0.109.0"/>
+  <package id="Microsoft.Guardian.Cli" version="0.199.0"/>
 </packages>

eng/common/templates/steps/vmr-sync.yml

@@ -0,0 +1,207 @@
+### These steps synchronize new code from product repositories into the VMR (https://github.com/dotnet/dotnet).
+### They initialize the darc CLI and pull the new updates.
+### Changes are applied locally onto the already cloned VMR (located in $vmrPath).
+
+parameters:
+- name: targetRef
+  displayName: Target revision in dotnet/<repo> to synchronize
+  type: string
+  default: $(Build.SourceVersion)
+
+- name: vmrPath
+  displayName: Path where the dotnet/dotnet is checked out to
+  type: string
+  default: $(Agent.BuildDirectory)/vmr
+
+- name: additionalSyncs
+  displayName: Optional list of package names whose repo's source will also be synchronized in the local VMR, e.g. NuGet.Protocol
+  type: object
+  default: []
+
+steps:
+- checkout: vmr
+  displayName: Clone dotnet/dotnet
+  path: vmr
+  clean: true
+
+- checkout: self
+  displayName: Clone $(Build.Repository.Name)
+  path: repo
+  fetchDepth: 0
+
+# This step is needed so that when we get a detached HEAD / shallow clone,
+# we still pull the commit into the temporary repo clone to use it during the sync.
+# Also unshallow the clone so that forwardflow command would work.
+- script: |
+    git branch repo-head
+    git rev-parse HEAD
+  displayName: Label PR commit
+  workingDirectory: $(Agent.BuildDirectory)/repo
+
+- script: |
+    vmr_sha=$(grep -oP '(?<=Sha=")[^"]*' $(Agent.BuildDirectory)/repo/eng/Version.Details.xml)
+    echo "##vso[task.setvariable variable=vmr_sha]$vmr_sha"
+  displayName: Obtain the vmr sha from Version.Details.xml (Unix)
+  condition: ne(variables['Agent.OS'], 'Windows_NT')
+  workingDirectory: $(Agent.BuildDirectory)/repo
+
+- powershell: |
+    [xml]$xml = Get-Content -Path $(Agent.BuildDirectory)/repo/eng/Version.Details.xml
+    $vmr_sha = $xml.SelectSingleNode("//Source").Sha
+    Write-Output "##vso[task.setvariable variable=vmr_sha]$vmr_sha"
+  displayName: Obtain the vmr sha from Version.Details.xml (Windows)
+  condition: eq(variables['Agent.OS'], 'Windows_NT')
+  workingDirectory: $(Agent.BuildDirectory)/repo
+
+- script: |
+    git fetch --all
+    git checkout $(vmr_sha)
+  displayName: Checkout VMR at correct sha for repo flow
+  workingDirectory: ${{ parameters.vmrPath }}
+
+- script: |
+    git config --global user.name "dotnet-maestro[bot]"
+    git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com"
+  displayName: Set git author to dotnet-maestro[bot]
+  workingDirectory: ${{ parameters.vmrPath }}
+
+- script: |
+    ./eng/common/vmr-sync.sh                         \
+      --vmr ${{ parameters.vmrPath }}         \
+      --tmp $(Agent.TempDirectory)            \
+      --azdev-pat '$(dn-bot-all-orgs-code-r)' \
+      --ci                                    \
+      --debug
+
+    if [ "$?" -ne 0 ]; then
+      echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
+      exit 1
+    fi
+  displayName: Sync repo into VMR (Unix)
+  condition: ne(variables['Agent.OS'], 'Windows_NT')
+  workingDirectory: $(Agent.BuildDirectory)/repo
+
+- script: |
+    git config --global diff.astextplain.textconv echo
+    git config --system core.longpaths true
+  displayName: Configure Windows git (longpaths, astextplain)
+  condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+- powershell: |
+    ./eng/common/vmr-sync.ps1                      `
+      -vmr ${{ parameters.vmrPath }}        `
+      -tmp $(Agent.TempDirectory)           `
+      -azdevPat '$(dn-bot-all-orgs-code-r)' `
+      -ci                                   `
+      -debugOutput
+
+    if ($LASTEXITCODE -ne 0) {
+      echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
+      exit 1
+    }
+  displayName: Sync repo into VMR (Windows)
+  condition: eq(variables['Agent.OS'], 'Windows_NT')
+  workingDirectory: $(Agent.BuildDirectory)/repo
+
+- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+  - task: CopyFiles@2
+    displayName: Collect failed patches
+    condition: failed()
+    inputs:
+      SourceFolder: '$(Agent.TempDirectory)'
+      Contents: '*.patch'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)/FailedPatches'
+
+  - publish: '$(Build.ArtifactStagingDirectory)/FailedPatches'
+    artifact: $(System.JobDisplayName)_FailedPatches
+    displayName: Upload failed patches
+    condition: failed()
+
+- ${{ each assetName in parameters.additionalSyncs }}:
+  # The vmr-sync script ends up staging files in the local VMR so we have to commit those
+  - script:
+      git commit --allow-empty -am "Forward-flow $(Build.Repository.Name)"
+    displayName: Commit local VMR changes
+    workingDirectory: ${{ parameters.vmrPath }}
+
+  - script: |
+      set -ex
+
+      echo "Searching for details of asset ${{ assetName }}..."
+
+      # Use darc to get dependencies information
+      dependencies=$(./.dotnet/dotnet darc get-dependencies --name '${{ assetName }}' --ci)
+
+      # Extract repository URL and commit hash
+      repository=$(echo "$dependencies" | grep 'Repo:' | sed 's/Repo:[[:space:]]*//' | head -1)
+
+      if [ -z "$repository" ]; then
+        echo "##vso[task.logissue type=error]Asset ${{ assetName }} not found in the dependency list"
+        exit 1
+      fi
+
+      commit=$(echo "$dependencies" | grep 'Commit:' | sed 's/Commit:[[:space:]]*//' | head -1)
+
+      echo "Updating the VMR from $repository / $commit..."
+      cd ..
+      git clone $repository ${{ assetName }}
+      cd ${{ assetName }}
+      git checkout $commit
+      git branch "sync/$commit"
+
+      ./eng/common/vmr-sync.sh                         \
+        --vmr ${{ parameters.vmrPath }}         \
+        --tmp $(Agent.TempDirectory)            \
+        --azdev-pat '$(dn-bot-all-orgs-code-r)' \
+        --ci                                    \
+        --debug
+
+      if [ "$?" -ne 0 ]; then
+        echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
+        exit 1
+      fi
+    displayName: Sync ${{ assetName }} into (Unix)
+    condition: ne(variables['Agent.OS'], 'Windows_NT')
+    workingDirectory: $(Agent.BuildDirectory)/repo
+
+  - powershell: |
+      $ErrorActionPreference = 'Stop'
+
+      Write-Host "Searching for details of asset ${{ assetName }}..."
+
+      $dependencies = .\.dotnet\dotnet darc get-dependencies --name '${{ assetName }}' --ci
+
+      $repository = $dependencies | Select-String -Pattern 'Repo:\s+([^\s]+)' | Select-Object -First 1
+      $repository -match 'Repo:\s+([^\s]+)' | Out-Null
+      $repository = $matches[1]
+
+      if ($repository -eq $null) {
+          Write-Error "Asset ${{ assetName }} not found in the dependency list"
+          exit 1
+      }
+
+      $commit = $dependencies | Select-String -Pattern 'Commit:\s+([^\s]+)' | Select-Object -First 1
+      $commit -match 'Commit:\s+([^\s]+)' | Out-Null
+      $commit = $matches[1]
+
+      Write-Host "Updating the VMR from $repository / $commit..."
+      cd ..
+      git clone $repository ${{ assetName }}
+      cd ${{ assetName }}
+      git checkout $commit
+      git branch "sync/$commit"
+
+      .\eng\common\vmr-sync.ps1 `
+        -vmr ${{ parameters.vmrPath }}             `
+        -tmp $(Agent.TempDirectory)                `
+        -azdevPat '$(dn-bot-all-orgs-code-r)'      `
+        -ci                                        `
+        -debugOutput
+
+      if ($LASTEXITCODE -ne 0) {
+        echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
+        exit 1
+      }
+    displayName: Sync ${{ assetName }} into (Windows)
+    condition: ne(variables['Agent.OS'], 'Windows_NT')
+    workingDirectory: $(Agent.BuildDirectory)/repo