Skip to content

Commit

Permalink
Refactor CI artifacts for release improvements (#10704)
Browse files Browse the repository at this point in the history
* Consolidate artifacts

* Disable sdl spotBugs

* Remove PrivateSiteExtension

* stage release_notes.md as artifact

* Add script to generate private site extension

* Copy release_notes.md in site extension

* Flatten SiteExtension artifacts, custom hash task

* Generate release.json during build

* modifyOutputPath: false

* Use SHA256 for the hash

* Default private siteext to 64bit

* Default private siteext to zipping

* Remove binlog artifact to save space

* Update SiteExtension readme

* Update src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1

Co-authored-by: Lilian Kasem <likasem@microsoft.com>

---------

Co-authored-by: Lilian Kasem <likasem@microsoft.com>
  • Loading branch information
jviau and liliankasem authored Jan 24, 2025
1 parent ef63b6b commit e6b3d16
Show file tree
Hide file tree
Showing 10 changed files with 299 additions and 89 deletions.
2 changes: 2 additions & 0 deletions eng/ci/official-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ extends:
codeql:
# Move codeql for source languages to source analysis stage
runSourceLanguagesInSourceAnalysis: true
spotBugs:
enabled: false # Explicitly disabling as our JavaWorker package causes this to light up and subsequently fail.

stages:
- stage: Build
Expand Down
24 changes: 9 additions & 15 deletions eng/ci/templates/official/jobs/build-artifacts-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,19 @@ jobs:
project: src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj
configuration: release
runtime: linux-x64
artifacts_path: $(Build.ArtifactStagingDirectory)
log_dir: $(artifacts_path)/log
zip_artifacts_path: $(artifacts_path)/Linux
drop_path: $(Build.ArtifactStagingDirectory)
linux_drop_path: $(drop_path)/linux
build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true'

templateContext:
outputParentDirectory: $(artifacts_path)
outputParentDirectory: $(drop_path)
outputs:
# TODO: onboard to Azure Artifacts Drops to allow accessing this from docker linux pipeline in msazure
# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/azure-artifacts/artifact-services-onboarding
- output: pipelineArtifact
displayName: Publish linux artifacts
path: $(zip_artifacts_path)
artifact: Linux
- output: pipelineArtifact
displayName: Publish logs
path: $(log_dir)
artifact: Linux_Log
sbomEnabled: false
condition: always()
path: $(linux_drop_path)
artifact: drop_linux

pool:
name: 1es-pool-azfunc
Expand All @@ -41,15 +34,15 @@ jobs:
command: custom
custom: restore
projects: $(project)
arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true -bl:$(log_dir)/restore.binlog' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701
arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701

- task: DotNetCoreCLI@2
displayName: Build
inputs:
command: custom
custom: build
projects: $(project)
arguments: '$(build_args) --no-restore -bl:$(log_dir)/build.binlog'
arguments: '$(build_args) --no-restore'

- task: DotNetCoreCLI@2
displayName: Publish
Expand All @@ -58,5 +51,6 @@ jobs:
custom: publish
publishWebProjects: false # we use our own publish logic
zipAfterPublish: false # we use our own zip logic
modifyOutputPath: false
projects: $(project)
arguments: '$(build_args) --no-build -p:ZipArtifactsPath=$(zip_artifacts_path) -bl:$(log_dir)/publish.binlog'
arguments: '$(build_args) --no-build -o $(linux_drop_path)/host'
37 changes: 13 additions & 24 deletions eng/ci/templates/official/jobs/build-artifacts-windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,45 +8,34 @@ jobs:
os: windows

variables:
nuget_package_path: $(Build.ArtifactStagingDirectory)/NugetPackages
log_dir: $(Build.ArtifactStagingDirectory)/log
drop_path: $(Build.ArtifactStagingDirectory)
nuget_drop_path: $(drop_path)/nuget
windows_drop_path: $(drop_path)/windows

templateContext:
outputParentDirectory: $(Build.ArtifactStagingDirectory)
outputParentDirectory: $(drop_path)
outputs:
- output: pipelineArtifact
displayName: Publish site extension
path: $(Build.ArtifactStagingDirectory)/SiteExtension
artifact: SiteExtension
- output: pipelineArtifact
displayName: Publish private site extension
path: $(Build.ArtifactStagingDirectory)/PrivateSiteExtension
artifact: PrivateSiteExtension
- output: pipelineArtifact
displayName: Publish site extension symbols
path: $(Build.ArtifactStagingDirectory)/SiteExtensionSymbols
artifact: Symbols
path: $(windows_drop_path)
artifact: drop_windows
- output: pipelineArtifact
displayName: Publish nuget packages
path: $(Build.ArtifactStagingDirectory)/NugetPackages
artifact: NugetPackages
- output: pipelineArtifact
displayName: Publish logs
path: $(log_dir)
artifact: Windows_Log
sbomEnabled: false
condition: always()
path: $(nuget_drop_path)
artifact: drop_nuget

steps:
- template: /eng/ci/templates/install-dotnet.yml@self
- template: /eng/ci/templates/steps/build-site-ext.yml@self
parameters:
publishDir: $(windows_drop_path)

- task: DotNetCoreCLI@2
displayName: Build host packages
inputs:
command: custom
custom: pack
arguments: -c release -o $(nuget_package_path)
arguments: -c release -o $(nuget_drop_path)
projects: |
**/WebJobs.Script.csproj
**/WebJobs.Script.WebHost.csproj
Expand Down Expand Up @@ -74,7 +63,7 @@ jobs:
inputs:
command: custom
custom: pack
arguments: '--no-build -c release -o $(nuget_package_path)'
arguments: '--no-build -c release -o $(nuget_drop_path)'
projects: |
**/WebJobs.Script.Abstractions.csproj
Expand All @@ -91,7 +80,7 @@ jobs:
inputs:
command: custom
custom: pack
arguments: '--no-build -c release -o $(nuget_package_path)'
arguments: '--no-build -c release -o $(nuget_drop_path)'
projects: |
**/ExtensionsMetadataGenerator.csproj
Expand Down
13 changes: 10 additions & 3 deletions eng/ci/templates/steps/build-site-ext.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ parameters:
- name: project
type: string
default: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj
- name: publishDir
type: string
default: ''

steps:
# Restore must be a separate step so we can pass in 'PublishReadyToRun=true'
Expand All @@ -11,15 +14,15 @@ steps:
command: custom
custom: restore
projects: ${{ parameters.project }}
arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/site_ext.restore.binlog'
arguments: '-v m -p:PublishReadyToRun=true'

- task: DotNetCoreCLI@2
displayName: Build site extension
inputs:
command: custom
custom: build
projects: ${{ parameters.project }}
arguments: '--no-restore -v m -c release -bl:$(log_dir)/site_ext.build.binlog'
arguments: '--no-restore -v m -c release'

- task: DotNetCoreCLI@2
displayName: Publish site extension
Expand All @@ -28,5 +31,9 @@ steps:
custom: publish
publishWebProjects: false # we use our own publish logic
zipAfterPublish: false # we use our own zip logic
modifyOutputPath: false
projects: ${{ parameters.project }}
arguments: '--no-build -v m -c release -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog'
${{ if eq(parameters.publishDir, '') }}:
arguments: '--no-build -v m -c release'
${{ else }}:
arguments: '--no-build -v m -c release -o ${{ parameters.publishDir }}'
130 changes: 130 additions & 0 deletions src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
<#
.SYNOPSIS
Produces a private site extension.
.DESCRIPTION
Takes in a published site extension and produces a private site extension.
.PARAMETER InputPath
The path of the published 'SiteExtension'. Leave null to scan for root in a child from here.
.PARAMETER OutputPath
The path to produce the private site extension to (either the zip file or folder). Leave null to compute this name.
.PARAMETER Bitness
The bitness to produce the private site extension with. Default is '64bit'.
.PARAMETER NoZip
[Switch] Include to produce site extension as a folder and not a zip.
.PARAMETER Force
[Switch] Include to overwrite existing files.
.INPUTS
None. You can't pipe objects to Update-Month.ps1.
.OUTPUTS
None. Update-Month.ps1 doesn't generate any output.
#>

param (
[string] $InputPath = $null,
[string] $OutputPath = $null,
[ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '64bit',
[switch] $NoZip,
[switch] $Force
)

$normalizeBitness = @{
'x64' = '64bit'
'64bit' = '64bit'
'x86' = '64bit'
'32bit' = '32bit'
}

$Bitness = $normalizeBitness[$Bitness]

if (-not $InputPath)
{
$InputPath = (Get-ChildItem -Path . -Filter "extension.xml" -Recurse).Directory.FullName
}

if (Test-Path (Join-Path $InputPath "WebJobs.Script.SiteExtension.csproj"))
{
Write-Error "This script should not be ran in the WebJobs.Script.SiteExtension project folder. Run this script in the root of the published site extension folder."
exit 1
}

if (-not (Join-Path $InputPath "extension.xml" | Test-Path))
{
Write-Error "Unable to find published site extension."
exit 1
}

if (-not $OutputPath)
{
$runtime = $Bitness -eq '32bit' ? 'win-x86' : 'win-x64'
$leaf = (Split-Path $InputPath -Leaf)
$split = $leaf.IndexOf('.')
$OutputPath = "$($leaf.Substring(0, $split)).Private.$($leaf.Substring($split + 1)).$runtime"
}

function New-TemporaryDirectory {
$tmp = [System.IO.Path]::GetTempPath()
$name = (New-Guid).ToString("N")
return New-Item -ItemType Directory -Path (Join-Path $tmp $name)
}

function Write-Zip ($outputPath)
{
if (Test-Path $outputPath) {
if ($Force) {
Remove-Item -Path $outputPath -Recurse -Force
} else {
Write-Error "OutputPath already exists. Use -Force to overwrite."
exit 1
}
}

$tempDir = New-TemporaryDirectory
Write-Folder $tempDir

Compress-Archive -Path "$tempDir/*" -DestinationPath $outputPath
Remove-Item -Path $tempDir -Recurse -Force
}

function Write-Folder ($outputPath)
{
if (Test-Path "$outputPath/*") {
if ($Force) {
Remove-Item -Path $outputPath -Recurse -Force
} else {
Write-Error "OutputPath already exists. Use -Force to overwrite."
exit 1
}
}

$inputPath = Get-ChildItem -Path $InputPath -Attributes Directory
$outputPath = Join-Path $outputPath "SiteExtensions" "Functions"
New-Item -ItemType Directory -Path $outputPath | Out-Null

Copy-Item "$inputPath/applicationHost.xdt" -Destination $outputPath | Out-Null

$filesDest = (Join-Path $outputPath "$Bitness")
Copy-Item "$inputPath/$Bitness/" -Destination "$filesDest/" -Container -Recurse | Out-Null

$workerDest = (Join-Path $filesDest "workers")
Copy-Item "$inputPath/workers/" -Destination "$workerDest/" -Container -Recurse | Out-Null
}

if ($NoZip) {
Write-Folder $OutputPath
} else {
if (-not $OutputPath.EndsWith(".zip")) {
$OutputPath = "$OutputPath.zip"
}

Write-Zip $OutputPath
}

Write-Host "Published private site extension to $OutputPath"
41 changes: 41 additions & 0 deletions src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
param(
[string] $Version = $null,
[string[]] $Artifacts = $null,
[string] $CommitId = $null,
[string] $OutputPath
)

if (-not $Version) {
$Version = dotnet build $PSScriptRoot --getProperty:Version
}

if (-Not $Artifacts) {
$Artifacts = @(
"Functions.$Version"
"Functions.Symbols.$Version.win-x64"
"Functions.Symbols.$Version.win-x86"
)
}

if (-not $CommitId) {
$CommitId = $env:BUILD_SOURCEVERSION
}

if (-not $CommitId) {
$CommitId = (git rev-parse HEAD).Trim()
}

$obj = @{
name = $Version
artifacts = $Artifacts
tag = "v$Version"
commitId = $CommitId
releaseNotesFile = "release_notes.md"
}

if ($OutputPath) {
Write-Host "Writing $obj to $OutputPath"
$obj | ConvertTo-Json | Out-File $OutputPath
}

return $obj
Loading

0 comments on commit e6b3d16

Please # to comment.