Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Improve yaml parsing ++ #313

Merged
merged 39 commits into from
Oct 29, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
774b112
Test yaml parser
Oct 29, 2021
d9279ae
trusting psgallery
Oct 29, 2021
ab88835
test without trusting psgallery
Oct 29, 2021
3fc1e18
update trigger
Oct 29, 2021
3687fae
adding -SkipPublisherCheck
Oct 29, 2021
c33c260
adding -SkipPublisherCheck
Oct 29, 2021
653ec2c
added force
Oct 29, 2021
9d38ec7
Merge branch 'ParseYAML' of https://github.com/Azure/ResourceModules …
Oct 29, 2021
549954f
Modified Get-WorkflowDefaultInput
Oct 29, 2021
d801ca4
added cmdlet binding to support verbose
Oct 29, 2021
abbcb17
Added output of cmd in aciton
Oct 29, 2021
001f53f
Testing env + yml parser
Oct 29, 2021
b715ad2
List available modules
Oct 29, 2021
2e300a9
Trigger push on budgets
Oct 29, 2021
b1720db
test push on budgets
Oct 29, 2021
c9b6e34
Skipping some prereqs on azcli, bicep cli
Oct 29, 2021
d55545e
Fix global module test
Oct 29, 2021
9b3394f
Testing on budget
Oct 29, 2021
24ba9ff
remove a single comment
Oct 29, 2021
ad25075
Set output to verbose
Oct 29, 2021
871df02
cleanup settings
Oct 29, 2021
968137a
Reverted back to string parsing
Oct 29, 2021
89dfdc3
Reverting to string parsing
Oct 29, 2021
f60fa74
Testing with budget module
Oct 29, 2021
d8d9851
Testing budgets
Oct 29, 2021
14a1865
Moved helper function into begin
Oct 29, 2021
7a2fbaf
Comment why installs are skipped
Oct 29, 2021
3d4429f
Fix output from Get-WorkflowDefaultInput
Oct 29, 2021
9cbf59b
Test deployment of budgets
Oct 29, 2021
6358f55
Cleanup verbose
Oct 29, 2021
91d08fc
Add dynamic check sections in yaml files
Oct 29, 2021
7500340
Test on budgets
Oct 29, 2021
425ce07
Test continue on further indented + exist on match
Oct 29, 2021
daadff9
Test stuff on budgets
Oct 29, 2021
cb1c719
Test parse
Oct 29, 2021
4d3db43
Bugfix
Oct 29, 2021
5c343a7
Reset budgets workflow
Oct 29, 2021
9360a3e
Update condition on for loop
Oct 29, 2021
afb1cac
fix for condition
Oct 29, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 38 additions & 19 deletions .github/actions/sharedScripts/Set-EnvironmentOnAgent.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ function Install-CustomModule {
if (Get-Module $Module -ErrorAction SilentlyContinue) {
try {
Remove-Module $Module -Force
}
catch {
} catch {
Write-Error ("Unable to remove module $($Module.Name) : $($_.Exception) found, $($_.ScriptStackTrace)")
}
}
Expand All @@ -51,22 +50,21 @@ function Install-CustomModule {

$localModuleVersions = Get-Module $foundModule.Name -ListAvailable
if ($localModuleVersions -and $localModuleVersions.Version -contains $foundModule.Version ) {
Write-Verbose ("Module [{0}] already installed with latest version [{1}]" -f $foundModule.Name, $foundModule.Version) -Verbose
Write-Verbose ('Module [{0}] already installed with latest version [{1}]' -f $foundModule.Name, $foundModule.Version) -Verbose
continue
}
if ($module.ExcludeModules -and $module.excludeModules.contains($foundModule.Name)) {
Write-Verbose ("Module {0} is configured to be ignored." -f $foundModule.Name) -Verbose
Write-Verbose ('Module {0} is configured to be ignored.' -f $foundModule.Name) -Verbose
continue
}

Write-Verbose ("Install module [{0}] with version [{1}]" -f $foundModule.Name, $foundModule.Version) -Verbose
if ($PSCmdlet.ShouldProcess("Module [{0}]" -f $foundModule.Name, "Install")) {
Write-Verbose ('Install module [{0}] with version [{1}]' -f $foundModule.Name, $foundModule.Version) -Verbose
if ($PSCmdlet.ShouldProcess('Module [{0}]' -f $foundModule.Name, 'Install')) {
$foundModule | Install-Module -Force -SkipPublisherCheck -AllowClobber
if ($installed = Get-Module -Name $foundModule.Name -ListAvailable) {
Write-Verbose ("Module [{0}] is installed with version [{1}]" -f $installed.Name, $installed.Version) -Verbose
}
else {
Write-Error ("Installation of module [{0}] failed" -f $foundModule.Name)
Write-Verbose ('Module [{0}] is installed with version [{1}]' -f $installed.Name, $installed.Version) -Verbose
} else {
Write-Error ('Installation of module [{0}] failed' -f $foundModule.Name)
}
}
}
Expand Down Expand Up @@ -106,13 +104,26 @@ function Set-EnvironmentOnAgent {
###########################
## Install Azure CLI ##
###########################

# AzCLI is pre-installed on GitHub hosted runners.
# https://github.com/actions/virtual-environments#available-environments

az --version
<#
Write-Verbose ("Install azure cli start") -Verbose
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
Write-Verbose ("Install azure cli end") -Verbose
#>

##############################
## Install Bicep for CLI #
##############################

# Bicep CLI is pre-installed on GitHub hosted runners.
# https://github.com/actions/virtual-environments#available-environments

bicep --version
<#
Write-Verbose ("Install bicep start") -Verbose
# Fetch the latest Bicep CLI binary
curl -Lo bicep 'https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64'
Expand All @@ -123,12 +134,19 @@ function Set-EnvironmentOnAgent {
# Add bicep to your PATH (requires admin)
sudo mv ./bicep /usr/local/bin/bicep
Write-Verbose ("Install bicep end") -Verbose
#>

###############################
## Install Extensions CLI #
###############################

Write-Verbose ("Install cli exentions start") -Verbose
# Azure CLI extension for DevOps is pre-installed on GitHub hosted runners.
# https://github.com/actions/virtual-environments#available-environments

az extension list | ConvertFrom-Json | Select-Object -Property name, version, preview, experimental

<#
Write-Verbose ('Install cli exentions start') -Verbose
$Extensions = @(
'azure-devops'
)
Expand All @@ -138,28 +156,29 @@ function Set-EnvironmentOnAgent {
az extension add --name $extension
}
}
Write-Verbose ("Install cli exentions end") -Verbose
Write-Verbose ('Install cli exentions end') -Verbose
#>

####################################
## Install PowerShell Modules ##
####################################

$count = 1
Write-Verbose ("Try installing:") -Verbose
Write-Verbose ('Try installing:') -Verbose
$modules | ForEach-Object {
Write-Verbose ("- {0}. [{1}]" -f $count, $_.Name) -Verbose
Write-Verbose ('- {0}. [{1}]' -f $count, $_.Name) -Verbose
$count++
}

Write-Verbose ("Install-CustomModule start") -Verbose
Write-Verbose ('Install-CustomModule start') -Verbose
$count = 1
Foreach ($Module in $Modules) {
Write-Verbose ("=====================") -Verbose
Write-Verbose ("HANDLING MODULE [{0}/{1}] [{2}] " -f $count, $Modules.Count, $Module.Name) -Verbose
Write-Verbose ("=====================") -Verbose
Write-Verbose ('=====================') -Verbose
Write-Verbose ('HANDLING MODULE [{0}/{1}] [{2}] ' -f $count, $Modules.Count, $Module.Name) -Verbose
Write-Verbose ('=====================') -Verbose
# Installing New Modules and Removing Old
$null = Install-CustomModule -Module $Module
$count++
}
Write-Verbose ("Install-CustomModule end") -Verbose
Write-Verbose ('Install-CustomModule end') -Verbose
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,40 +14,117 @@ Get-WorkflowDefaultInput -workflowPath 'path/to/workflow' -verbose
Retrieve input parameter default values for the 'path/to/workflow' workflow.
#>
function Get-WorkflowDefaultInput {

[CmdletBinding()]
param (
[Parameter(Mandatory)]
[string] $workflowPath

)

begin {
Write-Debug ('{0} entered' -f $MyInvocation.MyCommand)

#region Helper Functions

<#
.SYNOPSIS
Retrieve indentation of a line.

.PARAMETER Line
Mandatory. The line to analyse for indentation.

.EXAMPLE
$Line = ' Test'
Get-LineIndentation -Line $Line
4

Retrieve indentation of a line.
#>
function Get-LineIndentation {
[CmdletBinding()]
param (
[Parameter()]
[string] $Line
)
begin {}
process {
$indentation = 0
for ($i = 0; $i -lt $Line.Length; $i++) {
$Char = $Line[$i]
switch -regex ($Char) {
'`t' {
$indentation += 2
}
' ' {
$indentation += 1
}
default {
return $indentation
}
}
}
return $indentation
}
end {}
}

<#
.SYNOPSIS
Retrieve default value for a specified input in a workflow.

.PARAMETER InputName
Mandatory. The name of the input to get the default value for.

.PARAMETER Content
Mandatory. The content of the GitHub workflow file.

.EXAMPLE
$content = Get-Content -Path .\workflow.yml
Get-DefaultValue -Text 'removeDeployment' -Content $Content

Retrieve input default values for the 'removeDeployment' in the workflow.yml file.
#>
function Get-DefaultValue {
[CmdletBinding()]
param (
[Parameter(Mandatory)]
[string] $InputName,
[Parameter(Mandatory)]
[string[]] $Content
)
$Content = $Content.Split([Environment]::NewLine)
$SectionStartLine = ((0..($Content.Count - 1)) | Where-Object { $Content[$_] -match "$InputName" })[0]
$SectionStartIndentation = Get-LineIndentation -Line $Content[$SectionStartLine]
$CurrentLineIndentation = $SectionStartIndentation
for ($i = $SectionStartLine + 1; $i -lt $Content.Count; $i++) {
$CurrentLineIndentation = Get-LineIndentation -Line $Content[$i]
if ($CurrentLineIndentation -le $SectionStartIndentation) {
# Outside of start section, jumping out
break
}
if ($CurrentLineIndentation -gt $SectionStartIndentation + 2) {
# In child section, ignoring
continue
}
if ($Content[$i] -match 'default:') {
$defaultValue = $Content[$i].trim().Split('#')[0].Split(':')[-1].Replace("'", '').Trim()
break
}
}
Write-Verbose "Default input value for $InputName`: $defaultValue"
return $defaultValue
}
#endregion
}

process {
$content = Get-Content $workflowPath

# Get 'removeDeployment' default input value
$removeDeploymentRowIndex = ((0..($content.Count - 1)) | Where-Object { $content[$_] -like '*removeDeployment:*' })[0]
$removeDeployment = $content[$removeDeploymentRowIndex + 3].trim().Split(':')[1].Trim().Replace("'", '').Replace('"', '')
Write-Verbose "Default input value for removeDeployment: $removeDeployment"

# Get 'versioningOption' default input value
$versioningOptionRowIndex = ((0..($content.Count - 1)) | Where-Object { $content[$_] -like '*versioningOption:*' })[0]
$versioningOption = $content[$versioningOptionRowIndex + 3].trim().Split(':')[1].Trim().Replace("'", '').Replace('"', '')
Write-Verbose "Default input value for versioningOption: $versioningOption"

# Get 'customVersion' default input value
$customVersionRowIndex = ((0..($content.Count - 1)) | Where-Object { $content[$_] -like '*customVersion:*' })[0]
$customVersion = $content[$customVersionRowIndex + 3].trim().Split(':')[1].Trim().Replace("'", '').Replace('"', '')
Write-Verbose "Default input value for customVersion: $customVersion"

# Define hashtable to contain workflow parameters
$workflowParameters = @{}
$workflowParameters.Add('removeDeployment', $removeDeployment)
$workflowParameters.Add('versioningOption', $versioningOption)
$workflowParameters.Add('customVersion', $customVersion)
$workflowContent = Get-Content -Path $workflowPath -Raw

$workflowParameters = @{
removeDeployment = Get-DefaultValue -InputName 'removeDeployment' -Content $workflowContent -Verbose
versioningOption = Get-DefaultValue -InputName 'versioningOption' -Content $workflowContent -Verbose
customVersion = Get-DefaultValue -InputName 'customVersion' -Content $workflowContent -Verbose
}

Write-Verbose 'Get workflow default input complete'

# Return hashtable
Expand All @@ -58,3 +135,11 @@ function Get-WorkflowDefaultInput {
Write-Debug ('{0} exited' -f $MyInvocation.MyCommand)
}
}


$Test = @'
One
Two
Three
Four
'@
2 changes: 1 addition & 1 deletion .github/actions/templates/validateModuleApis/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ runs:
Invoke-Pester -Configuration @{
Run = @{
Container = New-PesterContainer -Path 'arm/.global/global.module.tests.ps1' -Data @{
moduleFolderPaths = "${{ inputs.modulePath }}"
moduleFolderPaths = Join-Path "$env:GITHUB_WORKSPACE" "${{ inputs.modulePath }}"
}
}
Filter = @{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ runs:
Invoke-Pester -Configuration @{
Run = @{
Container = New-PesterContainer -Path 'arm/.global/global.module.tests.ps1' -Data @{
moduleFolderPaths = "${{ inputs.modulePath }}"
moduleFolderPaths = Join-Path "$env:GITHUB_WORKSPACE" "${{ inputs.modulePath }}"
}
}
Filter = @{
Expand Down
16 changes: 8 additions & 8 deletions arm/.global/global.module.tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@ Describe 'File/folder tests' -Tag Modules {

$moduleFolderTestCases = [System.Collections.ArrayList] @()
foreach ($moduleFolderPath in $moduleFolderPaths) {

$moduleFolderTestCases += @{
moduleFolderName = $moduleFolderPath.Split('\arm\')[1]
moduleFolderName = $moduleFolderPath.Replace('\', '/').Split('/arm/')[1]
moduleFolderPath = $moduleFolderPath
}
}

It '[<moduleFolderName>] Module should contain a [deploy.json/deploy.bicep] file' -TestCases $moduleFolderTestCases {
param( [string] $moduleFolderPath )

$hasARM = (Test-Path (Join-Path -Path $moduleFolderPath 'deploy.json'))
$hasBicep = (Test-Path (Join-Path -Path $moduleFolderPath 'deploy.bicep'))
($hasARM -or $hasBicep) | Should -Be $true
Expand All @@ -53,7 +53,7 @@ Describe 'File/folder tests' -Tag Modules {
$folderTestCases = [System.Collections.ArrayList]@()
foreach ($moduleFolderPath in $moduleFolderPaths) {
$folderTestCases += @{
moduleFolderName = $moduleFolderPath.Split('\arm\')[1]
moduleFolderName = $moduleFolderPath.Replace('\', '/').Split('/arm/')[1]
moduleFolderPath = $moduleFolderPath
}
}
Expand All @@ -73,7 +73,7 @@ Describe 'File/folder tests' -Tag Modules {
if (Test-Path $parameterFolderPath) {
foreach ($parameterFile in (Get-ChildItem $parameterFolderPath -Filter '*parameters.json')) {
$parameterFolderFilesTestCases += @{
moduleFolderName = Split-Path $moduleFolderPath -Leaf
moduleFolderName = $moduleFolderPath.Replace('\', '/').Split('/arm/')[1]
parameterFilePath = $parameterFile.FullName
}
}
Expand Down Expand Up @@ -106,7 +106,7 @@ Describe 'Readme tests' -Tag Readme {
}

$readmeFolderTestCases += @{
moduleFolderName = $moduleFolderPath.Split('\arm\')[1]
moduleFolderName = $moduleFolderPath.Replace('\', '/').Split('/arm/')[1]
moduleFolderPath = $moduleFolderPath
templateContent = $templateContent
readMeContent = Get-Content (Join-Path -Path $moduleFolderPath 'readme.md')
Expand Down Expand Up @@ -405,14 +405,14 @@ Describe 'Deployment template tests' -Tag Template {

# Test file setup
$deploymentFolderTestCases += @{
moduleFolderName = Split-Path $moduleFolderPath -Leaf
moduleFolderName = $moduleFolderPath.Replace('\', '/').Split('/arm/')[1]
templateContent = $templateContent
parameterFileTestCases = $parameterFileTestCases
}
}
foreach ($moduleFolderPath in $moduleFolderPathsFiltered) {
$deploymentFolderTestCasesException += @{
moduleFolderNameException = Split-Path $moduleFolderPath -Leaf
moduleFolderNameException = $moduleFolderPath.Replace('\', '/').Split('/arm/')[1]
templateContentException = $templateContent
}
}
Expand Down Expand Up @@ -743,7 +743,7 @@ Describe "Api version tests [All apiVersions in the template should be 'recent']
$ApiVersions = Get-AzResourceProvider -ListAvailable
foreach ($moduleFolderPath in $moduleFolderPathsFiltered) {

$moduleFolderName = $moduleFolderPath.Split('\arm\')[1]
$moduleFolderName = $moduleFolderPath.Replace('\', '/').Split('/arm/')[1]

if (Test-Path (Join-Path $moduleFolderPath 'deploy.bicep')) {
$templateContent = az bicep build --file (Join-Path $moduleFolderPath 'deploy.bicep') --stdout | ConvertFrom-Json -AsHashtable
Expand Down