From 9a3a17e099703f6463faf0ad523ea309aa7a3279 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 4 Mar 2021 08:48:56 +0000 Subject: [PATCH 1/6] On windows, pass the changed files in a text file instead of the CLI Fixes https://github.com/saltstack/salt/issues/59667 --- lib/kitchen/verifier/nox.rb | 72 ++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/lib/kitchen/verifier/nox.rb b/lib/kitchen/verifier/nox.rb index 2c560ca8..3216dcdc 100644 --- a/lib/kitchen/verifier/nox.rb +++ b/lib/kitchen/verifier/nox.rb @@ -15,12 +15,17 @@ class Nox < Kitchen::Verifier::Base default_config :tests, [] default_config :save, {} default_config :windows, nil + default_config :windows do |verifier| + verifier.windows_os? ? true : false + end default_config :verbose, false default_config :run_destructive, false default_config :runtests, false default_config :coverage, false default_config :junitxml, false default_config :from_filenames, [] + default_config :from_filenames_basename, "changed-files-list.txt" + default_config :from_filenames_path, false default_config :enable_filenames, false default_config :passthrough_opts, [] default_config :output_columns, 120 @@ -29,15 +34,33 @@ class Nox < Kitchen::Verifier::Base default_config :environment_vars, {} default_config :zip_windows_artifacts, false - def call(state) - if config[:windows].nil? - # Since windows is not set, lets try and guess since kitchen actually knows this infomation - if instance.platform.os_type == 'windows' - config[:windows] = true - else - config[:windows] = false - end + def initialize(config = {}) + super(config) + debug("Running kitchen-salt nox verifier initialize method") + + if ENV['NOX_ENABLE_FROM_FILENAMES'] + config[:enable_filenames] = true end + + if config[:enable_filenames] and ENV['CHANGE_TARGET'] and ENV['BRANCH_NAME'] and ENV['FORCE_FULL'] != 'true' + require 'git' + repo = Git.open(Dir.pwd) + config[:from_filenames] = repo.diff("origin/#{ENV['CHANGE_TARGET']}", + "origin/#{ENV['BRANCH_NAME']}").name_status.keys.select{|file| file.end_with?('.py')} + debug("Populating `from_filenames` with: #{config[:from_filenames]}") + end + if config[:windows] && config[:from_filenames].any? + # On windows, if the changed files list is too big, it will error. + # Let's then pass an absolute path to a text file which contains the list of changed + # files, one per line. + config[:from_filenames_path] = "#{Dir.pwd}\\#{config[:from_filenames_basename]}" + from_filenames_contents = "#{config[:from_filenames].join('\n')}" + File.open(config[:from_filenames_path], "w") { |f| f.write from_filenames_contents } + debug("Created #{config[:from_filenames_path]} with contents:\n#{from_filenames_contents}") + end + end + + def call(state) debug("Detected platform for instance #{instance.name}: #{instance.platform.os_type}. Config's windows setting value: #{config[:windows]}") if (ENV['ONLY_DOWNLOAD_ARTEFACTS'] || '') == '1' only_download_artefacts = true @@ -64,10 +87,6 @@ def call(state) ENV['KITCHEN_TESTS'].split(' ').each{|test| config[:tests].push(test)} end - if ENV['NOX_ENABLE_FROM_FILENAMES'] - config[:enable_filenames] = true - end - if ENV['NOX_PASSTHROUGH_OPTS'] ENV['NOX_PASSTHROUGH_OPTS'].split(' ').each{|opt| config[:passthrough_opts].push(opt)} end @@ -108,13 +127,6 @@ def call(state) sys_stats = '' end - if config[:enable_filenames] and ENV['CHANGE_TARGET'] and ENV['BRANCH_NAME'] and ENV['FORCE_FULL'] != 'true' - require 'git' - repo = Git.open(Dir.pwd) - config[:from_filenames] = repo.diff("origin/#{ENV['CHANGE_TARGET']}", - "origin/#{ENV['BRANCH_NAME']}").name_status.keys.select{|file| file.end_with?('.py')} - end - if config[:junitxml] junitxml = File.join(root_path, config[:testingdir], 'artifacts', 'xml-unittests-output') if noxenv.include? "pytest" @@ -150,11 +162,18 @@ def call(state) if tests.nil? || tests.empty? # If we're not targetting specific tests... - extra_command = [ - (config[:from_filenames].any? ? "--from-filenames=#{config[:from_filenames].join(',')}" : ''), - (config[:windows] ? "--names-file=#{root_path}\\testing\\tests\\whitelist.txt" : ''), - ].join(' ') - command = "#{command} #{extra_command}" + extra_command = [] + if config[:windows] + extra_command.push("--names-file=#{root_path}\\testing\\tests\\whitelist.txt") + if config[:from_filenames_path] + extra_command.push("--from-filenames=#{root_path}\\testing\\#{config[:from_filenames_basename]}") + end + else + if config[:from_filenames].any? + extra_command.push("--from-filenames=#{config[:from_filenames].join(',')}") + end + end + command = "#{command} #{extra_command.join(' ')}" else command = "#{command} #{tests}" end @@ -167,6 +186,9 @@ def call(state) # the one from config environment_vars.merge!(config[:environment_vars]) + # Strip trailing whitespace + command = command.rstrip + if config[:windows] command = "cmd.exe /c --% \"#{command}\" 2>&1" end @@ -209,7 +231,7 @@ def call(state) info("7z.exe failed, attempting zip with powershell Compress-Archive") conn.execute("powershell Compress-Archive #{remote} #{remote}artifacts.zip -Force") rescue => e2 - error("Failed to create zip") + error("Failed to create zip: #{e2}") end end end From b0c4f7530ba5a1d987ed8b534c95b5710c645d92 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 4 Mar 2021 10:36:59 +0000 Subject: [PATCH 2/6] `pip` and `setuptools` have released versions which are no longer Py3.5 compatible --- lib/kitchen/provisioner/install.erb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/kitchen/provisioner/install.erb b/lib/kitchen/provisioner/install.erb index 14a3c0ff..30df68ec 100644 --- a/lib/kitchen/provisioner/install.erb +++ b/lib/kitchen/provisioner/install.erb @@ -40,7 +40,12 @@ then elif [ -z "${SALT_VERSION}" -a "#{salt_install}" = "pip" ] then echo "Make sure setuptools and pip are new enough" - #{sudo(salt_pip_bin)} install "setuptools>=30" "pip>=9" + export REQUIREMENTS_FILE=$(mktemp) + echo "setuptools >=30,<54.* ; python_version < '3.6'" > $REQUIREMENTS_FILE + echo "setuptools >=30,!=50.*,!=51.*,!=52.*; python_version >= '3.6'" > $REQUIREMENTS_FILE + echo "pip >=9,<21.0.0 ; python_version < '3.6'" >> $REQUIREMENTS_FILE + echo "pip >=9 ; python_version >= '3.6'" >> $REQUIREMENTS_FILE + #{sudo(salt_pip_bin)} install -r $REQUIREMENTS_FILE echo #{sudo(salt_pip_bin)} #{salt_pip_install_command} #{salt_pip_pkg} #{sudo(salt_pip_bin)} #{salt_pip_install_command} #{salt_pip_pkg} elif [ -z "${SALT_VERSION}" -a "#{salt_install}" = "apt" ] From 845e04d4a9cf3a9140dcd21d60c02b0ce06a844e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 4 Mar 2021 10:37:43 +0000 Subject: [PATCH 3/6] Start running on Ubuntu 20.04 --- .github/workflows/tests.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c31c1915..d761f09c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: Tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: os: @@ -14,28 +14,37 @@ jobs: - ubuntu-18 steps: - uses: actions/checkout@v2 + + - name: Set Python Cache Key + run: echo "PY=$(python --version --version | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + - uses: actions/cache@v2 with: path: | bundle virtualenv - key: Tests-${{ matrix.os }}-${{ hashFiles('Gemfile', 'test-requirements.txt') }} + key: Tests-${{ matrix.os }}-${{ env.PY }}-${{ hashFiles('Gemfile', 'test-requirements.txt') }} + - name: Install System Deps run: | sudo apt-get update sudo apt-get install -y curl git python3-pip ruby-dev ruby-bundler shellcheck virtualenv + - name: Install Python Deps run: | if [ ! -e virtualenv/bin/activate ]; then /usr/bin/virtualenv -p /usr/bin/python3 virtualenv; fi . virtualenv/bin/activate pip3 install -U -r test-requirements.txt + - name: Install Ruby Deps run: | /usr/bin/bundle install --without vagrant windows --path bundle + - name: Install GPG Key run: | install -d -m 700 gpgkeys gpg --no-default-keyring --homedir gpgkeys --import < tests/gpgkeys/test.txt + - name: Test run: | . virtualenv/bin/activate @@ -43,6 +52,7 @@ jobs: shellcheck assets/*.sh* LANG=en_US.UTF-8 /usr/bin/bundle exec rake "integration:verify[${{ matrix.os }}]" /usr/bin/bundle exec kitchen list ${{ matrix.os }} + - name: Cleanup run: | /usr/bin/bundle exec rake "integration:destroy[${{ matrix.os }}]" From bb644f9bd244cf122ee41db10b7cee6e18488b54 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 4 Mar 2021 10:48:04 +0000 Subject: [PATCH 4/6] Stop using deprecated flags --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d761f09c..c6478867 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,9 @@ jobs: - name: Install Ruby Deps run: | - /usr/bin/bundle install --without vagrant windows --path bundle + /usr/bin/bundle config set --local path 'bundle' + /usr/bin/bundle config set --local without 'vagrant windows' + /usr/bin/bundle install - name: Install GPG Key run: | From 0b5b5698ba9ca70acc34bd0bafe002e277237adf Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 4 Mar 2021 10:59:33 +0000 Subject: [PATCH 5/6] Don't fail fast --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c6478867..bc919279 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,7 @@ jobs: Tests: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: os: - centos From 23a173aa7eb6f14e1cc86c15a599c29cede24393 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 4 Mar 2021 11:00:03 +0000 Subject: [PATCH 6/6] One version per platform --- .github/workflows/tests.yml | 3 +-- .kitchen.yml | 21 ++++++--------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bc919279..d3a86cd7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,8 +11,7 @@ jobs: os: - centos - debian - - ubuntu-16 - - ubuntu-18 + - ubuntu steps: - uses: actions/checkout@v2 diff --git a/.kitchen.yml b/.kitchen.yml index 1ea777a1..98673648 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -86,20 +86,11 @@ platforms: sudo apt-get update DEBIAN_FRONTEND=noninteractive sudo apt-get install -y software-properties-common salt_bootstrap_options: -X -x python3 -p git -p curl -p sudo -p software-properties-common stable 3000.6 - - name: ubuntu-16.04 + - name: debian-10 provisioner: - salt_apt_repo: 'https://repo.saltproject.io/py3/ubuntu/16.04/amd64/archive' - salt_apt_repo_key: 'https://repo.saltproject.io/py3/ubuntu/16.04/amd64/latest/SALTSTACK-GPG-KEY.pub' - init_environment: | - sudo mkdir -p /tmp/kitchen/var/cache/salt/master - sudo apt-get update - DEBIAN_FRONTEND=noninteractive sudo apt-get install -y software-properties-common - salt_bootstrap_options: -X -x python3 -p git -p curl -p sudo -p software-properties-common stable 3000.6 - - name: debian-9 - provisioner: - salt_apt_repo: 'https://repo.saltproject.io/py3/debian/9/amd64/archive' - salt_apt_repo_key: 'https://repo.saltproject.io/py3/debian/9/amd64/latest/SALTSTACK-GPG-KEY.pub' - - name: centos-7 + salt_apt_repo: 'https://repo.saltproject.io/py3/debian/10/amd64/archive' + salt_apt_repo_key: 'https://repo.saltproject.io/py3/debian/10/amd64/latest/SALTSTACK-GPG-KEY.pub' + - name: centos-8 driver_config: provision_command: - echo . /etc/profile >> ~kitchen/.bashrc @@ -108,8 +99,8 @@ platforms: - yum install -y epel-release - yum install -y python3-pip python3-devel gcc git gcc-c++ provisioner: - salt_yum_repo: 'https://repo.saltproject.io/py3/redhat/7/x86_64/archive/%s' - salt_yum_rpm_key: 'https://repo.saltproject.io/py3/redhat/7/x86_64/latest/SALTSTACK-GPG-KEY.pub' + salt_yum_repo: 'https://repo.saltproject.io/py3/redhat/8/x86_64/archive/%s' + salt_yum_rpm_key: 'https://repo.saltproject.io/py3/redhat/8/x86_64/latest/SALTSTACK-GPG-KEY.pub' <% if @vagrant != false %> - name: windows-2012r2 driver: