From ab23a2a9c54fed2acbd553407a761f19af6e8ed4 Mon Sep 17 00:00:00 2001 From: Kristofer Rye Date: Thu, 16 Jul 2020 13:32:46 -0500 Subject: [PATCH 1/3] ci: Set `shell: bash` as a default, remove duplicates A follow-up to #74406, this commit merely removes the `shell: bash` lines where they are explicitly added in favor of setting defaults for *all* "run" steps. Signed-off-by: Kristofer Rye --- .github/workflows/ci.yml | 92 +++++------------------------------- src/ci/github-actions/ci.yml | 23 ++------- 2 files changed, 15 insertions(+), 100 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99140bec17029..9cf0f4dbd1e4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,10 +51,12 @@ jobs: os: ubuntu-latest-xl timeout-minutes: 600 runs-on: "${{ matrix.os }}" + defaults: + run: + shell: bash steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 with: @@ -66,77 +68,59 @@ jobs: if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" @@ -144,7 +128,6 @@ jobs: if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" @@ -170,10 +153,12 @@ jobs: env: {} timeout-minutes: 600 runs-on: "${{ matrix.os }}" + defaults: + run: + shell: bash steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 with: @@ -185,77 +170,59 @@ jobs: if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" @@ -263,7 +230,6 @@ jobs: if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" @@ -507,10 +473,12 @@ jobs: os: windows-latest-xl timeout-minutes: 600 runs-on: "${{ matrix.os }}" + defaults: + run: + shell: bash steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 with: @@ -522,77 +490,59 @@ jobs: if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" @@ -600,7 +550,6 @@ jobs: if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" @@ -652,10 +601,12 @@ jobs: os: macos-latest timeout-minutes: 600 runs-on: "${{ matrix.os }}" + defaults: + run: + shell: bash steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 with: @@ -667,77 +618,59 @@ jobs: if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" @@ -745,7 +678,6 @@ jobs: if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 0b3b77f217dbd..e8036b8c12075 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -75,11 +75,13 @@ x--expand-yaml-anchors--remove: - &base-ci-job timeout-minutes: 600 runs-on: "${{ matrix.os }}" + defaults: + run: + shell: bash env: *shared-ci-variables steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 @@ -95,7 +97,6 @@ x--expand-yaml-anchors--remove: - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: # Since it's not possible to merge `${{ matrix.env }}` with the other # variables in `job..env`, the variables defined in the matrix @@ -106,67 +107,54 @@ x--expand-yaml-anchors--remove: - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash <<: *step - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash <<: *step - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash <<: *step - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash <<: *step - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash <<: *step - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash <<: *step - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash <<: *step - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash <<: *step - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash <<: *step - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash <<: *step - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash <<: *step - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash <<: *step - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash <<: *step # Disable automatic line ending conversion (again). On Windows, when we're @@ -176,22 +164,18 @@ x--expand-yaml-anchors--remove: # appropriate line endings. - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash <<: *step - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash <<: *step - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash <<: *step - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} @@ -200,7 +184,6 @@ x--expand-yaml-anchors--remove: - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} From 49f50780480421e524adf6262a285003d6f8281e Mon Sep 17 00:00:00 2001 From: Kristofer Rye Date: Fri, 17 Jul 2020 08:29:01 -0500 Subject: [PATCH 2/3] ci: Stop setting CI_OVERRIDE_SHELL environment variable This will render the src/ci/exec-with-shell.py script more or less useless, but we're going to replace that by just using the system bash instead. Signed-off-by: Kristofer Rye --- src/ci/scripts/install-msys2.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh index 3a0c965a67710..185d361582505 100755 --- a/src/ci/scripts/install-msys2.sh +++ b/src/ci/scripts/install-msys2.sh @@ -12,9 +12,6 @@ if isWindows; then mkdir -p "${msys2Path}/home/${USERNAME}" ciCommandAddPath "${msys2Path}/usr/bin" - echo "switching shell to use our own bash" - ciCommandSetEnv CI_OVERRIDE_SHELL "${msys2Path}/usr/bin/bash.exe" - # Detect the native Python version installed on the agent. On GitHub # Actions, the C:\hostedtoolcache\windows\Python directory contains a # subdirectory for each installed Python version. From 586629c0b607f34981a893cf856bb35279f58542 Mon Sep 17 00:00:00 2001 From: Kristofer Rye Date: Fri, 17 Jul 2020 08:44:14 -0500 Subject: [PATCH 3/3] ci: Replace exec-with-shell wrapper with "plain bash" Also, promote defaults.run.shell from inside only the primary jobs to the top level. The src/ci/exec-with-shell.py wrapper script was formerly used to change out the shell mid-job by intercepting a CI_OVERRIDE_SHELL environment variable. Now, instead, we just set `bash` as the global default across all jobs, and we also delete the exec-with-shell.py script. Signed-off-by: Kristofer Rye --- .github/workflows/ci.yml | 14 +------------- src/ci/exec-with-shell.py | 16 ---------------- src/ci/github-actions/ci.yml | 19 ++++--------------- 3 files changed, 5 insertions(+), 44 deletions(-) delete mode 100755 src/ci/exec-with-shell.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cf0f4dbd1e4c..86de37820003a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ name: CI - "**" defaults: run: - shell: "python src/ci/exec-with-shell.py {0}" + shell: bash jobs: pr: name: PR @@ -51,9 +51,6 @@ jobs: os: ubuntu-latest-xl timeout-minutes: 600 runs-on: "${{ matrix.os }}" - defaults: - run: - shell: bash steps: - name: disable git crlf conversion run: git config --global core.autocrlf false @@ -153,9 +150,6 @@ jobs: env: {} timeout-minutes: 600 runs-on: "${{ matrix.os }}" - defaults: - run: - shell: bash steps: - name: disable git crlf conversion run: git config --global core.autocrlf false @@ -473,9 +467,6 @@ jobs: os: windows-latest-xl timeout-minutes: 600 runs-on: "${{ matrix.os }}" - defaults: - run: - shell: bash steps: - name: disable git crlf conversion run: git config --global core.autocrlf false @@ -601,9 +592,6 @@ jobs: os: macos-latest timeout-minutes: 600 runs-on: "${{ matrix.os }}" - defaults: - run: - shell: bash steps: - name: disable git crlf conversion run: git config --global core.autocrlf false diff --git a/src/ci/exec-with-shell.py b/src/ci/exec-with-shell.py deleted file mode 100755 index 26ce69e33d9c3..0000000000000 --- a/src/ci/exec-with-shell.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -# A simple wrapper that forwards the arguments to bash, unless the -# CI_OVERRIDE_SHELL environment variable is present: in that case the content -# of that environment variable is used as the shell path. - -import os -import sys -import subprocess - -try: - shell = os.environ["CI_OVERRIDE_SHELL"] -except KeyError: - shell = "bash" - -res = subprocess.call([shell] + sys.argv[1:]) -sys.exit(res) diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index e8036b8c12075..5573d87aa2e55 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -75,9 +75,6 @@ x--expand-yaml-anchors--remove: - &base-ci-job timeout-minutes: 600 runs-on: "${{ matrix.os }}" - defaults: - run: - shell: bash env: *shared-ci-variables steps: - name: disable git crlf conversion @@ -233,18 +230,10 @@ on: defaults: run: - # While on Linux and macOS builders it just forwards the arguments to the - # system bash, this wrapper allows switching from the host's bash.exe to - # the one we install along with MSYS2 mid-build on Windows. - # - # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL - # environment variable is set pointing to our MSYS2's bash.exe. From that - # moment the host's bash.exe will not be called anymore. - # - # This is needed because we can't launch our own bash.exe from the host - # bash.exe, as that would load two different cygwin1.dll in memory, causing - # "cygwin heap mismatch" errors. - shell: python src/ci/exec-with-shell.py {0} + # On Linux, macOS, and Windows, use the system-provided bash as the default + # shell. (This should only make a difference on Windows, where the default + # shell is PowerShell.) + shell: bash jobs: pr: