From d3f321f7a0ccdb52d82a0e143ae026ecda808a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 8 Nov 2024 19:39:30 +0000 Subject: [PATCH 1/8] chore(ci): fix interop tests test-plans region also --- .github/workflows/interop-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/interop-test.yml b/.github/workflows/interop-test.yml index e9446d013d7..c6e68eefe1d 100644 --- a/.github/workflows/interop-test.yml +++ b/.github/workflows/interop-test.yml @@ -38,6 +38,7 @@ jobs: s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }} s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }} s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }} + aws-region: ap-southeast-2 worker-count: 16 run-holepunching-interop: name: Run hole-punch interoperability tests @@ -56,4 +57,5 @@ jobs: s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }} s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }} s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }} + aws-region: ap-southeast-2 worker-count: 16 From 57758a1d6e71c9d5847749404e5616837e7b6e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 8 Nov 2024 19:42:21 +0000 Subject: [PATCH 2/8] remove interop conditions --- .github/workflows/interop-test.yml | 5 ++--- scripts/build-interop-image.sh | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/interop-test.yml b/.github/workflows/interop-test.yml index c6e68eefe1d..9d0b8100365 100644 --- a/.github/workflows/interop-test.yml +++ b/.github/workflows/interop-test.yml @@ -12,7 +12,6 @@ concurrency: jobs: run-transport-interop: name: Run transport interoperability tests - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ fromJSON(github.repository == 'libp2p/rust-libp2p' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} strategy: matrix: @@ -28,6 +27,7 @@ jobs: AWS_BUCKET_NAME: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }} AWS_ACCESS_KEY_ID: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_REGION }} FLAVOUR: ${{ matrix.flavour }} - name: Run ${{ matrix.flavour }} tests @@ -38,11 +38,10 @@ jobs: s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }} s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }} s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }} - aws-region: ap-southeast-2 + aws-region: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_REGION }} worker-count: 16 run-holepunching-interop: name: Run hole-punch interoperability tests - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ fromJSON(github.repository == 'libp2p/rust-libp2p' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} steps: - uses: actions/checkout@v4 diff --git a/scripts/build-interop-image.sh b/scripts/build-interop-image.sh index 4b96e353f9a..ff6f344976c 100755 --- a/scripts/build-interop-image.sh +++ b/scripts/build-interop-image.sh @@ -6,13 +6,13 @@ CACHE_TO="" # If we have credentials, write to cache if [[ -n "${AWS_SECRET_ACCESS_KEY}" ]]; then - CACHE_TO="--cache-to type=s3,mode=max,bucket=${AWS_BUCKET_NAME},region=ap-southeast-2,prefix=buildCache,name=${FLAVOUR}-rust-libp2p-head" + CACHE_TO="--cache-to type=s3,mode=max,bucket=${AWS_BUCKET_NAME},region=${AWS_REGION},prefix=buildCache,name=${FLAVOUR}-rust-libp2p-head" fi docker buildx build \ --load \ $CACHE_TO \ - --cache-from type=s3,mode=max,bucket=${AWS_BUCKET_NAME},region=ap-southeast-2,prefix=buildCache,name=${FLAVOUR}-rust-libp2p-head \ + --cache-from type=s3,mode=max,bucket=${AWS_BUCKET_NAME},region=${AWS_REGION},prefix=buildCache,name=${FLAVOUR}-rust-libp2p-head \ -t ${FLAVOUR}-rust-libp2p-head \ . \ -f interop-tests/Dockerfile.${FLAVOUR} From c5baf967163d47970ce694d2361cc712c123672c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 8 Nov 2024 19:39:30 +0000 Subject: [PATCH 3/8] chore(ci): fix interop tests test-plans region also --- .github/workflows/interop-test.yml | 5 +++-- scripts/build-interop-image.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/interop-test.yml b/.github/workflows/interop-test.yml index e9446d013d7..83b14841abc 100644 --- a/.github/workflows/interop-test.yml +++ b/.github/workflows/interop-test.yml @@ -12,7 +12,6 @@ concurrency: jobs: run-transport-interop: name: Run transport interoperability tests - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ fromJSON(github.repository == 'libp2p/rust-libp2p' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} strategy: matrix: @@ -28,6 +27,7 @@ jobs: AWS_BUCKET_NAME: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }} AWS_ACCESS_KEY_ID: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_REGION }} FLAVOUR: ${{ matrix.flavour }} - name: Run ${{ matrix.flavour }} tests @@ -38,10 +38,10 @@ jobs: s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }} s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }} s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_REGION }} worker-count: 16 run-holepunching-interop: name: Run hole-punch interoperability tests - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ fromJSON(github.repository == 'libp2p/rust-libp2p' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} steps: - uses: actions/checkout@v4 @@ -56,4 +56,5 @@ jobs: s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }} s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }} s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_REGION }} worker-count: 16 diff --git a/scripts/build-interop-image.sh b/scripts/build-interop-image.sh index 4b96e353f9a..ff6f344976c 100755 --- a/scripts/build-interop-image.sh +++ b/scripts/build-interop-image.sh @@ -6,13 +6,13 @@ CACHE_TO="" # If we have credentials, write to cache if [[ -n "${AWS_SECRET_ACCESS_KEY}" ]]; then - CACHE_TO="--cache-to type=s3,mode=max,bucket=${AWS_BUCKET_NAME},region=ap-southeast-2,prefix=buildCache,name=${FLAVOUR}-rust-libp2p-head" + CACHE_TO="--cache-to type=s3,mode=max,bucket=${AWS_BUCKET_NAME},region=${AWS_REGION},prefix=buildCache,name=${FLAVOUR}-rust-libp2p-head" fi docker buildx build \ --load \ $CACHE_TO \ - --cache-from type=s3,mode=max,bucket=${AWS_BUCKET_NAME},region=ap-southeast-2,prefix=buildCache,name=${FLAVOUR}-rust-libp2p-head \ + --cache-from type=s3,mode=max,bucket=${AWS_BUCKET_NAME},region=${AWS_REGION},prefix=buildCache,name=${FLAVOUR}-rust-libp2p-head \ -t ${FLAVOUR}-rust-libp2p-head \ . \ -f interop-tests/Dockerfile.${FLAVOUR} From 04f11c3687cb6be5b8713c07c8e818090e3c6730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 14 Nov 2024 19:20:40 +0000 Subject: [PATCH 4/8] Update .github/workflows/interop-test.yml Co-authored-by: Guillaume Michel --- .github/workflows/interop-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/interop-test.yml b/.github/workflows/interop-test.yml index 9d0b8100365..83b14841abc 100644 --- a/.github/workflows/interop-test.yml +++ b/.github/workflows/interop-test.yml @@ -56,5 +56,5 @@ jobs: s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }} s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }} s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }} - aws-region: ap-southeast-2 + aws-region: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_REGION }} worker-count: 16 From 0d76f7e039bf5fd3dd1c680ebc3745b3609805cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 20 Nov 2024 10:15:21 +0000 Subject: [PATCH 5/8] keep if conditions as variables and secrets are not exposed to external PRs --- .github/workflows/interop-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/interop-test.yml b/.github/workflows/interop-test.yml index 83b14841abc..c88579be68a 100644 --- a/.github/workflows/interop-test.yml +++ b/.github/workflows/interop-test.yml @@ -12,6 +12,7 @@ concurrency: jobs: run-transport-interop: name: Run transport interoperability tests + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ fromJSON(github.repository == 'libp2p/rust-libp2p' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} strategy: matrix: @@ -42,6 +43,7 @@ jobs: worker-count: 16 run-holepunching-interop: name: Run hole-punch interoperability tests + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ fromJSON(github.repository == 'libp2p/rust-libp2p' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} steps: - uses: actions/checkout@v4 From d7ef940a87abc4f557dcbd7df965746bc25e4aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 20 Nov 2024 10:17:56 +0000 Subject: [PATCH 6/8] keep if conditions as variables and secrets are not exposed to external PRs --- .github/workflows/interop-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/interop-test.yml b/.github/workflows/interop-test.yml index 83b14841abc..c88579be68a 100644 --- a/.github/workflows/interop-test.yml +++ b/.github/workflows/interop-test.yml @@ -12,6 +12,7 @@ concurrency: jobs: run-transport-interop: name: Run transport interoperability tests + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ fromJSON(github.repository == 'libp2p/rust-libp2p' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} strategy: matrix: @@ -42,6 +43,7 @@ jobs: worker-count: 16 run-holepunching-interop: name: Run hole-punch interoperability tests + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ fromJSON(github.repository == 'libp2p/rust-libp2p' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} steps: - uses: actions/checkout@v4 From 40f1d0c849ee0abffec01bdff7dd06f8620c48b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 20 Nov 2024 11:52:30 +0000 Subject: [PATCH 7/8] add batch queue to mergify for interop tests --- .github/mergify.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/mergify.yml b/.github/mergify.yml index 91cd8881237..50a0526e544 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -32,6 +32,16 @@ pull_request_rules: - base=master actions: queue: + name: default + + - name: Add to batch merge queue + conditions: + # All branch protection rules are implicit: https://docs.mergify.com/conditions/#about-branch-protection + - label=send-it-batch + - base=master + actions: + queue: + name: external_prs - name: Add approved dependabot PRs to merge queue conditions: @@ -40,6 +50,7 @@ pull_request_rules: - base=master actions: queue: + name: default - name: Remove reviews on updates after PR is queued for merging conditions: @@ -74,3 +85,8 @@ pull_request_rules: queue_rules: - name: default conditions: [] + # External PR's don't have access to secrets and variables, therefore they don't run the interop tests. + # using a batch queue allows to circumvent that as mergify creates it from an internal branch. + - name: external_prs + conditions: [] + batch_size: 1 From a96adf12e731018732771266c8b55d8e024e35a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 20 Nov 2024 15:36:03 +0000 Subject: [PATCH 8/8] add doc to the send-it-batch label --- .github/mergify.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/mergify.yml b/.github/mergify.yml index 50a0526e544..38f025c7814 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -34,6 +34,7 @@ pull_request_rules: queue: name: default + # Adds the Pr to the batch queue, so that we can run the interop tests. See the `external_prs` queue for more info. - name: Add to batch merge queue conditions: # All branch protection rules are implicit: https://docs.mergify.com/conditions/#about-branch-protection