diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6b5d3dabef743..cd255e237adb1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,6 +37,7 @@ jobs:
   pr:
     name: "PR - ${{ matrix.name }}"
     env:
+      DOWNLOAD_RUSTC: 1
       PR_CI_JOB: 1
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
@@ -268,8 +269,9 @@ jobs:
             os: ubuntu-20.04-8core-32gb
             env: {}
           - name: x86_64-gnu
+            env:
+              DOWNLOAD_RUSTC: 1
             os: ubuntu-20.04-4core-16gb
-            env: {}
           - name: x86_64-gnu-stable
             env:
               IMAGE: x86_64-gnu
diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index 4b218d5772769..f67e4fe57508e 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -254,6 +254,7 @@ docker \
   --env DEPLOY \
   --env DEPLOY_ALT \
   --env CI \
+  --env DOWNLOAD_RUSTC \
   --env GITHUB_ACTIONS \
   --env GITHUB_REF \
   --env TOOLSTATE_REPO_ACCESS_TOKEN \
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index f87e2f3745841..362a9ae89ed67 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -310,6 +310,7 @@ jobs:
     <<: *base-ci-job
     name: PR - ${{ matrix.name }}
     env:
+      DOWNLOAD_RUSTC: 1
       <<: [*shared-ci-variables, *public-variables]
       PR_CI_JOB: 1
     if: github.event_name == 'pull_request'
@@ -433,6 +434,8 @@ jobs:
             <<: *job-linux-8c
 
           - name: x86_64-gnu
+            env:
+              DOWNLOAD_RUSTC: 1
             <<: *job-linux-4c
 
           # This job ensures commits landing on nightly still pass the full
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 48fb40d6a6dd8..803e13bd9817e 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -121,6 +121,10 @@ else
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
   fi
 
+  if [ "$DOWNLOAD_RUSTC" = 1 ]; then
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
+  fi
+
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 
   # We enable this for non-dist builders, since those aren't trying to produce