Re-use testing clusters up to 3 days in CI (#390)

This PR updates our testing script for `eksctl` and `kOps` clusters to: 1. Not delete clusters after running tests on them if they're not older than 3 days 2. Re-use existing cluster if the existing cluster's spec matches with the expected one Hopefully this should speed up the CI time greatly. --- By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
awslabs · Feb 25, 2025 · 3f790e0 · 3f790e0
1 parent 53349be
commit 3f790e0
Show file tree

Hide file tree

Showing 2 changed files with 118 additions and 3 deletions.
diff --git a/tests/e2e-kubernetes/scripts/eksctl.sh b/tests/e2e-kubernetes/scripts/eksctl.sh
@@ -2,6 +2,9 @@
 
 set -euox pipefail
 
+# If the cluster is not older than this, it will be re-used.
+MAX_CLUSTER_AGE_SECONDS=$((3 * 24 * 60 * 60)) # 3 days
+
 function eksctl_install() {
   INSTALL_PATH=${1}
   EKSCTL_VERSION=${2}
@@ -12,6 +15,37 @@ function eksctl_install() {
   fi
 }
 
+function is_cluster_too_old() {
+  CLUSTER_NAME=${1}
+  REGION=${2}
+
+  CREATED_TIME=$(aws eks describe-cluster --name "${CLUSTER_NAME}" --region "${REGION}" --query 'cluster.createdAt' --output text)
+  CURRENT_TIME=$(date +%s)
+  CLUSTER_TIME=$(date -d "${CREATED_TIME}" +%s)
+
+  [ $((CURRENT_TIME - CLUSTER_TIME)) -gt ${MAX_CLUSTER_AGE_SECONDS} ]
+  return $?
+}
+
+function compute_cluster_spec_hash() {
+  NODE_TYPE=${1}
+  ZONES=${2}
+  EKSCTL_PATCH_SELINUX_ENFORCING_FILE=${3}
+
+  echo -n "${NODE_TYPE}-${ZONES}-${EKSCTL_PATCH_SELINUX_ENFORCING_FILE}" | sha256sum | cut -d' ' -f1
+}
+
+# Checks whether existing cluster matches with expected specs to decide whether to re-use it.
+function cluster_matches_specs() {
+  CLUSTER_NAME=${1}
+  REGION=${2}
+  DESIRED_HASH=${3}
+  CURRENT_HASH=$(aws eks describe-cluster --name "${CLUSTER_NAME}" --region "${REGION}" --query 'cluster.tags.ClusterSpecHash' --output text)
+
+  [ "${DESIRED_HASH}" = "${CURRENT_HASH}" ]
+  return $?
+}
+
 function eksctl_create_cluster() {
   CLUSTER_NAME=${1}
   REGION=${2}
@@ -27,7 +61,19 @@ function eksctl_create_cluster() {
   K8S_VERSION=${12}
   EKSCTL_PATCH_SELINUX_ENFORCING_FILE=${13}
 
-  eksctl_delete_cluster "$BIN" "$CLUSTER_NAME" "$REGION"
+  CLUSTER_SPEC_HASH=$(compute_cluster_spec_hash "${NODE_TYPE}" "${ZONES}" "${EKSCTL_PATCH_SELINUX_ENFORCING_FILE}")
+
+  # Check if cluster exists and matches our specs
+  if eksctl_cluster_exists "${BIN}" "${CLUSTER_NAME}"; then
+    if ! is_cluster_too_old "${CLUSTER_NAME}" "${REGION}" && \
+       cluster_matches_specs "${CLUSTER_NAME}" "${REGION}" "${CLUSTER_SPEC_HASH}"; then
+      echo "Reusing existing cluster ${CLUSTER_NAME} as it matches specifications and it is not too old"
+      return 0
+    fi
+
+    echo "Existing cluster ${CLUSTER_NAME} is either too old or doesn't match specifications. Re-creating..."
+    eksctl_delete_cluster "$BIN" "$CLUSTER_NAME" "$REGION"
+  fi
 
   # CAUTION: this may fail with "the targeted availability zone, does not currently have sufficient capacity to support the cluster" error, we may require a fix for that
   ${BIN} create cluster \
@@ -38,6 +84,7 @@ function eksctl_create_cluster() {
     --with-oidc \
     --zones $ZONES \
     --version $K8S_VERSION \
+    --tags ClusterSpecHash=${CLUSTER_SPEC_HASH} \
     --dry-run > $CLUSTER_FILE
 
   CLUSTER_FILE_TMP="${CLUSTER_FILE}.tmp"
@@ -62,9 +109,18 @@ function eksctl_delete_cluster() {
   BIN=${1}
   CLUSTER_NAME=${2}
   REGION=${3}
-  if eksctl_cluster_exists "${BIN}" "${CLUSTER_NAME}"; then
-    ${BIN} delete cluster "${CLUSTER_NAME}"
+
+  if ! eksctl_cluster_exists "${BIN}" "${CLUSTER_NAME}"; then
+    return 0
+  fi
+
+  # Skip deletion if cluster is not too old, so we can re-use it
+  if ! is_cluster_too_old "${CLUSTER_NAME}" "${REGION}"; then
+    echo "Skipping deletion of cluster ${CLUSTER_NAME} to re-use it"
+    return 0
   fi
+
+  ${BIN} delete cluster "${CLUSTER_NAME}"
   STACK_NAME="eksctl-${CLUSTER_NAME}-cluster"
   aws cloudformation delete-stack --region ${REGION} --stack-name ${STACK_NAME}
 

diff --git a/tests/e2e-kubernetes/scripts/kops.sh b/tests/e2e-kubernetes/scripts/kops.sh
@@ -2,6 +2,9 @@
 
 set -euox pipefail
 
+# If the cluster is not older than this, it will be re-used.
+MAX_CLUSTER_AGE_SECONDS=$((3 * 24 * 60 * 60)) # 3 days
+
 OS_ARCH=$(go env GOOS)-amd64
 
 function kops_install() {
@@ -19,6 +22,40 @@ function kops_install() {
   chmod +x "${INSTALL_PATH}"/kops
 }
 
+function is_cluster_too_old() {
+  CLUSTER_NAME=${1}
+  BIN=${2}
+  KOPS_STATE_FILE=${3}
+
+  CREATED_TIME=$(${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o json | jq -r '.metadata.creationTimestamp')
+  CURRENT_TIME=$(date +%s)
+  CLUSTER_TIME=$(date -d "${CREATED_TIME}" +%s)
+
+  [ $((CURRENT_TIME - CLUSTER_TIME)) -gt ${MAX_CLUSTER_AGE_SECONDS} ]
+  return $?
+}
+
+function compute_cluster_spec_hash() {
+  INSTANCE_TYPE=${1}
+  ZONES=${2}
+  AMI_ID=${3}
+  KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${4}
+
+  echo -n "${INSTANCE_TYPE}-${ZONES}-${AMI_ID}-${KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE}" | sha256sum | cut -d' ' -f1
+}
+
+# Checks whether existing cluster matches with expected specs to decide whether to re-use it.
+function cluster_matches_specs() {
+  CLUSTER_NAME=${1}
+  BIN=${2}
+  KOPS_STATE_FILE=${3}
+  DESIRED_HASH=${4}
+  CURRENT_HASH=$(${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o json | jq -r '.spec.cloudLabels.ClusterSpecHash // empty')
+
+  [ "${DESIRED_HASH}" = "${CURRENT_HASH}" ]
+  return $?
+}
+
 function kops_create_cluster() {
   CLUSTER_NAME=${1}
   BIN=${2}
@@ -35,7 +72,17 @@ function kops_create_cluster() {
   SSH_KEY=${13}
   KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${14}
 
+  CLUSTER_SPEC_HASH=$(compute_cluster_spec_hash "${INSTANCE_TYPE}" "${ZONES}" "${AMI_ID}" "${KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE}")
+
+  # Check if cluster exists and matches our specs
   if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
+    if ! is_cluster_too_old "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}" && \
+       cluster_matches_specs "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}" "${CLUSTER_SPEC_HASH}"; then
+      echo "Reusing existing cluster ${CLUSTER_NAME} as it matches specifications and it is not too old"
+      return 0
+    fi
+
+    echo "Existing cluster ${CLUSTER_NAME} is either too old or doesn't match specifications. Re-creating..."
     kops_delete_cluster "$BIN" "$CLUSTER_NAME" "$KOPS_STATE_FILE"
   fi
 
@@ -52,6 +99,7 @@ function kops_create_cluster() {
     --kubernetes-version="${K8S_VERSION}" \
     --dry-run \
     --cloud aws \
+    --cloud-labels="ClusterSpecHash=${CLUSTER_SPEC_HASH}" \
     -o yaml \
     ${ARGS[@]+"${ARGS[@]}"} \
     "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
@@ -87,6 +135,17 @@ function kops_delete_cluster() {
   BIN=${1}
   CLUSTER_NAME=${2}
   KOPS_STATE_FILE=${3}
+
+  if ! kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
+    return 0
+  fi
+
+  # Skip deletion if cluster is not too old, so we can re-use it
+  if ! is_cluster_too_old "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
+    echo "Skipping deletion of cluster ${CLUSTER_NAME} to re-use it"
+    return 0
+  fi
+
   echo "Deleting cluster ${CLUSTER_NAME}"
   ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
 }