diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml
index 8cacfceeb0..f56ba9624b 100644
--- a/.semaphore/cp_dockerfile_build.yml
+++ b/.semaphore/cp_dockerfile_build.yml
@@ -7,7 +7,7 @@ version: v1.0
name: build-test-release
agent:
machine:
- type: s1-prod-ubuntu20-04-amd64-1
+ type: s1-prod-ubuntu24-04-amd64-1
fail_fast:
cancel:
@@ -78,6 +78,28 @@ global_job_config:
- export LATEST_TAG=$BRANCH_TAG-latest
- export DOCKER_UPSTREAM_TAG="$LATEST_TAG"
- export DOCKER_REPOS="confluentinc/kafka-streams-examples"
+ - export COMMUNITY_DOCKER_REPOS=""
+ - |
+ if [[ $SKIP_COMMUNITY == "True" ]]; then
+ # Filter out community repos from DOCKER_REPOS
+ DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs)
+ export DOCKER_REPOS
+ echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS"
+
+ # Set Maven arguments for skipping community modules
+ export MAVEN_EXTRA_ARGS=""
+
+ # Check if current DOCKER_IMAGE is in community repos, skip job execution
+ for skip_repo in $COMMUNITY_DOCKER_REPOS; do
+ if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
+ echo "Skipping build for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
+ export SEMAPHORE_JOB_RESULT=passed
+ return 130
+ fi
+ done
+ else
+ export MAVEN_EXTRA_ARGS=""
+ fi
- export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
- export AMD_ARCH=.amd64
- export ARM_ARCH=.arm64
@@ -107,7 +129,7 @@ blocks:
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$PACKAGES_URL"
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
- $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
+ $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- >-
for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
@@ -128,6 +150,9 @@ blocks:
task:
jobs:
- name: Deploy AMD confluentinc/kafka-streams-examples ubi8
+ env_vars:
+ - name: DOCKER_IMAGE
+ value: confluentinc/kafka-streams-examples
commands:
- export OS_TAG="-ubi8"
- export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/kafka-streams-examples
@@ -156,7 +181,7 @@ blocks:
task:
agent:
machine:
- type: s1-prod-ubuntu20-04-arm64-1
+ type: s1-prod-ubuntu24-04-arm64-1
jobs:
- name: Build & Test ubi8
commands:
@@ -167,7 +192,7 @@ blocks:
- ci-tools ci-update-version
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
- $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
+ $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
epilogue:
@@ -183,9 +208,12 @@ blocks:
task:
agent:
machine:
- type: s1-prod-ubuntu20-04-arm64-1
+ type: s1-prod-ubuntu24-04-arm64-1
jobs:
- name: Deploy ARM confluentinc/kafka-streams-examples ubi8
+ env_vars:
+ - name: DOCKER_IMAGE
+ value: confluentinc/kafka-streams-examples
commands:
- export OS_TAG="-ubi8"
- export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/kafka-streams-examples
@@ -244,7 +272,7 @@ after_pipeline:
task:
agent:
machine:
- type: s1-prod-ubuntu20-04-arm64-0
+ type: s1-prod-ubuntu24-04-arm64-0
jobs:
- name: Metrics
commands:
diff --git a/.semaphore/cp_dockerfile_promote.yml b/.semaphore/cp_dockerfile_promote.yml
index 19a2df4180..951f9911d2 100644
--- a/.semaphore/cp_dockerfile_promote.yml
+++ b/.semaphore/cp_dockerfile_promote.yml
@@ -7,7 +7,7 @@ version: v1.0
name: build-test-release
agent:
machine:
- type: s1-prod-ubuntu20-04-amd64-1
+ type: s1-prod-ubuntu24-04-amd64-1
fail_fast:
cancel:
@@ -46,6 +46,17 @@ global_job_config:
- docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY
- export AMD_ARCH=.amd64
- export ARM_ARCH=.arm64
+ - export COMMUNITY_DOCKER_REPOS=""
+ - |
+ if [[ $SKIP_COMMUNITY == "True" ]]; then
+ for skip_repo in $COMMUNITY_DOCKER_REPOS; do
+ if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
+ echo "Skipping promotion for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
+ export SEMAPHORE_JOB_RESULT=passed
+ return 130
+ fi
+ done
+ fi
blocks:
- name: Promote AMD
@@ -53,6 +64,9 @@ blocks:
task:
jobs:
- name: Promote confluentinc/kafka-streams-examples ubi8 AMD
+ env_vars:
+ - name: DOCKER_IMAGE
+ value: confluentinc/kafka-streams-examples
commands:
- export OS_TYPE="ubi8"
- export DOCKER_REPO="confluentinc/kafka-streams-examples"
@@ -85,6 +99,9 @@ blocks:
task:
jobs:
- name: Promote confluentinc/kafka-streams-examples ubi8 ARM
+ env_vars:
+ - name: DOCKER_IMAGE
+ value: confluentinc/kafka-streams-examples
commands:
- export OS_TYPE="ubi8"
- export DOCKER_REPO="confluentinc/kafka-streams-examples"
@@ -117,6 +134,9 @@ blocks:
task:
jobs:
- name: Create Manifest confluentinc/kafka-streams-examples ubi8
+ env_vars:
+ - name: DOCKER_IMAGE
+ value: confluentinc/kafka-streams-examples
commands:
- export OS_TYPE="ubi8"
- if [[ ! "$OS_TYPE" ]]; then export OS_TAG=""; elif [[ "$OS_TYPE" =~ $PROMOTE_OS_TYPE* ]]; then export OS_TAG="-$OS_TYPE"; fi
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index b1ac914c31..07bb145e8f 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -7,7 +7,7 @@ version: v1.0
name: build-test-release
agent:
machine:
- type: s1-prod-ubuntu20-04-amd64-1
+ type: s1-prod-ubuntu24-04-amd64-1
fail_fast:
cancel:
@@ -71,6 +71,28 @@ global_job_config:
- export LATEST_TAG=$BRANCH_TAG-latest
- export DOCKER_UPSTREAM_TAG="$LATEST_TAG"
- export DOCKER_REPOS="confluentinc/kafka-streams-examples"
+ - export COMMUNITY_DOCKER_REPOS=""
+ - |
+ if [[ $SKIP_COMMUNITY == "True" ]]; then
+ # Filter out community repos from DOCKER_REPOS
+ DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs)
+ export DOCKER_REPOS
+ echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS"
+
+ # Set Maven arguments for skipping community modules
+ export MAVEN_EXTRA_ARGS=""
+
+ # Check if current DOCKER_IMAGE is in community repos, skip job execution
+ for skip_repo in $COMMUNITY_DOCKER_REPOS; do
+ if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
+ echo "Skipping build for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
+ export SEMAPHORE_JOB_RESULT=passed
+ return 130
+ fi
+ done
+ else
+ export MAVEN_EXTRA_ARGS=""
+ fi
- export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
- export AMD_ARCH=.amd64
- export ARM_ARCH=.arm64
@@ -99,7 +121,7 @@ blocks:
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$PACKAGES_URL"
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
- $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
+ $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- >-
for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
@@ -120,7 +142,7 @@ blocks:
task:
agent:
machine:
- type: s1-prod-ubuntu20-04-arm64-1
+ type: s1-prod-ubuntu24-04-arm64-1
jobs:
- name: Build & Test ubi8
commands:
@@ -131,7 +153,7 @@ blocks:
- ci-tools ci-update-version
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
- $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
+ $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
epilogue:
@@ -144,7 +166,7 @@ after_pipeline:
task:
agent:
machine:
- type: s1-prod-ubuntu20-04-arm64-0
+ type: s1-prod-ubuntu24-04-arm64-0
jobs:
- name: Metrics
commands:
diff --git a/service.yml b/service.yml
index a468daf2e2..9abd757b3c 100644
--- a/service.yml
+++ b/service.yml
@@ -9,6 +9,8 @@ semaphore:
enable: true
pipeline_type: cp-dockerfile
docker_repos: ['confluentinc/kafka-streams-examples']
+ community_docker_repos: []
+ community_maven_modules: []
maven_phase: 'package' # streams examples integration-test needs host-based networking, won't work in CI as-is
maven_skip_deploy: true
build_arm: true
@@ -40,6 +42,13 @@ semaphore:
- name: CONFLUENT_DEB_VERSION
required: false
default_value: '1'
+ - name: SKIP_COMMUNITY
+ required: false
+ default_value: 'False'
+ description: 'Skip building community docker images'
+ options:
+ - 'True'
+ - 'False'
- name: cp-dockerfile-promote
branch: master
pipeline_file: .semaphore/cp_dockerfile_promote.yml
@@ -58,3 +67,10 @@ semaphore:
options:
- 'deb'
- 'ubi'
+ - name: SKIP_COMMUNITY
+ required: false
+ default_value: 'False'
+ description: 'Skip publishing community docker images'
+ options:
+ - 'True'
+ - 'False'
\ No newline at end of file