Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

CPBR-2445 Introduce SKIP_COMMUNITY flag to skip building and publishing community docker images #536

Merged
merged 2 commits into from
Apr 3, 2025
Merged

CPBR-2445 Introduce SKIP_COMMUNITY flag to skip building and publishing community docker images #536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
31c147a
Introduce SKIP_COMMUNITY flag to skip building and publishing communi…
wadhwa1 Mar 28, 2025
d87fc28
apply service bot changes
wadhwa1 Apr 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 34 additions & 6 deletions .semaphore/cp_dockerfile_build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ version: v1.0
name: build-test-release
agent:
machine:
type: s1-prod-ubuntu20-04-amd64-1
type: s1-prod-ubuntu24-04-amd64-1

fail_fast:
cancel:
Expand Down Expand Up @@ -78,6 +78,28 @@ global_job_config:
- export LATEST_TAG=$BRANCH_TAG-latest
- export DOCKER_UPSTREAM_TAG="$LATEST_TAG"
- export DOCKER_REPOS="confluentinc/kafka-streams-examples"
- export COMMUNITY_DOCKER_REPOS=""
- |
if [[ $SKIP_COMMUNITY == "True" ]]; then
# Filter out community repos from DOCKER_REPOS
DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs)
export DOCKER_REPOS
echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS"

# Set Maven arguments for skipping community modules
export MAVEN_EXTRA_ARGS=""

# Check if current DOCKER_IMAGE is in community repos, skip job execution
for skip_repo in $COMMUNITY_DOCKER_REPOS; do
if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
echo "Skipping build for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
export SEMAPHORE_JOB_RESULT=passed
return 130
fi
done
else
export MAVEN_EXTRA_ARGS=""
fi
- export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
- export AMD_ARCH=.amd64
- export ARM_ARCH=.arm64
Expand Down Expand Up @@ -107,7 +129,7 @@ blocks:
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$PACKAGES_URL"
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
$PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
$PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- >-
for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
Expand All @@ -128,6 +150,9 @@ blocks:
task:
jobs:
- name: Deploy AMD confluentinc/kafka-streams-examples ubi8
env_vars:
- name: DOCKER_IMAGE
value: confluentinc/kafka-streams-examples
commands:
- export OS_TAG="-ubi8"
- export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/kafka-streams-examples
Expand Down Expand Up @@ -156,7 +181,7 @@ blocks:
task:
agent:
machine:
type: s1-prod-ubuntu20-04-arm64-1
type: s1-prod-ubuntu24-04-arm64-1
jobs:
- name: Build & Test ubi8
commands:
Expand All @@ -167,7 +192,7 @@ blocks:
- ci-tools ci-update-version
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
$PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
$PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
epilogue:
Expand All @@ -183,9 +208,12 @@ blocks:
task:
agent:
machine:
type: s1-prod-ubuntu20-04-arm64-1
type: s1-prod-ubuntu24-04-arm64-1
jobs:
- name: Deploy ARM confluentinc/kafka-streams-examples ubi8
env_vars:
- name: DOCKER_IMAGE
value: confluentinc/kafka-streams-examples
commands:
- export OS_TAG="-ubi8"
- export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/kafka-streams-examples
Expand Down Expand Up @@ -244,7 +272,7 @@ after_pipeline:
task:
agent:
machine:
type: s1-prod-ubuntu20-04-arm64-0
type: s1-prod-ubuntu24-04-arm64-0
jobs:
- name: Metrics
commands:
Expand Down
22 changes: 21 additions & 1 deletion .semaphore/cp_dockerfile_promote.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ version: v1.0
name: build-test-release
agent:
machine:
type: s1-prod-ubuntu20-04-amd64-1
type: s1-prod-ubuntu24-04-amd64-1

fail_fast:
cancel:
Expand Down Expand Up @@ -46,13 +46,27 @@ global_job_config:
- docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY
- export AMD_ARCH=.amd64
- export ARM_ARCH=.arm64
- export COMMUNITY_DOCKER_REPOS=""
- |
if [[ $SKIP_COMMUNITY == "True" ]]; then
for skip_repo in $COMMUNITY_DOCKER_REPOS; do
if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
echo "Skipping promotion for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
export SEMAPHORE_JOB_RESULT=passed
return 130
fi
done
fi

blocks:
- name: Promote AMD
dependencies: []
task:
jobs:
- name: Promote confluentinc/kafka-streams-examples ubi8 AMD
env_vars:
- name: DOCKER_IMAGE
value: confluentinc/kafka-streams-examples
commands:
- export OS_TYPE="ubi8"
- export DOCKER_REPO="confluentinc/kafka-streams-examples"
Expand Down Expand Up @@ -85,6 +99,9 @@ blocks:
task:
jobs:
- name: Promote confluentinc/kafka-streams-examples ubi8 ARM
env_vars:
- name: DOCKER_IMAGE
value: confluentinc/kafka-streams-examples
commands:
- export OS_TYPE="ubi8"
- export DOCKER_REPO="confluentinc/kafka-streams-examples"
Expand Down Expand Up @@ -117,6 +134,9 @@ blocks:
task:
jobs:
- name: Create Manifest confluentinc/kafka-streams-examples ubi8
env_vars:
- name: DOCKER_IMAGE
value: confluentinc/kafka-streams-examples
commands:
- export OS_TYPE="ubi8"
- if [[ ! "$OS_TYPE" ]]; then export OS_TAG=""; elif [[ "$OS_TYPE" =~ $PROMOTE_OS_TYPE* ]]; then export OS_TAG="-$OS_TYPE"; fi
Expand Down
32 changes: 27 additions & 5 deletions .semaphore/semaphore.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ version: v1.0
name: build-test-release
agent:
machine:
type: s1-prod-ubuntu20-04-amd64-1
type: s1-prod-ubuntu24-04-amd64-1

fail_fast:
cancel:
Expand Down Expand Up @@ -71,6 +71,28 @@ global_job_config:
- export LATEST_TAG=$BRANCH_TAG-latest
- export DOCKER_UPSTREAM_TAG="$LATEST_TAG"
- export DOCKER_REPOS="confluentinc/kafka-streams-examples"
- export COMMUNITY_DOCKER_REPOS=""
- |
if [[ $SKIP_COMMUNITY == "True" ]]; then
# Filter out community repos from DOCKER_REPOS
DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs)
export DOCKER_REPOS
echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS"

# Set Maven arguments for skipping community modules
export MAVEN_EXTRA_ARGS=""

# Check if current DOCKER_IMAGE is in community repos, skip job execution
for skip_repo in $COMMUNITY_DOCKER_REPOS; do
if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
echo "Skipping build for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
export SEMAPHORE_JOB_RESULT=passed
return 130
fi
done
else
export MAVEN_EXTRA_ARGS=""
fi
- export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
- export AMD_ARCH=.amd64
- export ARM_ARCH=.arm64
Expand Down Expand Up @@ -99,7 +121,7 @@ blocks:
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$PACKAGES_URL"
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
$PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
$PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- >-
for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
Expand All @@ -120,7 +142,7 @@ blocks:
task:
agent:
machine:
type: s1-prod-ubuntu20-04-arm64-1
type: s1-prod-ubuntu24-04-arm64-1
jobs:
- name: Build & Test ubi8
commands:
Expand All @@ -131,7 +153,7 @@ blocks:
- ci-tools ci-update-version
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
$PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
$PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
epilogue:
Expand All @@ -144,7 +166,7 @@ after_pipeline:
task:
agent:
machine:
type: s1-prod-ubuntu20-04-arm64-0
type: s1-prod-ubuntu24-04-arm64-0
jobs:
- name: Metrics
commands:
Expand Down
16 changes: 16 additions & 0 deletions service.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ semaphore:
enable: true
pipeline_type: cp-dockerfile
docker_repos: ['confluentinc/kafka-streams-examples']
community_docker_repos: []
community_maven_modules: []
maven_phase: 'package' # streams examples integration-test needs host-based networking, won't work in CI as-is
maven_skip_deploy: true
build_arm: true
Expand Down Expand Up @@ -40,6 +42,13 @@ semaphore:
- name: CONFLUENT_DEB_VERSION
required: false
default_value: '1'
- name: SKIP_COMMUNITY
required: false
default_value: 'False'
description: 'Skip building community docker images'
options:
- 'True'
- 'False'
- name: cp-dockerfile-promote
branch: master
pipeline_file: .semaphore/cp_dockerfile_promote.yml
Expand All @@ -58,3 +67,10 @@ semaphore:
options:
- 'deb'
- 'ubi'
- name: SKIP_COMMUNITY
required: false
default_value: 'False'
description: 'Skip publishing community docker images'
options:
- 'True'
- 'False'