Merge branch 'main' into chore/remove-deprecated-hdfs-pre-25.7.0

Author: razvan

.github/ISSUE_TEMPLATE/update-product-airflow.md

@@ -31,6 +31,7 @@ Add/Change/Remove anything that isn't applicable anymore
 - [ ] Download new constraints file (see `airflow/download_constraints.sh`).
 - [ ] Update other dependencies if applicable (eg: python, statsd_exporter, etc).
 - [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR(s) to the list below.
+- [ ] Ensure prerequisites are up to date (required-external-components.adoc).
 - [ ] Update the version in demos. Add the PR(s) to the list below.
 
 ## Related Pull Requests

.github/ISSUE_TEMPLATE/update-product-druid.md

@@ -34,6 +34,7 @@ Add/Change/Remove anything that isn't applicable anymore
 - [ ] Update `versions.py` to the latest supported version of JVM (base and devel).
 - [ ] Update the [druid-opa-authorizer](https://github.com/stackabletech/druid-opa-authorizer/) with the new set of versions.
 - [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR(s) to the list below.
+- [ ] Ensure prerequisites are up to date (required-external-components.adoc).
 - [ ] Update the version in demos. Add the PR(s) to the list below.
 
 ## Related Pull Requests

.github/ISSUE_TEMPLATE/update-product-hive.md

@@ -32,6 +32,7 @@ Add/Change/Remove anything that isn't applicable anymore
 - [ ] Update `versions.py` to the latest supported version of JVM (base and devel).
 - [ ] Update other dependencies if applicable (eg: jmx_exporter, aws_java_sdk_bundle, etc).
 - [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR(s) to the list below.
+- [ ] Ensure prerequisites are up to date (required-external-components.adoc).
 - [ ] Update the version in demos. Add the PR(s) to the list below.
 
 ## Related Pull Requests

.github/ISSUE_TEMPLATE/update-product-superset.md

@@ -33,6 +33,7 @@ Add/Change/Remove anything that isn't applicable anymore
 - [ ] Delete old constraint files and patch directories.
 - [ ] Update other dependencies if applicable (eg: python, auth_lib, etc).
 - [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR(s) to the list below.
+- [ ] Ensure prerequisites are up to date (required-external-components.adoc).
 - [ ] Update the version in demos. Add the PR(s) to the list below.
 
 ## Related Pull Requests

.github/workflows/mirror.yaml

@@ -14,6 +14,7 @@ on:
           - registry.k8s.io/sig-storage/csi-node-driver-registrar
           - registry.k8s.io/sig-storage/csi-provisioner
           - registry.k8s.io/git-sync/git-sync
+          - registry-1.docker.io/library/golang
       image-index-manifest-tag:
         description: |
           The image index manifest tag, like 1.0.14 or v1.0.14
@@ -35,7 +36,7 @@ jobs:
           - amd64
           - arm64
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
 
@@ -56,7 +57,7 @@ jobs:
           echo "IMAGE_REPOSITORY=$(.scripts/get_repo_name.sh)" | tee -a "$GITHUB_ENV"
 
       - name: Publish Container Image on oci.stackable.tech
-        uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
+        uses: stackabletech/actions/publish-image@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$sdp+github-action-build
@@ -72,7 +73,7 @@ jobs:
       id-token: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
 
@@ -83,14 +84,17 @@ jobs:
           echo "IMAGE_REPOSITORY=$(.scripts/get_repo_name.sh)" | tee -a "$GITHUB_ENV"
 
       - name: Publish and Sign Image Index Manifest to oci.stackable.tech
-        uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
+        uses: stackabletech/actions/publish-index-manifest@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$sdp+github-action-build
           image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
           image-repository: ${{ format('sdp/{0}', env.IMAGE_REPOSITORY) }}
           image-index-manifest-tag: ${{ inputs.image-index-manifest-tag }}
 
+  # NOTE (@Techassi) It is currently not possible to use our own action here, because the inputs
+  # assume it is used to report on image build results, not mirror results. The action needs to be
+  # adjusted to support other use-cases.
   notify:
     name: Failure Notification
     needs: [mirror-image, publish_manifests]

.github/workflows/pr_pre-commit.yaml

@@ -12,11 +12,11 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
           fetch-depth: 0
-      - uses: stackabletech/actions/run-pre-commit@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
+      - uses: stackabletech/actions/run-pre-commit@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           hadolint: ${{ env.HADOLINT_VERSION }}

.github/workflows/preflight.yaml

@@ -74,7 +74,7 @@ jobs:
     env:
       GITHUB_REF_NAME: ${{ github.ref_name }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
       - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0

.github/workflows/reusable_build_image.yaml

@@ -23,11 +23,11 @@ jobs:
     name: Generate Version List
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
       - id: shard
-        uses: stackabletech/actions/shard@2d3d7ddad981ae09901d45a0f6bf30c2658b1b78 # 0.7.0
+        uses: stackabletech/actions/shard@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1
         with:
           product-name: ${{ inputs.product-name }}
     outputs:
@@ -48,23 +48,23 @@ jobs:
         versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
 
       - name: Free Disk Space
-        uses: stackabletech/actions/free-disk-space@2d3d7ddad981ae09901d45a0f6bf30c2658b1b78 # 0.7.0
+        uses: stackabletech/actions/free-disk-space@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1
 
       - name: Build Product Image
         id: build
-        uses: stackabletech/actions/build-product-image@2d3d7ddad981ae09901d45a0f6bf30c2658b1b78 # 0.7.0
+        uses: stackabletech/actions/build-product-image@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1
         with:
           product-name: ${{ inputs.product-name }}
           product-version: ${{ matrix.versions }}
           sdp-version: ${{ inputs.sdp-version }}
 
       - name: Publish Container Image on oci.stackable.tech
-        uses: stackabletech/actions/publish-image@2d3d7ddad981ae09901d45a0f6bf30c2658b1b78 # 0.7.0
+        uses: stackabletech/actions/publish-image@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$${{ inputs.registry-namespace }}+github-action-build
@@ -85,12 +85,12 @@ jobs:
         versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
 
       - name: Publish and Sign Image Index Manifest to oci.stackable.tech
-        uses: stackabletech/actions/publish-index-manifest@2d3d7ddad981ae09901d45a0f6bf30c2658b1b78 # 0.7.0
+        uses: stackabletech/actions/publish-index-manifest@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$${{ inputs.registry-namespace }}+github-action-build
@@ -102,44 +102,11 @@ jobs:
     name: Failure Notification
     needs: [generate_matrix, build, publish_manifests]
     runs-on: ubuntu-latest
-    if: failure()
+    if: failure() || github.run_attempt > 1
     steps:
-      - uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+      - name: Send Notification
+        uses: stackabletech/actions/send-slack-notification@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # v0.8.1
         with:
-          channel-id: "C07UG6JH44F" # notifications-container-images
-          payload: |
-            {
-              "text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
-              "attachments": [
-                {
-                  "pretext": "See the details below for a summary of which job(s) failed.",
-                  "color": "#aa0000",
-                  "fields": [
-                    {
-                      "title": "Generate Version List",
-                      "short": true,
-                      "value": "${{ needs.generate_matrix.result }}"
-                    },
-                    {
-                      "title": "Build/Publish Image",
-                      "short": true,
-                      "value": "${{ needs.build.result }}"
-                    },
-                    {
-                      "title": "Build/Publish Manifests",
-                      "short": true,
-                      "value": "${{ needs.publish_manifests.result }}"
-                    }
-                  ],
-                  "actions": [
-                    {
-                      "type": "button",
-                      "text": "Go to workflow run",
-                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
-                    }
-                  ]
-                }
-              ]
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.slack-token }}
+          publish-manifests-result: ${{ needs.publish_manifests.result }}
+          build-result: ${{ needs.build.result }}
+          slack-token: ${{ secrets.slack-token }}

.github/workflows/ubi-rust-builder.yml

@@ -22,7 +22,7 @@ jobs:
         ubi-version: ["ubi9"]
     runs-on: ${{ matrix.runner }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
       - name: Login to Stackable Harbor
@@ -63,7 +63,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: ["build"]
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
       - name: Login to Stackable Harbor

CHANGELOG.md

@@ -8,8 +8,10 @@ All notable changes to this project will be documented in this file.
 
 - airflow: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1054]).
+- airflow: Add `2.10.5` ([#1108]).
 - druid: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1039]).
+- druid: Add `33.0.0` ([#1110]).
 - hadoop: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1029]).
 - hbase: check for correct permissions and ownerships in /stackable folder via
@@ -23,20 +25,26 @@ All notable changes to this project will be documented in this file.
   `check-permissions-ownership.sh` provided in stackable-base image ([#1027]).
 - opa: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1038]).
+- opa: Add `1.4.2` ([#1103]).
 - spark-k8s: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1055]).
 - superset: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1053]).
+- superset: Add version `4.1.2` ([#1102]).
 - trino: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1025]).
 - zookeeper: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1043]).
+- nifi: Add [nifi-iceberg-bundle](https://github.com/stackabletech/nifi-iceberg-bundle) for NiFi `2.2.0` ([#1060], [#1106]).
 - java: Add JDK 24 ([#1097]).
+- ci: Add golang image to mirror workflow ([#1103]).
 
 ### Changed
 
+- airflow,superset: Use `uv` to build the product ([#1116]).
 - ubi-rust-builder: Bump Rust toolchain to 1.85.0, cargo-cyclonedx to 0.5.7, and cargo-auditable to 0.6.6 ([#1050]).
 - ubi9-rust-builder: Bump base image and update protoc to `30.2` ([#1091]).
+- stackable-devel: Bump ubi9 base image ([#1103]).
 - spark-k8s: Include spark-connect jars, replace OpenJDK with Temurin JDK, cleanup ([#1034]).
 - spark-connect-client: Image is now completely based on spark-k8s and includes JupyterLab and other demo dependencies ([#1071]).
 - jmx_exporter: Bump products to use `1.2.0` ([#1090]).
@@ -47,6 +55,7 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+- airflow: Pin Cython version ([#1116]).
 - druid: reduce docker image size by removing the recursive chown/chmods in the final image ([#1039]).
 - hadoop: reduce docker image size by removing the recursive chown/chmods in the final image ([#1029]).
 - hbase: reduce docker image size by removing the recursive chown/chmods in the final image ([#1028]).
@@ -55,7 +64,9 @@ All notable changes to this project will be documented in this file.
 - Add `--locked` flag to `cargo install` commands for reproducible builds ([#1044]).
 - nifi: reduce docker image size by removing the recursive chown/chmods in the final image ([#1027]).
 - opa: reduce docker image size by removing the recursive chown/chmods in the final image ([#1038]).
+- opa: Manually install Go 1.23.9 ([#1103]).
 - spark-k8s: reduce docker image size by removing the recursive chown/chmods in the final image ([#1042]).
+- superset: Pin Cython version ([#1116]).
 - trino: reduce docker image size by removing the recursive chown/chmods in the final image ([#1025]).
 - zookeeper: reduce docker image size by removing the recursive chown/chmods in the final image ([#1043]).
 - Fixed two hardcoded username references ([#1052]).
@@ -67,6 +78,9 @@ All notable changes to this project will be documented in this file.
 - zookeeper: Remove 3.9.2 ([#1093]).
 - Remove ubi8-rust-builder image ([#1091]).
 - hadoop: Remove `3.3.4` and `3.4.0` ([#1099]).
+- opa: Remove `0.67.1` ([#1103]).
+- opa: Remove legacy bundle-builder from container build ([#1103]).
+- druid: Remove `30.0.0` ([#1110]).
 
 [#1025]: https://github.com/stackabletech/docker-images/pull/1025
 [#1027]: https://github.com/stackabletech/docker-images/pull/1027
@@ -86,12 +100,19 @@ All notable changes to this project will be documented in this file.
 [#1054]: https://github.com/stackabletech/docker-images/pull/1054
 [#1055]: https://github.com/stackabletech/docker-images/pull/1055
 [#1056]: https://github.com/stackabletech/docker-images/pull/1056
+[#1060]: https://github.com/stackabletech/docker-images/pull/1060
 [#1090]: https://github.com/stackabletech/docker-images/pull/1090
 [#1091]: https://github.com/stackabletech/docker-images/pull/1091
 [#1093]: https://github.com/stackabletech/docker-images/pull/1093
 [#1097]: https://github.com/stackabletech/docker-images/pull/1097
 [#1098]: https://github.com/stackabletech/docker-images/pull/1098
 [#1099]: https://github.com/stackabletech/docker-images/pull/1099
+[#1102]: https://github.com/stackabletech/docker-images/pull/1102
+[#1103]: https://github.com/stackabletech/docker-images/pull/1103
+[#1106]: https://github.com/stackabletech/docker-images/pull/1106
+[#1108]: https://github.com/stackabletech/docker-images/pull/1108
+[#1110]: https://github.com/stackabletech/docker-images/pull/1110
+[#1116]: https://github.com/stackabletech/docker-images/pull/1116
 
 ## [25.3.0] - 2025-03-21
 

Cargo.lock

@@ -4,7 +4,7 @@ resolver = "2"
 
 [workspace.dependencies]
 clap = { version = "4.5.27", features = ["derive"] }
-git2 = "0.20.0"
+git2 = "0.20.1"
 serde = { version = "1.0.217", features = ["derive"] }
 snafu = "0.8.5"
 tempfile = "3.16.0"

Cargo.toml

@@ -31,9 +31,7 @@ ARG TARGETARCH
 ARG STACKABLE_USER_UID
 ARG S3FS
 ARG CYCLONEDX_BOM
-
-COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt
-COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/
+ARG UV
 
 # The mysql provider is currently excluded.
 # Requires implementation of https://github.com/apache/airflow/blob/2.2.5/scripts/docker/install_mysql.sh
@@ -62,19 +60,27 @@ RUN microdnf update && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
+COPY airflow/stackable/constraints/${PRODUCT}/constraints-python${PYTHON}.txt /tmp/constraints.txt
+COPY airflow/stackable/constraints/${PRODUCT}/build-constraints-python${PYTHON}.txt /tmp/build-constraints.txt
+COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/
+
 RUN <<EOF
 python${PYTHON} -m venv --system-site-packages /stackable/app
 
 source /stackable/app/bin/activate
 
+# Upgrade pip to the latest version
+# Also install uv to get support for build constraints
 pip install --no-cache-dir --upgrade pip
-pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt
+pip install --no-cache-dir uv==${UV}
+
+uv pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt
 # Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
-pip install --no-cache-dir s3fs==${S3FS} cyclonedx-bom==${CYCLONEDX_BOM}
+uv pip install --no-cache-dir s3fs==${S3FS} cyclonedx-bom==${CYCLONEDX_BOM}
 # Needed for OIDC
-pip install --no-cache-dir Flask_OIDC==2.2.0 Flask-OpenID==1.3.1
+uv pip install --no-cache-dir Flask_OIDC==2.2.0 Flask-OpenID==1.3.1
 
-pip install --no-cache-dir /tmp/opa_auth_manager-0.1.0-py3-none-any.whl
+uv pip install --no-cache-dir /tmp/opa_auth_manager-0.1.0-py3-none-any.whl
 
 # Create the SBOM for Airflow
 # Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete

airflow/Dockerfile

@@ -0,0 +1 @@
+Cython==3.1.0