From b1503a871aa3c45eb331c0fc0af3cab1de0add91 Mon Sep 17 00:00:00 2001
From: Mayank Vadariya <48036907+mayankvadariya@users.noreply.github.com>
Date: Sun, 8 Dec 2024 11:09:30 -0500
Subject: [PATCH] Add hive 4.0 image

---
 .github/workflows/ci.yml                    |  4 ++++
 .github/workflows/release.yml               |  1 +
 bin/test.sh                                 | 19 +++++++++++++++
 etc/compose/hive4.0-hive/docker-compose.yml |  7 ++++++
 testing/hive4.0-hive/Dockerfile             | 26 +++++++++++++++++++++
 5 files changed, 57 insertions(+)
 create mode 100644 etc/compose/hive4.0-hive/docker-compose.yml
 create mode 100644 testing/hive4.0-hive/Dockerfile

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9b3d2b36..3c1c087a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,6 +35,10 @@ jobs:
           - image: hive3.1-hive
             platforms: linux/amd64,linux/arm64
             test: hive3.1-hive
+          - image: hive4.0-hive
+            # Haven't added `linux/arm64` platform as test image fails with `The requested image's platform (linux/arm64) does not match the detected host platform (linux/amd64/v3) and no specific platform was requested`
+            platforms: linux/amd64
+            test: hive4.0-hive
           - image: hdp3.1-hive-kerberized
             test: hdp3.1-hive
           # TODO add test https://github.com/trinodb/trino/issues/14543
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 041bb5fa..7f9139e9 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -91,6 +91,7 @@ jobs:
           testing/spark3-hudi
           testing/polaris-catalog
           testing/unity-catalog
+          testing/hive4.0-hive
         )
         referenced_images=("${skipped_images[@]}" "${single_arch[@]}" "${multi_arch[@]}")
         make meta
diff --git a/bin/test.sh b/bin/test.sh
index be62213a..f1204fd8 100755
--- a/bin/test.sh
+++ b/bin/test.sh
@@ -57,6 +57,15 @@ function run_hive_transactional_tests() {
         true
 }
 
+function check_hive4() {
+    environment_compose exec hiveserver2 beeline -u jdbc:hive2://localhost:10000 -e 'SELECT 1;' >/dev/null 2>&1
+}
+
+function run_hive4_tests() {
+    environment_compose exec hiveserver2 beeline -u jdbc:hive2://localhost:10000 -e 'SHOW DATABASES;' &&
+    true
+}
+
 function check_spark() {
     environment_compose exec spark curl --http0.9 -f http://localhost:10213 -o /dev/null
 }
@@ -186,6 +195,16 @@ for ARCH in "${platforms[@]}"; do
         test true
     elif [[ ${ENVIRONMENT} == "kerberos" ]]; then
         run_kerberos_tests
+    elif [[ ${ENVIRONMENT} == *"hive4"* ]]; then
+        # wait until hiveserver is started
+        retry check_hive4
+
+        # run tests
+        set -x
+        set +e
+        sleep 10
+        run_hive4_tests
+        # TODO add transactional hive tests
     elif [[ ${ENVIRONMENT} == *"hive"* ]]; then
         # wait until hadoop processes is started
         retry check_hadoop
diff --git a/etc/compose/hive4.0-hive/docker-compose.yml b/etc/compose/hive4.0-hive/docker-compose.yml
new file mode 100644
index 00000000..ba16ba36
--- /dev/null
+++ b/etc/compose/hive4.0-hive/docker-compose.yml
@@ -0,0 +1,7 @@
+version: '2.0'
+services:
+  hiveserver2:
+    hostname: hiveserver2
+    image: testing/hive4.0-hive:latest$ARCH
+    environment:
+      - SERVICE_NAME=hiveserver2
diff --git a/testing/hive4.0-hive/Dockerfile b/testing/hive4.0-hive/Dockerfile
new file mode 100644
index 00000000..b9cd6a1a
--- /dev/null
+++ b/testing/hive4.0-hive/Dockerfile
@@ -0,0 +1,26 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM apache/hive:4.0.0
+
+# TODO replace with aws sdk v2 by following https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html
+ARG AWS_JAVA_SDK_BUNDLE_VERSION=1.12.367
+ARG HADOOP_AWS_VERSION=3.3.6
+
+USER root
+RUN apt-get -y update
+RUN apt install curl -y
+
+# Install AWS SDK so we can access S3; the version must match the hadoop-aws* jars which are part of SPARK distribution
+RUN mkdir -p /opt/hive/auxlib && \
+    curl -fLsS -o /opt/hive/auxlib/aws-java-sdk-bundle-$AWS_JAVA_SDK_BUNDLE_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/$AWS_JAVA_SDK_BUNDLE_VERSION/aws-java-sdk-bundle-$AWS_JAVA_SDK_BUNDLE_VERSION.jar && \
+    curl -fLsS -o /opt/hive/auxlib/hadoop-aws-$HADOOP_AWS_VERSION.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/$HADOOP_AWS_VERSION/hadoop-aws-$HADOOP_AWS_VERSION.jar