diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b3d2b36..3c1c087a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,10 @@ jobs: - image: hive3.1-hive platforms: linux/amd64,linux/arm64 test: hive3.1-hive + - image: hive4.0-hive + # Haven't added `linux/arm64` platform as test image fails with `The requested image's platform (linux/arm64) does not match the detected host platform (linux/amd64/v3) and no specific platform was requested` + platforms: linux/amd64 + test: hive4.0-hive - image: hdp3.1-hive-kerberized test: hdp3.1-hive # TODO add test https://github.com/trinodb/trino/issues/14543 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 041bb5fa..7f9139e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,6 +91,7 @@ jobs: testing/spark3-hudi testing/polaris-catalog testing/unity-catalog + testing/hive4.0-hive ) referenced_images=("${skipped_images[@]}" "${single_arch[@]}" "${multi_arch[@]}") make meta diff --git a/bin/test.sh b/bin/test.sh index be62213a..f1204fd8 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -57,6 +57,15 @@ function run_hive_transactional_tests() { true } +function check_hive4() { + environment_compose exec hiveserver2 beeline -u jdbc:hive2://localhost:10000 -e 'SELECT 1;' >/dev/null 2>&1 +} + +function run_hive4_tests() { + environment_compose exec hiveserver2 beeline -u jdbc:hive2://localhost:10000 -e 'SHOW DATABASES;' && + true +} + function check_spark() { environment_compose exec spark curl --http0.9 -f http://localhost:10213 -o /dev/null } @@ -186,6 +195,16 @@ for ARCH in "${platforms[@]}"; do test true elif [[ ${ENVIRONMENT} == "kerberos" ]]; then run_kerberos_tests + elif [[ ${ENVIRONMENT} == *"hive4"* ]]; then + # wait until hiveserver is started + retry check_hive4 + + # run tests + set -x + set +e + sleep 10 + run_hive4_tests + # TODO add transactional hive tests elif [[ ${ENVIRONMENT} == *"hive"* ]]; then # wait until hadoop processes is started retry check_hadoop diff --git a/etc/compose/hive4.0-hive/docker-compose.yml b/etc/compose/hive4.0-hive/docker-compose.yml new file mode 100644 index 00000000..ba16ba36 --- /dev/null +++ b/etc/compose/hive4.0-hive/docker-compose.yml @@ -0,0 +1,7 @@ +version: '2.0' +services: + hiveserver2: + hostname: hiveserver2 + image: testing/hive4.0-hive:latest$ARCH + environment: + - SERVICE_NAME=hiveserver2 diff --git a/testing/hive4.0-hive/Dockerfile b/testing/hive4.0-hive/Dockerfile new file mode 100644 index 00000000..b9cd6a1a --- /dev/null +++ b/testing/hive4.0-hive/Dockerfile @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM apache/hive:4.0.0 + +# TODO replace with aws sdk v2 by following https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html +ARG AWS_JAVA_SDK_BUNDLE_VERSION=1.12.367 +ARG HADOOP_AWS_VERSION=3.3.6 + +USER root +RUN apt-get -y update +RUN apt install curl -y + +# Install AWS SDK so we can access S3; the version must match the hadoop-aws* jars which are part of SPARK distribution +RUN mkdir -p /opt/hive/auxlib && \ + curl -fLsS -o /opt/hive/auxlib/aws-java-sdk-bundle-$AWS_JAVA_SDK_BUNDLE_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/$AWS_JAVA_SDK_BUNDLE_VERSION/aws-java-sdk-bundle-$AWS_JAVA_SDK_BUNDLE_VERSION.jar && \ + curl -fLsS -o /opt/hive/auxlib/hadoop-aws-$HADOOP_AWS_VERSION.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/$HADOOP_AWS_VERSION/hadoop-aws-$HADOOP_AWS_VERSION.jar