diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ef6c79b..cabaa946 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,9 +23,9 @@ jobs: - image: spark3-iceberg platforms: linux/amd64,linux/arm64 test: spark3-iceberg - - image: spark3-delta + - image: spark4-delta platforms: linux/amd64,linux/arm64 - test: spark3-delta + test: spark4-delta - image: spark3-hudi platforms: linux/amd64,linux/arm64 test: spark3-hudi diff --git a/etc/compose/spark3-delta/docker-compose.yml b/etc/compose/spark3-delta/docker-compose.yml deleted file mode 100644 index f04dcf9c..00000000 --- a/etc/compose/spark3-delta/docker-compose.yml +++ /dev/null @@ -1,4 +0,0 @@ -version: '2.0' -services: - spark: - image: testing/spark3-delta:latest$ARCH diff --git a/etc/compose/spark4-delta/docker-compose.yml b/etc/compose/spark4-delta/docker-compose.yml new file mode 100644 index 00000000..711470bc --- /dev/null +++ b/etc/compose/spark4-delta/docker-compose.yml @@ -0,0 +1,4 @@ +version: '2.0' +services: + spark: + image: testing/spark4-delta:latest$ARCH diff --git a/testing/spark3-delta/Dockerfile b/testing/spark4-delta/Dockerfile similarity index 78% rename from testing/spark3-delta/Dockerfile rename to testing/spark4-delta/Dockerfile index 8d3e4a86..576ed709 100644 --- a/testing/spark3-delta/Dockerfile +++ b/testing/spark4-delta/Dockerfile @@ -11,12 +11,12 @@ # limitations under the License. ARG ARCH -FROM testing/centos7-oj11:unlabelled$ARCH +FROM testing/centos7-oj17:unlabelled$ARCH -ARG SPARK_VERSION=3.5.0 +ARG SPARK_VERSION=4.0.0-preview1 ARG HADOOP_VERSION=3 -ARG DELTA_VERSION=3.2.0 -ARG SCALA_VERSION=2.12 +ARG DELTA_VERSION=4.0.0rc1 +ARG SCALA_VERSION=2.13 ARG SPARK_ARTIFACT="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" @@ -35,8 +35,8 @@ RUN wget -nv "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/ RUN wget -nv "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.319/aws-java-sdk-bundle-1.12.319.jar" # install Delta -RUN wget -nv "https://repo1.maven.org/maven2/io/delta/delta-spark_${SCALA_VERSION}/${DELTA_VERSION}/delta-spark_${SCALA_VERSION}-${DELTA_VERSION}.jar" -RUN wget -nv "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" +RUN wget -nv "https://oss.sonatype.org/content/repositories/iodelta-1147/io/delta/delta-spark_${SCALA_VERSION}/${DELTA_VERSION}/delta-spark_${SCALA_VERSION}-${DELTA_VERSION}.jar" +RUN wget -nv "https://oss.sonatype.org/content/repositories/iodelta-1147/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" # Create hive user to match Hive container RUN adduser hive