From 9445a366c1a3ffa4e273aa555bdb6a2d2d0f11d8 Mon Sep 17 00:00:00 2001 From: Guilherme Torres Castro <1149991+guitcastro@users.noreply.github.com> Date: Wed, 31 Jul 2024 17:17:04 -0300 Subject: [PATCH 1/5] Create Dockerfile --- Dockerfile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..bae63bdf8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +# syntax=docker/dockerfile:1 + +FROM maven:3.9.8-eclipse-temurin-11-focal as package + +WORKDIR /build + +COPY ./ ./ +RUN --mount=type=cache,target=/root/.m2 \ + MAVEN_OPTS=-Dorg.slf4j.simpleLogger.defaultLogLevel=warn mvn -B package -DskipTests +RUN mv xtable-utilities/target/xtable-utilities-$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)-bundled.jar target/app.jar + +FROM eclipse-temurin:17-jre-jammy AS final + +# Create a non-privileged user that the app will run under. +# See https://docs.docker.com/go/dockerfile-user-best-practices/ +ARG UID=10001 +RUN adduser \ + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + appuser +USER appuser + +# Copy the executable from the "package" stage. +COPY --from=package build/target/app.jar ./app.jar + +ENTRYPOINT [ + "java", \ + "--add-opens=java.base/sun.nio.hb=ALL-UNNAMED", \ + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", \ + "--add-opens=java.base/java.nio=ALL-UNNAMED", \ + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", \ + "--add-opens=java.base/java.util=ALL-UNNAMED", \ + "--add-opens=java.base/java.lang=ALL-UNNAMED", \ + "-jar", \ + "./app.jar" \ + ] From 763173075711b7ab34956b23ad50b7cf89a26bab Mon Sep 17 00:00:00 2001 From: Guilherme Torres Castro <1149991+guitcastro@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:53:18 -0300 Subject: [PATCH 2/5] Added about add-opens --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index bae63bdf8..580bd6c49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,8 @@ USER appuser # Copy the executable from the "package" stage. COPY --from=package build/target/app.jar ./app.jar +# It's necessary to added opens to run spark on java >= 17 +# https://stackoverflow.com/questions/73465937/apache-spark-3-3-0-breaks-on-java-17-with-cannot-access-class-sun-nio-ch-direct ENTRYPOINT [ "java", \ "--add-opens=java.base/sun.nio.hb=ALL-UNNAMED", \ From fc4cc027a26996ce4018f0b1f51e133125d95eb2 Mon Sep 17 00:00:00 2001 From: Guilherme Torres Castro <1149991+guitcastro@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:41:09 -0300 Subject: [PATCH 3/5] Update README.md --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index a150af499..0a0835aa7 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,19 @@ The custom hadoop configurations can be passed in with the `--hadoopConfig [cust The config in custom hadoop config file will override the default hadoop configurations. For an example of a custom hadoop config file, see [hadoop.xml](https://xtable.apache.org/docs/fabric#step-2-translate-source-table-to-delta-lake-format-using-apache-xtable-incubating). +# Running using docker + +In order to run using docker, you must mount the config files on the container. The follow commands how to: + +``` +docker run \ + -v ./xtable/config.yml:/xtable/config.yml \ + -v ./xtable/core-site.xml:/xtable/core-site.xml \ + -v ./xtable/catalog.yml:/xtable/catalog.yml \ + registry.gitlab.com/pedalin/xtable-docker-image:4420d2b6 \ + --datasetConfig /xtable/config.yml --hadoopConfig /xtable/core-site.xml --icebergCatalogConfig xtable/catalog.yml +``` + # Contributing ## Setup For setting up the repo on IntelliJ, open the project and change the java version to Java11 in File->ProjectStructure From fcb82767f098d1f08b793b267d987e856e361d00 Mon Sep 17 00:00:00 2001 From: Guilherme Torres Castro <1149991+guitcastro@users.noreply.github.com> Date: Mon, 5 Aug 2024 23:58:24 -0300 Subject: [PATCH 4/5] added instructions to build the image --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a0835aa7..5c32cdb5c 100644 --- a/README.md +++ b/README.md @@ -119,14 +119,15 @@ of a custom hadoop config file, see [hadoop.xml](https://xtable.apache.org/docs/ # Running using docker -In order to run using docker, you must mount the config files on the container. The follow commands how to: +1. Build the docker image using `docker builld . -t xtable` +2. Mount the config files on the container and run the container: ``` docker run \ -v ./xtable/config.yml:/xtable/config.yml \ -v ./xtable/core-site.xml:/xtable/core-site.xml \ -v ./xtable/catalog.yml:/xtable/catalog.yml \ - registry.gitlab.com/pedalin/xtable-docker-image:4420d2b6 \ + xtable \ --datasetConfig /xtable/config.yml --hadoopConfig /xtable/core-site.xml --icebergCatalogConfig xtable/catalog.yml ``` From 6c1d3bc01c97071f7097171661564617dffdb16b Mon Sep 17 00:00:00 2001 From: Guilherme Torres Castro <1149991+guitcastro@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:30:30 -0300 Subject: [PATCH 5/5] Update Dockerfile added apache license --- Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Dockerfile b/Dockerfile index 580bd6c49..c749651ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # syntax=docker/dockerfile:1 FROM maven:3.9.8-eclipse-temurin-11-focal as package