From 59e2d7cdda73171ad5b3f0952a610c77927421a0 Mon Sep 17 00:00:00 2001 From: seung-00 Date: Sat, 5 Oct 2024 21:30:31 +0900 Subject: [PATCH 1/3] feat: Write a Dockerfile for python interpreter image build --- python/Dockerfile | 50 +++++++++++++++++++++++++++++++++++++++++++++++ python/README.md | 32 ++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 python/Dockerfile diff --git a/python/Dockerfile b/python/Dockerfile new file mode 100644 index 00000000000..596a71fa5d0 --- /dev/null +++ b/python/Dockerfile @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM openjdk:11 as builder + +COPY . /zeppelin/ + +WORKDIR /zeppelin + +RUN chmod +x ./mvnw + +RUN ./mvnw clean package -am -pl zeppelin-interpreter-shaded,zeppelin-interpreter,python -DskipTests + + +FROM openjdk:11 + +COPY --from=builder /zeppelin/bin /zeppelin/bin/ +COPY --from=builder /zeppelin/conf /zeppelin/conf + +COPY --from=builder /zeppelin/interpreter/python /zeppelin/interpreter/python +COPY --from=builder /zeppelin/zeppelin-interpreter-shaded/target /zeppelin/zeppelin-interpreter-shaded/target + +WORKDIR /zeppelin + +ENV PYTHON_INTERPRETER_PORT=8084 + +RUN chmod +x ./bin/interpreter.sh + +CMD ./bin/interpreter.sh \ + -d ./interpreter/python \ + -c host.docker.internal \ + -p "${INTERPRETER_EVENT_SERVER_PORT}" \ + -r "${PYTHON_INTERPRETER_PORT}:${PYTHON_INTERPRETER_PORT}" \ + -i python-shared_process \ + -l ./local-repo \ + -g python diff --git a/python/README.md b/python/README.md index bde6e8e6a22..74cd20cd027 100644 --- a/python/README.md +++ b/python/README.md @@ -64,3 +64,35 @@ Current interpreter delegate the whole work to ipython kernel via `jupyter_clien Zeppelin interpreter process will communicate with the python process via `grpc`. Ideally every feature works in IPython should work in Zeppelin as well. +## Run the interpreter with docker +You can run the python interpreter as a standalone docker container. + +### Step 1. Specify the configuration for the interpreter +```bash + # conf/interpreter.json + + "python": { + ... + "option": + } { + "remote": true, + "port": {INTERPRETER_PROCESS_PORT_IN_HOST}, + "isExistingProcess": true, + "host": "localhost", + ... + } +```` + +### Step 2. Build and run the interpreter +```bash +zeppelin $ ./mvnw clean install -DskipTests + +zeppelin $ ./bin/zeppelin-daemon.sh start # start zeppelin server. +# check the port of the interpreter event server. you can find it by looking for the log that starts with "InterpreterEventServer is starting at" + +zeppelin $ docker build -f ./python/Dockerfile -t python-interpreter . + +zeppelin $ docker run -p {INTERPRETER_PROCESS_PORT_IN_HOST}:8084 \ + -e INTERPRETER_EVENT_SERVER_PORT={INTERPRETER_EVENT_SERVER_PORT} \ + python-interpreter +``` From 439447683318ece7a5ababf8f0535eeaf8043364 Mon Sep 17 00:00:00 2001 From: seung-00 Date: Sun, 6 Oct 2024 19:56:31 +0900 Subject: [PATCH 2/3] feat: Add dependencies for python interpreter container --- python/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/Dockerfile b/python/Dockerfile index 596a71fa5d0..44305a0bd06 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -28,6 +28,11 @@ RUN ./mvnw clean package -am -pl zeppelin-interpreter-shaded,zeppelin-interprete FROM openjdk:11 +RUN apt-get update && \ + apt-get install -y python3 python3-pip && \ + ln -s /usr/bin/python3 /usr/bin/python && \ + rm -rf /var/lib/apt/lists/* + COPY --from=builder /zeppelin/bin /zeppelin/bin/ COPY --from=builder /zeppelin/conf /zeppelin/conf From 60fbca93385156cb2db9c185329fa0540440ca0b Mon Sep 17 00:00:00 2001 From: seung-00 Date: Wed, 9 Oct 2024 18:04:36 +0900 Subject: [PATCH 3/3] feat: Add dependencies for python interpreter container --- python/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/python/Dockerfile b/python/Dockerfile index 44305a0bd06..eaa8b54542b 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -30,6 +30,7 @@ FROM openjdk:11 RUN apt-get update && \ apt-get install -y python3 python3-pip && \ + pip3 install jupyter-client grpcio protobuf~=3.20 ipython ipykernel && \ ln -s /usr/bin/python3 /usr/bin/python && \ rm -rf /var/lib/apt/lists/*