From d4f0209240ca6de2df128fc362469d514b566d3c Mon Sep 17 00:00:00 2001 From: Eitan Sela Date: Thu, 6 Oct 2022 14:49:24 +0300 Subject: [PATCH 1/7] Fix issue #22 - conda-env-kernel-image example is broken Fix issue #22 - conda-env-kernel-image example is broken --- .gitignore | 5 +++++ examples/conda-env-kernel-image/.idea/.gitignore | 8 ++++++++ examples/conda-env-kernel-image/Dockerfile | 2 +- .../conda-env-kernel-image/app-image-config-input.json | 4 ++-- examples/conda-env-kernel-image/environment.yml | 4 ++-- 5 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 .gitignore create mode 100644 examples/conda-env-kernel-image/.idea/.gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5dcaadb --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ + +examples/conda-env-kernel-image/.idea/conda-env-kernel-image.iml +examples/conda-env-kernel-image/.idea/inspectionProfiles/profiles_settings.xml +examples/conda-env-kernel-image/.idea/modules.xml +examples/conda-env-kernel-image/.idea/vcs.xml diff --git a/examples/conda-env-kernel-image/.idea/.gitignore b/examples/conda-env-kernel-image/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/examples/conda-env-kernel-image/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/examples/conda-env-kernel-image/Dockerfile b/examples/conda-env-kernel-image/Dockerfile index 3e40c00..01ec540 100644 --- a/examples/conda-env-kernel-image/Dockerfile +++ b/examples/conda-env-kernel-image/Dockerfile @@ -1,4 +1,4 @@ FROM continuumio/miniconda3:4.9.2 COPY environment.yml . -RUN conda env create -f environment.yml +RUN conda env update -f environment.yml --prune diff --git a/examples/conda-env-kernel-image/app-image-config-input.json b/examples/conda-env-kernel-image/app-image-config-input.json index 284f2a0..99db336 100644 --- a/examples/conda-env-kernel-image/app-image-config-input.json +++ b/examples/conda-env-kernel-image/app-image-config-input.json @@ -3,12 +3,12 @@ "KernelGatewayImageConfig": { "KernelSpecs": [ { - "Name": "conda-env-myenv-py", + "Name": "python3", "DisplayName": "Python [conda env: myenv]" } ], "FileSystemConfig": { - "MountPath": "/root", + "MountPath": "/home/sagemaker-user", "DefaultUid": 0, "DefaultGid": 0 } diff --git a/examples/conda-env-kernel-image/environment.yml b/examples/conda-env-kernel-image/environment.yml index 3a0ab8e..e382920 100644 --- a/examples/conda-env-kernel-image/environment.yml +++ b/examples/conda-env-kernel-image/environment.yml @@ -1,8 +1,8 @@ -name: myenv +name: base channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 - numpy - awscli - boto3 From 85f3031f88092d983355b52a136b0bae6856f7ca Mon Sep 17 00:00:00 2001 From: Eitan Sela Date: Thu, 6 Oct 2022 14:50:41 +0300 Subject: [PATCH 2/7] Delete .gitignore --- examples/conda-env-kernel-image/.idea/.gitignore | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 examples/conda-env-kernel-image/.idea/.gitignore diff --git a/examples/conda-env-kernel-image/.idea/.gitignore b/examples/conda-env-kernel-image/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/examples/conda-env-kernel-image/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml From bcfadda260c182639ffbca85661971b7e1ab8240 Mon Sep 17 00:00:00 2001 From: Eitan Sela Date: Thu, 6 Oct 2022 14:52:13 +0300 Subject: [PATCH 3/7] Update README.md --- examples/conda-env-kernel-image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/conda-env-kernel-image/README.md b/examples/conda-env-kernel-image/README.md index 1dadaff..ae4e86f 100644 --- a/examples/conda-env-kernel-image/README.md +++ b/examples/conda-env-kernel-image/README.md @@ -82,7 +82,7 @@ docker push ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom:${IMAG ``` -Create new App Image Version. +Create new App Image Version ``` aws --region ${REGION} sagemaker create-image-version \ --image-name ${IMAGE_NAME} \ From f630b9728d141fb6a89ee5caa285b6d43646eb3f Mon Sep 17 00:00:00 2001 From: Eitan Sela Date: Thu, 6 Oct 2022 16:38:36 +0300 Subject: [PATCH 4/7] Update README.md --- examples/conda-env-kernel-image/README.md | 28 ++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/conda-env-kernel-image/README.md b/examples/conda-env-kernel-image/README.md index ae4e86f..cf810fb 100644 --- a/examples/conda-env-kernel-image/README.md +++ b/examples/conda-env-kernel-image/README.md @@ -9,7 +9,7 @@ The Conda environment must have the appropriate kernel package installed, for e. ### Building the image -Build the Docker image and push to Amazon ECR. +Build the Docker image. ``` # Modify these as required. The Docker registry endpoint can be tuned based on your current region from https://docs.aws.amazon.com/general/latest/gr/ecr.html#ecr-docker-endpoints REGION= @@ -26,6 +26,32 @@ docker build . -t ${IMAGE_NAME} -t ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com docker push ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom:${IMAGE_NAME} ``` +### Local testing + +Run the image locally to verify that the kernels in the image are visible to a Kernel Gateway. +``` +docker run -it "$IMAGE_NAME" bash +``` + +Run the container with a KernelGateway to validate that the kernels are visible from the REST endpoint exposed to the host. + +``` +docker run -it -p 8888:8888 "$IMAGE_NAME" bash -c 'pip install jupyter_kernel_gateway && jupyter-kernelgateway --ip 0.0.0.0 --debug --port 8888' +``` + +Verify the Kernel Gateway is started successfully (e.g., *[KernelGatewayApp] Jupyter Kernel Gateway at http://0.0.0.0:8888* in the Docker logs) and validate that you can list the kernelspecs in the the running container + +``` +curl http://0.0.0.0:8888/api/kernelspecs +``` + + +### Pushing the image +Push the Docker image to Amazon ECR +``` +docker push ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom:${IMAGE_NAME} +``` + ### Using with SageMaker Studio Create a SageMaker Image (SMI) with the image in ECR. Request parameter RoleArn value is used to get ECR image information when and Image version is created. After creating Image, create an Image Version during which From 8b32c8f3f68f3f98df7c26471dc2d7267ca863bc Mon Sep 17 00:00:00 2001 From: Eitan Sela Date: Thu, 6 Oct 2022 16:39:14 +0300 Subject: [PATCH 5/7] Update README.md --- examples/conda-env-kernel-image/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/conda-env-kernel-image/README.md b/examples/conda-env-kernel-image/README.md index cf810fb..be35d19 100644 --- a/examples/conda-env-kernel-image/README.md +++ b/examples/conda-env-kernel-image/README.md @@ -23,7 +23,6 @@ aws --region ${REGION} ecr create-repository --repository-name smstudio-custom # Build and push the image aws --region ${REGION} ecr get-login-password | docker login --username AWS --password-stdin ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom docker build . -t ${IMAGE_NAME} -t ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom:${IMAGE_NAME} -docker push ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom:${IMAGE_NAME} ``` ### Local testing From aa7c5d81da9cc4b58e07dab6757a4c232016b56c Mon Sep 17 00:00:00 2001 From: Eitan Sela Date: Thu, 6 Oct 2022 16:58:51 +0300 Subject: [PATCH 6/7] Update README.md --- examples/conda-env-kernel-image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/conda-env-kernel-image/README.md b/examples/conda-env-kernel-image/README.md index be35d19..0e28804 100644 --- a/examples/conda-env-kernel-image/README.md +++ b/examples/conda-env-kernel-image/README.md @@ -86,7 +86,7 @@ Create a Domain, providing the SageMaker Image and AppImageConfig in the Domain aws --region ${REGION} sagemaker create-domain --cli-input-json file://create-domain-input.json ``` -If you have an existing Domain, you can also use the `update-domain` +If you have an existing Domain, you can also use the `update-domain`. Replace the placeholder for Domain ID in `update-domain-input.json` ```bash aws --region ${REGION} sagemaker update-domain --cli-input-json file://update-domain-input.json From 08533277abf8604a2eb838c3003c600211c739f1 Mon Sep 17 00:00:00 2001 From: Eitan Sela Date: Thu, 17 Nov 2022 09:55:51 +0200 Subject: [PATCH 7/7] Update app-image-config-input.json --- examples/conda-env-kernel-image/app-image-config-input.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/conda-env-kernel-image/app-image-config-input.json b/examples/conda-env-kernel-image/app-image-config-input.json index 99db336..cb4ed2a 100644 --- a/examples/conda-env-kernel-image/app-image-config-input.json +++ b/examples/conda-env-kernel-image/app-image-config-input.json @@ -8,7 +8,7 @@ } ], "FileSystemConfig": { - "MountPath": "/home/sagemaker-user", + "MountPath": "/root", "DefaultUid": 0, "DefaultGid": 0 }