From aee3d6dfee588c1973a894f10890e114d07c9d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C2=A9?= Date: Mon, 13 Mar 2023 08:47:01 +0000 Subject: [PATCH 1/6] add new workflow files --- .github/config.json | 5 +++ .github/workflows/binder.yaml | 30 +++++++++++++++ .github/workflows/build.yaml | 47 +++++++++++++++++++++++ .github/workflows/render.yaml | 70 +++++++++++++++++++++++++++++++++++ .github/workflows/test.yaml | 44 ++++++++++++++++++++++ 5 files changed, 196 insertions(+) create mode 100644 .github/config.json create mode 100644 .github/workflows/binder.yaml create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/render.yaml create mode 100644 .github/workflows/test.yaml diff --git a/.github/config.json b/.github/config.json new file mode 100644 index 0000000..f9d54dc --- /dev/null +++ b/.github/config.json @@ -0,0 +1,5 @@ +{ + "notebook_name": "general-exploration-landcover_io.ipynb", + "kernel": "python", + "quay_image": "edsbook/general-exploration-landcover_io" +} \ No newline at end of file diff --git a/.github/workflows/binder.yaml b/.github/workflows/binder.yaml new file mode 100644 index 0000000..d5571b3 --- /dev/null +++ b/.github/workflows/binder.yaml @@ -0,0 +1,30 @@ +# Reference https://mybinder.readthedocs.io/en/latest/howto/gh-actions-badges.html +name: Test this PR on Binder Badge +on: + pull_request_target: + types: [opened] + +permissions: + pull-requests: + write + +jobs: + binder: + runs-on: ubuntu-latest + steps: + - name: comment on PR with Binder link + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + var PR_HEAD_USERREPO = process.env.PR_HEAD_USERREPO; + var PR_HEAD_REF = process.env.PR_HEAD_REF; + github.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/${PR_HEAD_USERREPO}/${PR_HEAD_REF}) :point_left: Test this PR on Binder` + }) + env: + PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_HEAD_USERREPO: ${{ github.event.pull_request.head.repo.full_name }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..3899055 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,47 @@ +name: Build and push container image + +on: + push: + branches: + - main + +jobs: + build-and-push: + runs-on: ubuntu-latest + if: github.repository_owner == 'Environmental-DS-Book' + steps: + # For biggish images, github actions runs out of disk space. + # So we cleanup some unwanted things in the disk image, and reclaim that space for our docker use + # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150 + # and https://github.com/easimon/maximize-build-space/blob/b4d02c14493a9653fe7af06cc89ca5298071c66e/action.yml#L104 + # This gives us a total of about 52G of free space, which should be enough for now + - name: cleanup disk space + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc + df -h + + - name: Checkout files in repo + uses: actions/checkout@main + + # Set quay image + - name: Get properties from the config file + id: config_quay + uses: notiz-dev/github-action-json-property@release + with: + path: '.github/config.json' + prop_path: 'quay_image' + + - name: Build and push the image to quay.io + uses: jupyterhub/repo2docker-action@master + with: + # Make sure username & password/token pair matches your registry credentials + DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + DOCKER_REGISTRY: "quay.io" + + # Uncomment and modify the following line with your image name, otherwise no push will happen + IMAGE_NAME: ${{steps.config_quay.outputs.prop}} + + # Lets us monitor disks getting full as images get bigger over time + - name: Show how much disk space is left + run: df -h \ No newline at end of file diff --git a/.github/workflows/render.yaml b/.github/workflows/render.yaml new file mode 100644 index 0000000..0d4e3f2 --- /dev/null +++ b/.github/workflows/render.yaml @@ -0,0 +1,70 @@ +name: Render + +on: + workflow_run: + workflows: ["Build and push container image"] + types: + - completed + +# This job installs dependencies, build the jupyter notebook, and pushes it to `render`, a new `branch` +jobs: + build: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + if: github.repository_owner == 'Environmental-DS-Book' + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v2 + # Set notebook name + - name: Get properties from the config file + id: config_file + uses: notiz-dev/github-action-json-property@release + with: + path: '.github/config.json' + prop_path: 'notebook_name' + # Set quay image + - name: Get properties from the config file + id: config_quay + uses: notiz-dev/github-action-json-property@release + with: + path: '.github/config.json' + prop_path: 'quay_image' + # Set kernel name + - name: Get properties from the config file + id: config_kernel + uses: notiz-dev/github-action-json-property@release + with: + path: '.github/config.json' + prop_path: 'kernel' + # Render the notebook + - name: Run the build process with Docker + uses: addnab/docker-run-action@v3 + with: + image: quay.io/${{steps.config_quay.outputs.prop}}:latest + options: --user root -v ${{ github.workspace }}:/tmp + shell: bash + run: | + v=$(jupyter kernelspec list | awk '{print $1}' | egrep ${{steps.config_kernel.outputs.prop}}) + jupyter-nbconvert --to notebook --ExecutePreprocessor.kernel_name="$v" --inplace --execute ${{steps.config_file.outputs.prop}} + cp ${{steps.config_file.outputs.prop}} /tmp + - name: Commit + run: | + ls + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git rm \* ':!${{steps.config_file.outputs.prop}}' + git add ${{steps.config_file.outputs.prop}} + git commit -m 'update' + - name: Push + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: render + force: true diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..b953632 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,44 @@ +name: Test container image build + +on: + pull_request: + +jobs: + test-build: + runs-on: ubuntu-latest + if: github.repository_owner == 'Environmental-DS-Book' + steps: + + # For biggish images, github actions runs out of disk space. + # So we cleanup some unwanted things in the disk image, and reclaim that space for our docker use + # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150 + # and https://github.com/easimon/maximize-build-space/blob/b4d02c14493a9653fe7af06cc89ca5298071c66e/action.yml#L104 + # This gives us a total of about 52G of free space, which should be enough for now + - name: cleanup disk space + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc + df -h + + - name: checkout files in repo + uses: actions/checkout@main + + # Set quay image name + - name: Get properties from the config file + id: config_file + uses: notiz-dev/github-action-json-property@release + with: + path: '.github/config.json' + prop_path: 'quay_image' + + - name: Build the image and push it if `NO_PUSH` disabled + uses: jupyterhub/repo2docker-action@master + with: # make sure username & password/token matches your registry + NO_PUSH: "true" + DOCKER_REGISTRY: ${{steps.config_quay.outputs.prop}} + + # Uncomment and modify the following line with your image name. a + IMAGE_NAME: "environmental_ds_book/test2i2c" + + # Lets us monitor disks getting full as images get bigger over time + - name: Show how much disk space is left + run: df -h From 9c4a2d25aad76fb7740945df5f4be5e34606cc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C2=A9?= Date: Mon, 13 Mar 2023 08:47:12 +0000 Subject: [PATCH 2/6] remove old config file --- config.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 config.json diff --git a/config.json b/config.json deleted file mode 100644 index 3e320ec..0000000 --- a/config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "notebook_name": "general-exploration-landcover_io.ipynb" -} From 5a934059ccdd96b8becf3263fac120120840c2f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C2=A9?= Date: Mon, 13 Mar 2023 08:47:24 +0000 Subject: [PATCH 3/6] remove old workflow file --- .github/workflows/publish.yml | 60 ----------------------------------- 1 file changed, 60 deletions(-) delete mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 261413d..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: publish - -on: - # Trigger the workflow on push to main/master branch - push: - branches: - - master - - main - -# This job installs dependencies, build the jupyter notebook, and pushes it to `render`, a new `branch` -jobs: - build: - name: ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v2 - - name: Set up conda and dependencies - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - activate-environment: repo - environment-file: .binder/environment.yml - # Install extra packages - - name: Install jupyter dependencies - run: | - conda install pip - pip install jinja2==3.0.2 - pip install jupyter - pip install nbconvert - # Set notebook anme - - name: Get properties from the config file - id: config_file - uses: notiz-dev/github-action-json-property@release - with: - path: 'config.json' - prop_path: 'notebook_name' - # Render the notebook - - name: Render the notebook - run: | - jupyter nbconvert --to notebook --ExecutePreprocessor.kernel_name=python3 --inplace --execute ${{steps.config_file.outputs.prop}} - - name: Commit - run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git rm \* ':!*.ipynb' - git add ${{steps.config_file.outputs.prop}} - git commit -m 'update' - - name: Push - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: render - force: true From c36dcf39fff569de7b16faa314e6efea480c956f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C2=A9?= Date: Mon, 13 Mar 2023 08:47:42 +0000 Subject: [PATCH 4/6] change Environmental-DS-Book to EDS book --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index f33b37f..db19226 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Environmental-DS-Book +Copyright (c) 2022 EDS book Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From cee94acfd634fe08a392002c9f8e47d4192dec76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C2=A9?= Date: Mon, 13 Mar 2023 08:47:57 +0000 Subject: [PATCH 5/6] Change to the render badge and add 2i2c in attribution --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4f9982..a4d63f7 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ Binder - - Continuous integration badge + + Render

@@ -62,3 +62,4 @@ You may also download the notebook from GitHub to run it locally: # Credits The **How to run** section was adapted from the [Project Pythia Cookbook](https://cookbooks.projectpythia.org/) project. +The workflow actions were adapted from [2i2c’s hub-user-image-template](https://github.com/2i2c-org/hub-user-image-template) released under BSD-3-Clause license. \ No newline at end of file From a023542f392d5b92e15aa5edb1d5c4b9817cadc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C2=A9?= Date: Mon, 13 Mar 2023 08:52:11 +0000 Subject: [PATCH 6/6] fix DOCKER_REGISTRY and IMAGE_NAME values --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b953632..c08cfc3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -34,10 +34,10 @@ jobs: uses: jupyterhub/repo2docker-action@master with: # make sure username & password/token matches your registry NO_PUSH: "true" - DOCKER_REGISTRY: ${{steps.config_quay.outputs.prop}} + DOCKER_REGISTRY: "quay.io" # Uncomment and modify the following line with your image name. a - IMAGE_NAME: "environmental_ds_book/test2i2c" + IMAGE_NAME: ${{steps.config_quay.outputs.prop}} # Lets us monitor disks getting full as images get bigger over time - name: Show how much disk space is left