From 9415152b309d8a9b62a50d1a0a035e1d4eee645b Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 15 Nov 2024 15:11:39 +0000 Subject: [PATCH 01/35] fix: clarify comment on deleting previous microreact cluster job results and update run_app script to start rq worker --- beebop/app.py | 2 +- scripts/run_app | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beebop/app.py b/beebop/app.py index b14bafef..dc55e49e 100644 --- a/beebop/app.py +++ b/beebop/app.py @@ -301,7 +301,7 @@ def run_poppunk_internal(sketches: dict, depends_on=job_assign, **queue_kwargs) redis.hset("beebop:hash:job:network", p_hash, job_network.id) # microreact - # delete all previous microreact cluster job results + # delete all previous microreact cluster job results for this project redis.delete(f"beebop:hash:job:microreact:{p_hash}") job_microreact = q.enqueue(visualise.microreact, args=(p_hash, diff --git a/scripts/run_app b/scripts/run_app index 40bbe940..784b99bb 100755 --- a/scripts/run_app +++ b/scripts/run_app @@ -10,4 +10,4 @@ function cleanup() { trap cleanup INT trap cleanup ERR -STORAGE_LOCATION=./storage DBS_LOCATION=./storage/dbs FLASK_APP=beebop/app.py poetry run flask run +rq worker & STORAGE_LOCATION=./storage DBS_LOCATION=./storage/dbs FLASK_APP=beebop/app.py poetry run flask run From 054bc41433c689e90271522cb2fadaaf24be9f83 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 27 Nov 2024 14:50:47 +0000 Subject: [PATCH 02/35] refactor: replace Dockerfile with separate production and development Dockerfiles, update build and push scripts --- docker/Dockerfile.dev | 44 ++++++++++++++++++++++++++ docker/{Dockerfile => Dockerfile.prod} | 0 docker/build | 10 +++++- docker/common | 8 ++++- docker/push | 3 ++ 5 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 docker/Dockerfile.dev rename docker/{Dockerfile => Dockerfile.prod} (100%) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 00000000..07f692eb --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,44 @@ +FROM continuumio/miniconda3 + +# Switch to root to handle system installations +USER root + +# Configure conda channels +RUN conda config --append channels conda-forge && \ + conda config --append channels bioconda + +# System dependencies - combined to reduce layers +RUN apt-get update && \ + apt-get install -y \ + libeigen3-dev \ + libegl1 \ + libopenblas-dev \ + libxcb.*-dev \ + libx11-xcb-dev \ + libglu1-mesa-dev \ + libxrender-dev \ + libxi-dev \ + libxkbcommon-dev \ + libxkbcommon-x11-dev && \ + rm -rf /var/lib/apt/lists/* + +# Install conda and pip packages. update poppunk desired version +RUN conda install python=3.10 && \ + conda install graph-tool && \ + conda install mandrake && \ + conda install rapidnj && \ + conda install -c bioconda pp-sketchlib=2.0.0 && \ + pip install joblib==1.1.0 && \ + pip3 install git+https://github.com/bacpop/PopPUNK@v2.7.1#egg=PopPUNK + +# Poetry setup +RUN pip install poetry +COPY *.toml *.lock / +RUN poetry config virtualenvs.create false && \ + poetry install + +COPY . /beebop +WORKDIR /beebop +EXPOSE 5000 + +CMD poetry run waitress-serve --port=5000 'beebop.app:app' \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile.prod similarity index 100% rename from docker/Dockerfile rename to docker/Dockerfile.prod diff --git a/docker/build b/docker/build index d91bf1b6..b52cd8db 100755 --- a/docker/build +++ b/docker/build @@ -4,11 +4,19 @@ set -ex HERE=$(dirname $0) . $HERE/common +# Build the production image docker build --pull \ --tag $TAG_SHA \ - -f docker/Dockerfile \ + -f docker/Dockerfile.prod \ + $PACKAGE_ROOT + +# Build the development image +docker build --pull \ + --tag $TAG_DEV_SHA \ + -f docker/Dockerfile.dev \ $PACKAGE_ROOT # We always push the SHA tagged versions, for debugging if the tests # after this step fail docker push $TAG_SHA +docker push $TAG_DEV_SHA diff --git a/docker/common b/docker/common index ad2f9552..0d06e286 100644 --- a/docker/common +++ b/docker/common @@ -4,6 +4,7 @@ set -e PACKAGE_ROOT=$(realpath $HERE/..) PACKAGE_NAME=beebop-py PACKAGE_ORG=mrcide +PACKAGE_DEV=dev # Buildkite doesn't check out a full history from the remote (just the # single commit) so you end up with a detached head and git rev-parse @@ -20,6 +21,11 @@ else GIT_BRANCH=$(git -C "$PACKAGE_ROOT" symbolic-ref --short HEAD) fi +# production images TAG_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}" TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}" -TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest" \ No newline at end of file +TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest" + +# development images +TAG_DEV_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}:${PACKAGE_DEV}" +TAG_DEV_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}:${PACKAGE_DEV}" \ No newline at end of file diff --git a/docker/push b/docker/push index 3fecfecb..b901dafd 100755 --- a/docker/push +++ b/docker/push @@ -10,6 +10,9 @@ HERE=$(dirname $0) docker tag $TAG_SHA $TAG_BRANCH docker push $TAG_BRANCH +docker tag $TAG_DEV_SHA $TAG_DEV_BRANCH +docker push $TAG_DEV_BRANCH + if [ $GIT_BRANCH == "main" ]; then docker tag $TAG_SHA $TAG_LATEST docker push $TAG_LATEST From e9acf2d7daacf48f3a93aeecb29ab9d6963f44e1 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 27 Nov 2024 14:56:35 +0000 Subject: [PATCH 03/35] fix: update development image tags to include SHA for better versioning --- docker/common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/common b/docker/common index 0d06e286..fcf7ae1f 100644 --- a/docker/common +++ b/docker/common @@ -27,5 +27,5 @@ TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}" TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest" # development images -TAG_DEV_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}:${PACKAGE_DEV}" -TAG_DEV_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}:${PACKAGE_DEV}" \ No newline at end of file +TAG_DEV_SHA="${TAG_SHA}-${PACKAGE_DEV}" +TAG_DEV_BRANCH="${TAG_BRANCH}-${PACKAGE_DEV}" \ No newline at end of file From f5b8ab62536d947d49c670d9c7a9a04e64d9cf09 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 27 Nov 2024 15:12:21 +0000 Subject: [PATCH 04/35] fix: streamline Dockerfile.dev by consolidating package installations and removing commented-out code --- docker/Dockerfile.dev | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 07f692eb..fdcbfa77 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -8,28 +8,29 @@ RUN conda config --append channels conda-forge && \ conda config --append channels bioconda # System dependencies - combined to reduce layers -RUN apt-get update && \ - apt-get install -y \ - libeigen3-dev \ - libegl1 \ - libopenblas-dev \ - libxcb.*-dev \ - libx11-xcb-dev \ - libglu1-mesa-dev \ - libxrender-dev \ - libxi-dev \ - libxkbcommon-dev \ - libxkbcommon-x11-dev && \ - rm -rf /var/lib/apt/lists/* +# RUN apt-get update && \ +# apt-get install -y \ +# libeigen3-dev \ +# libegl1 \ +# libopenblas-dev \ +# libxcb.*-dev \ +# libx11-xcb-dev \ +# libglu1-mesa-dev \ +# libxrender-dev \ +# libxi-dev \ +# libxkbcommon-dev \ +# libxkbcommon-x11-dev && \ +# rm -rf /var/lib/apt/lists/* +RUN conda install graph-tool && \ +conda install mandrake && \ +conda install rapidnj && \ +conda install -c bioconda pp-sketchlib=2.0.0 && \ +pip install joblib==1.1.0 + + # Install conda and pip packages. update poppunk desired version -RUN conda install python=3.10 && \ - conda install graph-tool && \ - conda install mandrake && \ - conda install rapidnj && \ - conda install -c bioconda pp-sketchlib=2.0.0 && \ - pip install joblib==1.1.0 && \ - pip3 install git+https://github.com/bacpop/PopPUNK@v2.7.1#egg=PopPUNK +RUN pip3 install git+https://github.com/bacpop/PopPUNK@v2.7.1#egg=PopPUNK # Poetry setup RUN pip install poetry From 996b8bba92a4a4d57846e57b42615bdd56715d7f Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 27 Nov 2024 15:39:07 +0000 Subject: [PATCH 05/35] fix: update Dockerfile.dev to install Python 3.10 --- docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index fdcbfa77..6c05cdb3 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -21,7 +21,7 @@ RUN conda config --append channels conda-forge && \ # libxkbcommon-dev \ # libxkbcommon-x11-dev && \ # rm -rf /var/lib/apt/lists/* - +RUN RUN conda install python=3.10 RUN conda install graph-tool && \ conda install mandrake && \ conda install rapidnj && \ From ae8e61df5edeca3ed3ddc317262bdc228e4942c5 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 27 Nov 2024 15:51:43 +0000 Subject: [PATCH 06/35] fix: correct Dockerfile.dev syntax by removing redundant RUN command --- docker/Dockerfile.dev | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 6c05cdb3..9fc90441 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -21,14 +21,14 @@ RUN conda config --append channels conda-forge && \ # libxkbcommon-dev \ # libxkbcommon-x11-dev && \ # rm -rf /var/lib/apt/lists/* -RUN RUN conda install python=3.10 +RUN conda install python=3.10 RUN conda install graph-tool && \ -conda install mandrake && \ -conda install rapidnj && \ -conda install -c bioconda pp-sketchlib=2.0.0 && \ -pip install joblib==1.1.0 - - + conda install mandrake && \ + conda install rapidnj && \ + conda install -c bioconda pp-sketchlib=2.0.0 && \ + pip install joblib==1.1.0 + + # Install conda and pip packages. update poppunk desired version RUN pip3 install git+https://github.com/bacpop/PopPUNK@v2.7.1#egg=PopPUNK From 180e9077efd0d17049d966c426cbb689c2c4616a Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 27 Nov 2024 16:17:27 +0000 Subject: [PATCH 07/35] fix: separate package installations in Dockerfile.dev for clarity and maintainability --- docker/Dockerfile.dev | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 9fc90441..2b460cfa 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -22,11 +22,11 @@ RUN conda config --append channels conda-forge && \ # libxkbcommon-x11-dev && \ # rm -rf /var/lib/apt/lists/* RUN conda install python=3.10 -RUN conda install graph-tool && \ - conda install mandrake && \ - conda install rapidnj && \ - conda install -c bioconda pp-sketchlib=2.0.0 && \ - pip install joblib==1.1.0 +RUN conda install graph-tool +RUN conda install mandrake +RUN conda install rapidnj +RUN conda install -c bioconda pp-sketchlib=2.0.0 +RUN pip install joblib==1.1.0 # Install conda and pip packages. update poppunk desired version From a96326971381b05ce277ff85a1aca1c7f37ef298 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 27 Nov 2024 16:40:50 +0000 Subject: [PATCH 08/35] fix: update mandrake installation in Dockerfile.dev to use classic solver --- docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 2b460cfa..5a6a355a 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -23,7 +23,7 @@ RUN conda config --append channels conda-forge && \ # rm -rf /var/lib/apt/lists/* RUN conda install python=3.10 RUN conda install graph-tool -RUN conda install mandrake +RUN conda install --solver=classic mandrake RUN conda install rapidnj RUN conda install -c bioconda pp-sketchlib=2.0.0 RUN pip install joblib==1.1.0 From 5d05f6c9ea62e7094bbdbefc6340559e62779b86 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 27 Nov 2024 16:46:04 +0000 Subject: [PATCH 09/35] fix: correct typo in Dockerfile.dev for rapidnj installation solver --- docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 5a6a355a..abc8159d 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -24,7 +24,7 @@ RUN conda config --append channels conda-forge && \ RUN conda install python=3.10 RUN conda install graph-tool RUN conda install --solver=classic mandrake -RUN conda install rapidnj +RUN conda install --solver=clasic rapidnj RUN conda install -c bioconda pp-sketchlib=2.0.0 RUN pip install joblib==1.1.0 From a2aa01ce54c3f8e3b985f27aa7c65d51bf1b7369 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 29 Nov 2024 10:08:07 +0000 Subject: [PATCH 10/35] fet workimg verison --- docker/Dockerfile.dev | 67 ++++++++++++++++++++++++++----------------- docker/build | 12 ++++---- 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index abc8159d..5e4ab37f 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,36 +1,51 @@ FROM continuumio/miniconda3 +RUN conda create -n beebop-env python=3.10 +SHELL ["conda", "run", "-n", "beebop-env", "/bin/bash", "-c"] +# RUN echo "source activate beebop-env" > ~/.bashrc +# ENV PATH /opt/conda/envs/beebop-env/bin:$PATH + # Switch to root to handle system installations USER root -# Configure conda channels +# System dependencies +RUN apt-get update && \ + apt-get install -y \ + build-essential \ + g++ \ + cmake \ + libeigen3-dev \ + libegl1 \ + libopenblas-dev \ + libxcb.*-dev \ + libx11-xcb-dev \ + libglu1-mesa-dev \ + libxrender-dev \ + libxi-dev \ + libxkbcommon-dev \ + libxkbcommon-x11-dev && \ + rm -rf /var/lib/apt/lists/* + +# RUN conda install python=3.10 +RUN pip install --upgrade pip +RUN pip install --upgrade setuptools +# RUN pip install cmake RUN conda config --append channels conda-forge && \ conda config --append channels bioconda -# System dependencies - combined to reduce layers -# RUN apt-get update && \ -# apt-get install -y \ -# libeigen3-dev \ -# libegl1 \ -# libopenblas-dev \ -# libxcb.*-dev \ -# libx11-xcb-dev \ -# libglu1-mesa-dev \ -# libxrender-dev \ -# libxi-dev \ -# libxkbcommon-dev \ -# libxkbcommon-x11-dev && \ -# rm -rf /var/lib/apt/lists/* -RUN conda install python=3.10 -RUN conda install graph-tool -RUN conda install --solver=classic mandrake -RUN conda install --solver=clasic rapidnj -RUN conda install -c bioconda pp-sketchlib=2.0.0 -RUN pip install joblib==1.1.0 - - -# Install conda and pip packages. update poppunk desired version -RUN pip3 install git+https://github.com/bacpop/PopPUNK@v2.7.1#egg=PopPUNK +RUN conda install -n base conda-forge::mamba +RUN conda config --set channel_priority flexible + +RUN mamba install conda-forge:graph-tool +RUN mamba install conda-forge:mandrake +# RUN mamba install rapidnj +# RUN mamba install bioconda:pp-sketchlib +RUN mamba install requests pandas networkx pp-sketchlib scikit-learn hdbscan biopython tqdm treeswift rapidnj +# RUN pip install joblib==1.1.0 + +# Update poppunk desired version +# RUN mamba install cmake +RUN pip install git+https://github.com/bacpop/PopPUNK@v2.6.7#egg=PopPUNK # Poetry setup RUN pip install poetry @@ -42,4 +57,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD poetry run waitress-serve --port=5000 'beebop.app:app' \ No newline at end of file +CMD /bin/bash -c "conda run --no-capture -n beebop-env rq worker > rq_output.txt 2>&1 & conda run --no-capture -n beebop-env poetry run waitress-serve --port=5000 'beebop.app:app'" \ No newline at end of file diff --git a/docker/build b/docker/build index b52cd8db..46523707 100755 --- a/docker/build +++ b/docker/build @@ -5,10 +5,10 @@ HERE=$(dirname $0) . $HERE/common # Build the production image -docker build --pull \ - --tag $TAG_SHA \ - -f docker/Dockerfile.prod \ - $PACKAGE_ROOT +# docker build --pull \ +# --tag $TAG_SHA \ +# -f docker/Dockerfile.prod \ +# $PACKAGE_ROOT # Build the development image docker build --pull \ @@ -18,5 +18,5 @@ docker build --pull \ # We always push the SHA tagged versions, for debugging if the tests # after this step fail -docker push $TAG_SHA -docker push $TAG_DEV_SHA +# docker push $TAG_SHA +# docker push $TAG_DEV_SHA From f4290685a7e81068de57ef065f506e0eb43f468b Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 29 Nov 2024 10:13:30 +0000 Subject: [PATCH 11/35] fix: uncomment Dockerfile.dev commands for improved installation process and streamline build script --- docker/Dockerfile.dev | 7 +------ docker/build | 12 ++++++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 5e4ab37f..c810b57a 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -26,7 +26,6 @@ RUN apt-get update && \ libxkbcommon-x11-dev && \ rm -rf /var/lib/apt/lists/* -# RUN conda install python=3.10 RUN pip install --upgrade pip RUN pip install --upgrade setuptools # RUN pip install cmake @@ -38,13 +37,9 @@ RUN conda config --set channel_priority flexible RUN mamba install conda-forge:graph-tool RUN mamba install conda-forge:mandrake -# RUN mamba install rapidnj -# RUN mamba install bioconda:pp-sketchlib RUN mamba install requests pandas networkx pp-sketchlib scikit-learn hdbscan biopython tqdm treeswift rapidnj -# RUN pip install joblib==1.1.0 # Update poppunk desired version -# RUN mamba install cmake RUN pip install git+https://github.com/bacpop/PopPUNK@v2.6.7#egg=PopPUNK # Poetry setup @@ -57,4 +52,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD /bin/bash -c "conda run --no-capture -n beebop-env rq worker > rq_output.txt 2>&1 & conda run --no-capture -n beebop-env poetry run waitress-serve --port=5000 'beebop.app:app'" \ No newline at end of file +CMD conda run --no-capture -n beebop-env rq worker > rq_output.txt 2>&1 & conda run --no-capture -n beebop-env poetry run waitress-serve --port=5000 'beebop.app:app' \ No newline at end of file diff --git a/docker/build b/docker/build index 46523707..b52cd8db 100755 --- a/docker/build +++ b/docker/build @@ -5,10 +5,10 @@ HERE=$(dirname $0) . $HERE/common # Build the production image -# docker build --pull \ -# --tag $TAG_SHA \ -# -f docker/Dockerfile.prod \ -# $PACKAGE_ROOT +docker build --pull \ + --tag $TAG_SHA \ + -f docker/Dockerfile.prod \ + $PACKAGE_ROOT # Build the development image docker build --pull \ @@ -18,5 +18,5 @@ docker build --pull \ # We always push the SHA tagged versions, for debugging if the tests # after this step fail -# docker push $TAG_SHA -# docker push $TAG_DEV_SHA +docker push $TAG_SHA +docker push $TAG_DEV_SHA From 35ed6782d9c6a1c3dbf9947c758081cc8260a05a Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 29 Nov 2024 10:37:19 +0000 Subject: [PATCH 12/35] fix: update CMD in Dockerfile.dev to use array syntax for improved command execution --- docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index c810b57a..eb464b4d 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -52,4 +52,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD conda run --no-capture -n beebop-env rq worker > rq_output.txt 2>&1 & conda run --no-capture -n beebop-env poetry run waitress-serve --port=5000 'beebop.app:app' \ No newline at end of file +CMD ["bash", "-c", "conda run --no-capture-output -n beebop-env rq worker > rq_output.txt 2>&1 & conda run --no-capture-output -n beebop-env poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file From 34e158fd0e4e6552d7067d1daa4736fc814de033 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 29 Nov 2024 10:58:45 +0000 Subject: [PATCH 13/35] fix: update Dockerfile.dev to use base environment and streamline command execution --- docker/Dockerfile.dev | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index eb464b4d..d6f1c9a9 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,9 +1,7 @@ FROM continuumio/miniconda3 -RUN conda create -n beebop-env python=3.10 -SHELL ["conda", "run", "-n", "beebop-env", "/bin/bash", "-c"] -# RUN echo "source activate beebop-env" > ~/.bashrc -# ENV PATH /opt/conda/envs/beebop-env/bin:$PATH +SHELL ["conda", "run", "-n", "base", "/bin/bash", "-c"] +RUN conda install python=3.10 # Switch to root to handle system installations USER root @@ -28,7 +26,6 @@ RUN apt-get update && \ RUN pip install --upgrade pip RUN pip install --upgrade setuptools -# RUN pip install cmake RUN conda config --append channels conda-forge && \ conda config --append channels bioconda @@ -52,4 +49,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD ["bash", "-c", "conda run --no-capture-output -n beebop-env rq worker > rq_output.txt 2>&1 & conda run --no-capture-output -n beebop-env poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file +CMD ["bash", "-c", "conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file From a1ac4e7a2173ba4f24d9c3da7d0c9368deb861c5 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 29 Nov 2024 12:31:27 +0000 Subject: [PATCH 14/35] docs: update README.md with deployment instructions and local development guidelines; refactor Dockerfile.dev for improved package management and environment setup --- README.md | 16 ++++++++++++ docker/Dockerfile.dev | 58 +++++++++++++++++++++++-------------------- docker/push | 3 ++- 3 files changed, 49 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ed3076ec..c7ce1031 100644 --- a/README.md +++ b/README.md @@ -101,3 +101,19 @@ Testing can be done in a second terminal (make sure to activate 'beebop_py') by ``` TESTING=True poetry run pytest ``` + +### Use/Deploy specific version of PopPUNK + +To use deploy a specific version/commit/branch of PopPUNK, you can update `RUN pip install git+https://github.com/bacpop/PopPUNK@v2.6.7#egg=PopPUNK ` +in `DockerFile.dev` with the desired version/commit/branch. + +The new images built with `/docker/build` will have a *-dev* postfix. + +### Local Development + +You can build the image with `/docker/build`, this new image can now be used by Beebop. + +### Deployment + +A pull request can be created so buildkite pushes the images to the docker hub. +Then on the `beebop-deploy` the api image can be updated with the new dev image. diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index d6f1c9a9..e5445f68 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,42 +1,46 @@ FROM continuumio/miniconda3 +# Make RUN commands use the new environment: SHELL ["conda", "run", "-n", "base", "/bin/bash", "-c"] + +# Set up conda environment RUN conda install python=3.10 +RUN conda config --append channels conda-forge && \ + conda config --append channels bioconda + +# Install mamba: which is a faster package manager than conda +RUN conda install -c conda-forge mamba +RUN conda config --set channel_priority flexible -# Switch to root to handle system installations -USER root +# Install PopPUNK conda dependencies +RUN mamba install -y -c conda-forge -y graph-tool mandrake +RUN mamba install -y \ + # Core data packages + pandas \ + requests \ + networkx \ + scikit-learn \ + # Bioinformatics tools + pp-sketchlib \ + biopython \ + treeswift \ + rapidnj \ + # Analysis tools + hdbscan \ + # Progress tracking + tqdm \ + && mamba clean -afy # System dependencies RUN apt-get update && \ apt-get install -y \ build-essential \ - g++ \ cmake \ libeigen3-dev \ - libegl1 \ - libopenblas-dev \ - libxcb.*-dev \ - libx11-xcb-dev \ - libglu1-mesa-dev \ - libxrender-dev \ - libxi-dev \ - libxkbcommon-dev \ - libxkbcommon-x11-dev && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install --upgrade pip -RUN pip install --upgrade setuptools -RUN conda config --append channels conda-forge && \ - conda config --append channels bioconda - -RUN conda install -n base conda-forge::mamba -RUN conda config --set channel_priority flexible - -RUN mamba install conda-forge:graph-tool -RUN mamba install conda-forge:mandrake -RUN mamba install requests pandas networkx pp-sketchlib scikit-learn hdbscan biopython tqdm treeswift rapidnj + libhdf5-dev \ + libopenblas-dev -# Update poppunk desired version +# Update PopPUNK to desired version RUN pip install git+https://github.com/bacpop/PopPUNK@v2.6.7#egg=PopPUNK # Poetry setup @@ -49,4 +53,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD ["bash", "-c", "conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file +CMD ["bash", "-c", "mamba run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file diff --git a/docker/push b/docker/push index b901dafd..b95fb370 100755 --- a/docker/push +++ b/docker/push @@ -7,9 +7,10 @@ HERE=$(dirname $0) # In case we switch agents between steps [ ! -z $(docker images -q $TAG_SHA) ] || docker pull $TAG_SHA +# production image docker tag $TAG_SHA $TAG_BRANCH docker push $TAG_BRANCH - +# development image docker tag $TAG_DEV_SHA $TAG_DEV_BRANCH docker push $TAG_DEV_BRANCH From b6e8555460ab4a7e9ac0f638b899bbb7a52db621 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Mon, 2 Dec 2024 19:55:31 +0000 Subject: [PATCH 15/35] fix lint --- beebop/app.py | 25 +++++++++++------- beebop/assignClusters.py | 56 ++++++++++++++++++++++++++++------------ beebop/visualise.py | 3 ++- tests/test_unit.py | 43 ++++++++++++++++++------------ 4 files changed, 83 insertions(+), 44 deletions(-) diff --git a/beebop/app.py b/beebop/app.py index 0c359610..7200e43f 100644 --- a/beebop/app.py +++ b/beebop/app.py @@ -305,16 +305,21 @@ def run_poppunk_internal(sketches: dict, # microreact # delete all previous microreact cluster job results redis.delete(f"beebop:hash:job:microreact:{p_hash}") - job_microreact = q.enqueue(visualise.microreact, - args=(p_hash, - fs, - full_db_fs, - args, - name_mapping, - species, - redis_host, - queue_kwargs), - depends_on=Dependency([job_assign, job_network], allow_failure=True), **queue_kwargs) + job_microreact = q.enqueue( + visualise.microreact, + args=( + p_hash, + fs, + full_db_fs, + args, + name_mapping, + species, + redis_host, + queue_kwargs, + ), + depends_on=Dependency([job_assign, job_network], allow_failure=True), + **queue_kwargs, + ) redis.hset("beebop:hash:job:microreact", p_hash, job_microreact.id) return jsonify( response_success( diff --git a/beebop/assignClusters.py b/beebop/assignClusters.py index fc779655..7c6d289d 100644 --- a/beebop/assignClusters.py +++ b/beebop/assignClusters.py @@ -196,8 +196,10 @@ def handle_external_clusters( config.external_clusters_prefix, ) if not_found_query_names: - queries_names, queries_clusters, not_found_query_clusters = filter_queries( - queries_names, queries_clusters, not_found_query_names + queries_names, queries_clusters, not_found_query_clusters = ( + filter_queries( + queries_names, queries_clusters, not_found_query_names + ) ) output_full_tmp = config.fs.output_tmp(config.p_hash) not_found_query_names_new, not_found_query_clusters_new = ( @@ -241,15 +243,19 @@ def handle_not_found_queries( """ [Handles queries that were not found in the initial external clusters file. - This function processes the sketches of the queries that were not found for external clusters from the reference db, - assigns clusters to them from the full db, and then summarizes the clusters. It also + This function processes the sketches of the queries that were not found + for external clusters from the reference db, + assigns clusters to them from the full db, and then + summarizes the clusters. It also handles all file manipulations needed] :param config: [ClusteringConfig with all necessary information] - :param sketches_dict: [dictionary with filehash (key) and sketch (value)] + :param sketches_dict: [dictionary + with filehash (key) and sketch (value)] :param not_found_query_names: [list of sample hashes that were not found] :param output_full_tmp: [path to temporary output directory] - :param not_found_query_clusters: [set of clusters assigned to initial not found samples] + :param not_found_query_clusters: [set of clusters assigned to + initial not found samples] :return tuple[list, list]: [list initial not found sample hashes, list of clusters assigned to initial not found samples] """ @@ -280,6 +286,7 @@ def handle_not_found_queries( return query_names, query_clusters + def handle_files_manipulation( config: ClusteringConfig, output_full_tmp: str, @@ -288,13 +295,16 @@ def handle_files_manipulation( """ [Handles file manipulations for queries that were not found in the initial external clusters file. - This function copies include files from the full assign output directory - to the output directory, deletes include files for queries that were not found, + This function copies include files from the + full assign output directory + to the output directory, deletes include files for queries + that were not found, and merges the partial query graph files.] - + :param config: [ClusteringConfig with all necessary information] :param output_full_tmp: [path to temporary output directory] - :param not_found_query_clusters: [set of clusters assigned to initial not found samples] + :param not_found_query_clusters: [set of clusters assigned + to initial not found samples] """ delete_include_files( config.fs, @@ -302,7 +312,11 @@ def handle_files_manipulation( not_found_query_clusters, ) copy_include_files(output_full_tmp, config.out_dir) - merge_txt_files(config.fs.partial_query_graph(config.p_hash), config.fs.partial_query_graph_tmp(config.p_hash)) + merge_txt_files( + config.fs.partial_query_graph(config.p_hash), + config.fs.partial_query_graph_tmp(config.p_hash), + ) + def update_external_clusters( config: ClusteringConfig, @@ -316,8 +330,10 @@ def update_external_clusters( using the full database. This function reads the external clusters from the new previous query clustering file - and updates the initial external clusters file on ref db with the clusters for samples - that were initially not found, and have now been assigned by the current query with the full database.] + and updates the initial external clusters + file on ref db with the clusters for samples + that were initially not found, and have now been + assigned by the current query with the full database.] :param config: [ClusteringConfig with all necessary information] @@ -399,10 +415,12 @@ def filter_queries( :param queries_names: [list of sample hashes] :param queries_clusters: [list of sample PopPUNK clusters] - :param not_found: [list of sample hashes that were not found] + :param not_found: [list of sample hashes + that were not found] :param config: [ClusteringConfig with all necessary information] :return tuple[list[str], list[str], set[str]]: [filtered sample hashes, - filtered sample PopPUNK clusters, set of clusters assigned to not found samples] + filtered sample PopPUNK clusters, + set of clusters assigned to not found samples] """ filtered_names = [name for name in queries_names if name not in not_found] filtered_clusters = [ @@ -410,8 +428,12 @@ def filter_queries( for name, cluster in zip(queries_names, queries_clusters) if name not in not_found ] - - return filtered_names, filtered_clusters, set(queries_clusters) - set(filtered_clusters) + + return ( + filtered_names, + filtered_clusters, + set(queries_clusters) - set(filtered_clusters), + ) def delete_include_files( diff --git a/beebop/visualise.py b/beebop/visualise.py index e22ef4b5..b2601650 100644 --- a/beebop/visualise.py +++ b/beebop/visualise.py @@ -39,7 +39,8 @@ def microreact( redis = Redis(host=redis_host) # get results from previous job current_job = get_current_job(redis) - assign_result = current_job.dependency.result # gets first dependency result (i.e assign_clusters) + # gets first dependency result (i.e assign_clusters) + assign_result = current_job.dependency.result external_to_poppunk_clusters = None try: diff --git a/tests/test_unit.py b/tests/test_unit.py index 68778e9f..6b03b58f 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -1130,9 +1130,7 @@ def test_handle_external_clusters_with_not_found(mocker, config): ) # not found function calls - mock_filter_queries.assert_called_once_with( - q_names, q_clusters, not_found - ) + mock_filter_queries.assert_called_once_with(q_names, q_clusters, not_found) mock_handle_not_found.assert_called_once_with( config, {}, not_found, tmp_output, not_found_q_clusters ) @@ -1184,6 +1182,7 @@ def test_handle_not_found_queries( assert query_names == ["hash1"] assert query_clusters == [10] + @patch("beebop.assignClusters.merge_txt_files") @patch("beebop.assignClusters.copy_include_files") @patch("beebop.assignClusters.delete_include_files") @@ -1192,12 +1191,18 @@ def test_handle_files_manipulation(mock_delete, mock_copy, mock_merge, config): not_found_query_clusters = {1234, 6969} config.fs.partial_query_graph.return_value = "partial_query_graph" config.fs.partial_query_graph_tmp.return_value = "partial_query_graph_tmp" - - assignClusters.handle_files_manipulation(config, outdir_tmp, not_found_query_clusters) - mock_delete.assert_called_once_with(config.fs, config.p_hash, not_found_query_clusters) + assignClusters.handle_files_manipulation( + config, outdir_tmp, not_found_query_clusters + ) + + mock_delete.assert_called_once_with( + config.fs, config.p_hash, not_found_query_clusters + ) mock_copy.assert_called_once_with(outdir_tmp, config.out_dir) - mock_merge.assert_called_once_with("partial_query_graph", "partial_query_graph_tmp") + mock_merge.assert_called_once_with( + "partial_query_graph", "partial_query_graph_tmp" + ) @patch("beebop.assignClusters.update_external_clusters_csv") @@ -1284,38 +1289,44 @@ def test_copy_include_files_no_conflict(tmp_path): assert (output_full_tmp / f).exists() # Still in original location assert not (outdir / f).exists() # Not in new location + def test_copy_include_file_conflict(tmp_path): output_full_tmp = tmp_path / "output_full_tmp" outdir = tmp_path / "outdir" output_full_tmp.mkdir() outdir.mkdir() - - include_files_tmp = ["include_1.txt",] + + include_files_tmp = [ + "include_1.txt", + ] include_files = ["include_1.txt"] - + # Create include files (output_full_tmp / include_files_tmp[0]).write_text("new content") (outdir / include_files[0]).write_text("original content") - + assignClusters.copy_include_files(str(output_full_tmp), str(outdir)) - - assert not (output_full_tmp / include_files_tmp[0]).exists() # Original removed + + assert not ( + output_full_tmp / include_files_tmp[0] + ).exists() # Original removed included_file_content = (outdir / include_files[0]).read_text() assert "new content" in included_file_content # New content assert "original content" in included_file_content # Original content + def test_filter_queries(): q_names = ["sample1", "sample2", "sample3"] q_clusters = [1, 2, 3] not_found = ["sample2"] - filtered_names, filtered_clusters, not_found_q_clusters = assignClusters.filter_queries( - q_names, q_clusters, not_found + filtered_names, filtered_clusters, not_found_q_clusters = ( + assignClusters.filter_queries(q_names, q_clusters, not_found) ) assert filtered_names == ["sample1", "sample3"] assert filtered_clusters == [1, 3] - assert not_found_q_clusters + assert not_found_q_clusters def test_delete_include_files(tmp_path): From 1214b4156a84dd7f5ac261fa47c7d50d9d0af071 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Mon, 2 Dec 2024 20:09:59 +0000 Subject: [PATCH 16/35] Add miscellaneous section to README with analysis process diagram --- README.md | 5 + diagrams/Beebop-PopPUNK-run.drawio | 521 +++++++++++++++++++++++++++++ 2 files changed, 526 insertions(+) create mode 100644 diagrams/Beebop-PopPUNK-run.drawio diff --git a/README.md b/README.md index d48211cd..865a8b6f 100644 --- a/README.md +++ b/README.md @@ -107,3 +107,8 @@ Testing can be done in a second terminal (make sure to activate 'beebop_py') by ``` TESTING=True poetry run pytest ``` + +### Miscellaneous + +- There is a .drawio graph in graphs folder illustrating the proccess of running a analysis. This includes +all the files created anf how they are used in each job diff --git a/diagrams/Beebop-PopPUNK-run.drawio b/diagrams/Beebop-PopPUNK-run.drawio new file mode 100644 index 00000000..ca3073a5 --- /dev/null +++ b/diagrams/Beebop-PopPUNK-run.drawio @@ -0,0 +1,521 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From e0203a72a0b07c080d7cd6d540c4f0542b467dfc Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Mon, 2 Dec 2024 20:41:43 +0000 Subject: [PATCH 17/35] docs: update README.md and build scripts to include --with-dev flag for specific version deployments --- README.md | 9 ++++----- buildkite/pipeline.yml | 5 +++-- docker/build | 20 +++++++++++--------- docker/push | 10 ++++++---- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c7ce1031..605cfd33 100644 --- a/README.md +++ b/README.md @@ -104,16 +104,15 @@ TESTING=True poetry run pytest ### Use/Deploy specific version of PopPUNK -To use deploy a specific version/commit/branch of PopPUNK, you can update `RUN pip install git+https://github.com/bacpop/PopPUNK@v2.6.7#egg=PopPUNK ` -in `DockerFile.dev` with the desired version/commit/branch. +To use a specific version, commit or branch of PopPUNK in a beebop_py deployment, you can update the line which installs PopPUNK in `DockerFile.dev to`RUN pip install git+https://github.com/bacpop/PopPUNK@[VERSION]#egg=PopPUNK `, replacing `[VERSION]` with the desired version/commit/branch. -The new images built with `/docker/build` will have a *-dev* postfix. +The new dev images built with `/docker/build --with-dev` will have a *-dev* postfix. ### Local Development -You can build the image with `/docker/build`, this new image can now be used by Beebop. +You can build the image with `/docker/build --with-dev`, this new image can now be used by Beebop. ### Deployment -A pull request can be created so buildkite pushes the images to the docker hub. +A pull request can be created so buildkite pushes the images to the docker hub. Add `--with-dev` to the build & push commands `pipeline.yaml`. Then on the `beebop-deploy` the api image can be updated with the new dev image. diff --git a/buildkite/pipeline.yml b/buildkite/pipeline.yml index 34b4fa6a..73082301 100644 --- a/buildkite/pipeline.yml +++ b/buildkite/pipeline.yml @@ -1,8 +1,9 @@ +# Add --with-dev to the build dev image for a specific version of PopPunk steps: - label: ":whale::python: Build" - command: docker/build + command: docker/build --with-dev - wait - label: ":shipit: Push images" - command: docker/push \ No newline at end of file + command: docker/push --with-dev \ No newline at end of file diff --git a/docker/build b/docker/build index b52cd8db..010750f2 100755 --- a/docker/build +++ b/docker/build @@ -4,19 +4,21 @@ set -ex HERE=$(dirname $0) . $HERE/common -# Build the production image + + docker build --pull \ --tag $TAG_SHA \ -f docker/Dockerfile.prod \ $PACKAGE_ROOT - -# Build the development image -docker build --pull \ - --tag $TAG_DEV_SHA \ - -f docker/Dockerfile.dev \ - $PACKAGE_ROOT - # We always push the SHA tagged versions, for debugging if the tests # after this step fail docker push $TAG_SHA -docker push $TAG_DEV_SHA + +# Build and push dev image only if --with-dev flag is set +if [ "$1" == "--with-dev" ]; then + docker build --pull \ + --tag $TAG_DEV_SHA \ + -f docker/Dockerfile.dev \ + $PACKAGE_ROOT + docker push $TAG_DEV_SHA +fi diff --git a/docker/push b/docker/push index b95fb370..9c7b5c7e 100755 --- a/docker/push +++ b/docker/push @@ -7,12 +7,14 @@ HERE=$(dirname $0) # In case we switch agents between steps [ ! -z $(docker images -q $TAG_SHA) ] || docker pull $TAG_SHA -# production image docker tag $TAG_SHA $TAG_BRANCH docker push $TAG_BRANCH -# development image -docker tag $TAG_DEV_SHA $TAG_DEV_BRANCH -docker push $TAG_DEV_BRANCH + +if [ "$1" == "--with-dev" ]; then + [ ! -z $(docker images -q $TAG_DEV_SHA) ] || docker pull $TAG_DEV_SHA + docker tag $TAG_DEV_SHA $TAG_DEV_BRANCH + docker push $TAG_DEV_BRANCH +fi if [ $GIT_BRANCH == "main" ]; then docker tag $TAG_SHA $TAG_LATEST From 912047a461759bdb36bba5613b33e235a5445200 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Mon, 2 Dec 2024 20:42:14 +0000 Subject: [PATCH 18/35] docs: add note to README.md to remove --with-dev flag before merging PR --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 605cfd33..f827fca9 100644 --- a/README.md +++ b/README.md @@ -115,4 +115,5 @@ You can build the image with `/docker/build --with-dev`, this new image can now ### Deployment A pull request can be created so buildkite pushes the images to the docker hub. Add `--with-dev` to the build & push commands `pipeline.yaml`. +**Ensure to remove the `--with-dev` flag before merging the PR.** Then on the `beebop-deploy` the api image can be updated with the new dev image. From a2dc8dda95fad8c5804b9d85b15d87a7b9424f0c Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 09:34:41 +0000 Subject: [PATCH 19/35] fix: update CMD in Dockerfile.dev to use conda instead of mamba for command execution --- docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index e5445f68..b70375ba 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -53,4 +53,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD ["bash", "-c", "mamba run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file +CMD ["bash", "-c", "conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file From 55b5c50185665a372513f33afaad8cc97c6c83e6 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 10:18:48 +0000 Subject: [PATCH 20/35] fix: update README.md and Dockerfile.dev to use POPPUNK_VERSION variable for versioning --- README.md | 2 +- docker/Dockerfile.dev | 6 ++++-- docker/build | 4 ++-- docker/common | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f827fca9..129ea2d5 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ TESTING=True poetry run pytest ### Use/Deploy specific version of PopPUNK -To use a specific version, commit or branch of PopPUNK in a beebop_py deployment, you can update the line which installs PopPUNK in `DockerFile.dev to`RUN pip install git+https://github.com/bacpop/PopPUNK@[VERSION]#egg=PopPUNK `, replacing `[VERSION]` with the desired version/commit/branch. +To use a specific version, commit or branch of PopPUNK in a beebop_py deployment, you can update `POPPUNK_VERSION` in `common`. The new dev images built with `/docker/build --with-dev` will have a *-dev* postfix. diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index b70375ba..0bbd3906 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,5 +1,7 @@ FROM continuumio/miniconda3 +ARG POPPUNK_VERSION="v2.6.7" + # Make RUN commands use the new environment: SHELL ["conda", "run", "-n", "base", "/bin/bash", "-c"] @@ -40,8 +42,8 @@ RUN apt-get update && \ libhdf5-dev \ libopenblas-dev -# Update PopPUNK to desired version -RUN pip install git+https://github.com/bacpop/PopPUNK@v2.6.7#egg=PopPUNK + +RUN pip install git+https://github.com/bacpop/PopPUNK@${POPPUNK_VERSION}#egg=PopPUNK # Poetry setup RUN pip install poetry diff --git a/docker/build b/docker/build index 010750f2..a3bd7d11 100755 --- a/docker/build +++ b/docker/build @@ -4,8 +4,7 @@ set -ex HERE=$(dirname $0) . $HERE/common - - +# Build and push the production image docker build --pull \ --tag $TAG_SHA \ -f docker/Dockerfile.prod \ @@ -17,6 +16,7 @@ docker push $TAG_SHA # Build and push dev image only if --with-dev flag is set if [ "$1" == "--with-dev" ]; then docker build --pull \ + --build-arg POPPUNK_VERSION=$POPPUNK_VERSION \ --tag $TAG_DEV_SHA \ -f docker/Dockerfile.dev \ $PACKAGE_ROOT diff --git a/docker/common b/docker/common index fcf7ae1f..f83c51d1 100644 --- a/docker/common +++ b/docker/common @@ -21,11 +21,12 @@ else GIT_BRANCH=$(git -C "$PACKAGE_ROOT" symbolic-ref --short HEAD) fi -# production images +# production image TAG_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}" TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}" TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest" -# development images +# development image TAG_DEV_SHA="${TAG_SHA}-${PACKAGE_DEV}" -TAG_DEV_BRANCH="${TAG_BRANCH}-${PACKAGE_DEV}" \ No newline at end of file +TAG_DEV_BRANCH="${TAG_BRANCH}-${PACKAGE_DEV}" +POPPUNK_VERSION=v2.6.7 # can be version, branch or commit \ No newline at end of file From 9fb3c5bcc985773d670ebd2bb7453424a3b23e4b Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 11:03:14 +0000 Subject: [PATCH 21/35] fix: simplify CMD in Dockerfile.dev by removing unnecessary bash invocation --- docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 0bbd3906..b69fc660 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -55,4 +55,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD ["bash", "-c", "conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file +CMD ["conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file From bfc77ac0bd28ebbc4c62d0e471ef2264cd66108e Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 11:21:27 +0000 Subject: [PATCH 22/35] fix: update CMD in Dockerfile.dev to use bash for command execution --- docker/Dockerfile.dev | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index b69fc660..6a4ca34d 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -14,7 +14,7 @@ RUN conda config --append channels conda-forge && \ RUN conda install -c conda-forge mamba RUN conda config --set channel_priority flexible -# Install PopPUNK conda dependencies +#Install PopPUNK conda dependencies RUN mamba install -y -c conda-forge -y graph-tool mandrake RUN mamba install -y \ # Core data packages @@ -55,4 +55,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD ["conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file +CMD ["bash", "-c", "conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file From e22937c61f176ca0a567233a11d07ae9178f17b3 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 12:41:04 +0000 Subject: [PATCH 23/35] fix: update CMD in Dockerfile.dev to use absolute path for bash --- docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 6a4ca34d..3501d4d0 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -55,4 +55,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD ["bash", "-c", "conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file +CMD ["/bin/bash", "-c", "conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file From fbfebb4dffc83f4710d86fcccdffcb23c40eef9f Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 12:49:14 +0000 Subject: [PATCH 24/35] fix: update CMD in Dockerfile.dev to streamline conda command execution --- docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 3501d4d0..dd91f0e5 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -55,4 +55,4 @@ COPY . /beebop WORKDIR /beebop EXPOSE 5000 -CMD ["/bin/bash", "-c", "conda run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"] \ No newline at end of file +CMD ["conda", "run" ,"--no-capture-output", "-n", "base", "poetry", "run", "waitress-serve", "--port=5000", "beebop.app:app"] \ No newline at end of file From 5cf6820d7ce48cdf26807e4d8f0e840edaed6d45 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 13:54:42 +0000 Subject: [PATCH 25/35] try github container registry --- .github/workflows/docker.yml | 25 +++++++++++++++++++++++++ buildkite/pipeline.yml | 14 +++++++------- docker/common | 12 ++++++------ 3 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..6956d0f2 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,25 @@ +name: build and push docker image, +on: + push: + branches: + - main + pull_request: + branches: + - '*' +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Login to GHCR (GitHub Packages) + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build docker image + run: ./docker/build --with-dev + - name: Push SHA docker image + run: ./docker/push --with-dev diff --git a/buildkite/pipeline.yml b/buildkite/pipeline.yml index 73082301..f527e64c 100644 --- a/buildkite/pipeline.yml +++ b/buildkite/pipeline.yml @@ -1,9 +1,9 @@ -# Add --with-dev to the build dev image for a specific version of PopPunk -steps: - - label: ":whale::python: Build" - command: docker/build --with-dev +# # Add --with-dev to the build dev image for a specific version of PopPunk +# steps: +# - label: ":whale::python: Build" +# command: docker/build --with-dev - - wait +# - wait - - label: ":shipit: Push images" - command: docker/push --with-dev \ No newline at end of file +# - label: ":shipit: Push images" +# command: docker/push --with-dev \ No newline at end of file diff --git a/docker/common b/docker/common index f83c51d1..0c40bf01 100644 --- a/docker/common +++ b/docker/common @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -e - +REGISTRY=ghcr.io PACKAGE_ROOT=$(realpath $HERE/..) PACKAGE_NAME=beebop-py PACKAGE_ORG=mrcide @@ -22,11 +22,11 @@ else fi # production image -TAG_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}" -TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}" -TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest" +TAG_SHA="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}" +TAG_BRANCH="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}" +TAG_LATEST="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:latest" # development image -TAG_DEV_SHA="${TAG_SHA}-${PACKAGE_DEV}" -TAG_DEV_BRANCH="${TAG_BRANCH}-${PACKAGE_DEV}" +TAG_DEV_SHA="${REGISTRY}/${TAG_SHA}-${PACKAGE_DEV}" +TAG_DEV_BRANCH="${REGISTRY}/${TAG_BRANCH}-${PACKAGE_DEV}" POPPUNK_VERSION=v2.6.7 # can be version, branch or commit \ No newline at end of file From 0200a4b0139693da145b36dfbd77a655245d4fd5 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 13:56:53 +0000 Subject: [PATCH 26/35] fix: streamline GIT_SHA and GIT_BRANCH retrieval in docker/common --- docker/common | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/docker/common b/docker/common index 0c40bf01..e0ef4f38 100644 --- a/docker/common +++ b/docker/common @@ -6,19 +6,11 @@ PACKAGE_NAME=beebop-py PACKAGE_ORG=mrcide PACKAGE_DEV=dev -# Buildkite doesn't check out a full history from the remote (just the -# single commit) so you end up with a detached head and git rev-parse -# doesn't work -if [ "$BUILDKITE" = "true" ]; then - GIT_SHA=${BUILDKITE_COMMIT:0:7} +GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD) +if [[ -v "BRANCH_NAME" ]]; then + GIT_BRANCH=${BRANCH_NAME} else - GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD) -fi - -if [ "$BUILDKITE" = "true" ]; then - GIT_BRANCH=$BUILDKITE_BRANCH -else - GIT_BRANCH=$(git -C "$PACKAGE_ROOT" symbolic-ref --short HEAD) + GIT_BRANCH=$(git symbolic-ref --short HEAD) fi # production image From b69e2a1e820ef66c70b82b87fa3ec436d101a8bd Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 14:08:42 +0000 Subject: [PATCH 27/35] fix: update PACKAGE_ORG in docker/common to use bacpop --- docker/common | 2 +- docker/push | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/common b/docker/common index e0ef4f38..a985b07e 100644 --- a/docker/common +++ b/docker/common @@ -3,7 +3,7 @@ set -e REGISTRY=ghcr.io PACKAGE_ROOT=$(realpath $HERE/..) PACKAGE_NAME=beebop-py -PACKAGE_ORG=mrcide +PACKAGE_ORG=bacpop PACKAGE_DEV=dev GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD) diff --git a/docker/push b/docker/push index 9c7b5c7e..adc42edb 100755 --- a/docker/push +++ b/docker/push @@ -5,7 +5,6 @@ HERE=$(dirname $0) . $HERE/common # In case we switch agents between steps -[ ! -z $(docker images -q $TAG_SHA) ] || docker pull $TAG_SHA docker tag $TAG_SHA $TAG_BRANCH docker push $TAG_BRANCH From cc6234470f8fa620980bdfe79da8f4df508310d6 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 19:22:35 +0000 Subject: [PATCH 28/35] remove buildkte --- .github/workflows/{docker.yml => build_and_push.yml} | 6 +++--- README.md | 2 +- buildkite/pipeline.yml | 9 --------- 3 files changed, 4 insertions(+), 13 deletions(-) rename .github/workflows/{docker.yml => build_and_push.yml} (72%) delete mode 100644 buildkite/pipeline.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/build_and_push.yml similarity index 72% rename from .github/workflows/docker.yml rename to .github/workflows/build_and_push.yml index 6956d0f2..d8adfe7f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/build_and_push.yml @@ -1,4 +1,4 @@ -name: build and push docker image, +name: build and push docker image on: push: branches: @@ -20,6 +20,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build docker image - run: ./docker/build --with-dev + run: ./docker/build --with-dev # TODO: remove --with_dev before merging - name: Push SHA docker image - run: ./docker/push --with-dev + run: ./docker/push --with-dev # TODO: remove --with_dev before merging diff --git a/README.md b/README.md index 129ea2d5..c8bd2005 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,6 @@ You can build the image with `/docker/build --with-dev`, this new image can now ### Deployment -A pull request can be created so buildkite pushes the images to the docker hub. Add `--with-dev` to the build & push commands `pipeline.yaml`. +A pull request can be created so GHA pushes the images to the docker hub. Add `--with-dev` to the build & push commands `pipeline.yaml`. **Ensure to remove the `--with-dev` flag before merging the PR.** Then on the `beebop-deploy` the api image can be updated with the new dev image. diff --git a/buildkite/pipeline.yml b/buildkite/pipeline.yml deleted file mode 100644 index f527e64c..00000000 --- a/buildkite/pipeline.yml +++ /dev/null @@ -1,9 +0,0 @@ -# # Add --with-dev to the build dev image for a specific version of PopPunk -# steps: -# - label: ":whale::python: Build" -# command: docker/build --with-dev - -# - wait - -# - label: ":shipit: Push images" -# command: docker/push --with-dev \ No newline at end of file From d84fb0981e1931dd6d484d546cd3c6ff8bec4436 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 3 Dec 2024 19:33:44 +0000 Subject: [PATCH 29/35] fix: update workflow name to reflect multiple docker images --- .github/workflows/build_and_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index d8adfe7f..879576e9 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -1,4 +1,4 @@ -name: build and push docker image +name: build and push docker images on: push: branches: From dfeeacb05d394b4e10e72de1fd5935bea21d268e Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Wed, 4 Dec 2024 15:24:47 +0000 Subject: [PATCH 30/35] fix: set CONDA_OVERRIDE_ARCHSPEC to ensure correct architecture for conda packages --- docker/Dockerfile.dev | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index dd91f0e5..bb4e2587 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,5 +1,8 @@ FROM continuumio/miniconda3 +# ensure conda does not install packages for the wrong architecture +ENV CONDA_OVERRIDE_ARCHSPEC=skylake + ARG POPPUNK_VERSION="v2.6.7" # Make RUN commands use the new environment: From 24f16110ccdb801eb747accb5af4c25722e80b0b Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Thu, 5 Dec 2024 10:09:43 +0000 Subject: [PATCH 31/35] fix: update branch pattern in workflow and improve README formatting --- .github/workflows/build_and_push.yml | 6 +++--- README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 879576e9..3fe4aecf 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -5,7 +5,7 @@ on: - main pull_request: branches: - - '*' + - "*" env: BRANCH_NAME: ${{ github.head_ref || github.ref_name }} jobs: @@ -21,5 +21,5 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build docker image run: ./docker/build --with-dev # TODO: remove --with_dev before merging - - name: Push SHA docker image - run: ./docker/push --with-dev # TODO: remove --with_dev before merging + - name: Push docker image + run: ./docker/push --with-dev # TODO: remove --with_dev before merging diff --git a/README.md b/README.md index c8bd2005..3e63bd8d 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Testing can be done in a second terminal (make sure to activate 'beebop_py') by TESTING=True poetry run pytest ``` -### Use/Deploy specific version of PopPUNK +## Use/Deploy specific version of PopPUNK To use a specific version, commit or branch of PopPUNK in a beebop_py deployment, you can update `POPPUNK_VERSION` in `common`. From f9b363b332103197f03ab21558f692856fdcb7ba Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Thu, 5 Dec 2024 10:18:27 +0000 Subject: [PATCH 32/35] fix: remove development flags from docker build and push commands --- .github/workflows/build_and_push.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 3fe4aecf..a2e6251b 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -19,7 +19,8 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # add --with-dev to below commands to build & push the dev image - name: Build docker image - run: ./docker/build --with-dev # TODO: remove --with_dev before merging + run: ./docker/build - name: Push docker image - run: ./docker/push --with-dev # TODO: remove --with_dev before merging + run: ./docker/push From 972c302eb5912f700b91ff61b52b244b599fed1a Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Thu, 5 Dec 2024 10:47:32 +0000 Subject: [PATCH 33/35] Update README diagrams section and remove unused parameter from filter_queries function --- README.md | 6 +++--- beebop/assignClusters.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 865a8b6f..830174b1 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Testing can be done in a second terminal (make sure to activate 'beebop_py') by TESTING=True poetry run pytest ``` -### Miscellaneous +### Diagrams -- There is a .drawio graph in graphs folder illustrating the proccess of running a analysis. This includes -all the files created anf how they are used in each job +- There is a .drawio graph in the `diagrams` folder illustrating the process of running a analysis. This includes +all the files created and how they are used in each job. You can open and view the diagram at [draw.io](https://draw.io). diff --git a/beebop/assignClusters.py b/beebop/assignClusters.py index 7c6d289d..3c0a6bd4 100644 --- a/beebop/assignClusters.py +++ b/beebop/assignClusters.py @@ -417,7 +417,6 @@ def filter_queries( :param queries_clusters: [list of sample PopPUNK clusters] :param not_found: [list of sample hashes that were not found] - :param config: [ClusteringConfig with all necessary information] :return tuple[list[str], list[str], set[str]]: [filtered sample hashes, filtered sample PopPUNK clusters, set of clusters assigned to not found samples] From 9c596de7bde43b42af5dd5af0740f022b14e288c Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Thu, 5 Dec 2024 13:42:09 +0000 Subject: [PATCH 34/35] update diagram as per pr comments --- diagrams/Beebop-PopPUNK-run.drawio | 182 ++++++++++++++++------------- 1 file changed, 102 insertions(+), 80 deletions(-) diff --git a/diagrams/Beebop-PopPUNK-run.drawio b/diagrams/Beebop-PopPUNK-run.drawio index ca3073a5..ab8bab82 100644 --- a/diagrams/Beebop-PopPUNK-run.drawio +++ b/diagrams/Beebop-PopPUNK-run.drawio @@ -4,10 +4,10 @@ - + - + @@ -22,9 +22,9 @@ - + - + @@ -38,13 +38,13 @@ - + - - + + @@ -53,7 +53,7 @@ - + @@ -63,7 +63,7 @@ - + @@ -78,9 +78,6 @@ - - - @@ -89,6 +86,9 @@ + + + @@ -99,13 +99,13 @@ - + - - + + @@ -127,30 +127,31 @@ - + + - + - - + + - + - - + + @@ -169,18 +170,23 @@ - + - - + + + + + + + @@ -203,7 +209,7 @@ - + @@ -218,8 +224,8 @@ - - + + @@ -259,7 +265,7 @@ - + @@ -284,11 +290,11 @@ - - + + - + @@ -297,7 +303,7 @@ - + @@ -311,7 +317,7 @@ - + @@ -350,10 +356,10 @@ - + - + @@ -363,18 +369,18 @@ - + - + - - + + - - + + @@ -402,22 +408,19 @@ - - + + - + - - - - + @@ -430,20 +433,17 @@ - - - - - - - - - - - + + + + + + + + - - + + @@ -457,18 +457,6 @@ - - - - - - - - - - - - @@ -483,26 +471,52 @@ - - + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + @@ -515,6 +529,14 @@ + + + + + + + + From 3e9d0d9d351c9356bc23bd79a9ebc23b4d007c5b Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 6 Dec 2024 15:02:14 +0000 Subject: [PATCH 35/35] update drawio --- diagrams/Beebop-PopPUNK-run.drawio | 1086 ++++++++++++++-------------- 1 file changed, 543 insertions(+), 543 deletions(-) diff --git a/diagrams/Beebop-PopPUNK-run.drawio b/diagrams/Beebop-PopPUNK-run.drawio index ab8bab82..ecd9b2ab 100644 --- a/diagrams/Beebop-PopPUNK-run.drawio +++ b/diagrams/Beebop-PopPUNK-run.drawio @@ -1,543 +1,543 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file