WIP Move to Poetry

thelastpickle · Nov 2, 2023 · 62fd201 · 62fd201
1 parent 01a49de
commit 62fd201
Show file tree

Hide file tree

Showing 15 changed files with 2,464 additions and 182 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,10 +13,8 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.8, 3.9, "3.10", "3.11"]
         include:
-          - python-version: 3.7
-            tox-py: py37
           - python-version: 3.8
             tox-py: py38
           - python-version: 3.9
@@ -32,27 +30,23 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Set up Poetry
+      uses: snok/install-poetry@v1
     - name: Install dependencies
       run: |
-        python -m venv venv
-        . venv/bin/activate
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install tox
+        poetry install
 
     - name: Lint with flake8
       run: |
-        pip install flake8
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=W503
+        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=W503
     - name: Run tox
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        . venv/bin/activate
-        tox -e ${{ matrix.tox-py }}
+        poetry run tox -e ${{ matrix.tox-py }}
 
     - uses: codecov/codecov-action@v1
       name: Report code coverage  
@@ -98,14 +92,19 @@ jobs:
         sudo dpkg -i cassandra-medusa_${version}-0~${{ matrix.suite }}0_amd64.deb
         medusa
         exit $?
+
+    - name: Setup tmate session
+      if: "failure()"
+      uses: mxschmitt/action-tmate@v3
+      timeout-minutes: 30
 
   integration-tests:
     needs: build
     continue-on-error: ${{ matrix.experimental }}
     strategy:
         fail-fast: false
         matrix:
-          python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+          python-version: [3.8, 3.9, "3.10", "3.11"]
           it-backend: [local, s3, gcs, minio, azure, azure-hierarchical]
           # IBM not included by default due to lite plan quota being easily exceeded
           #it-backend: [local, s3, gcs, minio, ibm, azure]
@@ -133,11 +132,6 @@ jobs:
               cassandra-version: 4.0.0
               it-backend: gcs
               experimental: false
-            # explicitly include tests against python 3.7 and one version of cassandra
-            - python-version: 3.7
-              cassandra-version: 4.0.0
-              it-backend: gcs
-              experimental: false
           exclude:
             # no tests against trunk
             - cassandra-version: 'github:apache/trunk'
@@ -153,7 +147,6 @@ jobs:
             - it-backend: azure-hierarchical
               cassandra-version: "3.11.11"
             # no tests against non-python3.9, except the explicitly allowed combinations
-            - python-version: 3.7
             - python-version: 3.8
             - python-version: "3.10"
             - python-version: "3.11"
@@ -181,23 +174,13 @@ jobs:
       with:
         java-version: '8.0.252'
         architecture: x64
+    - name: Setup Poetry
+      uses: snok/install-poetry@v1
     - name: Install dependencies
       run: |
-        python -m venv venv
-        . venv/bin/activate
-        sudo apt-get remove azure-cli
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements-test.txt
+        poetry install
+        sudo apt-get remove azure-cli || true
         pip install ccm
-        case '${{ matrix.it-backend }}' in
-          'azure'|'azure-hierarchical')
-            echo "No extra requirements for now."
-            ;;
-          'ibm'|'minio'|'s3')
-            echo "No extra requirements for now."
-            ;;
-        esac
 
     - name: Check if integration tests can run
       env:
@@ -224,7 +207,6 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.BUCKET_SECRET }}
       run: |
         set -e
-        . venv/bin/activate
         if [[ "${{ matrix.it-backend }}" == "ibm" ]];
         then
           # Prevent awscli from using AWS secrets in case we're running against IBM cloud
@@ -300,7 +282,7 @@ jobs:
 
         # Move and convert the coverage analysis file to XML
         mv tests/integration/.coverage .
-        coverage xml
+        poetry run coverage xml
 
     - uses: codecov/codecov-action@v1
       name: Report code coverage

diff --git a/debian/cassandra-medusa.links b/debian/cassandra-medusa.links
@@ -1,2 +1,2 @@
 usr/share/cassandra-medusa/bin/medusa usr/bin/medusa
-usr/share/cassandra-medusa/bin/medusa-wrapper usr/bin/medusa-wrapper
+usr/share/cassandra-medusa/bin/medusa-wrapper usr/bin/medusa-wrapper
diff --git a/debian/rules b/debian/rules
@@ -21,6 +21,7 @@ PACKAGEVERSION = $(VERSION)-0~$(DISTRIBUTION)0
 PY3VER = $(shell py3versions -d)
 SSH2_LIBS_SUFFIX = debian/cassandra-medusa/usr/share/cassandra-medusa/lib/$(PY3VER)/site-packages/ssh2_python.libs/
 SSH_LIBS_SUFFIX = debian/cassandra-medusa/usr/share/cassandra-medusa/lib/$(PY3VER)/site-packages/ssh_python.libs/
+POETRY_VIRTUALENVS_IN_PROJECT = true
 
 export DH_ALWAYS_EXCLUDE = .git
 export DH_VIRTUALENV_INSTALL_ROOT = /usr/share
@@ -29,8 +30,8 @@ export DH_VIRTUALENV_INSTALL_ROOT = /usr/share
 
 override_dh_virtualenv:
 	dh_virtualenv \
-	  --extra-pip-arg "--no-cache-dir" \
-	  --python /usr/bin/python3 --preinstall=setuptools==40.3.0 --preinstall=pip==21.3.1 --preinstall=wheel --builtin-venv
+	--python /usr/bin/python3 --preinstall=setuptools==40.3.0 --preinstall=pip==21.3.1 --preinstall=wheel --builtin-venv \
+	--preinstall=poetry==1.6.1 --preinstall=dh-poetry==0.2.0 --pip-tool=dh-poetry
 
 override_dh_strip:
 	dh_strip --no-automatic-dbgsym -X libssh2 -X libssh -X libgssapi_krb5 -X libcrypto -X libkrb5 -X libk5crypto

diff --git a/k8s/Dockerfile b/k8s/Dockerfile
@@ -27,12 +27,11 @@ ENV PATH=/root/.local/bin:$PATH
 COPY . /build/
 
 # General requirements
-RUN python3 -m pip install -U pip && pip3 install --ignore-installed --user \
-    -r /build/requirements.txt \
-    -r /build/requirements-grpc-runtime.txt
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+RUN python3 -m pip install -U pip && pip3 install --ignore-installed --user poetry
 
 # Build medusa itself so we can add the executables in the final image
-RUN pip3 install --ignore-installed --user /build
+RUN cd /build && poetry install
 
 # Could be python:slim, but we have a .sh entrypoint
 FROM ubuntu:22.04

diff --git a/k8s/docker-entrypoint.sh b/k8s/docker-entrypoint.sh
@@ -41,14 +41,14 @@ restore() {
         echo "Skipping restore operation"    
     else
         echo "Restoring backup $BACKUP_NAME"
-        python3 -m medusa.service.grpc.restore -- "/etc/medusa/medusa.ini" $RESTORE_KEY
+        poetry --directory /build run python -m medusa.service.grpc.restore -- "/etc/medusa/medusa.ini" $RESTORE_KEY
         echo $RESTORE_KEY > $last_restore_file
     fi
 }
 
 grpc() {
     echo "Starting Medusa gRPC service"
-    exec python3 -m medusa.service.grpc.server server.py
+    exec poetry --directory /build run python -m medusa.service.grpc.server server.py
 }
 
 echo "sleeping for $DEBUG_SLEEP sec"

diff --git a/packaging/docker-build/Dockerfile b/packaging/docker-build/Dockerfile
@@ -33,8 +33,8 @@ RUN apt-get update && \
 # install dependencies
 RUN apt-get update \
     && apt-get install -y \
-        libffi-dev \   
-        libssl-dev \  
+        libffi-dev \
+        libssl-dev \
         debhelper \
         gdebi-core \
         gcc \