From 60d07576c724c94f2d32ddb8d35386b8d4adb4de Mon Sep 17 00:00:00 2001 From: ad-daniel <44834743+ad-daniel@users.noreply.github.com> Date: Mon, 24 Jan 2022 14:45:18 +0100 Subject: [PATCH] Automatic test of pip and docker (#200) * Test pip installation * Trigger * Fix * Minor * Updated wheel test * Added libopenblas install * Temporary test disable * Temporary test disable * Added libsndfile1 dependency * Restored disabled test * New wheel building pipeline * Update build_wheel.sh * Added libboost-dev dep * Update installation.md * Switch to sdist * sdist workflow * Update tests_suite.yml * Added numpy include dirs * Disabled two 3d detection tests * Disabled three 3d detection tests * Disabled five 3d detection tests * Disabled all 3d detection tests * Disabled model download - 3d test * Disabled everything? * Removed imports from 3d object detection test * Import test * Import test * Import test * Import test * Import test * Import test * Restored test * Disabled object detection 3d test * Refactor workflow * disable temporarely * Fix docker mobile manipulation * Re-enable test * add badge * Fix changelog * remove test install reference * Restored pip installation Co-authored-by: Nikolaos Passalis Co-authored-by: Nikolaos --- .github/workflows/test_packages.yml | 103 ++++++++++++++++++++++++++++ .github/workflows/tests_suite.yml | 1 + CHANGELOG.md | 3 +- MANIFEST.in | 2 + README.md | 1 + bin/build_wheel.sh | 19 +++++ bin/install.sh | 8 --- docs/reference/installation.md | 2 +- setup.py | 20 +++++- 9 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/test_packages.yml create mode 100644 MANIFEST.in create mode 100755 bin/build_wheel.sh diff --git a/.github/workflows/test_packages.yml b/.github/workflows/test_packages.yml new file mode 100644 index 0000000000..e7a3b718ea --- /dev/null +++ b/.github/workflows/test_packages.yml @@ -0,0 +1,103 @@ +name: Test Packages +# This workflow tests the latest (third-party hosted) available builds + +on: + pull_request: + types: [opened, reopened, synchronize, labeled, unlabeled] + schedule: + - cron: '0 23 * * *' + +defaults: + run: + shell: bash + + +jobs: + cleanup-runs: + if: ${{ contains(github.event.pull_request.labels.*.name, 'test packages') || github.event_name == 'schedule' }} + runs-on: ubuntu-latest + steps: + - uses: rokroskar/workflow-run-cleanup-action@master + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + test-wheel: + needs: cleanup-runs + if: ${{ contains(github.event.pull_request.labels.*.name, 'test packages') || github.event_name == 'schedule' }} + strategy: + matrix: + os: [ubuntu-20.04] + package: + - engine + - utils + - perception/activity_recognition + - perception/compressive_learning + - perception/face_recognition + - perception/heart_anomaly_detection + - perception/multimodal_human_centric + - perception/object_tracking_2d + - perception/pose_estimation + - perception/speech_recognition + - perception/skeleton_based_action_recognition + - perception/semantic_segmentation + - perception/object_detection_2d + - perception/facial_expression_recognition + # - perception/object_detection_3d + # - control/mobile_manipulation + # - simulation/human_model_generation + # - control/single_demo_grasp + # - perception/object_tracking_3d + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Test Wheel + run: | + export DISABLE_BCOLZ_AVX2=true + sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev python3-dev + python3 -m venv venv + source venv/bin/activate + wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt + cat pip_requirements.txt | xargs -n 1 -L 1 pip install + # Test new package + pip install opendr-toolkit + python -m unittest discover -s tests/sources/tools/${{ matrix.package }} + test-docker: + needs: cleanup-runs + if: ${{ contains(github.event.pull_request.labels.*.name, 'test packages') || github.event_name == 'schedule' }} + strategy: + matrix: + os: [ubuntu-20.04] + package: + - engine + - utils + - perception/activity_recognition + - perception/compressive_learning + - perception/face_recognition + - perception/heart_anomaly_detection + - perception/multimodal_human_centric + - perception/object_tracking_2d + - perception/pose_estimation + - perception/speech_recognition + - perception/skeleton_based_action_recognition + - perception/semantic_segmentation + - perception/object_detection_2d + - perception/facial_expression_recognition + - perception/object_detection_3d + - control/mobile_manipulation + - simulation/human_model_generation + - control/single_demo_grasp + #- perception/object_tracking_3d + runs-on: ${{ matrix.os }} + steps: + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Test Docker + run: | + docker run --name toolkit -i opendr/opendr-toolkit:cpu_latest bash + docker start toolkit + docker exec -i toolkit bash -c "source bin/activate.sh && source tests/sources/tools/control/mobile_manipulation/run_ros.sh && python -m unittest discover -s tests/sources/tools/${{ matrix.package }}" diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 7d0503a429..82b80e7e96 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -106,3 +106,4 @@ jobs: source tests/sources/tools/control/mobile_manipulation/run_ros.sh python -m unittest discover -s tests/sources/tools/${{ matrix.package }} fi + diff --git a/CHANGELOG.md b/CHANGELOG.md index 2de83a228b..7e0bec9ded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ Released on XX, XXth, 2022. - Enhancements: - None. - Bug Fixes: - - None. + - Updated wheel building pipeline to include missing files and removed unnecessary dependencies ([#200](https://github.com/opendr-eu/opendr/pull/200)). - Dependency Updates: - `heart anomaly detection`: upgraded scikit-learn runtime dependency from 0.21.3 to 0.22 ([#198](https://github.com/opendr-eu/opendr/pull/198)). + ## Version 1.0 Released on December 31th, 2021. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..2c2972093c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include src/opendr * +include requirements.txt diff --git a/README.md b/README.md index e91b500193..b3102f0580 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ ______________________________________________________________________ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Test Suite (master)](https://github.com/opendr-eu/opendr/actions/workflows/tests_suite.yml/badge.svg)](https://github.com/opendr-eu/opendr/actions/workflows/tests_suite.yml) +[![Test Packages](https://github.com/opendr-eu/opendr/actions/workflows/test_packages.yml/badge.svg)](https://github.com/opendr-eu/opendr/actions/workflows/test_packages.yml) ## About diff --git a/bin/build_wheel.sh b/bin/build_wheel.sh new file mode 100755 index 0000000000..c0564a5d0c --- /dev/null +++ b/bin/build_wheel.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Build all OpenDR dependecies +./bin/install.sh + +# Activate OpenDR +source ./bin/activate.sh + +# Prepare requirements.txt for wheel distributions +pip3 freeze > requirements.txt + +# Remove detectron and git repositories (installation not supported through PyPI) +sed -i '/detectron2/d' requirements.txt +sed -i '/git/d' requirements.txt +sed -i '/pkg_resources/d' requirements.txt +sed -i '/auditwheel/d' requirements.txt + +# Build binary wheel and repair it +python3 setup.py sdist diff --git a/bin/install.sh b/bin/install.sh index d0f3b6c269..e819c7a100 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -40,13 +40,5 @@ make install_compilation_dependencies make install_runtime_dependencies make libopendr -# Prepare requirements.txt for wheel distributions -pip3 freeze > requirements.txt - -# Remove detectron, since it was installed from git repo -sed -i '/detectron2/d' requirements.txt -sed -i '/git/d' requirements.txt -python3 setup.py bdist_wheel - deactivate diff --git a/docs/reference/installation.md b/docs/reference/installation.md index 1b36477f82..8ede23d12e 100644 --- a/docs/reference/installation.md +++ b/docs/reference/installation.md @@ -74,7 +74,7 @@ You can directly install OpenDR toolkit for CPU-only inference using pip. First, install the required dependencies: ```bash export DISABLE_BCOLZ_AVX2=true -sudo apt install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget +sudo apt install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev python3 -m venv venv source venv/bin/activate wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt diff --git a/setup.py b/setup.py index 5eff9c2b0c..c2d988b385 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,33 @@ +import os +from os.path import join from setuptools import setup from setuptools import find_packages from Cython.Build import cythonize +import numpy + packages = find_packages(where="./src") +# Get the requirements with open('requirements.txt') as fp: install_requires = fp.read().splitlines() +# Retrieve version exec(open('src/opendr/_version.py').read()) - try: __version__ except NameError: __version__ = '0.0' +# Gather all files +data_files = [] +for root, dirs, files in os.walk("src/opendr"): + for file in files: + file_extension = file.split(".")[-1] + # Add all files except from shared libraries + if file_extension != "so" and file_extension != "py": + data_files.append(join(root.replace("src/opendr/", ""), file)) + setup( name='opendr-toolkit', version=__version__, @@ -49,5 +63,7 @@ license='LICENSE', package_dir={"": "src"}, install_requires=install_requires, - ext_modules=cythonize(["src/opendr/perception/object_detection_2d/retinaface/algorithm/cython/*.pyx"]) + package_data={'': data_files}, + ext_modules=cythonize(["src/opendr/perception/object_detection_2d/retinaface/algorithm/cython/*.pyx"]), + include_dirs=[numpy.get_include()] )