From c916dd843fd65d8fbc2c07f916883e4d6a8230b7 Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Mon, 17 Jan 2022 10:58:03 +0100 Subject: [PATCH 01/41] Test pip installation --- .github/workflows/tests_suite.yml | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 7d0503a429..852d051827 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -106,3 +106,46 @@ jobs: source tests/sources/tools/control/mobile_manipulation/run_ros.sh python -m unittest discover -s tests/sources/tools/${{ matrix.package }} fi + test-install: + needs: cleanup-runs + if: ${{ contains(github.event.pull_request.labels.*.name, 'test install') || github.event_name == 'schedule' }} + strategy: + matrix: + os: [ubuntu-20.04] + package: + - engine + - utils + - perception/activity_recognition + - perception/compressive_learning + - perception/face_recognition + - perception/heart_anomaly_detection + - perception/multimodal_human_centric + - perception/object_tracking_2d + - perception/object_detection_3d + - perception/pose_estimation + - perception/speech_recognition + - perception/skeleton_based_action_recognition + - perception/semantic_segmentation + - perception/object_detection_2d + - perception/facial_expression_recognition + #- control/mobile_manipulation + #- simulation/human_model_generation + #- control/single_demo_grasp + # - perception/object_tracking_3d + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Test Install + run: | + export DISABLE_BCOLZ_AVX2=true + sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget + python3 -m venv venv + source venv/bin/activate + wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt + cat pip_requirements.txt | xargs -n 1 -L 1 pip install + pip install opendr-toolkit + python -m unittest discover -s tests/sources/tools/${{ matrix.package }} \ No newline at end of file From b134ac52036560b6c14ce282c848f567c9baa3c7 Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Mon, 17 Jan 2022 11:02:45 +0100 Subject: [PATCH 02/41] Trigger --- .github/workflows/tests_suite.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 852d051827..d9def70beb 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -12,7 +12,7 @@ defaults: jobs: cleanup-runs: - if: ${{ contains(github.event.pull_request.labels.*.name, 'test sources') || contains(github.event.pull_request.labels.*.name, 'test tools') || github.event_name == 'schedule' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'test sources') || contains(github.event.pull_request.labels.*.name, 'test tools') || contains(github.event.pull_request.labels.*.name, 'test install') || github.event_name == 'schedule' }} runs-on: ubuntu-latest steps: - uses: rokroskar/workflow-run-cleanup-action@master @@ -128,9 +128,9 @@ jobs: - perception/semantic_segmentation - perception/object_detection_2d - perception/facial_expression_recognition - #- control/mobile_manipulation - #- simulation/human_model_generation - #- control/single_demo_grasp + # - control/mobile_manipulation + # - simulation/human_model_generation + # - control/single_demo_grasp # - perception/object_tracking_3d runs-on: ${{ matrix.os }} steps: From 6a42f3f7a12d323ef6b381f4e40babd0d707572d Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Mon, 17 Jan 2022 11:29:32 +0100 Subject: [PATCH 03/41] Fix --- .github/workflows/tests_suite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index d9def70beb..bce3deed6c 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -145,6 +145,7 @@ jobs: sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget python3 -m venv venv source venv/bin/activate + pip3 install --upgrade setuptools pip wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt cat pip_requirements.txt | xargs -n 1 -L 1 pip install pip install opendr-toolkit From 23cc3e463da392fadc9622ce15c29d19b66559eb Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Mon, 17 Jan 2022 11:31:26 +0100 Subject: [PATCH 04/41] Minor --- .github/workflows/tests_suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index bce3deed6c..1e73d9b46b 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -145,8 +145,8 @@ jobs: sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget python3 -m venv venv source venv/bin/activate - pip3 install --upgrade setuptools pip wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt cat pip_requirements.txt | xargs -n 1 -L 1 pip install + pip uninstall pkg-resources==0.0.0 pip install opendr-toolkit - python -m unittest discover -s tests/sources/tools/${{ matrix.package }} \ No newline at end of file + python -m unittest discover -s tests/sources/tools/${{ matrix.package }} From f11129fbdec2e0e0321d4b2946e2ac941281d7db Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Wed, 19 Jan 2022 15:24:57 +0200 Subject: [PATCH 05/41] Updated wheel test --- .github/workflows/tests_suite.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 1e73d9b46b..691863b40d 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -148,5 +148,8 @@ jobs: wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt cat pip_requirements.txt | xargs -n 1 -L 1 pip install pip uninstall pkg-resources==0.0.0 - pip install opendr-toolkit + # Test new package + wget https://raw.githubusercontent.com/passalis/test_repo/main/opendr_toolkit-1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_24_x86_64.whl + pip install opendr_toolkit-1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_24_x86_64.whl + #pip install opendr-toolkit python -m unittest discover -s tests/sources/tools/${{ matrix.package }} From 5466a015a622876b118f6287bc8e2bf02470dc2d Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Wed, 19 Jan 2022 16:36:20 +0200 Subject: [PATCH 06/41] Added libopenblas install --- .github/workflows/tests_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 691863b40d..919d6421b2 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -142,7 +142,7 @@ jobs: - name: Test Install run: | export DISABLE_BCOLZ_AVX2=true - sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget + sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev python3 -m venv venv source venv/bin/activate wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt From f9fbe9fe8c5fd7b2c2cc55be65805b60df23b86e Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Wed, 19 Jan 2022 19:04:54 +0200 Subject: [PATCH 07/41] Temporary test disable --- .github/workflows/tests_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 919d6421b2..e6cddf23f1 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -65,7 +65,7 @@ jobs: - perception/heart_anomaly_detection - perception/multimodal_human_centric - perception/object_tracking_2d - - perception/object_detection_3d + # - perception/object_detection_3d # temporary disabling this test - perception/pose_estimation - perception/speech_recognition - perception/skeleton_based_action_recognition From ff7351ea9e118f36cc70f09eb690351fe642ddd3 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Wed, 19 Jan 2022 19:05:34 +0200 Subject: [PATCH 08/41] Temporary test disable --- .github/workflows/tests_suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index e6cddf23f1..f249243a4e 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -65,7 +65,7 @@ jobs: - perception/heart_anomaly_detection - perception/multimodal_human_centric - perception/object_tracking_2d - # - perception/object_detection_3d # temporary disabling this test + - perception/object_detection_3d - perception/pose_estimation - perception/speech_recognition - perception/skeleton_based_action_recognition @@ -121,7 +121,7 @@ jobs: - perception/heart_anomaly_detection - perception/multimodal_human_centric - perception/object_tracking_2d - - perception/object_detection_3d + # - perception/object_detection_3d # temporary disabling this test - perception/pose_estimation - perception/speech_recognition - perception/skeleton_based_action_recognition From 2849f754269e98ec66ed0fed61e6f83dae32f2a5 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Wed, 19 Jan 2022 20:18:40 +0200 Subject: [PATCH 09/41] Added libsndfile1 dependency --- .github/workflows/tests_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index f249243a4e..4fc8a1dcd4 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -142,7 +142,7 @@ jobs: - name: Test Install run: | export DISABLE_BCOLZ_AVX2=true - sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev + sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 python3 -m venv venv source venv/bin/activate wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt From 6c709ef7ee99a0a4ab40f8dcd3b1b92c19561946 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Wed, 19 Jan 2022 22:35:27 +0200 Subject: [PATCH 10/41] Restored disabled test --- .github/workflows/tests_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 4fc8a1dcd4..269302fadf 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -121,7 +121,7 @@ jobs: - perception/heart_anomaly_detection - perception/multimodal_human_centric - perception/object_tracking_2d - # - perception/object_detection_3d # temporary disabling this test + - perception/object_detection_3d - perception/pose_estimation - perception/speech_recognition - perception/skeleton_based_action_recognition From c1878f801ed353430a78bfb9362b624ae94a6e95 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Wed, 19 Jan 2022 22:44:52 +0200 Subject: [PATCH 11/41] New wheel building pipeline --- CHANGELOG.md | 3 ++- bin/build_wheel.sh | 21 +++++++++++++++++++++ bin/install.sh | 8 -------- docs/reference/installation.md | 2 +- setup.py | 15 ++++++++++++++- 5 files changed, 38 insertions(+), 11 deletions(-) create mode 100755 bin/build_wheel.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 2de83a228b..8742a39734 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ Released on XX, XXth, 2022. - Enhancements: - None. - Bug Fixes: - - None. + - Updated wheel building pipeline to include missing files and removed unnecessary dependencies. - Dependency Updates: - `heart anomaly detection`: upgraded scikit-learn runtime dependency from 0.21.3 to 0.22 ([#198](https://github.com/opendr-eu/opendr/pull/198)). + ## Version 1.0 Released on December 31th, 2021. \ No newline at end of file diff --git a/bin/build_wheel.sh b/bin/build_wheel.sh new file mode 100755 index 0000000000..cdccaa387a --- /dev/null +++ b/bin/build_wheel.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Build all OpenDR dependecies +#./bin/install.sh + +# Activate OpenDR +source ./bin/activate.sh + +# Prepare requirements.txt for wheel distributions +pip3 freeze > requirements.txt + +# Remove detectron and git repositories (installation not supported through PyPI) +sed -i '/detectron2/d' requirements.txt +sed -i '/git/d' requirements.txt +sed -i '/pkg_resources/d' requirements.txt +sed -i '/auditwheel/d' requirements.txt + +# Build binary wheel and repair it +python3 setup.py bdist_wheel +pip3 install auditwheel +auditwheel repair dist/*.whl -w dist/ --plat manylinux_2_24_x86_64 diff --git a/bin/install.sh b/bin/install.sh index d0f3b6c269..e819c7a100 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -40,13 +40,5 @@ make install_compilation_dependencies make install_runtime_dependencies make libopendr -# Prepare requirements.txt for wheel distributions -pip3 freeze > requirements.txt - -# Remove detectron, since it was installed from git repo -sed -i '/detectron2/d' requirements.txt -sed -i '/git/d' requirements.txt -python3 setup.py bdist_wheel - deactivate diff --git a/docs/reference/installation.md b/docs/reference/installation.md index 1b36477f82..b5ebf31d8e 100644 --- a/docs/reference/installation.md +++ b/docs/reference/installation.md @@ -74,7 +74,7 @@ You can directly install OpenDR toolkit for CPU-only inference using pip. First, install the required dependencies: ```bash export DISABLE_BCOLZ_AVX2=true -sudo apt install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget +sudo apt install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 python3 -m venv venv source venv/bin/activate wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt diff --git a/setup.py b/setup.py index 5eff9c2b0c..9d73d6c3ec 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,31 @@ +import os +from os.path import join from setuptools import setup from setuptools import find_packages from Cython.Build import cythonize packages = find_packages(where="./src") +# Get the requirements with open('requirements.txt') as fp: install_requires = fp.read().splitlines() +# Retrieve version exec(open('src/opendr/_version.py').read()) - try: __version__ except NameError: __version__ = '0.0' +# Gather all files +data_files = [] +for root, dirs, files in os.walk("src/opendr"): + for file in files: + file_extension = file.split(".")[-1] + # Add all files except from shared libraries + if file_extension != "so" and file_extension != "py": + data_files.append(join(root.replace("src/opendr/", ""), file)) + setup( name='opendr-toolkit', version=__version__, @@ -49,5 +61,6 @@ license='LICENSE', package_dir={"": "src"}, install_requires=install_requires, + package_data={'': data_files}, ext_modules=cythonize(["src/opendr/perception/object_detection_2d/retinaface/algorithm/cython/*.pyx"]) ) From 758d95e882e2fc1fdafeff2a9237cbda98fbbb42 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Wed, 19 Jan 2022 22:46:14 +0200 Subject: [PATCH 12/41] Update build_wheel.sh --- bin/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build_wheel.sh b/bin/build_wheel.sh index cdccaa387a..3f1a3233ec 100755 --- a/bin/build_wheel.sh +++ b/bin/build_wheel.sh @@ -1,7 +1,7 @@ #!/bin/bash # Build all OpenDR dependecies -#./bin/install.sh +./bin/install.sh # Activate OpenDR source ./bin/activate.sh From 2140e32bbf1a4939c9d42e7b42ce375dc30096f0 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Thu, 20 Jan 2022 06:17:49 +0200 Subject: [PATCH 13/41] Added libboost-dev dep --- .github/workflows/tests_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 269302fadf..fe160b0fca 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -142,7 +142,7 @@ jobs: - name: Test Install run: | export DISABLE_BCOLZ_AVX2=true - sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 + sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev python3 -m venv venv source venv/bin/activate wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt From 19033ea90433321997128a612fdf400e9d314361 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Thu, 20 Jan 2022 06:18:12 +0200 Subject: [PATCH 14/41] Update installation.md --- docs/reference/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/installation.md b/docs/reference/installation.md index b5ebf31d8e..8ede23d12e 100644 --- a/docs/reference/installation.md +++ b/docs/reference/installation.md @@ -74,7 +74,7 @@ You can directly install OpenDR toolkit for CPU-only inference using pip. First, install the required dependencies: ```bash export DISABLE_BCOLZ_AVX2=true -sudo apt install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 +sudo apt install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev python3 -m venv venv source venv/bin/activate wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt From 116fc14d3f8db7e5f9e03d0898795ec1af0494f2 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Thu, 20 Jan 2022 07:20:07 +0200 Subject: [PATCH 15/41] Switch to sdist --- .github/workflows/tests_suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index fe160b0fca..ac723f35c6 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -149,7 +149,7 @@ jobs: cat pip_requirements.txt | xargs -n 1 -L 1 pip install pip uninstall pkg-resources==0.0.0 # Test new package - wget https://raw.githubusercontent.com/passalis/test_repo/main/opendr_toolkit-1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_24_x86_64.whl - pip install opendr_toolkit-1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_24_x86_64.whl + wget https://raw.githubusercontent.com/passalis/test_repo/main/opendr-toolkit-1.0.tar.gz + pip install opendr-toolkit-1.0.tar.gz #pip install opendr-toolkit python -m unittest discover -s tests/sources/tools/${{ matrix.package }} From dec5392c8fea587a3e01ba0392856dffe4f7a337 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 07:35:25 +0200 Subject: [PATCH 16/41] sdist workflow --- MANIFEST.in | 2 ++ bin/build_wheel.sh | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..322d0e3262 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include src/opendr * +include requirements.txt \ No newline at end of file diff --git a/bin/build_wheel.sh b/bin/build_wheel.sh index 3f1a3233ec..7fbf1d9565 100755 --- a/bin/build_wheel.sh +++ b/bin/build_wheel.sh @@ -16,6 +16,4 @@ sed -i '/pkg_resources/d' requirements.txt sed -i '/auditwheel/d' requirements.txt # Build binary wheel and repair it -python3 setup.py bdist_wheel -pip3 install auditwheel -auditwheel repair dist/*.whl -w dist/ --plat manylinux_2_24_x86_64 +python3 setup.py sdist \ No newline at end of file From 20a3f95891159b22de516e6cca9516c38f9ec124 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Thu, 20 Jan 2022 07:36:05 +0200 Subject: [PATCH 17/41] Update tests_suite.yml --- .github/workflows/tests_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index ac723f35c6..62ba1aab77 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -142,7 +142,7 @@ jobs: - name: Test Install run: | export DISABLE_BCOLZ_AVX2=true - sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev + sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev python3-dev python3 -m venv venv source venv/bin/activate wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt From 79285e5ce0a09f0182713dbb7fd28e35c8c123d9 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 07:49:54 +0200 Subject: [PATCH 18/41] Added numpy include dirs --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9d73d6c3ec..c2d988b385 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,8 @@ from setuptools import setup from setuptools import find_packages from Cython.Build import cythonize +import numpy + packages = find_packages(where="./src") @@ -62,5 +64,6 @@ package_dir={"": "src"}, install_requires=install_requires, package_data={'': data_files}, - ext_modules=cythonize(["src/opendr/perception/object_detection_2d/retinaface/algorithm/cython/*.pyx"]) + ext_modules=cythonize(["src/opendr/perception/object_detection_2d/retinaface/algorithm/cython/*.pyx"]), + include_dirs=[numpy.get_include()] ) From 535c69117707c6087efb8ed20b180de6ce8a1fa1 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 08:28:46 +0200 Subject: [PATCH 19/41] Disabled two 3d detection tests --- .../test_object_detection_3d.py | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 2372294671..741febc050 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -228,68 +228,68 @@ def test_model(name, config): for name, config in self.car_configs.items(): test_model(name, config) - def test_save(self): - def test_model(name, config): - print("Save", name, "start", file=sys.stderr) - model_path = os.path.join(self.temp_dir, "test_save_" + name) - save_path = os.path.join(model_path, "save") - - learner = VoxelObjectDetection3DLearner( - model_config_path=config, device=DEVICE - ) - learner.save(save_path, True) - starting_param_1 = list(learner.model.parameters())[0].clone() - - learner2 = VoxelObjectDetection3DLearner( - model_config_path=config, device=DEVICE - ) - starting_param_2 = list(learner2.model.parameters())[0].clone() - learner2.load(save_path) - - new_param = list(learner2.model.parameters())[0].clone() - self.assertFalse(torch.equal(starting_param_1, starting_param_2)) - self.assertTrue(torch.equal(starting_param_1, new_param)) - - del learner - del learner2 - print("Save", name, "ok", file=sys.stderr) - - for name, config in self.car_configs.items(): - test_model(name, config) - - def test_optimize(self): - def test_model(name, config): - print("Optimize", name, "start", file=sys.stderr) - model_path = os.path.join(self.temp_dir, "test_optimize_" + name) - - dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") - - learner = VoxelObjectDetection3DLearner( - model_config_path=config, device=DEVICE, - temp_path=self.temp_dir - ) - learner.optimize() - - result = learner.infer( - dataset[0] - ) - self.assertTrue(len(result) > 0) - - learner.save(model_path) - - learner2 = VoxelObjectDetection3DLearner( - model_config_path=config, device=DEVICE - ) - learner2.load(model_path, True) - - self.assertTrue(learner2.model.rpn_ort_session is not None) - - del learner - del learner2 - print("Optimize", name, "ok", file=sys.stderr) - - for name, config in self.car_configs.items(): - test_model(name, config) + # def test_save(self): + # def test_model(name, config): + # print("Save", name, "start", file=sys.stderr) + # model_path = os.path.join(self.temp_dir, "test_save_" + name) + # save_path = os.path.join(model_path, "save") + # + # learner = VoxelObjectDetection3DLearner( + # model_config_path=config, device=DEVICE + # ) + # learner.save(save_path, True) + # starting_param_1 = list(learner.model.parameters())[0].clone() + # + # learner2 = VoxelObjectDetection3DLearner( + # model_config_path=config, device=DEVICE + # ) + # starting_param_2 = list(learner2.model.parameters())[0].clone() + # learner2.load(save_path) + # + # new_param = list(learner2.model.parameters())[0].clone() + # self.assertFalse(torch.equal(starting_param_1, starting_param_2)) + # self.assertTrue(torch.equal(starting_param_1, new_param)) + # + # del learner + # del learner2 + # print("Save", name, "ok", file=sys.stderr) + # + # for name, config in self.car_configs.items(): + # test_model(name, config) + # + # def test_optimize(self): + # def test_model(name, config): + # print("Optimize", name, "start", file=sys.stderr) + # model_path = os.path.join(self.temp_dir, "test_optimize_" + name) + # + # dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") + # + # learner = VoxelObjectDetection3DLearner( + # model_config_path=config, device=DEVICE, + # temp_path=self.temp_dir + # ) + # learner.optimize() + # + # result = learner.infer( + # dataset[0] + # ) + # self.assertTrue(len(result) > 0) + # + # learner.save(model_path) + # + # learner2 = VoxelObjectDetection3DLearner( + # model_config_path=config, device=DEVICE + # ) + # learner2.load(model_path, True) + # + # self.assertTrue(learner2.model.rpn_ort_session is not None) + # + # del learner + # del learner2 + # print("Optimize", name, "ok", file=sys.stderr) + # + # for name, config in self.car_configs.items(): + # test_model(name, config) if __name__ == "__main__": From 21d75754149d6e5dfaaca301fcc96f38ca0ceb7e Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 09:47:03 +0200 Subject: [PATCH 20/41] Disabled three 3d detection tests --- .../test_object_detection_3d.py | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 741febc050..b645793531 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -200,33 +200,33 @@ def test_model(name, config): for name, config in self.car_configs.items(): test_model(name, config) - def test_infer(self): - def test_model(name, config): - print("Infer", name, "start", file=sys.stderr) - - dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") - - learner = VoxelObjectDetection3DLearner( - model_config_path=config, device=DEVICE - ) - - result = learner.infer( - dataset[0] - ) - - self.assertTrue(len(result) > 0) - - result = learner.infer( - [dataset[0], dataset[1], dataset[2]] - ) - self.assertTrue(len(result) == 3) - self.assertTrue(len(result[0]) > 0) - - del learner - print("Infer", name, "ok", file=sys.stderr) - - for name, config in self.car_configs.items(): - test_model(name, config) + # def test_infer(self): + # def test_model(name, config): + # print("Infer", name, "start", file=sys.stderr) + # + # dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") + # + # learner = VoxelObjectDetection3DLearner( + # model_config_path=config, device=DEVICE + # ) + # + # result = learner.infer( + # dataset[0] + # ) + # + # self.assertTrue(len(result) > 0) + # + # result = learner.infer( + # [dataset[0], dataset[1], dataset[2]] + # ) + # self.assertTrue(len(result) == 3) + # self.assertTrue(len(result[0]) > 0) + # + # del learner + # print("Infer", name, "ok", file=sys.stderr) + # + # for name, config in self.car_configs.items(): + # test_model(name, config) # def test_save(self): # def test_model(name, config): From 724ecd22bf0014314a47d93099b39dc92e5d8a9d Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 10:29:54 +0200 Subject: [PATCH 21/41] Disabled five 3d detection tests --- .../test_object_detection_3d.py | 106 +++++++++--------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index b645793531..31559c6dea 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -145,60 +145,60 @@ def test_model(name, config): for name, config in self.car_configs.items(): test_model(name, config) - def test_fit_iterator(self): - def test_model(name, config): - print("Fit iterator", name, "start", file=sys.stderr) - model_path = os.path.join(self.temp_dir, "test_fit_iterator_" + name) - dataset = LabeledPointCloudsDatasetIterator( - self.dataset_path + "/training/velodyne_reduced", - self.dataset_path + "/training/label_2", - self.dataset_path + "/training/calib", - ) - - val_dataset = LabeledPointCloudsDatasetIterator( - self.dataset_path + "/training/velodyne_reduced", - self.dataset_path + "/training/label_2", - self.dataset_path + "/training/calib", - ) - - learner = VoxelObjectDetection3DLearner( - model_config_path=config, device=DEVICE, - checkpoint_after_iter=90, - ) - - starting_param = list(learner.model.parameters())[0].clone() - learner.fit( - dataset, - val_dataset=val_dataset, - model_dir=model_path, - evaluate=False, - ) - new_param = list(learner.model.parameters())[0].clone() - self.assertFalse(torch.equal(starting_param, new_param)) - - del learner - print("Fit iterator", name, "ok", file=sys.stderr) - - for name, config in self.car_configs.items(): - test_model(name, config) - - def test_eval(self): - def test_model(name, config): - print("Eval", name, "start", file=sys.stderr) - model_path = os.path.join(self.temp_dir, self.download_model_names[name]) - dataset = KittiDataset(self.dataset_path, self.subsets_path) - - learner = VoxelObjectDetection3DLearner(model_config_path=config, device=DEVICE) - learner.load(model_path) - mAPbbox, mAPbev, mAP3d, mAPaos = learner.eval(dataset, count=2) - - self.assertTrue(mAPbbox[0][0][0] > 1 and mAPbbox[0][0][0] < 95, msg=mAPbbox[0][0][0]) - - del learner - print("Eval", name, "ok", file=sys.stderr) + # def test_fit_iterator(self): + # def test_model(name, config): + # print("Fit iterator", name, "start", file=sys.stderr) + # model_path = os.path.join(self.temp_dir, "test_fit_iterator_" + name) + # dataset = LabeledPointCloudsDatasetIterator( + # self.dataset_path + "/training/velodyne_reduced", + # self.dataset_path + "/training/label_2", + # self.dataset_path + "/training/calib", + # ) + # + # val_dataset = LabeledPointCloudsDatasetIterator( + # self.dataset_path + "/training/velodyne_reduced", + # self.dataset_path + "/training/label_2", + # self.dataset_path + "/training/calib", + # ) + # + # learner = VoxelObjectDetection3DLearner( + # model_config_path=config, device=DEVICE, + # checkpoint_after_iter=90, + # ) + # + # starting_param = list(learner.model.parameters())[0].clone() + # learner.fit( + # dataset, + # val_dataset=val_dataset, + # model_dir=model_path, + # evaluate=False, + # ) + # new_param = list(learner.model.parameters())[0].clone() + # self.assertFalse(torch.equal(starting_param, new_param)) + # + # del learner + # print("Fit iterator", name, "ok", file=sys.stderr) + # + # for name, config in self.car_configs.items(): + # test_model(name, config) - for name, config in self.car_configs.items(): - test_model(name, config) + # def test_eval(self): + # def test_model(name, config): + # print("Eval", name, "start", file=sys.stderr) + # model_path = os.path.join(self.temp_dir, self.download_model_names[name]) + # dataset = KittiDataset(self.dataset_path, self.subsets_path) + # + # learner = VoxelObjectDetection3DLearner(model_config_path=config, device=DEVICE) + # learner.load(model_path) + # mAPbbox, mAPbev, mAP3d, mAPaos = learner.eval(dataset, count=2) + # + # self.assertTrue(mAPbbox[0][0][0] > 1 and mAPbbox[0][0][0] < 95, msg=mAPbbox[0][0][0]) + # + # del learner + # print("Eval", name, "ok", file=sys.stderr) + # + # for name, config in self.car_configs.items(): + # test_model(name, config) # def test_infer(self): # def test_model(name, config): From 693ea960c74ef41b6456b5d2911e9db6c4e9d003 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 10:47:34 +0200 Subject: [PATCH 22/41] Disabled all 3d detection tests --- .../test_object_detection_3d.py | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 31559c6dea..ec3442f67a 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -117,33 +117,33 @@ def tearDownClass(cls): rmdir(os.path.join(cls.temp_dir)) pass - def test_fit(self): - - def test_model(name, config): - print("Fit", name, "start", file=sys.stderr) - model_path = os.path.join(self.temp_dir, "test_fit_" + name) - dataset = KittiDataset(self.dataset_path, self.subsets_path) - - learner = VoxelObjectDetection3DLearner( - model_config_path=config, device=DEVICE, - checkpoint_after_iter=2, - ) - - starting_param = list(learner.model.parameters())[0].clone() - learner.fit( - dataset, - model_dir=model_path, - verbose=True, - evaluate=False, - ) - new_param = list(learner.model.parameters())[0].clone() - self.assertFalse(torch.equal(starting_param, new_param)) - - del learner - print("Fit", name, "ok", file=sys.stderr) - - for name, config in self.car_configs.items(): - test_model(name, config) + # def test_fit(self): + # + # def test_model(name, config): + # print("Fit", name, "start", file=sys.stderr) + # model_path = os.path.join(self.temp_dir, "test_fit_" + name) + # dataset = KittiDataset(self.dataset_path, self.subsets_path) + # + # learner = VoxelObjectDetection3DLearner( + # model_config_path=config, device=DEVICE, + # checkpoint_after_iter=2, + # ) + # + # starting_param = list(learner.model.parameters())[0].clone() + # learner.fit( + # dataset, + # model_dir=model_path, + # verbose=True, + # evaluate=False, + # ) + # new_param = list(learner.model.parameters())[0].clone() + # self.assertFalse(torch.equal(starting_param, new_param)) + # + # del learner + # print("Fit", name, "ok", file=sys.stderr) + # + # for name, config in self.car_configs.items(): + # test_model(name, config) # def test_fit_iterator(self): # def test_model(name, config): From 7e30ee98ccad737d56f45b246e24addeaff36a28 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 11:15:41 +0200 Subject: [PATCH 23/41] Disabled model download - 3d test --- .../voxel_object_detection_3d/test_object_detection_3d.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index ec3442f67a..3763e43a10 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -102,11 +102,9 @@ def setUpClass(cls): ).path print("Dataset downloaded", file=sys.stderr) - - for model_name in cls.download_model_names.values(): - VoxelObjectDetection3DLearner.download( - model_name, cls.temp_dir - ) + # learner = VoxelObjectDetection3DLearner() + # for model_name in cls.download_model_names.values(): + # learner.download(model_name, cls.temp_dir) print("Models downloaded", file=sys.stderr) From 1b09f3e3dae64dd15eb6017684816a815bd648fe Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 11:55:55 +0200 Subject: [PATCH 24/41] Disabled everything? --- .../test_object_detection_3d.py | 112 +++++++++--------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 3763e43a10..cde78b7cf9 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -47,72 +47,72 @@ class TestVoxelObjectDetection3DLearner(unittest.TestCase): def setUpClass(cls): print("\n\n****************************************\nTEST Voxel Object Detection 3D Learner\n" "****************************************") - cls.temp_dir = os.path.join("tests", "sources", "tools", - "perception", "object_detection_3d", - "voxel_object_detection_3d", - "voxel_object_detection_3d_temp") + # cls.temp_dir = os.path.join("tests", "sources", "tools", + # "perception", "object_detection_3d", + # "voxel_object_detection_3d", + # "voxel_object_detection_3d_temp") + # + # cls.config_tanet_car = os.path.join(".", "src", "opendr", "perception", + # "object_detection_3d", + # "voxel_object_detection_3d", + # "second_detector", "configs", "tanet", + # "car", "test_short.proto") + # + # cls.config_tanet_ped_cycle = os.path.join(".", "src", "opendr", "perception", + # "object_detection_3d", + # "voxel_object_detection_3d", + # "second_detector", "configs", "tanet", + # "ped_cycle", + # "test_short.proto") + # + # cls.config_pointpillars_car = os.path.join( + # ".", "src", "opendr", "perception", "object_detection_3d", + # "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", + # "car", "test_short.proto") + # + # cls.config_pointpillars_ped_cycle = os.path.join( + # ".", "src", "opendr", "perception", "object_detection_3d", + # "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", + # "ped_cycle", "test_short.proto") + # + # cls.subsets_path = os.path.join( + # ".", "src", "opendr", "perception", "object_detection_3d", + # "datasets", "nano_kitti_subsets") + # + # cls.download_model_names = { + # "tanet_car": "tanet_car_xyres_16", + # "tanet_ped_cycle": "tanet_ped_cycle_xyres_16", + # "pointpillars_car": "pointpillars_car_xyres_16", + # "pointpillars_ped_cycle": "pointpillars_ped_cycle_xyres_16", + # } + # + # cls.all_configs = { + # "tanet_car": cls.config_tanet_car, + # "tanet_ped_cycle": cls.config_tanet_ped_cycle, + # "pointpillars_car": cls.config_pointpillars_car, + # "pointpillars_ped_cycle": cls.config_pointpillars_ped_cycle, + # } + # cls.car_configs = { + # "tanet_car": cls.config_tanet_car, + # "pointpillars_car": cls.config_pointpillars_car, + # } + # + # cls.dataset_path = KittiDataset.download_nano_kitti( + # cls.temp_dir, True, cls.subsets_path + # ).path - cls.config_tanet_car = os.path.join(".", "src", "opendr", "perception", - "object_detection_3d", - "voxel_object_detection_3d", - "second_detector", "configs", "tanet", - "car", "test_short.proto") - - cls.config_tanet_ped_cycle = os.path.join(".", "src", "opendr", "perception", - "object_detection_3d", - "voxel_object_detection_3d", - "second_detector", "configs", "tanet", - "ped_cycle", - "test_short.proto") - - cls.config_pointpillars_car = os.path.join( - ".", "src", "opendr", "perception", "object_detection_3d", - "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", - "car", "test_short.proto") - - cls.config_pointpillars_ped_cycle = os.path.join( - ".", "src", "opendr", "perception", "object_detection_3d", - "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", - "ped_cycle", "test_short.proto") - - cls.subsets_path = os.path.join( - ".", "src", "opendr", "perception", "object_detection_3d", - "datasets", "nano_kitti_subsets") - - cls.download_model_names = { - "tanet_car": "tanet_car_xyres_16", - "tanet_ped_cycle": "tanet_ped_cycle_xyres_16", - "pointpillars_car": "pointpillars_car_xyres_16", - "pointpillars_ped_cycle": "pointpillars_ped_cycle_xyres_16", - } - - cls.all_configs = { - "tanet_car": cls.config_tanet_car, - "tanet_ped_cycle": cls.config_tanet_ped_cycle, - "pointpillars_car": cls.config_pointpillars_car, - "pointpillars_ped_cycle": cls.config_pointpillars_ped_cycle, - } - cls.car_configs = { - "tanet_car": cls.config_tanet_car, - "pointpillars_car": cls.config_pointpillars_car, - } - - cls.dataset_path = KittiDataset.download_nano_kitti( - cls.temp_dir, True, cls.subsets_path - ).path - - print("Dataset downloaded", file=sys.stderr) + # print("Dataset downloaded", file=sys.stderr) # learner = VoxelObjectDetection3DLearner() # for model_name in cls.download_model_names.values(): # learner.download(model_name, cls.temp_dir) - print("Models downloaded", file=sys.stderr) + # print("Models downloaded", file=sys.stderr) @classmethod def tearDownClass(cls): # Clean up downloaded files - rmdir(os.path.join(cls.temp_dir)) + # rmdir(os.path.join(cls.temp_dir)) pass # def test_fit(self): From 1aeb2754a29e55631bdb8af1762be667eec95325 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 12:15:00 +0200 Subject: [PATCH 25/41] Removed imports from 3d object detection test --- .../test_object_detection_3d.py | 538 +++++++++--------- 1 file changed, 269 insertions(+), 269 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index cde78b7cf9..89851cbec0 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -17,277 +17,277 @@ import shutil import os import torch -from opendr.engine.datasets import PointCloudsDatasetIterator +# from opendr.engine.datasets import PointCloudsDatasetIterator from opendr.perception.object_detection_3d import VoxelObjectDetection3DLearner -from opendr.perception.object_detection_3d import KittiDataset, LabeledPointCloudsDatasetIterator +# from opendr.perception.object_detection_3d import KittiDataset, LabeledPointCloudsDatasetIterator - -DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu" - -print("Using device:", DEVICE) -print("Using device:", DEVICE, file=sys.stderr) - - -def rmfile(path): - try: - os.remove(path) - except OSError as e: - print("Error: %s - %s." % (e.filename, e.strerror)) - - -def rmdir(_dir): - try: - shutil.rmtree(_dir) - except OSError as e: - print("Error: %s - %s." % (e.filename, e.strerror)) - - -class TestVoxelObjectDetection3DLearner(unittest.TestCase): - @classmethod - def setUpClass(cls): - print("\n\n****************************************\nTEST Voxel Object Detection 3D Learner\n" - "****************************************") - # cls.temp_dir = os.path.join("tests", "sources", "tools", - # "perception", "object_detection_3d", - # "voxel_object_detection_3d", - # "voxel_object_detection_3d_temp") - # - # cls.config_tanet_car = os.path.join(".", "src", "opendr", "perception", - # "object_detection_3d", - # "voxel_object_detection_3d", - # "second_detector", "configs", "tanet", - # "car", "test_short.proto") - # - # cls.config_tanet_ped_cycle = os.path.join(".", "src", "opendr", "perception", - # "object_detection_3d", - # "voxel_object_detection_3d", - # "second_detector", "configs", "tanet", - # "ped_cycle", - # "test_short.proto") - # - # cls.config_pointpillars_car = os.path.join( - # ".", "src", "opendr", "perception", "object_detection_3d", - # "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", - # "car", "test_short.proto") - # - # cls.config_pointpillars_ped_cycle = os.path.join( - # ".", "src", "opendr", "perception", "object_detection_3d", - # "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", - # "ped_cycle", "test_short.proto") - # - # cls.subsets_path = os.path.join( - # ".", "src", "opendr", "perception", "object_detection_3d", - # "datasets", "nano_kitti_subsets") - # - # cls.download_model_names = { - # "tanet_car": "tanet_car_xyres_16", - # "tanet_ped_cycle": "tanet_ped_cycle_xyres_16", - # "pointpillars_car": "pointpillars_car_xyres_16", - # "pointpillars_ped_cycle": "pointpillars_ped_cycle_xyres_16", - # } - # - # cls.all_configs = { - # "tanet_car": cls.config_tanet_car, - # "tanet_ped_cycle": cls.config_tanet_ped_cycle, - # "pointpillars_car": cls.config_pointpillars_car, - # "pointpillars_ped_cycle": cls.config_pointpillars_ped_cycle, - # } - # cls.car_configs = { - # "tanet_car": cls.config_tanet_car, - # "pointpillars_car": cls.config_pointpillars_car, - # } - # - # cls.dataset_path = KittiDataset.download_nano_kitti( - # cls.temp_dir, True, cls.subsets_path - # ).path - - # print("Dataset downloaded", file=sys.stderr) - # learner = VoxelObjectDetection3DLearner() - # for model_name in cls.download_model_names.values(): - # learner.download(model_name, cls.temp_dir) - - # print("Models downloaded", file=sys.stderr) - - @classmethod - def tearDownClass(cls): - # Clean up downloaded files - - # rmdir(os.path.join(cls.temp_dir)) - pass - - # def test_fit(self): - # - # def test_model(name, config): - # print("Fit", name, "start", file=sys.stderr) - # model_path = os.path.join(self.temp_dir, "test_fit_" + name) - # dataset = KittiDataset(self.dataset_path, self.subsets_path) - # - # learner = VoxelObjectDetection3DLearner( - # model_config_path=config, device=DEVICE, - # checkpoint_after_iter=2, - # ) - # - # starting_param = list(learner.model.parameters())[0].clone() - # learner.fit( - # dataset, - # model_dir=model_path, - # verbose=True, - # evaluate=False, - # ) - # new_param = list(learner.model.parameters())[0].clone() - # self.assertFalse(torch.equal(starting_param, new_param)) - # - # del learner - # print("Fit", name, "ok", file=sys.stderr) - # - # for name, config in self.car_configs.items(): - # test_model(name, config) - - # def test_fit_iterator(self): - # def test_model(name, config): - # print("Fit iterator", name, "start", file=sys.stderr) - # model_path = os.path.join(self.temp_dir, "test_fit_iterator_" + name) - # dataset = LabeledPointCloudsDatasetIterator( - # self.dataset_path + "/training/velodyne_reduced", - # self.dataset_path + "/training/label_2", - # self.dataset_path + "/training/calib", - # ) - # - # val_dataset = LabeledPointCloudsDatasetIterator( - # self.dataset_path + "/training/velodyne_reduced", - # self.dataset_path + "/training/label_2", - # self.dataset_path + "/training/calib", - # ) - # - # learner = VoxelObjectDetection3DLearner( - # model_config_path=config, device=DEVICE, - # checkpoint_after_iter=90, - # ) - # - # starting_param = list(learner.model.parameters())[0].clone() - # learner.fit( - # dataset, - # val_dataset=val_dataset, - # model_dir=model_path, - # evaluate=False, - # ) - # new_param = list(learner.model.parameters())[0].clone() - # self.assertFalse(torch.equal(starting_param, new_param)) - # - # del learner - # print("Fit iterator", name, "ok", file=sys.stderr) - # - # for name, config in self.car_configs.items(): - # test_model(name, config) - - # def test_eval(self): - # def test_model(name, config): - # print("Eval", name, "start", file=sys.stderr) - # model_path = os.path.join(self.temp_dir, self.download_model_names[name]) - # dataset = KittiDataset(self.dataset_path, self.subsets_path) - # - # learner = VoxelObjectDetection3DLearner(model_config_path=config, device=DEVICE) - # learner.load(model_path) - # mAPbbox, mAPbev, mAP3d, mAPaos = learner.eval(dataset, count=2) - # - # self.assertTrue(mAPbbox[0][0][0] > 1 and mAPbbox[0][0][0] < 95, msg=mAPbbox[0][0][0]) - # - # del learner - # print("Eval", name, "ok", file=sys.stderr) - # - # for name, config in self.car_configs.items(): - # test_model(name, config) - - # def test_infer(self): - # def test_model(name, config): - # print("Infer", name, "start", file=sys.stderr) - # - # dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") - # - # learner = VoxelObjectDetection3DLearner( - # model_config_path=config, device=DEVICE - # ) - # - # result = learner.infer( - # dataset[0] - # ) - # - # self.assertTrue(len(result) > 0) - # - # result = learner.infer( - # [dataset[0], dataset[1], dataset[2]] - # ) - # self.assertTrue(len(result) == 3) - # self.assertTrue(len(result[0]) > 0) - # - # del learner - # print("Infer", name, "ok", file=sys.stderr) - # - # for name, config in self.car_configs.items(): - # test_model(name, config) - - # def test_save(self): - # def test_model(name, config): - # print("Save", name, "start", file=sys.stderr) - # model_path = os.path.join(self.temp_dir, "test_save_" + name) - # save_path = os.path.join(model_path, "save") - # - # learner = VoxelObjectDetection3DLearner( - # model_config_path=config, device=DEVICE - # ) - # learner.save(save_path, True) - # starting_param_1 = list(learner.model.parameters())[0].clone() - # - # learner2 = VoxelObjectDetection3DLearner( - # model_config_path=config, device=DEVICE - # ) - # starting_param_2 = list(learner2.model.parameters())[0].clone() - # learner2.load(save_path) - # - # new_param = list(learner2.model.parameters())[0].clone() - # self.assertFalse(torch.equal(starting_param_1, starting_param_2)) - # self.assertTrue(torch.equal(starting_param_1, new_param)) - # - # del learner - # del learner2 - # print("Save", name, "ok", file=sys.stderr) - # - # for name, config in self.car_configs.items(): - # test_model(name, config) - # - # def test_optimize(self): - # def test_model(name, config): - # print("Optimize", name, "start", file=sys.stderr) - # model_path = os.path.join(self.temp_dir, "test_optimize_" + name) - # - # dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") - # - # learner = VoxelObjectDetection3DLearner( - # model_config_path=config, device=DEVICE, - # temp_path=self.temp_dir - # ) - # learner.optimize() - # - # result = learner.infer( - # dataset[0] - # ) - # self.assertTrue(len(result) > 0) - # - # learner.save(model_path) - # - # learner2 = VoxelObjectDetection3DLearner( - # model_config_path=config, device=DEVICE - # ) - # learner2.load(model_path, True) - # - # self.assertTrue(learner2.model.rpn_ort_session is not None) - # - # del learner - # del learner2 - # print("Optimize", name, "ok", file=sys.stderr) - # - # for name, config in self.car_configs.items(): - # test_model(name, config) +# +# DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu" +# +# print("Using device:", DEVICE) +# print("Using device:", DEVICE, file=sys.stderr) +# +# +# def rmfile(path): +# try: +# os.remove(path) +# except OSError as e: +# print("Error: %s - %s." % (e.filename, e.strerror)) +# +# +# def rmdir(_dir): +# try: +# shutil.rmtree(_dir) +# except OSError as e: +# print("Error: %s - %s." % (e.filename, e.strerror)) +# +# +# class TestVoxelObjectDetection3DLearner(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# print("\n\n****************************************\nTEST Voxel Object Detection 3D Learner\n" +# "****************************************") +# # cls.temp_dir = os.path.join("tests", "sources", "tools", +# # "perception", "object_detection_3d", +# # "voxel_object_detection_3d", +# # "voxel_object_detection_3d_temp") +# # +# # cls.config_tanet_car = os.path.join(".", "src", "opendr", "perception", +# # "object_detection_3d", +# # "voxel_object_detection_3d", +# # "second_detector", "configs", "tanet", +# # "car", "test_short.proto") +# # +# # cls.config_tanet_ped_cycle = os.path.join(".", "src", "opendr", "perception", +# # "object_detection_3d", +# # "voxel_object_detection_3d", +# # "second_detector", "configs", "tanet", +# # "ped_cycle", +# # "test_short.proto") +# # +# # cls.config_pointpillars_car = os.path.join( +# # ".", "src", "opendr", "perception", "object_detection_3d", +# # "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", +# # "car", "test_short.proto") +# # +# # cls.config_pointpillars_ped_cycle = os.path.join( +# # ".", "src", "opendr", "perception", "object_detection_3d", +# # "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", +# # "ped_cycle", "test_short.proto") +# # +# # cls.subsets_path = os.path.join( +# # ".", "src", "opendr", "perception", "object_detection_3d", +# # "datasets", "nano_kitti_subsets") +# # +# # cls.download_model_names = { +# # "tanet_car": "tanet_car_xyres_16", +# # "tanet_ped_cycle": "tanet_ped_cycle_xyres_16", +# # "pointpillars_car": "pointpillars_car_xyres_16", +# # "pointpillars_ped_cycle": "pointpillars_ped_cycle_xyres_16", +# # } +# # +# # cls.all_configs = { +# # "tanet_car": cls.config_tanet_car, +# # "tanet_ped_cycle": cls.config_tanet_ped_cycle, +# # "pointpillars_car": cls.config_pointpillars_car, +# # "pointpillars_ped_cycle": cls.config_pointpillars_ped_cycle, +# # } +# # cls.car_configs = { +# # "tanet_car": cls.config_tanet_car, +# # "pointpillars_car": cls.config_pointpillars_car, +# # } +# # +# # cls.dataset_path = KittiDataset.download_nano_kitti( +# # cls.temp_dir, True, cls.subsets_path +# # ).path +# +# # print("Dataset downloaded", file=sys.stderr) +# # learner = VoxelObjectDetection3DLearner() +# # for model_name in cls.download_model_names.values(): +# # learner.download(model_name, cls.temp_dir) +# +# # print("Models downloaded", file=sys.stderr) +# +# @classmethod +# def tearDownClass(cls): +# # Clean up downloaded files +# +# # rmdir(os.path.join(cls.temp_dir)) +# pass +# +# # def test_fit(self): +# # +# # def test_model(name, config): +# # print("Fit", name, "start", file=sys.stderr) +# # model_path = os.path.join(self.temp_dir, "test_fit_" + name) +# # dataset = KittiDataset(self.dataset_path, self.subsets_path) +# # +# # learner = VoxelObjectDetection3DLearner( +# # model_config_path=config, device=DEVICE, +# # checkpoint_after_iter=2, +# # ) +# # +# # starting_param = list(learner.model.parameters())[0].clone() +# # learner.fit( +# # dataset, +# # model_dir=model_path, +# # verbose=True, +# # evaluate=False, +# # ) +# # new_param = list(learner.model.parameters())[0].clone() +# # self.assertFalse(torch.equal(starting_param, new_param)) +# # +# # del learner +# # print("Fit", name, "ok", file=sys.stderr) +# # +# # for name, config in self.car_configs.items(): +# # test_model(name, config) +# +# # def test_fit_iterator(self): +# # def test_model(name, config): +# # print("Fit iterator", name, "start", file=sys.stderr) +# # model_path = os.path.join(self.temp_dir, "test_fit_iterator_" + name) +# # dataset = LabeledPointCloudsDatasetIterator( +# # self.dataset_path + "/training/velodyne_reduced", +# # self.dataset_path + "/training/label_2", +# # self.dataset_path + "/training/calib", +# # ) +# # +# # val_dataset = LabeledPointCloudsDatasetIterator( +# # self.dataset_path + "/training/velodyne_reduced", +# # self.dataset_path + "/training/label_2", +# # self.dataset_path + "/training/calib", +# # ) +# # +# # learner = VoxelObjectDetection3DLearner( +# # model_config_path=config, device=DEVICE, +# # checkpoint_after_iter=90, +# # ) +# # +# # starting_param = list(learner.model.parameters())[0].clone() +# # learner.fit( +# # dataset, +# # val_dataset=val_dataset, +# # model_dir=model_path, +# # evaluate=False, +# # ) +# # new_param = list(learner.model.parameters())[0].clone() +# # self.assertFalse(torch.equal(starting_param, new_param)) +# # +# # del learner +# # print("Fit iterator", name, "ok", file=sys.stderr) +# # +# # for name, config in self.car_configs.items(): +# # test_model(name, config) +# +# # def test_eval(self): +# # def test_model(name, config): +# # print("Eval", name, "start", file=sys.stderr) +# # model_path = os.path.join(self.temp_dir, self.download_model_names[name]) +# # dataset = KittiDataset(self.dataset_path, self.subsets_path) +# # +# # learner = VoxelObjectDetection3DLearner(model_config_path=config, device=DEVICE) +# # learner.load(model_path) +# # mAPbbox, mAPbev, mAP3d, mAPaos = learner.eval(dataset, count=2) +# # +# # self.assertTrue(mAPbbox[0][0][0] > 1 and mAPbbox[0][0][0] < 95, msg=mAPbbox[0][0][0]) +# # +# # del learner +# # print("Eval", name, "ok", file=sys.stderr) +# # +# # for name, config in self.car_configs.items(): +# # test_model(name, config) +# +# # def test_infer(self): +# # def test_model(name, config): +# # print("Infer", name, "start", file=sys.stderr) +# # +# # dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") +# # +# # learner = VoxelObjectDetection3DLearner( +# # model_config_path=config, device=DEVICE +# # ) +# # +# # result = learner.infer( +# # dataset[0] +# # ) +# # +# # self.assertTrue(len(result) > 0) +# # +# # result = learner.infer( +# # [dataset[0], dataset[1], dataset[2]] +# # ) +# # self.assertTrue(len(result) == 3) +# # self.assertTrue(len(result[0]) > 0) +# # +# # del learner +# # print("Infer", name, "ok", file=sys.stderr) +# # +# # for name, config in self.car_configs.items(): +# # test_model(name, config) +# +# # def test_save(self): +# # def test_model(name, config): +# # print("Save", name, "start", file=sys.stderr) +# # model_path = os.path.join(self.temp_dir, "test_save_" + name) +# # save_path = os.path.join(model_path, "save") +# # +# # learner = VoxelObjectDetection3DLearner( +# # model_config_path=config, device=DEVICE +# # ) +# # learner.save(save_path, True) +# # starting_param_1 = list(learner.model.parameters())[0].clone() +# # +# # learner2 = VoxelObjectDetection3DLearner( +# # model_config_path=config, device=DEVICE +# # ) +# # starting_param_2 = list(learner2.model.parameters())[0].clone() +# # learner2.load(save_path) +# # +# # new_param = list(learner2.model.parameters())[0].clone() +# # self.assertFalse(torch.equal(starting_param_1, starting_param_2)) +# # self.assertTrue(torch.equal(starting_param_1, new_param)) +# # +# # del learner +# # del learner2 +# # print("Save", name, "ok", file=sys.stderr) +# # +# # for name, config in self.car_configs.items(): +# # test_model(name, config) +# # +# # def test_optimize(self): +# # def test_model(name, config): +# # print("Optimize", name, "start", file=sys.stderr) +# # model_path = os.path.join(self.temp_dir, "test_optimize_" + name) +# # +# # dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") +# # +# # learner = VoxelObjectDetection3DLearner( +# # model_config_path=config, device=DEVICE, +# # temp_path=self.temp_dir +# # ) +# # learner.optimize() +# # +# # result = learner.infer( +# # dataset[0] +# # ) +# # self.assertTrue(len(result) > 0) +# # +# # learner.save(model_path) +# # +# # learner2 = VoxelObjectDetection3DLearner( +# # model_config_path=config, device=DEVICE +# # ) +# # learner2.load(model_path, True) +# # +# # self.assertTrue(learner2.model.rpn_ort_session is not None) +# # +# # del learner +# # del learner2 +# # print("Optimize", name, "ok", file=sys.stderr) +# # +# # for name, config in self.car_configs.items(): +# # test_model(name, config) if __name__ == "__main__": From 52a993b65b80843965c4974d86b5c9d65cc70577 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 12:32:10 +0200 Subject: [PATCH 26/41] Import test --- .../test_object_detection_3d.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 89851cbec0..775ab7a875 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -18,9 +18,34 @@ import os import torch # from opendr.engine.datasets import PointCloudsDatasetIterator -from opendr.perception.object_detection_3d import VoxelObjectDetection3DLearner +# from opendr.perception.object_detection_3d import VoxelObjectDetection3DLearner # from opendr.perception.object_detection_3d import KittiDataset, LabeledPointCloudsDatasetIterator +import onnxruntime as ort +from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.load import ( + create_model as second_create_model, + load_from_checkpoint, +) +from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.run import ( + compute_lidar_kitti_output, evaluate, example_convert_to_torch, train +) +from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.builder import ( + input_reader_builder, ) +from opendr.perception.object_detection_3d.voxel_object_detection_3d.logger import ( + Logger, ) +from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.models.tanet import ( + set_tanet_config +) +from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.data.preprocess import ( + _prep_v9, _prep_v9_infer +) +from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.builder.dataset_builder import ( + create_prep_func +) +from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.data.preprocess import ( + merge_second_batch, +) + # # DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu" # From d28155c868e6edf70a2874a2978c7c701f0e4a2c Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 12:50:24 +0200 Subject: [PATCH 27/41] Import test --- .../test_object_detection_3d.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 775ab7a875..a9264452eb 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -14,7 +14,7 @@ import sys import unittest -import shutil +import shutilge import os import torch # from opendr.engine.datasets import PointCloudsDatasetIterator @@ -26,26 +26,28 @@ create_model as second_create_model, load_from_checkpoint, ) -from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.run import ( - compute_lidar_kitti_output, evaluate, example_convert_to_torch, train -) -from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.builder import ( - input_reader_builder, ) -from opendr.perception.object_detection_3d.voxel_object_detection_3d.logger import ( - Logger, ) -from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.models.tanet import ( - set_tanet_config -) -from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.data.preprocess import ( - _prep_v9, _prep_v9_infer -) -from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.builder.dataset_builder import ( - create_prep_func -) -from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.data.preprocess import ( - merge_second_batch, +# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.run import ( +# compute_lidar_kitti_output, evaluate, example_convert_to_torch, train +# ) +# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.models.tanet import ( +# set_tanet_config ) +# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.builder import ( +# input_reader_builder, ) +# from opendr.perception.object_detection_3d.voxel_object_detection_3d.logger import ( +# Logger, ) + +# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.data.preprocess import ( +# _prep_v9, _prep_v9_infer +# ) +# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.builder.dataset_builder import ( +# create_prep_func +# ) +# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.data.preprocess import ( +# merge_second_batch, +# ) + # # DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu" # From dd2c7784987d36a40f9912ea7bae8d5c3d2e98a9 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 12:50:44 +0200 Subject: [PATCH 28/41] Import test --- .../voxel_object_detection_3d/test_object_detection_3d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index a9264452eb..1cd52ece03 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -14,7 +14,7 @@ import sys import unittest -import shutilge +import shutil import os import torch # from opendr.engine.datasets import PointCloudsDatasetIterator From 68dd58df61a4f1315720a831a0a5ba28048178c4 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 12:50:58 +0200 Subject: [PATCH 29/41] Import test --- .../voxel_object_detection_3d/test_object_detection_3d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 1cd52ece03..452770201c 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -31,7 +31,7 @@ # ) # from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.models.tanet import ( # set_tanet_config -) +# ) # from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.builder import ( # input_reader_builder, ) From 51f9b0405be4121a002e9876b13abda6954dd8b7 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 13:22:36 +0200 Subject: [PATCH 30/41] Import test --- .../test_object_detection_3d.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 452770201c..61b253b130 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -22,10 +22,11 @@ # from opendr.perception.object_detection_3d import KittiDataset, LabeledPointCloudsDatasetIterator import onnxruntime as ort -from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.load import ( - create_model as second_create_model, - load_from_checkpoint, -) + +# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.load import ( +# create_model as second_create_model, +# load_from_checkpoint, +# ) # from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.run import ( # compute_lidar_kitti_output, evaluate, example_convert_to_torch, train # ) From 0a511beadfff8244b1c8bea70a85ceeab67f3346 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 14:37:10 +0200 Subject: [PATCH 31/41] Import test --- .../test_object_detection_3d.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index 61b253b130..f14e1f9187 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -21,12 +21,12 @@ # from opendr.perception.object_detection_3d import VoxelObjectDetection3DLearner # from opendr.perception.object_detection_3d import KittiDataset, LabeledPointCloudsDatasetIterator -import onnxruntime as ort +# import onnxruntime as ort -# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.load import ( -# create_model as second_create_model, -# load_from_checkpoint, -# ) +from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.load import ( + create_model as second_create_model, + load_from_checkpoint, +) # from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.run import ( # compute_lidar_kitti_output, evaluate, example_convert_to_torch, train # ) From b8f941ce0fa75fe029e301231e5789bdecc382f6 Mon Sep 17 00:00:00 2001 From: Nikolaos Date: Thu, 20 Jan 2022 15:03:02 +0200 Subject: [PATCH 32/41] Restored test --- .../test_object_detection_3d.py | 570 +++++++++--------- 1 file changed, 272 insertions(+), 298 deletions(-) diff --git a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py index f14e1f9187..2372294671 100644 --- a/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py +++ b/tests/sources/tools/perception/object_detection_3d/voxel_object_detection_3d/test_object_detection_3d.py @@ -17,305 +17,279 @@ import shutil import os import torch -# from opendr.engine.datasets import PointCloudsDatasetIterator -# from opendr.perception.object_detection_3d import VoxelObjectDetection3DLearner -# from opendr.perception.object_detection_3d import KittiDataset, LabeledPointCloudsDatasetIterator - -# import onnxruntime as ort - -from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.load import ( - create_model as second_create_model, - load_from_checkpoint, -) -# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.run import ( -# compute_lidar_kitti_output, evaluate, example_convert_to_torch, train -# ) -# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.models.tanet import ( -# set_tanet_config -# ) - -# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.pytorch.builder import ( -# input_reader_builder, ) -# from opendr.perception.object_detection_3d.voxel_object_detection_3d.logger import ( -# Logger, ) - -# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.data.preprocess import ( -# _prep_v9, _prep_v9_infer -# ) -# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.builder.dataset_builder import ( -# create_prep_func -# ) -# from opendr.perception.object_detection_3d.voxel_object_detection_3d.second_detector.data.preprocess import ( -# merge_second_batch, -# ) +from opendr.engine.datasets import PointCloudsDatasetIterator +from opendr.perception.object_detection_3d import VoxelObjectDetection3DLearner +from opendr.perception.object_detection_3d import KittiDataset, LabeledPointCloudsDatasetIterator -# -# DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu" -# -# print("Using device:", DEVICE) -# print("Using device:", DEVICE, file=sys.stderr) -# -# -# def rmfile(path): -# try: -# os.remove(path) -# except OSError as e: -# print("Error: %s - %s." % (e.filename, e.strerror)) -# -# -# def rmdir(_dir): -# try: -# shutil.rmtree(_dir) -# except OSError as e: -# print("Error: %s - %s." % (e.filename, e.strerror)) -# -# -# class TestVoxelObjectDetection3DLearner(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# print("\n\n****************************************\nTEST Voxel Object Detection 3D Learner\n" -# "****************************************") -# # cls.temp_dir = os.path.join("tests", "sources", "tools", -# # "perception", "object_detection_3d", -# # "voxel_object_detection_3d", -# # "voxel_object_detection_3d_temp") -# # -# # cls.config_tanet_car = os.path.join(".", "src", "opendr", "perception", -# # "object_detection_3d", -# # "voxel_object_detection_3d", -# # "second_detector", "configs", "tanet", -# # "car", "test_short.proto") -# # -# # cls.config_tanet_ped_cycle = os.path.join(".", "src", "opendr", "perception", -# # "object_detection_3d", -# # "voxel_object_detection_3d", -# # "second_detector", "configs", "tanet", -# # "ped_cycle", -# # "test_short.proto") -# # -# # cls.config_pointpillars_car = os.path.join( -# # ".", "src", "opendr", "perception", "object_detection_3d", -# # "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", -# # "car", "test_short.proto") -# # -# # cls.config_pointpillars_ped_cycle = os.path.join( -# # ".", "src", "opendr", "perception", "object_detection_3d", -# # "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", -# # "ped_cycle", "test_short.proto") -# # -# # cls.subsets_path = os.path.join( -# # ".", "src", "opendr", "perception", "object_detection_3d", -# # "datasets", "nano_kitti_subsets") -# # -# # cls.download_model_names = { -# # "tanet_car": "tanet_car_xyres_16", -# # "tanet_ped_cycle": "tanet_ped_cycle_xyres_16", -# # "pointpillars_car": "pointpillars_car_xyres_16", -# # "pointpillars_ped_cycle": "pointpillars_ped_cycle_xyres_16", -# # } -# # -# # cls.all_configs = { -# # "tanet_car": cls.config_tanet_car, -# # "tanet_ped_cycle": cls.config_tanet_ped_cycle, -# # "pointpillars_car": cls.config_pointpillars_car, -# # "pointpillars_ped_cycle": cls.config_pointpillars_ped_cycle, -# # } -# # cls.car_configs = { -# # "tanet_car": cls.config_tanet_car, -# # "pointpillars_car": cls.config_pointpillars_car, -# # } -# # -# # cls.dataset_path = KittiDataset.download_nano_kitti( -# # cls.temp_dir, True, cls.subsets_path -# # ).path -# -# # print("Dataset downloaded", file=sys.stderr) -# # learner = VoxelObjectDetection3DLearner() -# # for model_name in cls.download_model_names.values(): -# # learner.download(model_name, cls.temp_dir) -# -# # print("Models downloaded", file=sys.stderr) -# -# @classmethod -# def tearDownClass(cls): -# # Clean up downloaded files -# -# # rmdir(os.path.join(cls.temp_dir)) -# pass -# -# # def test_fit(self): -# # -# # def test_model(name, config): -# # print("Fit", name, "start", file=sys.stderr) -# # model_path = os.path.join(self.temp_dir, "test_fit_" + name) -# # dataset = KittiDataset(self.dataset_path, self.subsets_path) -# # -# # learner = VoxelObjectDetection3DLearner( -# # model_config_path=config, device=DEVICE, -# # checkpoint_after_iter=2, -# # ) -# # -# # starting_param = list(learner.model.parameters())[0].clone() -# # learner.fit( -# # dataset, -# # model_dir=model_path, -# # verbose=True, -# # evaluate=False, -# # ) -# # new_param = list(learner.model.parameters())[0].clone() -# # self.assertFalse(torch.equal(starting_param, new_param)) -# # -# # del learner -# # print("Fit", name, "ok", file=sys.stderr) -# # -# # for name, config in self.car_configs.items(): -# # test_model(name, config) -# -# # def test_fit_iterator(self): -# # def test_model(name, config): -# # print("Fit iterator", name, "start", file=sys.stderr) -# # model_path = os.path.join(self.temp_dir, "test_fit_iterator_" + name) -# # dataset = LabeledPointCloudsDatasetIterator( -# # self.dataset_path + "/training/velodyne_reduced", -# # self.dataset_path + "/training/label_2", -# # self.dataset_path + "/training/calib", -# # ) -# # -# # val_dataset = LabeledPointCloudsDatasetIterator( -# # self.dataset_path + "/training/velodyne_reduced", -# # self.dataset_path + "/training/label_2", -# # self.dataset_path + "/training/calib", -# # ) -# # -# # learner = VoxelObjectDetection3DLearner( -# # model_config_path=config, device=DEVICE, -# # checkpoint_after_iter=90, -# # ) -# # -# # starting_param = list(learner.model.parameters())[0].clone() -# # learner.fit( -# # dataset, -# # val_dataset=val_dataset, -# # model_dir=model_path, -# # evaluate=False, -# # ) -# # new_param = list(learner.model.parameters())[0].clone() -# # self.assertFalse(torch.equal(starting_param, new_param)) -# # -# # del learner -# # print("Fit iterator", name, "ok", file=sys.stderr) -# # -# # for name, config in self.car_configs.items(): -# # test_model(name, config) -# -# # def test_eval(self): -# # def test_model(name, config): -# # print("Eval", name, "start", file=sys.stderr) -# # model_path = os.path.join(self.temp_dir, self.download_model_names[name]) -# # dataset = KittiDataset(self.dataset_path, self.subsets_path) -# # -# # learner = VoxelObjectDetection3DLearner(model_config_path=config, device=DEVICE) -# # learner.load(model_path) -# # mAPbbox, mAPbev, mAP3d, mAPaos = learner.eval(dataset, count=2) -# # -# # self.assertTrue(mAPbbox[0][0][0] > 1 and mAPbbox[0][0][0] < 95, msg=mAPbbox[0][0][0]) -# # -# # del learner -# # print("Eval", name, "ok", file=sys.stderr) -# # -# # for name, config in self.car_configs.items(): -# # test_model(name, config) -# -# # def test_infer(self): -# # def test_model(name, config): -# # print("Infer", name, "start", file=sys.stderr) -# # -# # dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") -# # -# # learner = VoxelObjectDetection3DLearner( -# # model_config_path=config, device=DEVICE -# # ) -# # -# # result = learner.infer( -# # dataset[0] -# # ) -# # -# # self.assertTrue(len(result) > 0) -# # -# # result = learner.infer( -# # [dataset[0], dataset[1], dataset[2]] -# # ) -# # self.assertTrue(len(result) == 3) -# # self.assertTrue(len(result[0]) > 0) -# # -# # del learner -# # print("Infer", name, "ok", file=sys.stderr) -# # -# # for name, config in self.car_configs.items(): -# # test_model(name, config) -# -# # def test_save(self): -# # def test_model(name, config): -# # print("Save", name, "start", file=sys.stderr) -# # model_path = os.path.join(self.temp_dir, "test_save_" + name) -# # save_path = os.path.join(model_path, "save") -# # -# # learner = VoxelObjectDetection3DLearner( -# # model_config_path=config, device=DEVICE -# # ) -# # learner.save(save_path, True) -# # starting_param_1 = list(learner.model.parameters())[0].clone() -# # -# # learner2 = VoxelObjectDetection3DLearner( -# # model_config_path=config, device=DEVICE -# # ) -# # starting_param_2 = list(learner2.model.parameters())[0].clone() -# # learner2.load(save_path) -# # -# # new_param = list(learner2.model.parameters())[0].clone() -# # self.assertFalse(torch.equal(starting_param_1, starting_param_2)) -# # self.assertTrue(torch.equal(starting_param_1, new_param)) -# # -# # del learner -# # del learner2 -# # print("Save", name, "ok", file=sys.stderr) -# # -# # for name, config in self.car_configs.items(): -# # test_model(name, config) -# # -# # def test_optimize(self): -# # def test_model(name, config): -# # print("Optimize", name, "start", file=sys.stderr) -# # model_path = os.path.join(self.temp_dir, "test_optimize_" + name) -# # -# # dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") -# # -# # learner = VoxelObjectDetection3DLearner( -# # model_config_path=config, device=DEVICE, -# # temp_path=self.temp_dir -# # ) -# # learner.optimize() -# # -# # result = learner.infer( -# # dataset[0] -# # ) -# # self.assertTrue(len(result) > 0) -# # -# # learner.save(model_path) -# # -# # learner2 = VoxelObjectDetection3DLearner( -# # model_config_path=config, device=DEVICE -# # ) -# # learner2.load(model_path, True) -# # -# # self.assertTrue(learner2.model.rpn_ort_session is not None) -# # -# # del learner -# # del learner2 -# # print("Optimize", name, "ok", file=sys.stderr) -# # -# # for name, config in self.car_configs.items(): -# # test_model(name, config) + +DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu" + +print("Using device:", DEVICE) +print("Using device:", DEVICE, file=sys.stderr) + + +def rmfile(path): + try: + os.remove(path) + except OSError as e: + print("Error: %s - %s." % (e.filename, e.strerror)) + + +def rmdir(_dir): + try: + shutil.rmtree(_dir) + except OSError as e: + print("Error: %s - %s." % (e.filename, e.strerror)) + + +class TestVoxelObjectDetection3DLearner(unittest.TestCase): + @classmethod + def setUpClass(cls): + print("\n\n****************************************\nTEST Voxel Object Detection 3D Learner\n" + "****************************************") + cls.temp_dir = os.path.join("tests", "sources", "tools", + "perception", "object_detection_3d", + "voxel_object_detection_3d", + "voxel_object_detection_3d_temp") + + cls.config_tanet_car = os.path.join(".", "src", "opendr", "perception", + "object_detection_3d", + "voxel_object_detection_3d", + "second_detector", "configs", "tanet", + "car", "test_short.proto") + + cls.config_tanet_ped_cycle = os.path.join(".", "src", "opendr", "perception", + "object_detection_3d", + "voxel_object_detection_3d", + "second_detector", "configs", "tanet", + "ped_cycle", + "test_short.proto") + + cls.config_pointpillars_car = os.path.join( + ".", "src", "opendr", "perception", "object_detection_3d", + "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", + "car", "test_short.proto") + + cls.config_pointpillars_ped_cycle = os.path.join( + ".", "src", "opendr", "perception", "object_detection_3d", + "voxel_object_detection_3d", "second_detector", "configs", "pointpillars", + "ped_cycle", "test_short.proto") + + cls.subsets_path = os.path.join( + ".", "src", "opendr", "perception", "object_detection_3d", + "datasets", "nano_kitti_subsets") + + cls.download_model_names = { + "tanet_car": "tanet_car_xyres_16", + "tanet_ped_cycle": "tanet_ped_cycle_xyres_16", + "pointpillars_car": "pointpillars_car_xyres_16", + "pointpillars_ped_cycle": "pointpillars_ped_cycle_xyres_16", + } + + cls.all_configs = { + "tanet_car": cls.config_tanet_car, + "tanet_ped_cycle": cls.config_tanet_ped_cycle, + "pointpillars_car": cls.config_pointpillars_car, + "pointpillars_ped_cycle": cls.config_pointpillars_ped_cycle, + } + cls.car_configs = { + "tanet_car": cls.config_tanet_car, + "pointpillars_car": cls.config_pointpillars_car, + } + + cls.dataset_path = KittiDataset.download_nano_kitti( + cls.temp_dir, True, cls.subsets_path + ).path + + print("Dataset downloaded", file=sys.stderr) + + for model_name in cls.download_model_names.values(): + VoxelObjectDetection3DLearner.download( + model_name, cls.temp_dir + ) + + print("Models downloaded", file=sys.stderr) + + @classmethod + def tearDownClass(cls): + # Clean up downloaded files + + rmdir(os.path.join(cls.temp_dir)) + pass + + def test_fit(self): + + def test_model(name, config): + print("Fit", name, "start", file=sys.stderr) + model_path = os.path.join(self.temp_dir, "test_fit_" + name) + dataset = KittiDataset(self.dataset_path, self.subsets_path) + + learner = VoxelObjectDetection3DLearner( + model_config_path=config, device=DEVICE, + checkpoint_after_iter=2, + ) + + starting_param = list(learner.model.parameters())[0].clone() + learner.fit( + dataset, + model_dir=model_path, + verbose=True, + evaluate=False, + ) + new_param = list(learner.model.parameters())[0].clone() + self.assertFalse(torch.equal(starting_param, new_param)) + + del learner + print("Fit", name, "ok", file=sys.stderr) + + for name, config in self.car_configs.items(): + test_model(name, config) + + def test_fit_iterator(self): + def test_model(name, config): + print("Fit iterator", name, "start", file=sys.stderr) + model_path = os.path.join(self.temp_dir, "test_fit_iterator_" + name) + dataset = LabeledPointCloudsDatasetIterator( + self.dataset_path + "/training/velodyne_reduced", + self.dataset_path + "/training/label_2", + self.dataset_path + "/training/calib", + ) + + val_dataset = LabeledPointCloudsDatasetIterator( + self.dataset_path + "/training/velodyne_reduced", + self.dataset_path + "/training/label_2", + self.dataset_path + "/training/calib", + ) + + learner = VoxelObjectDetection3DLearner( + model_config_path=config, device=DEVICE, + checkpoint_after_iter=90, + ) + + starting_param = list(learner.model.parameters())[0].clone() + learner.fit( + dataset, + val_dataset=val_dataset, + model_dir=model_path, + evaluate=False, + ) + new_param = list(learner.model.parameters())[0].clone() + self.assertFalse(torch.equal(starting_param, new_param)) + + del learner + print("Fit iterator", name, "ok", file=sys.stderr) + + for name, config in self.car_configs.items(): + test_model(name, config) + + def test_eval(self): + def test_model(name, config): + print("Eval", name, "start", file=sys.stderr) + model_path = os.path.join(self.temp_dir, self.download_model_names[name]) + dataset = KittiDataset(self.dataset_path, self.subsets_path) + + learner = VoxelObjectDetection3DLearner(model_config_path=config, device=DEVICE) + learner.load(model_path) + mAPbbox, mAPbev, mAP3d, mAPaos = learner.eval(dataset, count=2) + + self.assertTrue(mAPbbox[0][0][0] > 1 and mAPbbox[0][0][0] < 95, msg=mAPbbox[0][0][0]) + + del learner + print("Eval", name, "ok", file=sys.stderr) + + for name, config in self.car_configs.items(): + test_model(name, config) + + def test_infer(self): + def test_model(name, config): + print("Infer", name, "start", file=sys.stderr) + + dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") + + learner = VoxelObjectDetection3DLearner( + model_config_path=config, device=DEVICE + ) + + result = learner.infer( + dataset[0] + ) + + self.assertTrue(len(result) > 0) + + result = learner.infer( + [dataset[0], dataset[1], dataset[2]] + ) + self.assertTrue(len(result) == 3) + self.assertTrue(len(result[0]) > 0) + + del learner + print("Infer", name, "ok", file=sys.stderr) + + for name, config in self.car_configs.items(): + test_model(name, config) + + def test_save(self): + def test_model(name, config): + print("Save", name, "start", file=sys.stderr) + model_path = os.path.join(self.temp_dir, "test_save_" + name) + save_path = os.path.join(model_path, "save") + + learner = VoxelObjectDetection3DLearner( + model_config_path=config, device=DEVICE + ) + learner.save(save_path, True) + starting_param_1 = list(learner.model.parameters())[0].clone() + + learner2 = VoxelObjectDetection3DLearner( + model_config_path=config, device=DEVICE + ) + starting_param_2 = list(learner2.model.parameters())[0].clone() + learner2.load(save_path) + + new_param = list(learner2.model.parameters())[0].clone() + self.assertFalse(torch.equal(starting_param_1, starting_param_2)) + self.assertTrue(torch.equal(starting_param_1, new_param)) + + del learner + del learner2 + print("Save", name, "ok", file=sys.stderr) + + for name, config in self.car_configs.items(): + test_model(name, config) + + def test_optimize(self): + def test_model(name, config): + print("Optimize", name, "start", file=sys.stderr) + model_path = os.path.join(self.temp_dir, "test_optimize_" + name) + + dataset = PointCloudsDatasetIterator(self.dataset_path + "/testing/velodyne_reduced") + + learner = VoxelObjectDetection3DLearner( + model_config_path=config, device=DEVICE, + temp_path=self.temp_dir + ) + learner.optimize() + + result = learner.infer( + dataset[0] + ) + self.assertTrue(len(result) > 0) + + learner.save(model_path) + + learner2 = VoxelObjectDetection3DLearner( + model_config_path=config, device=DEVICE + ) + learner2.load(model_path, True) + + self.assertTrue(learner2.model.rpn_ort_session is not None) + + del learner + del learner2 + print("Optimize", name, "ok", file=sys.stderr) + + for name, config in self.car_configs.items(): + test_model(name, config) if __name__ == "__main__": From a4dd43fda18491f21539b4a4209ff60d6882c4b2 Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Thu, 20 Jan 2022 15:03:59 +0200 Subject: [PATCH 33/41] Disabled object detection 3d test --- .github/workflows/tests_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 62ba1aab77..82fc420ea1 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -121,13 +121,13 @@ jobs: - perception/heart_anomaly_detection - perception/multimodal_human_centric - perception/object_tracking_2d - - perception/object_detection_3d - perception/pose_estimation - perception/speech_recognition - perception/skeleton_based_action_recognition - perception/semantic_segmentation - perception/object_detection_2d - perception/facial_expression_recognition + # - perception/object_detection_3d # - control/mobile_manipulation # - simulation/human_model_generation # - control/single_demo_grasp From fca7395f52efe997705be8d0e00b4904d83d394e Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Fri, 21 Jan 2022 08:18:19 +0100 Subject: [PATCH 34/41] Refactor workflow --- .github/workflows/test_packages.yml | 105 ++++++++++++++++++++++++++++ .github/workflows/tests_suite.yml | 48 +------------ 2 files changed, 106 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/test_packages.yml diff --git a/.github/workflows/test_packages.yml b/.github/workflows/test_packages.yml new file mode 100644 index 0000000000..b8f9ba04e3 --- /dev/null +++ b/.github/workflows/test_packages.yml @@ -0,0 +1,105 @@ +name: Test Packages +# This workflow tests the latest (third-party hosted) available builds + +on: + pull_request: + types: [opened, reopened, synchronize, labeled, unlabeled] + schedule: + - cron: '0 23 * * *' + +defaults: + run: + shell: bash + + +jobs: + cleanup-runs: + if: ${{ contains(github.event.pull_request.labels.*.name, 'test packages') || github.event_name == 'schedule' }} + runs-on: ubuntu-latest + steps: + - uses: rokroskar/workflow-run-cleanup-action@master + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + test-wheel: + needs: cleanup-runs + if: ${{ contains(github.event.pull_request.labels.*.name, 'test packages') || github.event_name == 'schedule' }} + strategy: + matrix: + os: [ubuntu-20.04] + package: + - engine + - utils + - perception/activity_recognition + - perception/compressive_learning + - perception/face_recognition + - perception/heart_anomaly_detection + - perception/multimodal_human_centric + - perception/object_tracking_2d + - perception/pose_estimation + - perception/speech_recognition + - perception/skeleton_based_action_recognition + - perception/semantic_segmentation + - perception/object_detection_2d + - perception/facial_expression_recognition + # - perception/object_detection_3d + # - control/mobile_manipulation + # - simulation/human_model_generation + # - control/single_demo_grasp + # - perception/object_tracking_3d + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Test Wheel + run: | + export DISABLE_BCOLZ_AVX2=true + sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev python3-dev + python3 -m venv venv + source venv/bin/activate + wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt + cat pip_requirements.txt | xargs -n 1 -L 1 pip install + # Test new package + wget https://raw.githubusercontent.com/passalis/test_repo/main/opendr-toolkit-1.0.tar.gz + pip install opendr-toolkit-1.0.tar.gz + # pip install opendr-toolkit + python -m unittest discover -s tests/sources/tools/${{ matrix.package }} + test-docker: + needs: cleanup-runs + if: ${{ contains(github.event.pull_request.labels.*.name, 'test packages') || github.event_name == 'schedule' }} + strategy: + matrix: + os: [ubuntu-20.04] + package: + - engine + - utils + - perception/activity_recognition + - perception/compressive_learning + - perception/face_recognition + - perception/heart_anomaly_detection + - perception/multimodal_human_centric + - perception/object_tracking_2d + - perception/pose_estimation + - perception/speech_recognition + - perception/skeleton_based_action_recognition + - perception/semantic_segmentation + - perception/object_detection_2d + - perception/facial_expression_recognition + - perception/object_detection_3d + - control/mobile_manipulation + - simulation/human_model_generation + - control/single_demo_grasp + - perception/object_tracking_3d + runs-on: ${{ matrix.os }} + steps: + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Test Docker + run: | + docker run --name toolkit -i opendr/opendr-toolkit:cpu_latest bash + docker start toolkit + docker exec -i toolkit bash -c "source bin/activate.sh && python -m unittest discover -s tests/sources/tools/${{ matrix.package }}" diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 82fc420ea1..57c03eff60 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -106,50 +106,4 @@ jobs: source tests/sources/tools/control/mobile_manipulation/run_ros.sh python -m unittest discover -s tests/sources/tools/${{ matrix.package }} fi - test-install: - needs: cleanup-runs - if: ${{ contains(github.event.pull_request.labels.*.name, 'test install') || github.event_name == 'schedule' }} - strategy: - matrix: - os: [ubuntu-20.04] - package: - - engine - - utils - - perception/activity_recognition - - perception/compressive_learning - - perception/face_recognition - - perception/heart_anomaly_detection - - perception/multimodal_human_centric - - perception/object_tracking_2d - - perception/pose_estimation - - perception/speech_recognition - - perception/skeleton_based_action_recognition - - perception/semantic_segmentation - - perception/object_detection_2d - - perception/facial_expression_recognition - # - perception/object_detection_3d - # - control/mobile_manipulation - # - simulation/human_model_generation - # - control/single_demo_grasp - # - perception/object_tracking_3d - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Test Install - run: | - export DISABLE_BCOLZ_AVX2=true - sudo apt -y install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev python3-dev - python3 -m venv venv - source venv/bin/activate - wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt - cat pip_requirements.txt | xargs -n 1 -L 1 pip install - pip uninstall pkg-resources==0.0.0 - # Test new package - wget https://raw.githubusercontent.com/passalis/test_repo/main/opendr-toolkit-1.0.tar.gz - pip install opendr-toolkit-1.0.tar.gz - #pip install opendr-toolkit - python -m unittest discover -s tests/sources/tools/${{ matrix.package }} + From c877aedcb6d3d9d51fdbed5f0c834ba46758112a Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Fri, 21 Jan 2022 08:26:33 +0100 Subject: [PATCH 35/41] disable temporarely --- .github/workflows/test_packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_packages.yml b/.github/workflows/test_packages.yml index b8f9ba04e3..f15f7e0e76 100644 --- a/.github/workflows/test_packages.yml +++ b/.github/workflows/test_packages.yml @@ -88,10 +88,10 @@ jobs: - perception/object_detection_2d - perception/facial_expression_recognition - perception/object_detection_3d - - control/mobile_manipulation + #- control/mobile_manipulation - simulation/human_model_generation - control/single_demo_grasp - - perception/object_tracking_3d + #- perception/object_tracking_3d runs-on: ${{ matrix.os }} steps: - name: Set up Python 3.8 From bee89755a5a32048d562a92f71fcae8e502f3a7d Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Fri, 21 Jan 2022 08:54:11 +0100 Subject: [PATCH 36/41] Fix docker mobile manipulation --- .github/workflows/test_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_packages.yml b/.github/workflows/test_packages.yml index f15f7e0e76..3ab05686cd 100644 --- a/.github/workflows/test_packages.yml +++ b/.github/workflows/test_packages.yml @@ -102,4 +102,4 @@ jobs: run: | docker run --name toolkit -i opendr/opendr-toolkit:cpu_latest bash docker start toolkit - docker exec -i toolkit bash -c "source bin/activate.sh && python -m unittest discover -s tests/sources/tools/${{ matrix.package }}" + docker exec -i toolkit bash -c "source bin/activate.sh && source tests/sources/tools/control/mobile_manipulation/run_ros.sh && python -m unittest discover -s tests/sources/tools/${{ matrix.package }}" From 073d2f1e75986e43afaa6e2bd1a2fa75d18dd9b6 Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Fri, 21 Jan 2022 08:57:08 +0100 Subject: [PATCH 37/41] Re-enable test --- .github/workflows/test_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_packages.yml b/.github/workflows/test_packages.yml index 3ab05686cd..c7aad3e99b 100644 --- a/.github/workflows/test_packages.yml +++ b/.github/workflows/test_packages.yml @@ -88,7 +88,7 @@ jobs: - perception/object_detection_2d - perception/facial_expression_recognition - perception/object_detection_3d - #- control/mobile_manipulation + - control/mobile_manipulation - simulation/human_model_generation - control/single_demo_grasp #- perception/object_tracking_3d From 2b39395690aaae47213d006197cf94ebe982d720 Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Fri, 21 Jan 2022 09:00:11 +0100 Subject: [PATCH 38/41] add badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e91b500193..b3102f0580 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ ______________________________________________________________________ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Test Suite (master)](https://github.com/opendr-eu/opendr/actions/workflows/tests_suite.yml/badge.svg)](https://github.com/opendr-eu/opendr/actions/workflows/tests_suite.yml) +[![Test Packages](https://github.com/opendr-eu/opendr/actions/workflows/test_packages.yml/badge.svg)](https://github.com/opendr-eu/opendr/actions/workflows/test_packages.yml) ## About From b28a1fcbd8bb99a1ee99718d4196719ba722a65d Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Fri, 21 Jan 2022 09:02:18 +0100 Subject: [PATCH 39/41] Fix changelog --- CHANGELOG.md | 2 +- MANIFEST.in | 2 +- bin/build_wheel.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8742a39734..7e0bec9ded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Released on XX, XXth, 2022. - Enhancements: - None. - Bug Fixes: - - Updated wheel building pipeline to include missing files and removed unnecessary dependencies. + - Updated wheel building pipeline to include missing files and removed unnecessary dependencies ([#200](https://github.com/opendr-eu/opendr/pull/200)). - Dependency Updates: - `heart anomaly detection`: upgraded scikit-learn runtime dependency from 0.21.3 to 0.22 ([#198](https://github.com/opendr-eu/opendr/pull/198)). diff --git a/MANIFEST.in b/MANIFEST.in index 322d0e3262..2c2972093c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ recursive-include src/opendr * -include requirements.txt \ No newline at end of file +include requirements.txt diff --git a/bin/build_wheel.sh b/bin/build_wheel.sh index 7fbf1d9565..c0564a5d0c 100755 --- a/bin/build_wheel.sh +++ b/bin/build_wheel.sh @@ -16,4 +16,4 @@ sed -i '/pkg_resources/d' requirements.txt sed -i '/auditwheel/d' requirements.txt # Build binary wheel and repair it -python3 setup.py sdist \ No newline at end of file +python3 setup.py sdist From 198c68a13257f36f8cd533aae5c34585ce1575ce Mon Sep 17 00:00:00 2001 From: ad-daniel Date: Fri, 21 Jan 2022 09:25:27 +0100 Subject: [PATCH 40/41] remove test install reference --- .github/workflows/tests_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml index 57c03eff60..82b80e7e96 100644 --- a/.github/workflows/tests_suite.yml +++ b/.github/workflows/tests_suite.yml @@ -12,7 +12,7 @@ defaults: jobs: cleanup-runs: - if: ${{ contains(github.event.pull_request.labels.*.name, 'test sources') || contains(github.event.pull_request.labels.*.name, 'test tools') || contains(github.event.pull_request.labels.*.name, 'test install') || github.event_name == 'schedule' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'test sources') || contains(github.event.pull_request.labels.*.name, 'test tools') || github.event_name == 'schedule' }} runs-on: ubuntu-latest steps: - uses: rokroskar/workflow-run-cleanup-action@master From fee44752184904d2e2d5bf5871662cc31d3f12bd Mon Sep 17 00:00:00 2001 From: Nikolaos Passalis Date: Fri, 21 Jan 2022 11:17:50 +0200 Subject: [PATCH 41/41] Restored pip installation --- .github/workflows/test_packages.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test_packages.yml b/.github/workflows/test_packages.yml index c7aad3e99b..e7a3b718ea 100644 --- a/.github/workflows/test_packages.yml +++ b/.github/workflows/test_packages.yml @@ -62,9 +62,7 @@ jobs: wget https://raw.githubusercontent.com/opendr-eu/opendr/master/dependencies/pip_requirements.txt cat pip_requirements.txt | xargs -n 1 -L 1 pip install # Test new package - wget https://raw.githubusercontent.com/passalis/test_repo/main/opendr-toolkit-1.0.tar.gz - pip install opendr-toolkit-1.0.tar.gz - # pip install opendr-toolkit + pip install opendr-toolkit python -m unittest discover -s tests/sources/tools/${{ matrix.package }} test-docker: needs: cleanup-runs