From dd0cff229807fe0543db9ed71ec792d1ddaa6a48 Mon Sep 17 00:00:00 2001 From: Adrien Cacciaguerra Date: Mon, 15 Apr 2024 18:51:11 -0400 Subject: [PATCH] feat: add CodSpeed to the project --- .github/workflows/benchmarks.yml | 54 ++++++++ .github/workflows/pr_test_cpu.yml | 128 ------------------ .github/workflows/pypi-release.yml | 68 ---------- .github/workflows/scheduled_test_cpu.yml | 117 ---------------- .../workflows/scheduled_test_pypi_package.yml | 30 ---- benchmarks/augmentation/conftest.py | 7 +- requirements/requirements-benchmarks.txt | 2 + 7 files changed, 61 insertions(+), 345 deletions(-) create mode 100644 .github/workflows/benchmarks.yml delete mode 100644 .github/workflows/pr_test_cpu.yml delete mode 100644 .github/workflows/pypi-release.yml delete mode 100644 .github/workflows/scheduled_test_cpu.yml delete mode 100644 .github/workflows/scheduled_test_pypi_package.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..0b60d55d74 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,54 @@ +name: Benchmark + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + benchmark_cpu: + name: CPU Pytest benchmark + runs-on: ubuntu-latest + + steps: + - uses: asottile/workflows/.github/actions/fast-checkout@v1.6.0 + + - name: Setup environment + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install pytorch + shell: bash + run: pip install torch==2.2.0 --index-url https://download.pytorch.org/whl/cpu + + - name: Install Kornia dev + shell: bash + run: pip install .[dev,x] + + - name: Check torch version + shell: bash + run: pip show torch | grep 2.2.0 || false + + - name: Print dependencies and kornia version + shell: bash + run: | + python -c "import torch;print('Pytorch version: ', torch.__version__)" + python -c "import kornia;print('Kornia version: ', kornia.__version__)" + + - name: Install benchmark requirements + run: pip install -r requirements/requirements-benchmarks.txt + + - name: Run benchmarks + uses: CodSpeedHQ/action@v2 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + # run only 4 benchmarks because they take a long time to run for the time being + run: | + pytest -vvv --codspeed -n auto \ + benchmarks/augmentation/2d_geometric_test.py::test_aug_2d_thin_plate_spline \ + benchmarks/augmentation/2d_geometric_test.py::test_aug_2d_elastic_transform \ + benchmarks/augmentation/2d_intensity_test.py::test_aug_2d_linear_corner_illumination \ + benchmarks/augmentation/2d_intensity_test.py::test_aug_2d_solarize diff --git a/.github/workflows/pr_test_cpu.yml b/.github/workflows/pr_test_cpu.yml deleted file mode 100644 index 858091b48a..0000000000 --- a/.github/workflows/pr_test_cpu.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Tests on CPU (PR) - -on: - push: - branches: [test-me-*] - pull_request: - branches: [main] - types: [opened, reopened, synchronize, ready_for_review] - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - - pre-tests: - runs-on: ubuntu-latest - outputs: - hash: ${{ steps.hashid.outputs.weights-hash }} - steps: - - uses: kornia/workflows/.github/actions/env@v1.12.1 - - uses: actions/cache@v4 - id: cache-weights - with: - path: weights/ - key: model-weights-${{ hashFiles('.github/download-models-weights.py') }} - enableCrossOsArchive: true - - - name: Download models weights... - if: steps.cache-weights.outputs.cache-hit != 'true' - run: python .github/download-models-weights.py -t weights/ - - - name: write hashid - id: hashid - run: echo "weights-hash=${{ hashFiles('.github/download-models-weights.py') }}" >> "$GITHUB_OUTPUT" - - tests-cpu: - needs: [pre-tests] - strategy: - fail-fast: true - matrix: - os: ['Ubuntu-latest', 'Windows-latest'] - pytorch-dtype: ['float32', 'float64'] - - uses: kornia/workflows/.github/workflows/tests.yml@v1.12.1 - with: - os: ${{ matrix.os }} - python-version: '["3.8", "3.12"]' - pytorch-version: '["1.9.1", "2.4.0"]' - pytorch-dtype: ${{ matrix.pytorch-dtype }} - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- - - tests-cpu-macos: - needs: [pre-tests] - uses: kornia/workflows/.github/workflows/tests.yml@v1.12.1 - with: - os: 'MacOS-latest' - python-version: '["3.8", "3.12"]' - pytorch-dtype: 'float32' - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- - - - coverage: - needs: [pre-tests] - uses: kornia/workflows/.github/workflows/coverage.yml@v1.12.1 - with: - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- - - typing: - uses: kornia/workflows/.github/workflows/mypy.yml@v1.12.1 - - tutorials: - uses: kornia/workflows/.github/workflows/tutorials.yml@v1.12.1 - - docs: - needs: [pre-tests] - uses: kornia/workflows/.github/workflows/docs.yml@v1.12.1 - with: - python-version: '["3.11"]' - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- - - collector: - needs: [coverage, tests-cpu, tutorials, typing, docs] - if: always() - runs-on: ubuntu-latest - steps: - - name: check for failures - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: echo job failed && exit 1 - - tests-nightly: - if: contains(github.event.pull_request.labels.*.name, 'nightly') - needs: [pre-tests] - name: ${{ matrix.os }} - torch-nightly, ${{ matrix.pytorch-dtype }} - - strategy: - fail-fast: false - matrix: - os: ['Ubuntu-latest', 'Windows-latest'] #, 'MacOS-latest'] add it when https://github.com/pytorch/pytorch/pull/89262 be merged - pytorch-dtype: ['float32', 'float64'] - - uses: kornia/workflows/.github/workflows/tests.yml@v1.12.1 - with: - os: ${{ matrix.os }} - pytorch-version: '["nightly"]' - pytorch-dtype: ${{ matrix.pytorch-dtype }} - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml deleted file mode 100644 index 6e7f3c5a26..0000000000 --- a/.github/workflows/pypi-release.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: PyPI Release - -# https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: # Trigger the workflow on push or pull request, but only for the master branch - push: - branches: [main] - release: - types: [published] - -jobs: - build-sdist: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependencies - run: python -m pip install --user --upgrade setuptools build - - name: Build source - run: | - python -m build --outdir dist/ - ls -lh dist/ - - - uses: actions/upload-artifact@v2 - with: - name: pypi-packages-${{ github.sha }} - path: dist - - upload-packages: - runs-on: ubuntu-20.04 - needs: build-sdist - steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 - with: - name: pypi-packages-${{ github.sha }} - path: dist - - run: ls -lh dist/ - - - name: Upload to release - uses: AButler/upload-release-assets@v2.0 - if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - with: - files: 'dist/*' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - publish-pypi: - runs-on: ubuntu-20.04 - needs: build-sdist - if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - uses: actions/download-artifact@v2 - with: - name: pypi-packages-${{ github.sha }} - path: dist - - run: ls -l dist/ - - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@v1.4.1 - with: - user: __token__ - password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/scheduled_test_cpu.yml b/.github/workflows/scheduled_test_cpu.yml deleted file mode 100644 index 2d13b1ea02..0000000000 --- a/.github/workflows/scheduled_test_cpu.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Tests on CPU (scheduled) - -on: - push: - branches: [main, test-me-*] - schedule: - - cron: "0 4 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - pre-tests: - runs-on: ubuntu-latest - outputs: - hash: ${{ steps.hashid.outputs.weights-hash }} - steps: - - uses: kornia/workflows/.github/actions/env@v1.12.1 - - uses: actions/cache@v4 - id: cache-weights - with: - path: weights/ - key: model-weights-${{ hashFiles('.github/download-models-weights.py') }} - enableCrossOsArchive: true - - - name: Download models weights... - if: steps.cache-weights.outputs.cache-hit != 'true' - run: python .github/download-models-weights.py -t weights/ - - - name: write hashid - id: hashid - run: echo "weights-hash=${{ hashFiles('.github/download-models-weights.py') }}" >> "$GITHUB_OUTPUT" - - tests-cpu-ubuntu: - needs: [pre-tests] - strategy: - fail-fast: false - matrix: - # os: ['Ubuntu-latest', 'Windows-latest', 'MacOS-latest'] - pytorch-dtype: ['float32', 'float64'] - - uses: kornia/workflows/.github/workflows/tests.yml@v1.12.1 - with: - os: 'Ubuntu-latest' - python-version: '["3.8", "3.9", "3.10", "3.11", "3.12"]' - pytorch-version: '["1.9.1", "1.10.2", "1.11.0", "1.12.1", "1.13.1", "2.0.1", "2.1.2", "2.2.2", "2.3.1", "2.4.0"]' - pytorch-dtype: ${{ matrix.pytorch-dtype }} - pytest-extra: '--runslow' - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- - - tests-cpu-windows: - needs: [pre-tests] - strategy: - fail-fast: true - matrix: - pytorch-dtype: ['float32', 'float64'] - - uses: kornia/workflows/.github/workflows/tests.yml@v1.12.1 - with: - os: 'Windows-latest' - python-version: '["3.12"]' - pytorch-version: '["1.9.1", "2.4.0"]' - pytorch-dtype: ${{ matrix.pytorch-dtype }} - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- - - tests-cpu-mac: - needs: [pre-tests] - strategy: - fail-fast: true - matrix: - pytorch-dtype: ['float32', 'float64'] - - uses: kornia/workflows/.github/workflows/tests.yml@v1.12.1 - with: - os: 'MacOS-latest' - pytorch-dtype: ${{ matrix.pytorch-dtype }} - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- - - coverage: - needs: [pre-tests] - uses: kornia/workflows/.github/workflows/coverage.yml@v1.12.1 - with: - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- - - typing: - uses: kornia/workflows/.github/workflows/mypy.yml@v1.12.1 - - tutorials: - uses: kornia/workflows/.github/workflows/tutorials.yml@v1.12.1 - - docs: - needs: [pre-tests] - uses: kornia/workflows/.github/workflows/docs.yml@v1.12.1 - with: - python-version: '["3.11"]' - cache-path: weights/ - cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} - cache-restore-keys: | - model-weights-${{ needs.pre-tests.outputs.hash }} - model-weights- diff --git a/.github/workflows/scheduled_test_pypi_package.yml b/.github/workflows/scheduled_test_pypi_package.yml deleted file mode 100644 index f02c7aae99..0000000000 --- a/.github/workflows/scheduled_test_pypi_package.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Test PyPI package - -on: - schedule: - - cron: '0 4 * * *' - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - - steps: - - name: Checkout kornia - uses: actions/checkout@v3 - - - name: Config Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Pip install Kornia - run: pip install kornia - - - name: Check dependencies and kornia version - run: | - python -c "import torch;print('Pytorch version: ', torch.__version__)" - python -c "import kornia;print('Kornia version: ', kornia.__version__)" diff --git a/benchmarks/augmentation/conftest.py b/benchmarks/augmentation/conftest.py index 7239aa0b3d..f36ae80612 100644 --- a/benchmarks/augmentation/conftest.py +++ b/benchmarks/augmentation/conftest.py @@ -9,8 +9,11 @@ def shape(B, C, H, W): def pytest_generate_tests(metafunc): - B = [1, 5] - C = [1, 3] + # B = [1, 5] + # C = [1, 3] + # use only 1x1x128x128 for now, to only have a single test case + B = [1] + C = [1] H = W = [128] # , 237, 512] params = list(product(B, C, H, W)) metafunc.parametrize("B, C, H, W", params) diff --git a/requirements/requirements-benchmarks.txt b/requirements/requirements-benchmarks.txt index ad5b816183..97fb946a05 100644 --- a/requirements/requirements-benchmarks.txt +++ b/requirements/requirements-benchmarks.txt @@ -1 +1,3 @@ pytest-benchmark +pytest-codspeed +pytest-xdist