Updated github actions

SainsburyWellcomeCentre · Aug 28, 2023 · ee35412 · ee35412
1 parent 2f2af11
commit ee35412
Show file tree

Hide file tree

Showing 6 changed files with 104 additions and 79 deletions.
diff --git a/.github/workflows/build_env_run_tests.yml b/.github/workflows/build_env_run_tests.yml
@@ -1,20 +1,48 @@
-# Builds the aeon environment, flakes via flake8, checks type annotations via mypy, tests via pytest, 
-# reports test coverage via pytest-cov and codecov, and reports security vulnerabilities via bandit.
+# Builds the aeon environment; lints formatting and smells via ruff; checks type annotations via pyright;
+# tests via pytest; reports test coverage via pytest-cov and codecov.
 
 name: build_env_run_tests
 on:
-  push:
-    branches: [ main, reorg, jai_dev, config ]
   pull_request:
     branches: [ main ]
-  workflow_dispatch:  # Allows running manually from Github's 'Actions' tab
+    types: [opened, reopened, synchronize]
+  workflow_dispatch:  # allows running manually from Github's 'Actions' tab
 jobs:
-  build_env_run_tests:
+  build_env_pip_pyproject:  # checks only for building env using pip and pyproject.toml
+    name: Build env using pip and pyproject.toml
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.11]
+        #include:  # test other python versions only on ubuntu
+          #- os: ubuntu-latest
+          #  python-version: [3.9, 3.10]
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash -l {0}  # reset shell for each step
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Create venv and install dependencies
+        run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install -e .[dev]
+          python -c "import aeon"
+  
+  build_env_run_tests:  # checks for building env using mamba and runs codebase checks and tests
     name: Build env and run tests on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.11]
       fail-fast: false
     defaults:
       run:
@@ -23,30 +51,34 @@ jobs:
       - name: checkout repo
         uses: actions/checkout@v2
       - name: set up conda env
-        uses: conda-incubator/setup-miniconda@v2  # from github marketplace
+        uses: conda-incubator/setup-miniconda@v2
         with:
-          activate-environment: aeon
-          environment-file: ./env_config/env.yml
           use-mamba: true
           miniforge-variant: Mambaforge
+          python-version: ${{ matrix.python-version }}
+          environment-file: ./env_config/env.yml
+          activate-environment: aeon
       - name: Update conda env with dev reqs
         run: mamba env update -f ./env_config/env_dev.yml
-      - name: flake
-        run: python -m flake8 .
-      - name: mypy
-        run: python -m mypy .
-      - name: bandit
-        run: python -m bandit . -r -ll
+
+      # Only run codebase checks and tests for ubuntu.
+      - name: ruff
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
+        run: python -m ruff check --config ./pyproject.toml .
+      - name: pyright
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
+        run: python -m pyright --level error --project ./pyproject.toml . 
       - name: pytest
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
         run: python -m pytest tests/
-      # Only run coverage report job and upload for ubuntu.
+
       - name: generate test coverage report
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
         run: |
           python -m pytest --cov=aeon ./tests/ --cov-report=xml:./tests/test_coverage/test_coverage_report.xml
-          python -m pytest --cov=aeon ./tests/ --cov-report=html:./tests/test_coverage/test_coverage_report_html
+          #python -m pytest --cov=aeon ./tests/ --cov-report=html:./tests/test_coverage/test_coverage_report_html
       - name: upload test coverage report to codecov
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
         uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

diff --git a/.github/workflows/squash_merge_to_prod.yml b/.github/workflows/squash_merge_to_prod.yml
@@ -1,4 +1,4 @@
-# Squashe merges commits from PRs to 'main' into 'prod'. 'prod' thus serves as a  more 
+# Squash merges commits from PRs to 'main' into 'prod'. 'prod' thus serves as a  more 
 # readable history of the project, while 'main' retains the full commit history.
 
 name: squash_merge_to_prod

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -9,7 +9,7 @@ repos:
       - id: identity
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.4.0
     hooks:
       - id: check-yaml
       - id: detect-private-key
@@ -18,55 +18,27 @@ repos:
       - id: no-commit-to-branch
 
   - repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 23.7.0
     hooks:
       - id: black
         args:
           - "--config"
           - "./pyproject.toml"
-
-  - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
-    hooks:
-      - id: isort
-        name: isort (python)
-        args:
-          - "--settings-file"
-          - "./pyproject.toml"
 
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.1.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.286
     hooks:
-      - id: pyupgrade
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --config, ./pyproject.toml]
 
-  - repo: https://github.com/pycqa/flake8
-      rev: 3.9.2
-      hooks:
-        - id: flake8
-          entry: pflake8
-          additional_dependencies: [pyproject-flake8]
-
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.982
+  - repo: https://github.com/RobertCraigie/pyright-python
+    rev: v1.1.324
     hooks:
-      - id: mypy
-        args:
-          - "--config-file"
-          - "./pyproject.toml"
+    - id: pyright
+      args: [--level, error, --project, ./pyproject.toml]
 
-  # Bandit and Pytest are expensive, so we show their set-up but leave them commented out.
-  # - repo: https://github.com/PyCQA/bandit
-  #   rev: 1.7.4
-  #   hooks:
-  #     - id: bandit
-  #       args:
-  #         - "--configfile"
-  #         - "./pyproject.toml"
-  #         - "--recursive"
-  #         - "-ll"
-  #       additional_dependencies:
-  #         - "bandit[toml]"
 
+  # Pytest is expensive, so we show its set-up but leave it commented out.
   # - repo: local
   #   hooks:
   #     - id: pytest-py

diff --git a/aeon/preprocessing/__init__.py b/aeon/preprocessing/__init__.py
@@ -0,0 +1,2 @@
+# This is where functions for common preprocessing done on raw data files will live
+# (e.g. 'calc_forage_dist', 'calc_forage_bout', 'collate_movie', 'calc_tube_test', etc.)
diff --git a/env_config/env_dev.yml b/env_config/env_dev.yml
@@ -10,6 +10,7 @@ dependencies:
   - black
   - gh
   - ipdb
+  - jellyfish
   - pre-commit 
   - pydantic
   - pyright 

diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,10 @@ build-backend = "setuptools.build_meta"
 name = "aeon_mecha"
 version = "0.1.0"
 requires-python = ">=3.9.4"
-description = "Code for managing acquired data from Project Aeon experiments. Includes general file IO, data QC, querying, and analysis modules."
+description = \"\"\"
+    Code for managing acquired data from Project Aeon experiments. Includes general file IO,
+    data QC, querying, and analysis modules.
+\"\"\"
 authors = [
   { name = "Jai Bhagat", email = "jkbhagatio@gmail.com" },
   { name = "Goncalo Lopes", email = "goncaloclopes@gmail.com" },
@@ -97,28 +100,43 @@ exclude = '''
 )/
 '''
 
-[tool.isort]
-profile = "black"
-color_output = false
-
-# Ignore the following errors and warnings:
-#     - whitespace after '(' (E201)
-#     - whitespace before ')' (E202)
-#     - whitespace before ':' (E203)
-#     - whitespace after ',' or ':' (E231)
-#     - assigning to a lambda expression (E371)
-#     - linebreaking before a binary operator (W503)
-#     - multiple statments on one line separated by ';' (E702)
-[tool.flake8]
-max-line-length = 108
-extend-ignore = ["E201", "E202", "E203", "E231", "E731", "W503", "E702"]
+[tool.ruff]
+select = ["E", "W", "F", "I", "D", "UP", "S", "B", "A", "C4", "ICN", "PIE", "PT", "SIM", "PL"]
+line-length = 108
+ignore = [
+    "E201", "E202", "E203", "E231", "E731", "E702", 
+    "S101", 
+    "PT013",
+    "PLR0912", "PLR0913", "PLR0915"
+]"
 extend-exclude = [".git", ".github", ".idea", ".vscode"]
-docstring-convention = "google"
 
-[tool.mypy]
-ignore_missing_imports = true
+[tool.ruff.pydocstyle]
+convention = "google"
 
-[tool.bandit]
+[tool.pyright]
+reportImportCycles = "error"
+reportUnusedImport = "error"
+reportUnusedClass = "error"
+reportUnusedfunction = "error"
+reportUnusedVariable = "error"
+reportDuplicateImport = "error"
+reportWildcardImportFromLibrary = "error"
+reportPrivateUsage = "error"
+reportCallInDefaultInitializer = "error"
+reportUnnecessaryIsInstance = "error"
+reportUnnecesaryCast = "error"
+reportUnnecesarryComparison = "error"
+reportUnnecessaryContains = "error"
+reportAssertAlwaysTrue = "error"
+reportSelfClsParameterName = "error"
+reportUnusedExpression = "error"
+reportMatchNotExhaustive = "error"
+reportImplicitOverride = "error"
+reportShadowedImports = "error"
+# *Note*: we may want to set all 'ReportOptional*' rules to "none", but leaving 'em default for now
+venvPath = "."
+venv = ".venv"
 
 [tool.pytest.ini_options]
 markers = [