From 8692ce43236c2bd578610c16d93d7df3bce491d8 Mon Sep 17 00:00:00 2001 From: tomtitherington Date: Sat, 10 Feb 2024 18:12:24 +0000 Subject: [PATCH 1/4] Configure black, isort, flake8 and mypy --- .dictionary/custom.txt | 2 ++ .vscode/settings.json | 6 ++++-- backend/pyproject.toml | 36 ++++++++++++++++++++++++++++++++++++ backend/requirements.txt | 9 +++++++++ backend/setup.cfg | 23 +++++++++++++++++++++++ 5 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 backend/pyproject.toml create mode 100644 backend/setup.cfg diff --git a/.dictionary/custom.txt b/.dictionary/custom.txt index d13e5a1..bd2a079 100644 --- a/.dictionary/custom.txt +++ b/.dictionary/custom.txt @@ -104,3 +104,5 @@ ZIZW zustand ifneq psql +pathspec +pyproject diff --git a/.vscode/settings.json b/.vscode/settings.json index 6188f0c..77e6da9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,8 +2,10 @@ "python.envFile": "${workspaceFolder}/backend/.env", "prettier.configPath": "./frontend/.prettierrc.js", "prettier.bracketSpacing": false, - "python.formatting.provider": "black", - "black-formatter.args": ["--line-length", "120"], + "black-formatter.args": ["--config=${workspaceFolder}/backend/pyproject.toml"], + "flake8.args": ["--config=${workspaceFolder}/backend/setup.cfg"], + "isort.args": ["--settings-path=${workspaceFolder}/backend/pyproject.toml"], + "python.analysis.typeCheckingMode": "off", "python.testing.unittestArgs": ["-v", "-s", "./backend", "-p", "test_*.py"], "python.testing.pytestEnabled": false, "python.testing.unittestEnabled": true diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 0000000..39d6058 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,36 @@ +# A Python code formatter +[tool.black] +exclude = ''' +/( + | .git + | .venv + | __pycache__ +)/ +''' +include = '\.pyi?$' # Include .py and .pyi files +line-length = 120 # Set max line length to 120 characters +target-version = ['py310'] # Target Python version (adjust as necessary) + +# A static type checker that helps to catch type errors +[tool.mypy] +disallow_untyped_defs = true # Disallow defining functions without type annotations +ignore_missing_imports = true # Ignore errors for missing imports +plugins = [ + "mypy_django_plugin.main", # Enable Django plugin for MyPy +] +python_version = 3.10 # Target Python version + +# A linting tool that helps find syntax errors, and enforces coding style. +[mypy.plugins.django-stubs] +django_settings_module = "backend.settings" # Point to the project's settings module + +# A tool for sorting imports alphabetically, and automatically separated into sections. +[tool.isort] +ensure_newline_before_comments = true # Ensure a newline before comments +force_grid_wrap = 0 # Don't force grid wrap +include_trailing_comma = true # Include a trailing comma on multi-line +line_length = 120 # Set max line length to 120 characters +multi_line_output = 3 # Use vertical hanging indent format +profile = "black" # Make isort compatible with black +skip = [".venv", "__pycache__", ".git"] # Directories to skip +use_parentheses = true # Use parentheses for line continuation diff --git a/backend/requirements.txt b/backend/requirements.txt index 06818a8..e187f11 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,6 +3,7 @@ asgiref==3.6.0 astroid==2.15.6 async-timeout==4.0.3 attrs==23.1.0 +black==24.1.1 certifi==2022.12.7 cfgv==3.3.1 channels==4.0.0 @@ -17,6 +18,8 @@ dj-database-url==1.2.0 Django==4.1.5 django-cors-headers==3.13.0 django-rest-swagger==2.2.0 +django-stubs==4.2.7 +django-stubs-ext==4.2.7 djangorestframework==3.14.0 djangorestframework-simplejwt==5.2.2 drf-extensions==0.7.1 @@ -39,9 +42,13 @@ lazy-object-proxy==1.9.0 MarkupSafe==2.1.2 mccabe==0.7.0 msgpack==1.0.7 +mypy==1.8.0 +mypy-extensions==1.0.0 nodeenv==1.7.0 openapi-codec==1.3.2 openpyxl==3.1.1 +packaging==23.2 +pathspec==0.12.1 platformdirs==3.0.0 pre-commit==3.0.4 psycopg2==2.9.7 @@ -64,6 +71,8 @@ sniffio==1.3.0 sqlparse==0.4.3 tomli==2.0.1 tomlkit==0.11.8 +types-pytz==2024.1.0.20240203 +types-PyYAML==6.0.12.12 typing_extensions==4.7.1 uritemplate==4.1.1 urllib3==1.26.14 diff --git a/backend/setup.cfg b/backend/setup.cfg new file mode 100644 index 0000000..2d4354d --- /dev/null +++ b/backend/setup.cfg @@ -0,0 +1,23 @@ +[flake8] +# Set the maximum allowed line length in your Python code +max-line-length = 120 + +# List of patterns to exclude from checks (mostly to speed up linting) +exclude = + .git, # Ignore all files in the .git directory + __pycache__, # Ignore compiled Python files in __pycache__ directories + .venv, # Exclude files in the .venv virtual environment directory + migrations, # Ignore all Django migration files + +# List of error codes to ignore +ignore = + E203, # Whitespace before ':' (conflicts with black) + E266, # Too many leading '#' for block comment + E501, # Line too long (handled by max-line-length) + W503, # Line break occurred before a binary operator (conflicts with PEP 8 recommendation) + F403, # ‘from module import *’ used; unable to detect undefined names + F401, # Module imported but unused + +# Special rules for specific files +per-file-ignores = + __init__.py:F401 # Ignore "imported but unused" in __init__.py files From bf1c68b7322ce9ab8374264c8edda5ab45c47edf Mon Sep 17 00:00:00 2001 From: tomtitherington Date: Sun, 11 Feb 2024 13:08:28 +0000 Subject: [PATCH 2/4] Delete runtime.txt --- backend/runtime.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 backend/runtime.txt diff --git a/backend/runtime.txt b/backend/runtime.txt deleted file mode 100644 index 396db58..0000000 --- a/backend/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.9.6 \ No newline at end of file From 93f1834d07f1ecbe66d91876975ed00e7389246e Mon Sep 17 00:00:00 2001 From: tomtitherington Date: Sun, 11 Feb 2024 13:10:23 +0000 Subject: [PATCH 3/4] Update tooling config and delete old config files --- .vscode/settings.json | 21 ++++++++++++++++++--- backend/.flake8 | 5 ----- backend/.vscode/settings.json | 6 ------ backend/requirements.txt | 3 +++ backend/setup.cfg | 27 ++++++++++++++++++++------- cspell.config.yaml | 1 + 6 files changed, 42 insertions(+), 21 deletions(-) delete mode 100644 backend/.flake8 delete mode 100644 backend/.vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 77e6da9..d72df7e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,26 @@ { - "python.envFile": "${workspaceFolder}/backend/.env", "prettier.configPath": "./frontend/.prettierrc.js", "prettier.bracketSpacing": false, + "python.defaultInterpreterPath": "./.venv/bin/python", + "python.terminal.activateEnvironment": true, "black-formatter.args": ["--config=${workspaceFolder}/backend/pyproject.toml"], "flake8.args": ["--config=${workspaceFolder}/backend/setup.cfg"], - "isort.args": ["--settings-path=${workspaceFolder}/backend/pyproject.toml"], - "python.analysis.typeCheckingMode": "off", + "flake8.importStrategy": "fromEnvironment", + "flake8.showNotification": "onError", + "flake8.lintOnChange": true, + "isort.args": [ + "--profile", + "black", + "--settings-file", + "${workspaceFolder}/backend/pyproject.toml" + ], + "[python]": { + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + } + }, + "python.terminal.activateEnvInCurrentTerminal": true, + "python.analysis.typeCheckingMode": "basic", "python.testing.unittestArgs": ["-v", "-s", "./backend", "-p", "test_*.py"], "python.testing.pytestEnabled": false, "python.testing.unittestEnabled": true diff --git a/backend/.flake8 b/backend/.flake8 deleted file mode 100644 index 74d0519..0000000 --- a/backend/.flake8 +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -ignore = W503 -max-line-length = 120 -exclude = migrations,settings.py,__init__.py -max-complexity = 10 diff --git a/backend/.vscode/settings.json b/backend/.vscode/settings.json deleted file mode 100644 index 7500c81..0000000 --- a/backend/.vscode/settings.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" - }, - "cSpell.words": ["biochar", "corsheaders", "Puro"] -} diff --git a/backend/requirements.txt b/backend/requirements.txt index e187f11..ef58366 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -22,6 +22,7 @@ django-stubs==4.2.7 django-stubs-ext==4.2.7 djangorestframework==3.14.0 djangorestframework-simplejwt==5.2.2 +djangorestframework-stubs==3.14.5 drf-extensions==0.7.1 et-xmlfile==1.1.0 exceptiongroup==1.1.3 @@ -73,6 +74,8 @@ tomli==2.0.1 tomlkit==0.11.8 types-pytz==2024.1.0.20240203 types-PyYAML==6.0.12.12 +types-requests==2.31.0.6 +types-urllib3==1.26.25.14 typing_extensions==4.7.1 uritemplate==4.1.1 urllib3==1.26.14 diff --git a/backend/setup.cfg b/backend/setup.cfg index 2d4354d..4b2ba0a 100644 --- a/backend/setup.cfg +++ b/backend/setup.cfg @@ -8,16 +8,29 @@ exclude = __pycache__, # Ignore compiled Python files in __pycache__ directories .venv, # Exclude files in the .venv virtual environment directory migrations, # Ignore all Django migration files + settings.py, # Ignore Django settings file # List of error codes to ignore ignore = - E203, # Whitespace before ':' (conflicts with black) - E266, # Too many leading '#' for block comment - E501, # Line too long (handled by max-line-length) - W503, # Line break occurred before a binary operator (conflicts with PEP 8 recommendation) - F403, # ‘from module import *’ used; unable to detect undefined names - F401, # Module imported but unused + E203 + E266 + W503 + E501 + F403 + F401 + DJ10 + DJ11 + +# E203 Whitespace before ':' (conflicts with black) +# E266 Too many leading '#' for block comment +# E501 Line too long (handled by max-line-length) +# W503 Line break occurred before a binary operator (conflicts with PEP 8 recommendation) +# F403 ‘from module import *’ used; unable to detect undefined names +# F401 Module imported but unused +# DJ10 Model should define verbose_name on its Meta inner class (flake8-django) +# DJ11 Model should define verbose_name_plural on its Meta inner class (flake8-django) # Special rules for specific files +# Ignore "imported but unused" in __init__.py files per-file-ignores = - __init__.py:F401 # Ignore "imported but unused" in __init__.py files + __init__.py:F401 diff --git a/cspell.config.yaml b/cspell.config.yaml index 02f0920..870b70f 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -7,3 +7,4 @@ dictionaries: ignorePaths: - "**/migrations/**" - "**/node_modules/**" + - "**/.vscode/**" From 4c2c2b17a4697144c1570a60df9f0d57b36f6907 Mon Sep 17 00:00:00 2001 From: tomtitherington Date: Sun, 11 Feb 2024 13:11:03 +0000 Subject: [PATCH 4/4] Resolve linting, type checks and other warnings --- backend/app/models/environment.py | 14 ++++++++------ backend/app/models/flow_instance.py | 10 +++++++++- backend/app/models/flow_schema.py | 11 +++++++++-- backend/app/models/person.py | 21 +++++++++++++-------- backend/app/models/project.py | 8 ++++++-- backend/app/models/segments.py | 8 ++++---- backend/app/models/user.py | 4 ++-- 7 files changed, 51 insertions(+), 25 deletions(-) diff --git a/backend/app/models/environment.py b/backend/app/models/environment.py index dd1bfab..4b7ee97 100644 --- a/backend/app/models/environment.py +++ b/backend/app/models/environment.py @@ -1,22 +1,24 @@ -from django.db import models - import uuid +from django.db import models + class Environment(models.Model): name = models.CharField(max_length=100, blank=False, null=False) - identifier = models.SlugField(max_length=100, blank=False, null=False) + identifier = models.SlugField(max_length=100, blank=False, null=False) project = models.ForeignKey("Project", on_delete=models.CASCADE, related_name="environments") is_default = models.BooleanField(default=False, blank=False, null=False) class Meta: unique_together = ["identifier", "project"] + verbose_name = "Environment" + verbose_name_plural = "Environments" + + def __str__(self): + return self.name def save(self, *args, **kwargs): if not self.identifier: # If identifier is not provided, assign a GUID. self.identifier = str(uuid.uuid4()) super(Environment, self).save(*args, **kwargs) - - def __str__(self): - return self.name diff --git a/backend/app/models/flow_instance.py b/backend/app/models/flow_instance.py index c72d2db..e117dd8 100644 --- a/backend/app/models/flow_instance.py +++ b/backend/app/models/flow_instance.py @@ -1,6 +1,12 @@ -from app.models import UUIDModel, Person, Environment, StepSchema, FlowSchemaVersion, TransitionSchema +from typing import TYPE_CHECKING + from django.db import models +from app.models import Environment, FlowSchemaVersion, Person, StepSchema, TransitionSchema, UUIDModel + +if TYPE_CHECKING: + from django.db.models.manager import RelatedManager + class FlowInstance(UUIDModel): """ @@ -22,6 +28,8 @@ class FlowState(models.TextChoices): state = models.CharField(max_length=20, choices=FlowState.choices, default=FlowState.ACTIVE) environment = models.ForeignKey(Environment, on_delete=models.CASCADE) + steps: "RelatedManager[StepInstance]" + @property def active_steps(self): """ diff --git a/backend/app/models/flow_schema.py b/backend/app/models/flow_schema.py index f66a4ed..fc6392f 100644 --- a/backend/app/models/flow_schema.py +++ b/backend/app/models/flow_schema.py @@ -1,7 +1,12 @@ -from typing import Optional -from app.models import UUIDModel, Environment +from typing import TYPE_CHECKING, Optional + from django.db import models +from app.models import Environment, UUIDModel + +if TYPE_CHECKING: + from django.db.models.manager import RelatedManager + class FlowSchema(UUIDModel): """ @@ -16,6 +21,8 @@ class FlowSchema(UUIDModel): current_version = models.ForeignKey("FlowSchemaVersion", on_delete=models.SET_NULL, null=True, related_name="+") environment = models.ForeignKey(Environment, on_delete=models.CASCADE, related_name="schemas") + versions: "RelatedManager[FlowSchemaVersion]" + def latest_version(self) -> Optional["FlowSchemaVersion"]: """Retrieves the most recent version of the flow schema based on the 'created_at' timestamp.""" return self.versions.order_by("-created_at").first() diff --git a/backend/app/models/person.py b/backend/app/models/person.py index e414abc..cc0c7d3 100644 --- a/backend/app/models/person.py +++ b/backend/app/models/person.py @@ -1,7 +1,9 @@ -from django.db import models +import jsonschema from django.core.exceptions import ValidationError +from django.db import models +from jsonschema.exceptions import SchemaError as JSONSchemaError +from jsonschema.exceptions import ValidationError as JSONValidationError -import jsonschema class Person(models.Model): email = models.EmailField(unique=True, primary_key=True) @@ -9,6 +11,13 @@ class Person(models.Model): schema = models.JSONField() environment = models.ForeignKey("Environment", on_delete=models.CASCADE, related_name="people") + def __str__(self): + return self.email + + def save(self, *args, **kwargs): + self.full_clean() # Make sure we call the clean method on save + super().save(*args, **kwargs) + def clean(self) -> None: """ Validates the JSON schema and data. @@ -18,15 +27,11 @@ def clean(self) -> None: """ try: jsonschema.Draft202012Validator.check_schema(self.schema) - except jsonschema.exceptions.SchemaError as e: + except JSONSchemaError as e: raise ValidationError({"schema": f"Invalid JSON schema: {e}"}) validator = jsonschema.Draft202012Validator(self.schema) try: validator.validate(self.data) - except jsonschema.exceptions.ValidationError as e: + except JSONValidationError as e: raise ValidationError({"data": f"Data does not adhere to the schema: {e}"}) - - def save(self, *args, **kwargs): - self.full_clean() # Make sure we call the clean method on save - super().save(*args, **kwargs) diff --git a/backend/app/models/project.py b/backend/app/models/project.py index 9a00d3e..cdaa38b 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -1,15 +1,16 @@ # from django.utils.crypto import get_random_string import uuid -from app.models import User from django.db import models +from app.models import User + class Project(models.Model): name = models.CharField(max_length=100, blank=False, null=False) members = models.ManyToManyField(User, through="ProjectMembership", related_name="projects") # TODO: Replace this with proper key generation - api_key = models.CharField(max_length=256, unique=True, default=uuid.uuid4, null=False, blank=False) + api_key = models.CharField(max_length=256, unique=True, default=uuid.uuid4, null=False, blank=False) # type: ignore def __str__(self): return self.name @@ -28,3 +29,6 @@ class MembershipType(models.TextChoices): choices=MembershipType.choices, default=MembershipType.MEMBER, ) + + def __str__(self): + return f"{self.user.email} - {self.project.name}" diff --git a/backend/app/models/segments.py b/backend/app/models/segments.py index 2b30c0d..f257c85 100644 --- a/backend/app/models/segments.py +++ b/backend/app/models/segments.py @@ -1,6 +1,6 @@ # TODO # Segments are groups of people who share similar characteristics. -# - They can define grouping conditions that use the underlying data (schema) and potentially -#  use something like JSON Path https://www.postgresql.org/docs/current/functions-json.html -# - They can define a name and description -# - They can be used to trigger flows => when person enters a segment, trigger a flow \ No newline at end of file +# - They can define grouping conditions that use the underlying data (schema) and potentially +# use something like JSON Path https://www.postgresql.org/docs/current/functions-json.html +# - They can define a name and description +# - They can be used to trigger flows => when person enters a segment, trigger a flow diff --git a/backend/app/models/user.py b/backend/app/models/user.py index 9084e5d..387d6e2 100644 --- a/backend/app/models/user.py +++ b/backend/app/models/user.py @@ -1,10 +1,10 @@ +from typing import List, Type + from django.contrib.auth.base_user import BaseUserManager from django.contrib.auth.models import AbstractUser from django.db import models from django.utils.translation import gettext as _ -from typing import List, Type - class UserManager(BaseUserManager): """