Improve performance, upgrade deps and logging, remove numpy, numba, p…

…andas (#816)

* Upgrade deps and remove numpy, numba, pandas

* Replace BaseHTTPMiddleware with much faster ASGI equalivalent

* Increase anyio threads to 100

* Remove broken etag support

* Improved logging

* Switch to a2wsgi

* Remove a2wsgi again

* Remove greenifier extension

* Bump FastAPI to 0.89.0

* Add pytest in settings.json

* Set SQLAlchemy version to >= 1.4.40 and <= 1.4.46 to allow SQLModel > 0.0.6 (#819)

* Loosen Flask and marshmallow-sqlalchemy versions for superset

* Removed flake8, isort, pep8 in favor of ruff

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Module",
+            "type": "python",
+            "request": "launch",
+            "module": "tdp_core",
+            "justMyCode": false
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "tdp_core"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

Makefile

@@ -1,9 +1,9 @@
 .DEFAULT_GOAL := help
 pkg_src = tdp_core
 
-flake8 = flake8 $(pkg_src) setup.py
-isort = isort $(pkg_src) setup.py
 black = black --line-length 140 $(pkg_src) setup.py
+pyright = pyright $(pkg_src) setup.py
+ruff = ruff $(pkg_src) setup.py --line-length 140 --select E,W,F,N,I,C,B,UP,PT,SIM,RUF --ignore E501,C901,B008
 
 .PHONY: start  ## Start the development server
 start:
@@ -17,25 +17,25 @@ ci: check-format lint test
 
 .PHONY: format  ## Auto-format the source code
 format:
-	$(isort)
+	$(ruff) --fix
 	$(black)
 
 .PHONY: check-format  ## Check the source code format without changes
 check-format:
-	$(isort) --check-only
 	$(black) --check
 
-.PHONY: lint  ## Run flake8
+.PHONY: lint  ## Run flake8 and pyright
 lint:
-	$(flake8)
+	$(ruff) --format=github
+	$(pyright)
 
 .PHONY: test  ## Run tests
 test:
 	pytest $(pkg_src)
 
 .PHONEY: documentation ## Generate docs
-documentation: 
-	mkdocs build
+documentation:
+	echo "TODO"
 
 .PHONY: install  ## Install the requirements
 install:
@@ -45,6 +45,20 @@ install:
 develop:
 	pip install -e .[develop]
 
+.PHONY: env_encrypt ## Encrypts the current ./<app>/.env
+env_encrypt:
+	openssl aes-256-cbc -pbkdf2 -in ./$(pkg_src)/.env -out ./$(pkg_src)/.env.enc
+
+.PHONY: env_decrypt ## Decrypts the ./<app>/.env.enc
+env_decrypt:
+	@if [ -z "${ENV_PASSWORD}" ]; then \
+		echo "No ENV_PASSWORD set, prompting for password..."; \
+		openssl aes-256-cbc -pbkdf2 -d -in ./$(pkg_src)/.env.enc -out ./$(pkg_src)/.env; \
+	else \
+		echo "ENV_PASSWORD set, using it..."; \
+		openssl aes-256-cbc -pbkdf2 -d -in ./$(pkg_src)/.env.enc -out ./$(pkg_src)/.env -pass env:ENV_PASSWORD; \
+	fi
+
 .PHONY: build  ## Build a wheel
 build:
 	python setup.py sdist bdist_wheel --dist-dir dist_python

pyrightconfig.json

@@ -0,0 +1,4 @@
+{
+  "typeCheckingMode": "basic",
+  "useLibraryCodeForTypes": true,
+}

requirements.txt

@@ -1,30 +1,27 @@
-alembic==1.7.7
-fastapi==0.75.0
-cachetools==5.0.0
-flask-smorest==0.37.0
+# a2wsgi==1.6.0 # This WSIGMiddleware is not compatible with starlette_context
+alembic==1.9.0
+cachetools==5.2.0
+fastapi[all]==0.89.0
 flask-swagger-ui==3.36.0
-Flask==2.0.3
+Flask[async]>=2.1.0,<=2.2.2
 json-cfg==0.4.2
-marshmallow-sqlalchemy==0.28.0
-marshmallow==3.15.0
-numba==0.55.1
-numpy==1.21.6
+marshmallow-sqlalchemy>=0.26.0,<=0.28.1
+marshmallow==3.19.0
 openpyxl==3.0.9
-pandas==1.4.1
-Pillow==9.0.1
-psycopg2==2.9.3
-pydantic==1.9.0
-pyjwt==2.4.0
+Pillow==9.3.0
+psycopg==3.1.7
+psycopg2==2.9.5
+pydantic==1.10.2
+pyjwt==2.6.0
 pymongo~=3.11.1
+pytest-postgresql==4.1.1
 python-dateutil==2.8.2
 python-memcached==1.59
 python-multipart==0.0.5
-requests==2.27.1
-SQLAlchemy==1.4.32
-sqlmodel==0.0.6
+rdkit==2022.3.5
+requests==2.28.1
+SQLAlchemy>=1.4.40,<=1.4.46
+starlette-context==0.3.5
 urllib3==1.26.9
-uvicorn[standard]==0.17.6
+uvicorn[standard]==0.20.0
 yamlreader==3.0.4
-pytest-postgresql==4.1.1
-psycopg==3.0.16
-rdkit==2022.3.5

requirements_dev.txt

@@ -1,9 +1,5 @@
-black~=22.3.0
-debugpy~=1.5.1
-flake8~=4.0.1
-isort~=5.10.1
-mkdocs-material~=8.2.8
-pep8-naming~=0.12.1
+black~=22.12.0
+pyright~=1.1.285
 pytest-runner~=6.0.0
-pytest~=7.1.1
-recommonmark~=0.7.1
+pytest~=7.2.0
+ruff==0.0.218

setup.py

@@ -55,7 +55,7 @@ def requirements(file):
     package_data={},
     # Although 'package_data' is the preferred approach, in some case you may
     # need to place data files outside of your packages. See:
-    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
+    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files
     # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
     data_files=[],  # [('my_data', ['data/data_file'])],
 )

src/lineup/structureImage/StructureImageRenderer.ts

@@ -13,7 +13,7 @@ import { abortAble } from 'lineupengine';
 import { StructureImageColumn } from './StructureImageColumn';
 import { I18nextManager } from '../../i18n';
 
-const template = '<a target="_blank" rel="noopener" style="background-size: contain; background-position: center; background-repeat: no-repeat;"></a>';
+const template = '<div style="background-size: contain; background-position: center; background-repeat: no-repeat;"></div>';
 
 function getImageURL(structure: string, substructure: string | null = null, align: string | null = null): string {
   return `/api/rdkit/?structure=${encodeURIComponent(structure)}${substructure ? `&substructure=${encodeURIComponent(substructure)}` : ''}${
@@ -94,7 +94,6 @@ export class StructureImageRenderer implements ICellRendererFactory {
             }
             n.style.backgroundImage = `url('${getImageURL(value, col.getFilter()?.filter, col.getAlign())}')`;
             n.title = value;
-            n.href = `https://pubchem.ncbi.nlm.nih.gov/#query=${value}`;
           });
         }
         return null;

tdp_core/__init__.py

@@ -10,6 +10,10 @@ def init_app(self, app: FastAPI):
         app.include_router(img_api.app)
 
     def register(self, registry: RegHelper):
+        import logging
+
+        _log = logging.getLogger(__name__)
+
         # phovea_server
         registry.append(
             "namespace",
@@ -23,7 +27,14 @@ def register(self, registry: RegHelper):
             "tdp_core.id_mapping.idtype_api",
             {"namespace": "/api/idtype", "factory": "create_idtype"},
         )
-        registry.append("json-encoder", "numpy", "tdp_core.encoder.json_encoder")
+
+        try:
+            import numpy  # noqa, type: ignore
+
+            registry.append("json-encoder", "numpy", "tdp_core.encoder.json_encoder")
+        except ImportError:
+            _log.info('numpy not available, skipping "numpy" json encoder')
+
         registry.append("json-encoder", "set-encoder", "tdp_core.encoder.set_encoder", {})
 
         registry.append(
@@ -63,8 +74,6 @@ def register(self, registry: RegHelper):
         registry.append_router("tdp_plugin_router", "tdp_core.plugin.router", {})
         registry.append("namespace", "tdp_xlsx2json", "tdp_core.xlsx", {"namespace": "/api/tdp/xlsx"})
         registry.append("mapping_provider", "tdp_core", "tdp_core.mapping_table")
-        # TODO: Check if this is still required?
-        registry.append("greenifier", "psycopg2", "tdp_core.sql_use_gevent", {})
 
         # DB migration plugins
         registry.append(

tdp_core/dataset/dataset.py

@@ -1,5 +1,4 @@
 import itertools
-from builtins import str
 
 from .. import manager
 
@@ -41,29 +40,33 @@ def get(dataset_id):
     return None
 
 
-def add(desc, files=[], id=None):
+def add(desc, files=None, id=None):
     """
     adds a new dataset to this storage
     :param desc: the dict description information
     :param files: a list of FileStorage
     :param id: optional the unique id to use
     :return: the newly created dataset or None if an error occurred
     """
+    if files is None:
+        files = []
     for p in _providers():
         r = p.upload(desc, files, id)
         if r:
             return r
     return None
 
 
-def update(dataset, desc, files=[]):
+def update(dataset, desc, files=None):
     """
     updates the given dataset
     :param dataset: a dataset or a dataset id
     :param desc: the dict description information
     :param files: a list of FileStorage
     :return:
     """
+    if files is None:
+        files = []
     old = get(dataset) if isinstance(dataset, str) else dataset
     if old is None:
         return add(desc, files)
@@ -80,7 +83,4 @@ def remove(dataset):
     old = get(dataset) if isinstance(dataset, str) else dataset
     if old is None:
         return False
-    for p in _providers():
-        if p.remove(old):
-            return True
-    return False
+    return any(p.remove(old) for p in _providers())