Update upload script

.flake8

@@ -1,4 +1,4 @@
-; Copyright 2021 Universität Tübingen, DKFZ and EMBL
+; Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
 ; for the German Human Genome-Phenome Archive (GHGA)
 ;
 ; Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +14,8 @@
 ; limitations under the License.
 
 [flake8]
-ignore = E, W # ignore all style checks from pycodestyle
-              # as they are already checked by black
-exclude = .git,__pycache__,db_migration,build,dist
+ignore = E, W
+    # ignore all style checks from pycodestyle
+    # as they are already checked by black
+exclude = .git,__pycache__,build,dist
 max-complexity = 10

.pre-commit-config.yaml

@@ -4,11 +4,11 @@
 default_language_version:
   python: python3.9
 
-minimum_pre_commit_version: 2.13.0
+minimum_pre_commit_version: 3.0.0
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -31,31 +31,33 @@ repos:
       - id: detect-private-key
       - id: mixed-line-ending
         args: [--fix=lf]
+      - id: no-commit-to-branch
+        args: [--branch, dev, --branch, int, --branch, main]
       - id: debug-statements
       - id: debug-statements
       - id: debug-statements
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.1.0
     hooks:
       - id: black
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
         args: [--profile, black]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.981
+    rev: v1.0.0
     hooks:
       - id: mypy
         args: [--no-warn-unused-ignores]
   - repo: https://github.com/PyCQA/pylint
-    rev: v2.13.3
+    rev: v2.16.4
     hooks:
       - id: pylint
         args: [--disable=E0401]
-        exclude: db_migration|tests|.devcontainer
+        exclude: tests|.devcontainer
   - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
+    rev: 6.0.0
     hooks:
       - id: flake8
         args: [--config, .flake8]

.pylintrc

@@ -70,9 +70,12 @@ disable=
       duplicate-code, # is behaving strangely sometimes and cannot
                      # be disabled on an individual basis:
                      # https://github.com/PyCQA/pylint/issues/214
-      too-few-public-methods # says that classes should always have methods
+
+      too-few-public-methods, # says that classes should always have methods
                              # but that is not true anymore (e.g. dataclasses)
 
+      unnecessary-ellipsis, # often used for interfaces/protocols
+
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
@@ -235,6 +238,7 @@ good-names=i,
            j,
            k,
            ex,
+           ok,
            Run,
            _,
            __,
@@ -537,6 +541,6 @@ valid-metaclass-classmethod-first-arg=cls
 [EXCEPTIONS]
 
 # Exceptions that will emit a warning when being caught. Defaults to
-# "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+# "builtins.BaseException, builtins.Exception".
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

requirements.txt

@@ -1,28 +1,25 @@
 # All your requirements go here. Please adapt as needed:
-pydantic==1.10.2
-PyYAML==5.4.1
-pytest==7.1.1
-pytest-asyncio==0.18.3
-pytest-cov==3.0.0
-mypy==0.981
-mypy-extensions==0.4.3
-pylint==2.13.3
-click==8.1.2
-black==22.3.0
-flake8==4.0.1
-isort==5.10.1
+pytest==7.2.0
+pytest-asyncio==0.20.3
+pytest-cov==4.0.0
+mypy==1.0.0
+mypy-extensions==1.0.0
+pylint==2.16.4
+click==8.1.3
+black==23.1.0
+flake8==6.0.0
+isort==5.12.0
 bandit==1.7.4
-pre-commit==2.17.0
-mkdocs==1.3.0
+pre-commit==3.1.1
+mkdocs==1.4.2
 mkdocs-autorefs==0.4.1
-mkdocs-material==8.2.8
-mkdocs-material-extensions==1.0.3
-mkdocstrings==0.18.1
-mkdocstrings-python-legacy==0.2.2
-testcontainers[kafka,rabbitmq,mongo,postgresql]==3.4.2
-typer==0.4.1
+mkdocs-material==9.0.3
+mkdocs-material-extensions==1.1.1
+mkdocstrings==0.19.1
+mkdocstrings-python-legacy==0.2.3
+testcontainers[kafka,mongo,postgresql]==3.4.2
+typer==0.7.0
+httpx==0.23.3
 boto3==1.24.93
-crypt4gh==1.5
-ghga-service-chassis-lib==0.15.0
-hexkit==0.4.0
-pycurl-requests==0.5.0
+hexkit==0.9.2
+ghga-connector==0.2.12

src/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Universität Tübingen, DKFZ and EMBL
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/otp_tsv_to_upload.py

@@ -27,7 +27,7 @@
 import typer
 from pydantic import BaseModel
 
-from s3_upload import load_config_yaml
+from .s3_upload import load_config_yaml
 
 HERE = Path(__file__).parent
 
@@ -133,7 +133,6 @@ def handle_file_uploads(  # noqa: C901
 
     try:
         while files_to_do or in_progress:
-
             # start new processes:
             while len(in_progress) < parallel_processes and files_to_do:
                 next_file = files_to_do.pop()