Merge pull request #182 from broadinstitute/sn_POD-2347_fix_import_path

Moving mock and request utils so imports work
broadinstitute · Dec 19, 2024 · 4809c3e · 4809c3e
2 parents 9e78cae + d8b6f83
commit 4809c3e
Show file tree

Hide file tree

Showing 28 changed files with 37 additions and 38 deletions.
diff --git a/python/azure_tdr_to_gcp_file_transfer.py b/python/azure_tdr_to_gcp_file_transfer.py
@@ -10,7 +10,7 @@
 from typing import Union
 
 from utils.tdr_utils.tdr_api_utils import TDR
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 
 

diff --git a/python/copy_az_terra_files_to_gcp_bucket.py b/python/copy_az_terra_files_to_gcp_bucket.py
@@ -4,7 +4,7 @@
 from utils.azure_utils import AzureBlobDetails, SasTokenUtil
 from utils.terra_utils.terra_util import TerraWorkspace
 from utils.gcp_utils import GCPCloudFunctions
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 
 

diff --git a/python/copy_dataset.py b/python/copy_dataset.py
@@ -5,7 +5,7 @@
 
 from utils.tdr_utils.tdr_api_utils import TDR
 from utils.tdr_utils.tdr_ingest_utils import FilterAndBatchIngest
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils import GCP, ARG_DEFAULTS
 

diff --git a/python/create_workspace_file_manifest.py b/python/create_workspace_file_manifest.py
@@ -1,6 +1,6 @@
 from utils.terra_utils.terra_util import TerraWorkspace
 from utils.gcp_utils import GCPCloudFunctions
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils import GCP, comma_separated_list
 import csv

diff --git a/python/delete_dataset.py b/python/delete_dataset.py
@@ -5,7 +5,7 @@
 from argparse import ArgumentParser, Namespace
 
 from utils.tdr_utils.tdr_api_utils import TDR
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils import GCP
 

diff --git a/python/delete_files_from_datasets_in_bad_state.py b/python/delete_files_from_datasets_in_bad_state.py
@@ -1,7 +1,7 @@
 import logging
 import requests
 import re
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils import GCP, ARG_DEFAULTS
 from argparse import ArgumentParser, Namespace

diff --git a/python/delete_snapshot.py b/python/delete_snapshot.py
@@ -4,7 +4,7 @@
 
 from utils.tdr_utils.tdr_api_utils import TDR
 from utils.tdr_utils.tdr_job_utils import MonitorTDRJob
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils import GCP
 from collections import defaultdict

diff --git a/python/export_data_from_snapshot_or_dataset.py b/python/export_data_from_snapshot_or_dataset.py
@@ -7,7 +7,7 @@
 from utils.gcp_utils import GCPCloudFunctions
 from utils.token_util import Token
 from utils.tdr_utils.tdr_api_utils import TDR
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils import GCP, ARG_DEFAULTS
 
 logging.basicConfig(

diff --git a/python/gcp_workspace_table_to_dataset_ingest.py b/python/gcp_workspace_table_to_dataset_ingest.py
@@ -11,7 +11,7 @@
 )
 from utils.tdr_utils.tdr_table_utils import SetUpTDRTables
 from utils.token_util import Token
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 
 
 logging.basicConfig(

diff --git a/python/get_and_delete_orphaned_files_from_dataset.py b/python/get_and_delete_orphaned_files_from_dataset.py
@@ -2,7 +2,7 @@
 import argparse
 
 from utils.tdr_utils.tdr_api_utils import TDR
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils import GCP, ARG_DEFAULTS
 

diff --git a/python/hard_clone_workspace.py b/python/hard_clone_workspace.py
@@ -8,7 +8,7 @@
 
 from utils import GCP, comma_separated_list, ARG_DEFAULTS
 from utils.terra_utils.terra_util import TerraWorkspace
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils.csv_util import Csv
 from utils.gcp_utils import GCPCloudFunctions

diff --git a/python/import_workflows.py b/python/import_workflows.py
@@ -2,7 +2,7 @@
 import argparse
 from typing import Any
 from utils.terra_utils.terra_util import TerraWorkspace
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils.terra_utils.terra_workflow_configs import WorkflowConfigs, GetWorkflowNames
 

diff --git a/python/rename_and_reingest_files.py b/python/rename_and_reingest_files.py
@@ -6,7 +6,7 @@
 from utils.tdr_utils.tdr_api_utils import TDR
 from utils.tdr_utils.tdr_ingest_utils import GetPermissionsForWorkspaceIngest
 from utils.token_util import Token
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.terra_utils.terra_util import TerraWorkspace
 from utils import GCP, ARG_DEFAULTS
 

diff --git a/python/rename_columns_and_files_in_dataset.py b/python/rename_columns_and_files_in_dataset.py
@@ -6,7 +6,7 @@
 from utils.tdr_utils.tdr_api_utils import TDR
 from utils.tdr_utils.tdr_ingest_utils import GetPermissionsForWorkspaceIngest
 from utils.token_util import Token
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.terra_utils.terra_util import TerraWorkspace
 from utils import GCP, ARG_DEFAULTS
 

diff --git a/python/set_up_staging_workspace_and_dataset.py b/python/set_up_staging_workspace_and_dataset.py
@@ -7,7 +7,7 @@
 from utils.tdr_utils.tdr_ingest_utils import StartAndMonitorIngest
 from utils.terra_utils.terra_util import TerraWorkspace, TerraGroups, MEMBER, ADMIN
 from utils.terra_utils.terra_workflow_configs import WorkflowConfigs, GetWorkflowNames
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils.gcp_utils import GCPCloudFunctions
 from utils import GCP, comma_separated_list

diff --git a/python/tdr_summary_statistics.py b/python/tdr_summary_statistics.py
@@ -4,7 +4,7 @@
 
 from pandas import DataFrame
 from utils.tdr_utils.tdr_api_utils import TDR
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils.csv_util import Csv
 from utils.tdr_utils.tdr_bq_utils import TdrBq, GetTdrAssetInfo

diff --git a/python/template_script.py b/python/template_script.py
@@ -9,7 +9,7 @@
 from utils.tdr_utils.tdr_api_utils import TDR
 from utils.tdr_utils.tdr_ingest_utils import BatchIngest
 from utils.terra_utils.terra_util import TerraWorkspace
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils import GCP
 

diff --git a/python/terra_summary_statistics.py b/python/terra_summary_statistics.py
@@ -7,7 +7,7 @@
 from argparse import ArgumentParser, Namespace
 from datetime import datetime
 
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils import GCP
 from utils.tdr_utils.tdr_schema_utils import InferTDRSchema

diff --git a/python/tests/integration_tests/test_request_util.py b/python/tests/integration_tests/test_request_util.py
@@ -3,7 +3,7 @@
 from unittest.mock import MagicMock
 from requests import HTTPError
 
-from python.utils.request_util import RunRequest
+from python.utils.requests_utils.request_util import RunRequest
 
 
 """The instance of the Token class here is mocked, so we can simply test the functionality of the interaction

diff --git a/python/tests/integration_tests/test_terra_util.py b/python/tests/integration_tests/test_terra_util.py
@@ -4,7 +4,7 @@
 import re
 
 from python.utils import GCP
-from python.utils.request_util import RunRequest
+from python.utils.requests_utils.request_util import RunRequest
 from python.utils.terra_utils.terra_util import TerraWorkspace, TerraGroups, MEMBER
 from python.utils.terra_utils.terra_workflow_configs import WorkflowConfigs
 from python.utils.token_util import Token

diff --git a/python/tests/requirements.txt b/python/tests/requirements.txt
diff --git a/python/tests/unit_tests/test_tdr_utils.py b/python/tests/unit_tests/test_tdr_utils.py
@@ -8,7 +8,7 @@
 from python.utils.tdr_utils.tdr_schema_utils import InferTDRSchema
 from python.utils.tdr_utils.tdr_ingest_utils import BatchIngest
 from python.utils.token_util import Token
-from python.utils.request_util import RunRequest
+from python.utils.requests_utils.request_util import RunRequest
 
 
 def mock_api_response(test_json):

diff --git a/python/utils/requests_utils/__init__.py b/python/utils/requests_utils/__init__.py
diff --git a/python/tests/utils/mock_util.py → python/utils/requests_utils/mock_util.py b/python/tests/utils/mock_util.py → python/utils/requests_utils/mock_util.py
@@ -2,24 +2,25 @@
 import functools
 import inspect
 from pathlib import Path
+from typing import Any
 import responses
 import responses._recorder
 import git
 
 
-def make_filename(func):
+def make_filename(func: Any) -> Path:
     module = inspect.getmodule(func)
 
-    file_path = Path(module.__file__)
+    file_path = Path(module.__file__)  # type: ignore[union-attr, arg-type]
     repo = git.Repo(file_path, search_parent_directories=True)
     git_root = repo.git.rev_parse("--show-toplevel")
     return Path(git_root).joinpath("mock_output.yaml")
 
 
-def activate_responses():
-    def outer_decorator(func):
+def activate_responses() -> Any:
+    def outer_decorator(func: Any) -> Any:
         @functools.wraps(func)
-        def wrapper(*args, **kwargs):
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
             with responses.RequestsMock() as rsp:
                 rsp._add_from_file(file_path=make_filename(func))
                 return func(*args, **kwargs)
@@ -29,10 +30,10 @@ def wrapper(*args, **kwargs):
     return outer_decorator
 
 
-def activate_recorder():
-    def outer_decorator(func):
+def activate_recorder() -> Any:
+    def outer_decorator(func: Any) -> Any:
         @functools.wraps(func)
-        def wrapper(*args, **kwargs):
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
             recorder = responses._recorder.Recorder()
             with recorder:
                 try:
@@ -49,7 +50,7 @@ def wrapper(*args, **kwargs):
     return outer_decorator
 
 
-def mock_responses(activate=False, update_results=False):
+def mock_responses(activate: bool = False, update_results: bool = False) -> Any:
     """Decorator to record then mock requests made with the requests module.
 
     When update_results is True, will store requests to a yaml file. When it
@@ -66,7 +67,7 @@ def test_mytest(self):
                 request.get("https://example.com)
                 ...
     """
-    def conditional_decorator(func):
+    def conditional_decorator(func: Any) -> Any:
         if activate:
             if update_results:
                 return activate_recorder()

diff --git a/python/utils/request_util.py → python/utils/requests_utils/request_util.py b/python/utils/request_util.py → python/utils/requests_utils/request_util.py
@@ -1,5 +1,5 @@
 from typing import Any, Optional
-from python.tests.utils.mock_util import mock_responses
+from .mock_util import mock_responses
 import requests
 import backoff
 
@@ -83,7 +83,7 @@ def run_request(
         )
 
         # Apply decorators to request execution
-        @mock_responses(activate=self.create_mocks, update_results=True)
+        @mock_responses(activate=self.create_mocks, update_results=True)  # type: ignore[no-untyped-call]
         @backoff_decorator
         def _make_request() -> requests.Response:
             if method == GET:

diff --git a/python/utils/tdr_utils/tdr_api_utils.py b/python/utils/tdr_utils/tdr_api_utils.py
@@ -7,7 +7,7 @@
 from urllib.parse import unquote
 from pydantic import ValidationError
 
-from ..request_util import GET, POST, DELETE
+from ..requests_utils.request_util import GET, POST, DELETE
 from ..tdr_api_schema.create_dataset_schema import CreateDatasetSchema
 from ..tdr_api_schema.update_dataset_schema import UpdateSchema
 from .tdr_job_utils import MonitorTDRJob, SubmitAndMonitorMultipleJobs

diff --git a/python/utils/terra_utils/terra_util.py b/python/utils/terra_utils/terra_util.py
@@ -6,7 +6,7 @@
 
 from .. import GCP
 
-from ..request_util import GET, POST, PATCH, PUT, DELETE
+from ..requests_utils.request_util import GET, POST, PATCH, PUT, DELETE
 
 TERRA_LINK = "https://api.firecloud.org/api"
 LEONARDO_LINK = "https://leonardo.dsde-prod.broadinstitute.org/api"

diff --git a/python/validate_tdr_file_export.py b/python/validate_tdr_file_export.py
@@ -2,7 +2,7 @@
 import binascii
 from argparse import ArgumentParser, Namespace
 from utils.tdr_utils.tdr_api_utils import TDR
-from utils.request_util import RunRequest
+from utils.requests_utils.request_util import RunRequest
 from utils.token_util import Token
 from utils.gcp_utils import GCPCloudFunctions
 from utils.csv_util import Csv