From 52f3a8456e8e66e7b9e26cd78375e87dc5a0a707 Mon Sep 17 00:00:00 2001 From: Sam Novod Date: Wed, 18 Dec 2024 13:27:11 -0500 Subject: [PATCH] Moving mock and request utils so imports work --- python/azure_tdr_to_gcp_file_transfer.py | 2 +- python/copy_az_terra_files_to_gcp_bucket.py | 2 +- python/copy_dataset.py | 2 +- python/create_workspace_file_manifest.py | 2 +- python/delete_dataset.py | 2 +- ...delete_files_from_datasets_in_bad_state.py | 2 +- python/delete_snapshot.py | 2 +- .../export_data_from_snapshot_or_dataset.py | 2 +- .../gcp_workspace_table_to_dataset_ingest.py | 2 +- ..._and_delete_orphaned_files_from_dataset.py | 2 +- python/hard_clone_workspace.py | 2 +- python/import_workflows.py | 2 +- python/rename_and_reingest_files.py | 2 +- python/rename_columns_and_files_in_dataset.py | 2 +- .../set_up_staging_workspace_and_dataset.py | 2 +- python/tdr_summary_statistics.py | 2 +- python/template_script.py | 2 +- python/terra_summary_statistics.py | 2 +- .../integration_tests/test_request_util.py | 2 +- .../integration_tests/test_terra_util.py | 2 +- python/tests/requirements.txt | 2 -- python/tests/unit_tests/test_tdr_utils.py | 2 +- python/utils/requests_utils/__init__.py | 0 .../requests_utils}/mock_util.py | 21 ++++++++++--------- .../{ => requests_utils}/request_util.py | 4 ++-- python/utils/tdr_utils/tdr_api_utils.py | 2 +- python/utils/terra_utils/terra_util.py | 2 +- python/validate_tdr_file_export.py | 2 +- 28 files changed, 37 insertions(+), 38 deletions(-) delete mode 100644 python/tests/requirements.txt create mode 100644 python/utils/requests_utils/__init__.py rename python/{tests/utils => utils/requests_utils}/mock_util.py (77%) rename python/utils/{ => requests_utils}/request_util.py (98%) diff --git a/python/azure_tdr_to_gcp_file_transfer.py b/python/azure_tdr_to_gcp_file_transfer.py index 4facd715..c7f88e3d 100755 --- a/python/azure_tdr_to_gcp_file_transfer.py +++ b/python/azure_tdr_to_gcp_file_transfer.py @@ -10,7 +10,7 @@ from typing import Union from utils.tdr_utils.tdr_api_utils import TDR -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token diff --git a/python/copy_az_terra_files_to_gcp_bucket.py b/python/copy_az_terra_files_to_gcp_bucket.py index 59511171..90529666 100644 --- a/python/copy_az_terra_files_to_gcp_bucket.py +++ b/python/copy_az_terra_files_to_gcp_bucket.py @@ -4,7 +4,7 @@ from utils.azure_utils import AzureBlobDetails, SasTokenUtil from utils.terra_utils.terra_util import TerraWorkspace from utils.gcp_utils import GCPCloudFunctions -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token diff --git a/python/copy_dataset.py b/python/copy_dataset.py index ff75738a..42dc6775 100644 --- a/python/copy_dataset.py +++ b/python/copy_dataset.py @@ -5,7 +5,7 @@ from utils.tdr_utils.tdr_api_utils import TDR from utils.tdr_utils.tdr_ingest_utils import FilterAndBatchIngest -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils import GCP, ARG_DEFAULTS diff --git a/python/create_workspace_file_manifest.py b/python/create_workspace_file_manifest.py index 3542f09d..cc806acd 100755 --- a/python/create_workspace_file_manifest.py +++ b/python/create_workspace_file_manifest.py @@ -1,6 +1,6 @@ from utils.terra_utils.terra_util import TerraWorkspace from utils.gcp_utils import GCPCloudFunctions -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils import GCP, comma_separated_list import csv diff --git a/python/delete_dataset.py b/python/delete_dataset.py index 55cb4717..a8c8a3ba 100644 --- a/python/delete_dataset.py +++ b/python/delete_dataset.py @@ -5,7 +5,7 @@ from argparse import ArgumentParser, Namespace from utils.tdr_utils.tdr_api_utils import TDR -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils import GCP diff --git a/python/delete_files_from_datasets_in_bad_state.py b/python/delete_files_from_datasets_in_bad_state.py index 6c40f677..9aede977 100644 --- a/python/delete_files_from_datasets_in_bad_state.py +++ b/python/delete_files_from_datasets_in_bad_state.py @@ -1,7 +1,7 @@ import logging import requests import re -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils import GCP, ARG_DEFAULTS from argparse import ArgumentParser, Namespace diff --git a/python/delete_snapshot.py b/python/delete_snapshot.py index 1e5234e2..f6906a0e 100644 --- a/python/delete_snapshot.py +++ b/python/delete_snapshot.py @@ -4,7 +4,7 @@ from utils.tdr_utils.tdr_api_utils import TDR from utils.tdr_utils.tdr_job_utils import MonitorTDRJob -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils import GCP from collections import defaultdict diff --git a/python/export_data_from_snapshot_or_dataset.py b/python/export_data_from_snapshot_or_dataset.py index 5a90791b..3ea4e3b8 100644 --- a/python/export_data_from_snapshot_or_dataset.py +++ b/python/export_data_from_snapshot_or_dataset.py @@ -7,7 +7,7 @@ from utils.gcp_utils import GCPCloudFunctions from utils.token_util import Token from utils.tdr_utils.tdr_api_utils import TDR -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils import GCP, ARG_DEFAULTS logging.basicConfig( diff --git a/python/gcp_workspace_table_to_dataset_ingest.py b/python/gcp_workspace_table_to_dataset_ingest.py index 0f67baf1..e7d9518e 100644 --- a/python/gcp_workspace_table_to_dataset_ingest.py +++ b/python/gcp_workspace_table_to_dataset_ingest.py @@ -11,7 +11,7 @@ ) from utils.tdr_utils.tdr_table_utils import SetUpTDRTables from utils.token_util import Token -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest logging.basicConfig( diff --git a/python/get_and_delete_orphaned_files_from_dataset.py b/python/get_and_delete_orphaned_files_from_dataset.py index 13593242..1aa5af74 100644 --- a/python/get_and_delete_orphaned_files_from_dataset.py +++ b/python/get_and_delete_orphaned_files_from_dataset.py @@ -2,7 +2,7 @@ import argparse from utils.tdr_utils.tdr_api_utils import TDR -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils import GCP, ARG_DEFAULTS diff --git a/python/hard_clone_workspace.py b/python/hard_clone_workspace.py index 0e496ad2..e7646947 100644 --- a/python/hard_clone_workspace.py +++ b/python/hard_clone_workspace.py @@ -8,7 +8,7 @@ from utils import GCP, comma_separated_list, ARG_DEFAULTS from utils.terra_utils.terra_util import TerraWorkspace -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils.csv_util import Csv from utils.gcp_utils import GCPCloudFunctions diff --git a/python/import_workflows.py b/python/import_workflows.py index 654c5a7d..dcb188e5 100755 --- a/python/import_workflows.py +++ b/python/import_workflows.py @@ -2,7 +2,7 @@ import argparse from typing import Any from utils.terra_utils.terra_util import TerraWorkspace -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils.terra_utils.terra_workflow_configs import WorkflowConfigs, GetWorkflowNames diff --git a/python/rename_and_reingest_files.py b/python/rename_and_reingest_files.py index c8053040..140ba48a 100644 --- a/python/rename_and_reingest_files.py +++ b/python/rename_and_reingest_files.py @@ -6,7 +6,7 @@ from utils.tdr_utils.tdr_api_utils import TDR from utils.tdr_utils.tdr_ingest_utils import GetPermissionsForWorkspaceIngest from utils.token_util import Token -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.terra_utils.terra_util import TerraWorkspace from utils import GCP, ARG_DEFAULTS diff --git a/python/rename_columns_and_files_in_dataset.py b/python/rename_columns_and_files_in_dataset.py index ce6dccc4..8bcdbca3 100644 --- a/python/rename_columns_and_files_in_dataset.py +++ b/python/rename_columns_and_files_in_dataset.py @@ -6,7 +6,7 @@ from utils.tdr_utils.tdr_api_utils import TDR from utils.tdr_utils.tdr_ingest_utils import GetPermissionsForWorkspaceIngest from utils.token_util import Token -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.terra_utils.terra_util import TerraWorkspace from utils import GCP, ARG_DEFAULTS diff --git a/python/set_up_staging_workspace_and_dataset.py b/python/set_up_staging_workspace_and_dataset.py index 296efeed..b83d899d 100644 --- a/python/set_up_staging_workspace_and_dataset.py +++ b/python/set_up_staging_workspace_and_dataset.py @@ -7,7 +7,7 @@ from utils.tdr_utils.tdr_ingest_utils import StartAndMonitorIngest from utils.terra_utils.terra_util import TerraWorkspace, TerraGroups, MEMBER, ADMIN from utils.terra_utils.terra_workflow_configs import WorkflowConfigs, GetWorkflowNames -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils.gcp_utils import GCPCloudFunctions from utils import GCP, comma_separated_list diff --git a/python/tdr_summary_statistics.py b/python/tdr_summary_statistics.py index aebb5501..20bf7318 100644 --- a/python/tdr_summary_statistics.py +++ b/python/tdr_summary_statistics.py @@ -4,7 +4,7 @@ from pandas import DataFrame from utils.tdr_utils.tdr_api_utils import TDR -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils.csv_util import Csv from utils.tdr_utils.tdr_bq_utils import TdrBq, GetTdrAssetInfo diff --git a/python/template_script.py b/python/template_script.py index bf185110..c58aabd7 100644 --- a/python/template_script.py +++ b/python/template_script.py @@ -9,7 +9,7 @@ from utils.tdr_utils.tdr_api_utils import TDR from utils.tdr_utils.tdr_ingest_utils import BatchIngest from utils.terra_utils.terra_util import TerraWorkspace -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils import GCP diff --git a/python/terra_summary_statistics.py b/python/terra_summary_statistics.py index 3c70ee15..afab3675 100644 --- a/python/terra_summary_statistics.py +++ b/python/terra_summary_statistics.py @@ -7,7 +7,7 @@ from argparse import ArgumentParser, Namespace from datetime import datetime -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils import GCP from utils.tdr_utils.tdr_schema_utils import InferTDRSchema diff --git a/python/tests/integration_tests/test_request_util.py b/python/tests/integration_tests/test_request_util.py index f9d4684f..f66f01eb 100644 --- a/python/tests/integration_tests/test_request_util.py +++ b/python/tests/integration_tests/test_request_util.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock from requests import HTTPError -from python.utils.request_util import RunRequest +from python.utils.requests_utils.request_util import RunRequest """The instance of the Token class here is mocked, so we can simply test the functionality of the interaction diff --git a/python/tests/integration_tests/test_terra_util.py b/python/tests/integration_tests/test_terra_util.py index fefea18c..5efd8c68 100644 --- a/python/tests/integration_tests/test_terra_util.py +++ b/python/tests/integration_tests/test_terra_util.py @@ -4,7 +4,7 @@ import re from python.utils import GCP -from python.utils.request_util import RunRequest +from python.utils.requests_utils.request_util import RunRequest from python.utils.terra_utils.terra_util import TerraWorkspace, TerraGroups, MEMBER from python.utils.terra_utils.terra_workflow_configs import WorkflowConfigs from python.utils.token_util import Token diff --git a/python/tests/requirements.txt b/python/tests/requirements.txt deleted file mode 100644 index 208ec64c..00000000 --- a/python/tests/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest -responses diff --git a/python/tests/unit_tests/test_tdr_utils.py b/python/tests/unit_tests/test_tdr_utils.py index efc0fde7..4e601119 100644 --- a/python/tests/unit_tests/test_tdr_utils.py +++ b/python/tests/unit_tests/test_tdr_utils.py @@ -8,7 +8,7 @@ from python.utils.tdr_utils.tdr_schema_utils import InferTDRSchema from python.utils.tdr_utils.tdr_ingest_utils import BatchIngest from python.utils.token_util import Token -from python.utils.request_util import RunRequest +from python.utils.requests_utils.request_util import RunRequest def mock_api_response(test_json): diff --git a/python/utils/requests_utils/__init__.py b/python/utils/requests_utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/tests/utils/mock_util.py b/python/utils/requests_utils/mock_util.py similarity index 77% rename from python/tests/utils/mock_util.py rename to python/utils/requests_utils/mock_util.py index 830fcdaa..8edda037 100644 --- a/python/tests/utils/mock_util.py +++ b/python/utils/requests_utils/mock_util.py @@ -2,24 +2,25 @@ import functools import inspect from pathlib import Path +from typing import Any import responses import responses._recorder import git -def make_filename(func): +def make_filename(func: Any) -> Path: module = inspect.getmodule(func) - file_path = Path(module.__file__) + file_path = Path(module.__file__) # type: ignore[union-attr, arg-type] repo = git.Repo(file_path, search_parent_directories=True) git_root = repo.git.rev_parse("--show-toplevel") return Path(git_root).joinpath("mock_output.yaml") -def activate_responses(): - def outer_decorator(func): +def activate_responses() -> Any: + def outer_decorator(func: Any) -> Any: @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: with responses.RequestsMock() as rsp: rsp._add_from_file(file_path=make_filename(func)) return func(*args, **kwargs) @@ -29,10 +30,10 @@ def wrapper(*args, **kwargs): return outer_decorator -def activate_recorder(): - def outer_decorator(func): +def activate_recorder() -> Any: + def outer_decorator(func: Any) -> Any: @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: recorder = responses._recorder.Recorder() with recorder: try: @@ -49,7 +50,7 @@ def wrapper(*args, **kwargs): return outer_decorator -def mock_responses(activate=False, update_results=False): +def mock_responses(activate: bool = False, update_results: bool = False) -> Any: """Decorator to record then mock requests made with the requests module. When update_results is True, will store requests to a yaml file. When it @@ -66,7 +67,7 @@ def test_mytest(self): request.get("https://example.com) ... """ - def conditional_decorator(func): + def conditional_decorator(func: Any) -> Any: if activate: if update_results: return activate_recorder() diff --git a/python/utils/request_util.py b/python/utils/requests_utils/request_util.py similarity index 98% rename from python/utils/request_util.py rename to python/utils/requests_utils/request_util.py index 9dc5e385..c600b480 100644 --- a/python/utils/request_util.py +++ b/python/utils/requests_utils/request_util.py @@ -1,5 +1,5 @@ from typing import Any, Optional -from python.tests.utils.mock_util import mock_responses +from .mock_util import mock_responses import requests import backoff @@ -83,7 +83,7 @@ def run_request( ) # Apply decorators to request execution - @mock_responses(activate=self.create_mocks, update_results=True) + @mock_responses(activate=self.create_mocks, update_results=True) # type: ignore[no-untyped-call] @backoff_decorator def _make_request() -> requests.Response: if method == GET: diff --git a/python/utils/tdr_utils/tdr_api_utils.py b/python/utils/tdr_utils/tdr_api_utils.py index 8afddf9c..108c37b0 100644 --- a/python/utils/tdr_utils/tdr_api_utils.py +++ b/python/utils/tdr_utils/tdr_api_utils.py @@ -7,7 +7,7 @@ from urllib.parse import unquote from pydantic import ValidationError -from ..request_util import GET, POST, DELETE +from ..requests_utils.request_util import GET, POST, DELETE from ..tdr_api_schema.create_dataset_schema import CreateDatasetSchema from ..tdr_api_schema.update_dataset_schema import UpdateSchema from .tdr_job_utils import MonitorTDRJob, SubmitAndMonitorMultipleJobs diff --git a/python/utils/terra_utils/terra_util.py b/python/utils/terra_utils/terra_util.py index 0e332d77..49d899ee 100644 --- a/python/utils/terra_utils/terra_util.py +++ b/python/utils/terra_utils/terra_util.py @@ -6,7 +6,7 @@ from .. import GCP -from ..request_util import GET, POST, PATCH, PUT, DELETE +from ..requests_utils.request_util import GET, POST, PATCH, PUT, DELETE TERRA_LINK = "https://api.firecloud.org/api" LEONARDO_LINK = "https://leonardo.dsde-prod.broadinstitute.org/api" diff --git a/python/validate_tdr_file_export.py b/python/validate_tdr_file_export.py index 6fc92a76..72544584 100644 --- a/python/validate_tdr_file_export.py +++ b/python/validate_tdr_file_export.py @@ -2,7 +2,7 @@ import binascii from argparse import ArgumentParser, Namespace from utils.tdr_utils.tdr_api_utils import TDR -from utils.request_util import RunRequest +from utils.requests_utils.request_util import RunRequest from utils.token_util import Token from utils.gcp_utils import GCPCloudFunctions from utils.csv_util import Csv