Add 409 to ingest response handling (#64)

.pre-commit-config.yaml

@@ -45,13 +45,13 @@ repos:
       - id: no-commit-to-branch
         args: [--branch, dev, --branch, int, --branch, main]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.1
+    rev: v0.7.4
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.1
+    rev: v1.13.0
     hooks:
       - id: mypy
         args: [--no-warn-unused-ignores]

.pyproject_generation/pyproject_custom.toml

@@ -1,10 +1,10 @@
 [project]
 name = "ghga_datasteward_kit"
-version = "4.4.0"
+version = "4.4.1"
 description = "GHGA Data Steward Kit - A utils package for GHGA data stewards."
 dependencies = [
     "crypt4gh >=1.6, <2",
-    "hexkit[s3] >=3, <4",
+    "hexkit[s3] >=3.7.0, <4",
     "ghga-transpiler >=2.1.2, <3",
     "metldata~=2.1.1",
     "tenacity >=9.0.0, <10",

pyproject.toml

@@ -21,11 +21,11 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "ghga_datasteward_kit"
-version = "4.4.0"
+version = "4.4.1"
 description = "GHGA Data Steward Kit - A utils package for GHGA data stewards."
 dependencies = [
     "crypt4gh >=1.6, <2",
-    "hexkit[s3] >=3, <4",
+    "hexkit[s3] >=3.7.0, <4",
     "ghga-transpiler >=2.1.2, <3",
     "metldata~=2.1.1",
     "tenacity >=9.0.0, <10",

src/ghga_datasteward_kit/cli/file.py

@@ -15,12 +15,15 @@
 
 """File related CLI"""
 
+import logging
 from pathlib import Path
 
 import typer
 
 from ghga_datasteward_kit import batch_s3_upload, file_deletion, file_ingest, s3_upload
 
+log = logging.getLogger(__name__)
+
 cli = typer.Typer()
 
 
@@ -101,12 +104,15 @@ def ingest_upload_metadata(
     config_path: Path = typer.Option(..., help="Path to a config YAML."),
 ):
     """Upload all output metadata files from the given directory to the file ingest service."""
-    errors = file_ingest.main(config_path=config_path)
+    errors, successes = file_ingest.main(config_path=config_path)
 
     if errors:
         print(f"Encountered {len(errors)} errors during processing.")
         for file_path, cause in errors.items():
             print(f" -{file_path}: {cause}")
+        print("\nSuccessfully ingested the following files:")
+        for file_path in sorted(successes):
+            print(f" -{file_path}")
     else:
         print("Successfully sent all file upload metadata for ingest.")
 

src/ghga_datasteward_kit/file_ingest.py

@@ -119,6 +119,7 @@ def main(
     token = utils.STEWARD_TOKEN.read_token()
 
     errors = {}
+    successes = set()
 
     for in_path in config.input_dir.iterdir():
         if in_path.suffix != ".json":
@@ -128,8 +129,10 @@ def main(
         except (ValidationError, ValueError) as error:
             errors[in_path.resolve()] = str(error)
             continue
+        else:
+            successes.add(in_path.resolve())
 
-    return errors
+    return errors, successes
 
 
 def file_ingest(
@@ -185,12 +188,20 @@ def file_ingest(
 
         if response.status_code != 202:
             if response.status_code == 403:
-                raise ValueError("Not authorized to access ingest endpoint.")
-            if response.status_code == 422:
-                raise ValueError("Could not decrypt received payload.")
-            if response.status_code == 500:
-                raise ValueError(
+                error = ValueError("Not authorized to access ingest endpoint.")
+            elif response.status_code == 409:
+                error = ValueError("Metadata has already been processed.")
+            elif response.status_code == 422:
+                error = ValueError("Could not decrypt received payload.")
+            elif response.status_code == 500:
+                error = ValueError(
                     "Internal file ingest service error or communication with vault failed."
                 )
+            else:
+                error = ValueError(
+                    f"Unexpected server response: {response.status_code}."
+                )
+            LOG.error(error)
+            raise error
 
-            raise ValueError(f"Unexpected server response: {response.status_code}.")
+    LOG.info("Succesfully ingested metdatada for %s", in_path)

tests/test_retries.py

@@ -27,6 +27,9 @@
 URL = "http://not-a-real-url/test"
 
 
+@pytest.mark.httpx_mock(
+    assert_all_responses_were_requested=False, can_send_already_matched_responses=True
+)
 @pytest.mark.asyncio
 async def test_retry_handler(legacy_config_fixture: LegacyConfig, httpx_mock):  # noqa: F811
     """Test if configuration is correctly applied to retry handler"""

tests/test_s3_upload.py

@@ -41,12 +41,12 @@
 BUCKET_ID = "test-bucket"
 
 
-@pytest.fixture
-def non_mocked_hosts() -> list[str]:
-    """Overwrite default behaviour"""
-    return ["localhost", "host.docker.internal"]
-
-
+@pytest.mark.httpx_mock(
+    assert_all_responses_were_requested=False,
+    can_send_already_matched_responses=True,
+    should_mock=lambda request: request.url.host
+    not in ("127.0.0.1", "localhost", "host.docker.internal"),
+)
 @pytest.mark.asyncio
 async def test_legacy_process(
     legacy_config_fixture: LegacyConfig,  # noqa: F811
@@ -81,6 +81,12 @@ async def test_legacy_process(
         assert (config.output_dir / ALIAS).with_suffix(".json").exists()
 
 
+@pytest.mark.httpx_mock(
+    assert_all_responses_were_requested=False,
+    can_send_already_matched_responses=True,
+    should_mock=lambda request: request.url.host
+    not in ("127.0.0.1", "localhost", "host.docker.internal"),
+)
 @pytest.mark.asyncio
 async def test_process(config_fixture: Config, monkeypatch, httpx_mock: HTTPXMock):  # noqa: F811
     """Test whole upload/download process for s3_upload script"""