Merge branch 'main' into 1831-bug-json-serialization-of-date-types-ca…

…uses-schema-validation-to-fail-in-target-postgres

.pre-commit-config.yaml

@@ -50,7 +50,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.5
+  rev: v0.5.6
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
@@ -65,7 +65,7 @@ repos:
       )$
 
 - repo: https://github.com/pycqa/flake8
-  rev: 7.1.0
+  rev: 7.1.1
   hooks:
   - id: flake8
     additional_dependencies:

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml

@@ -18,19 +18,19 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.28.6
+  rev: 0.29.1
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.0
+  rev: v0.5.6
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.10.1
+  rev: v1.11.1
   hooks:
   - id: mypy

cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml

@@ -18,20 +18,20 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.28.6
+  rev: 0.29.1
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.0
+  rev: v0.5.6
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.10.1
+  rev: v1.11.1
   hooks:
   - id: mypy
     additional_dependencies:

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py

@@ -6,9 +6,8 @@
 {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
 from functools import cached_property
 {%- endif %}
-from typing import TYPE_CHECKING, Any, Callable, Iterable
+from typing import TYPE_CHECKING, Any, Iterable
 
-import requests
 {% if cookiecutter.auth_method  == "API Key" -%}
 from singer_sdk.authenticators import APIKeyAuthenticator
 from singer_sdk.helpers.jsonpath import extract_jsonpath
@@ -47,11 +46,14 @@
     import importlib_resources
 
 if TYPE_CHECKING:
+    import requests
+    {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
+    from singer_sdk.helpers.types import Auth, Context
+    {%- else %}
     from singer_sdk.helpers.types import Context
+    {%- endif %}
 
 
-_Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest]
-
 # TODO: Delete this is if not using json files for schema definition
 SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
@@ -74,7 +76,7 @@ def url_base(self) -> str:
 {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
 
     @cached_property
-    def authenticator(self) -> _Auth:
+    def authenticator(self) -> Auth:
         """Return a new authenticator object.
 
         Returns:

cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml

@@ -18,20 +18,20 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.28.6
+  rev: 0.29.1
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.0
+  rev: v0.5.6
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.10.1
+  rev: v1.11.1
   hooks:
   - id: mypy
     additional_dependencies:

noxfile.py

@@ -25,6 +25,8 @@
 
 RUFF_OVERRIDES = """\
 extend = "./pyproject.toml"
+
+[lint]
 extend-ignore = ["TD002", "TD003", "FIX002"]
 """
 
@@ -254,7 +256,7 @@ def test_cookiecutter(session: Session, replay_file_path: str) -> None:
     session.run("poetry", "lock", external=True)
     session.run("poetry", "install", external=True)
 
-    session.run("git", "init", external=True)
+    session.run("git", "init", "-b", "main", external=True)
     session.run("git", "add", ".", external=True)
     session.run("pre-commit", "run", "--all-files", external=True)
 

singer_sdk/_singerlib/json.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+from __future__ import annotations  # noqa: A005
 
 import datetime
 import decimal

singer_sdk/connectors/sql.py

@@ -974,8 +974,8 @@ def _get_column_type(
 
         return column.type
 
-    @staticmethod
     def get_column_add_ddl(
+        self,
         table_name: str,
         column_name: str,
         column_type: sa.types.TypeEngine,
@@ -998,11 +998,12 @@ def get_column_add_ddl(
                 column_type,
             ),
         )
+        compiled = create_column_clause.compile(self._engine).string
         return sa.DDL(
             "ALTER TABLE %(table_name)s ADD COLUMN %(create_column_clause)s",
             {
                 "table_name": table_name,
-                "create_column_clause": create_column_clause,
+                "create_column_clause": compiled,
             },
         )
 

singer_sdk/helpers/capabilities.py

@@ -239,7 +239,7 @@ def emit_warning(self) -> None:
 class DeprecatedEnumMeta(EnumMeta):
     """Metaclass for enumeration with deprecation support."""
 
-    def __getitem__(self, name: str) -> t.Any:  # noqa: ANN401
+    def __getitem__(cls, name: str) -> t.Any:  # noqa: ANN401
         """Retrieve mapping item.
 
         Args:
@@ -253,7 +253,7 @@ def __getitem__(self, name: str) -> t.Any:  # noqa: ANN401
             obj.emit_warning()
         return obj
 
-    def __getattribute__(cls, name: str) -> t.Any:  # noqa: ANN401, N805
+    def __getattribute__(cls, name: str) -> t.Any:  # noqa: ANN401
         """Retrieve enum attribute.
 
         Args:
@@ -267,7 +267,7 @@ def __getattribute__(cls, name: str) -> t.Any:  # noqa: ANN401, N805
             obj.emit_warning()
         return obj
 
-    def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:  # noqa: ANN401
+    def __call__(cls, *args: t.Any, **kwargs: t.Any) -> t.Any:  # noqa: ANN401
         """Call enum member.
 
         Args:

singer_sdk/helpers/types.py

@@ -1,10 +1,12 @@
-"""Type aliases for use in the SDK."""
+"""Type aliases for use in the SDK."""  # noqa: A005
 
 from __future__ import annotations
 
 import sys
 import typing as t
 
+import requests
+
 if sys.version_info < (3, 9):
     from typing import Mapping  # noqa: ICN003
 else:
@@ -15,10 +17,12 @@
 else:
     from typing import TypeAlias  # noqa: ICN003
 
+
 __all__ = [
     "Context",
     "Record",
 ]
 
 Context: TypeAlias = Mapping[str, t.Any]
 Record: TypeAlias = t.Dict[str, t.Any]
+Auth: TypeAlias = t.Callable[[requests.PreparedRequest], requests.PreparedRequest]

singer_sdk/streams/rest.py

@@ -26,25 +26,18 @@
 from singer_sdk.streams.core import Stream
 
 if t.TYPE_CHECKING:
-    import sys
     from datetime import datetime
 
     from backoff.types import Details
 
     from singer_sdk._singerlib import Schema
-    from singer_sdk.helpers.types import Context
+    from singer_sdk.helpers.types import Auth, Context
     from singer_sdk.tap_base import Tap
 
-    if sys.version_info >= (3, 10):
-        from typing import TypeAlias  # noqa: ICN003
-    else:
-        from typing_extensions import TypeAlias
-
 DEFAULT_PAGE_SIZE = 1000
 DEFAULT_REQUEST_TIMEOUT = 300  # 5 minutes
 
 _TToken = t.TypeVar("_TToken")
-_Auth: TypeAlias = t.Callable[[requests.PreparedRequest], requests.PreparedRequest]
 
 
 class RESTStream(Stream, t.Generic[_TToken], metaclass=abc.ABCMeta):  # noqa: PLR0904
@@ -610,7 +603,7 @@ def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
     # Abstract methods:
 
     @property
-    def authenticator(self) -> _Auth:
+    def authenticator(self) -> Auth:
         """Return or set the authenticator for managing HTTP auth headers.
 
         If an authenticator is not specified, REST-based taps will simply pass

singer_sdk/testing/target_test_streams/camelcase_complex_schema.singer

@@ -1,2 +1,2 @@
 {"type": "SCHEMA", "stream": "ForecastingTypeToCategory", "schema": {"properties": {"Id": {"type": "string"}, "IsDeleted": {"type": ["null", "boolean"]}, "CreatedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "CreatedById": {"type": ["null", "string"]}, "LastModifiedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "LastModifiedById": {"type": ["null", "string"]}, "SystemModstamp": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "ForecastingTypeId": {"type": ["null", "string"]}, "ForecastingItemCategory": {"type": ["null", "string"]}, "DisplayPosition": {"type": ["null", "integer"]}, "IsAdjustable": {"type": ["null", "boolean"]}, "IsOwnerAdjustable": {"type": ["null", "boolean"]}}, "type": "object", "additionalProperties": false}, "key_properties": ["Id"]}
-{"type": "SCHEMA", "stream": "ForecastingTypeToCategory", "schema": {"properties": {"Id": {"type": "string"}, "IsDeleted": {"type": ["null", "boolean"]}, "CreatedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "CreatedById": {"type": ["null", "string"]}, "LastModifiedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "LastModifiedById": {"type": ["null", "string"]}, "SystemModstamp": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "ForecastingTypeId": {"type": ["null", "string"]}, "ForecastingItemCategory": {"type": ["null", "string"]}, "DisplayPosition": {"type": ["null", "integer"]}, "IsAdjustable": {"type": ["null", "boolean"]}, "IsOwnerAdjustable": {"type": ["null", "boolean"]}, "age": {"type": "integer"}, "NewCamelCasedAttribute": {"type": "string"}}, "type": "object", "additionalProperties": false}, "key_properties": ["Id"]}
+{"type": "SCHEMA", "stream": "ForecastingTypeToCategory", "schema": {"properties": {"Id": {"type": "string"}, "IsDeleted": {"type": ["null", "boolean"]}, "CreatedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "CreatedById": {"type": ["null", "string"]}, "LastModifiedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "LastModifiedById": {"type": ["null", "string"]}, "SystemModstamp": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "ForecastingTypeId": {"type": ["null", "string"]}, "ForecastingItemCategory": {"type": ["null", "string"]}, "DisplayPosition": {"type": ["null", "integer"]}, "IsAdjustable": {"type": ["null", "boolean"]}, "IsOwnerAdjustable": {"type": ["null", "boolean"]}, "age": {"type": "integer"}, "NewCamelCasedAttribute": {"type": "string"}, "_attribute_startswith_underscore": {"type": "string"}}, "type": "object", "additionalProperties": false}, "key_properties": ["Id"]}

singer_sdk/typing.py

@@ -48,7 +48,7 @@
   other valid implementations which are not syntactically identical to those generated
   here.
 
-"""
+"""  # noqa: A005
 
 from __future__ import annotations
 

tests/samples/test_target_sqlite.py

@@ -182,7 +182,9 @@ def test_sqlite_column_addition(sqlite_sample_target: SQLTarget):
     props_a: dict[str, dict] = {"col_a": th.StringType().to_dict()}
     props_b = deepcopy(props_a)
     props_b["col_b"] = th.IntegerType().to_dict()
-    schema_msg_a, schema_msg_b = (
+    props_c = deepcopy(props_b)
+    props_c["_col_c"] = th.IntegerType().to_dict()
+    schema_msg_a, schema_msg_b, schema_msg_c = (
         {
             "type": "SCHEMA",
             "stream": test_tbl,
@@ -191,7 +193,7 @@ def test_sqlite_column_addition(sqlite_sample_target: SQLTarget):
                 "properties": props,
             },
         }
-        for props in [props_a, props_b]
+        for props in [props_a, props_b, props_c]
     )
     tap_output_a = "\n".join(
         json.dumps(msg)
@@ -211,8 +213,20 @@ def test_sqlite_column_addition(sqlite_sample_target: SQLTarget):
             },
         ]
     )
+    tap_output_c = "\n".join(
+        json.dumps(msg)
+        for msg in [
+            schema_msg_c,
+            {
+                "type": "RECORD",
+                "stream": test_tbl,
+                "record": {"col_a": "samplerow2", "col_b": 2, "_col_c": 3},
+            },
+        ]
+    )
     target_sync_test(sqlite_sample_target, input=StringIO(tap_output_a), finalize=True)
     target_sync_test(sqlite_sample_target, input=StringIO(tap_output_b), finalize=True)
+    target_sync_test(sqlite_sample_target, input=StringIO(tap_output_c), finalize=True)
 
 
 def test_sqlite_activate_version(