diff --git a/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst b/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst index 70e9a127f..1b3b608c5 100644 --- a/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst +++ b/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst @@ -5,4 +5,4 @@ .. autoclass:: APIAuthenticatorBase :members: - :special-members: __init__, __call__ + :special-members: __init__, __call__ \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index d47fcab8b..1b91724f0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # Show typehints in the description, along with parameter descriptions -autodoc_typehints = "signature" +autodoc_typehints = "description" autodoc_class_signature = "separated" autodoc_member_order = "groupwise" diff --git a/docs/testing.md b/docs/testing.md index 19ad02ef0..1f8c97e33 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -10,7 +10,7 @@ The Meltano SDK test framework consists of 4 main components: 1. A runner class (`TapTestRunner` and `TargetTestRunner`), responsible for executing Taps/Targets and capturing their output. 1. A suite dataclass, containing a list of tests. 1. A test template classes (`TapTestTemplate`, `StreamTestTemplate`, `AttributeTestTemplate` and `TargetTestTemplate`), with methods to `.setup()`, `.test()`, `.validate()` and `.teardown()` (called in that order using `.run()`). -1. `get_tap_test_class` and `get_target_test_class` factory methods. These wrap a `get_test_class` factory method, which takes a runner and a list of suites and return a `pytest` test class. +1. {func}`get_tap_test_class ` and {func}`get_target_test_class ` factory methods. These wrap a `get_test_class` factory method, which takes a runner and a list of suites and return a `pytest` test class. ## Example Usage @@ -76,7 +76,7 @@ class TestTargetExample(StandardTargetTests): ## Configuring Tests -Test suite behaviors can be configured by passing a `SuiteConfig` instance to the `get_test_class` functions: +Test suite behaviors can be configured by passing a {func}`SuiteConfig ` instance to the `get_test_class` functions: ```python from singer_sdk.testing import SuiteConfig, get_tap_test_class @@ -101,7 +101,7 @@ TestTapStackExchange = get_tap_test_class( ) ``` -Check out [`singer_sdk/testing/config.py`](https://github.com/meltano/sdk/tree/main/singer_sdk/testing/config.py) for available config options. +Check out [the reference](#reference) for more information on the available configuration options. ## Writing New Tests @@ -127,6 +127,21 @@ my_custom_tap_tests = TestSuite( ) ``` -This suite can now be passed to `get_tap_test_class` or `get_target_test_class` in a list of `custom_suites` along with any other suites, to generate your custom test class. +This suite can now be passed to {func}`get_tap_test_class ` or {func}`get_target_test_class ` in a list of `custom_suites` along with any other suites, to generate your custom test class. If your new test covers a common or general case, consider contributing to the standard test library via a pull request to [meltano/sdk](https://github.com/meltano/sdk). + +## Reference + +```{eval-rst} +.. autofunction:: singer_sdk.testing.get_tap_test_class +``` + +```{eval-rst} +.. autofunction:: singer_sdk.testing.get_target_test_class +``` + +```{eval-rst} +.. autoclass:: singer_sdk.testing.SuiteConfig + :members: +``` diff --git a/poetry.lock b/poetry.lock index e21215e4f..0cecf51f1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -592,13 +592,13 @@ files = [ [[package]] name = "duckdb-engine" -version = "0.11.0" +version = "0.11.1" description = "SQLAlchemy driver for duckdb" optional = false python-versions = ">=3.7" files = [ - {file = "duckdb_engine-0.11.0-py3-none-any.whl", hash = "sha256:b19c408f601d0a3af25f2d27cf78e2b9d981243443c5de5c258a3ccc586990f1"}, - {file = "duckdb_engine-0.11.0.tar.gz", hash = "sha256:4c808bc71873beef0b336b782d2e6809a152c1c5b2f7589db15893f75ec8ec9b"}, + {file = "duckdb_engine-0.11.1-py3-none-any.whl", hash = "sha256:ab69a00472fb34b0c57329461f1765e6fc528a0b42edbd2d566dad9b760a0d61"}, + {file = "duckdb_engine-0.11.1.tar.gz", hash = "sha256:ea7004c689e1f17b2e2d1734c522aa8a7d56366bfbeeddbe8a02dc6a6c36dc15"}, ] [package.dependencies] @@ -621,13 +621,13 @@ test = ["pytest (>=6)"] [[package]] name = "faker" -version = "22.6.0" +version = "23.1.0" description = "Faker is a Python package that generates fake data for you." optional = true python-versions = ">=3.8" files = [ - {file = "Faker-22.6.0-py3-none-any.whl", hash = "sha256:2b57f0256da6b45b7851dca87836ef5e2ae2fbb64d63d8697f1e47830d7b505d"}, - {file = "Faker-22.6.0.tar.gz", hash = "sha256:fa6d969728ef3da6229da91267a1bd4e6b902044c4822012d4fc46c71bb92b26"}, + {file = "Faker-23.1.0-py3-none-any.whl", hash = "sha256:60e89e5c0b584e285a7db05eceba35011a241954afdab2853cb246c8a56700a2"}, + {file = "Faker-23.1.0.tar.gz", hash = "sha256:b7f76bb1b2ac4cdc54442d955e36e477c387000f31ce46887fb9722a041be60b"}, ] [package.dependencies] @@ -2626,7 +2626,7 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] [extras] -docs = ["furo", "myst-parser", "sphinx", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-notfound-page", "sphinx-reredirects"] +docs = ["furo", "myst-parser", "pytest", "sphinx", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-notfound-page", "sphinx-reredirects"] faker = ["faker"] parquet = ["numpy", "numpy", "pyarrow"] s3 = ["fs-s3fs"] @@ -2635,4 +2635,4 @@ testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" python-versions = ">=3.8" -content-hash = "cf326e02c7b02fbe1d12a80b3d16a4485357799c06585b3a36b025ad02bde08f" +content-hash = "e9747e01321a2fd07fb58447bced9656dca2b3602f9b124a9fbd68792c47c30b" diff --git a/pyproject.toml b/pyproject.toml index 1c288463e..302919343 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,14 +91,15 @@ pytest = {version=">=7.2.1", optional = true} pytest-durations = {version = ">=1.2.0", optional = true} # installed as optional 'faker' extra -faker = {version = "~=22.5", optional = true} +faker = {version = ">=22.5,<24.0", optional = true} [tool.poetry.extras] docs = [ - "sphinx", "furo", - "sphinx-copybutton", "myst-parser", + "pytest", + "sphinx", + "sphinx-copybutton", "sphinx-autobuild", "sphinx-inline-tabs", "sphinx-notfound-page", diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index 02c8e8d16..e097a4ce9 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -464,7 +464,8 @@ def discover_catalog_entry( th.Property( name=column_name, wrapped=th.CustomType(jsonschema_type), - required=not is_nullable, + nullable=is_nullable, + required=column_name in key_properties if key_properties else False, ), ) schema = table_schema.to_dict() diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 80e553574..7a412fe81 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -517,7 +517,7 @@ class Property(JSONTypeHelper[T], t.Generic[T]): """Generic Property. Should be nested within a `PropertiesList`.""" # TODO: Make some of these arguments keyword-only. This is a breaking change. - def __init__( + def __init__( # noqa: PLR0913 self, name: str, wrapped: JSONTypeHelper[T] | type[JSONTypeHelper[T]], @@ -527,6 +527,8 @@ def __init__( secret: bool | None = False, # noqa: FBT002 allowed_values: list[T] | None = None, examples: list[T] | None = None, + *, + nullable: bool | None = None, ) -> None: """Initialize Property object. @@ -547,6 +549,7 @@ def __init__( are permitted. This will define the type as an 'enum'. examples: Optional. A list of one or more sample values. These may be displayed to the user as hints of the expected format of inputs. + nullable: If True, the property may be null. """ self.name = name self.wrapped = wrapped @@ -556,6 +559,7 @@ def __init__( self.secret = secret self.allowed_values = allowed_values or None self.examples = examples or None + self.nullable = nullable @property def type_dict(self) -> dict: # type: ignore[override] @@ -585,7 +589,7 @@ def to_dict(self) -> dict: A JSON Schema dictionary describing the object. """ type_dict = self.type_dict - if self.optional: + if self.nullable or self.optional: type_dict = append_type(type_dict, "null") if self.default is not None: type_dict.update({"default": self.default}) diff --git a/tests/samples/conftest.py b/tests/samples/conftest.py index b9ce33319..52c0857be 100644 --- a/tests/samples/conftest.py +++ b/tests/samples/conftest.py @@ -25,7 +25,14 @@ def _sqlite_sample_db(sqlite_connector): for t in range(3): conn.execute(sa.text(f"DROP TABLE IF EXISTS t{t}")) conn.execute( - sa.text(f"CREATE TABLE t{t} (c1 int PRIMARY KEY, c2 varchar(10))"), + sa.text( + f""" + CREATE TABLE t{t} ( + c1 int PRIMARY KEY NOT NULL, + c2 varchar(10) NOT NULL + ) + """ + ), ) for x in range(100): conn.execute( diff --git a/tests/samples/test_tap_sqlite.py b/tests/samples/test_tap_sqlite.py index b5ed7b549..2c1094b75 100644 --- a/tests/samples/test_tap_sqlite.py +++ b/tests/samples/test_tap_sqlite.py @@ -80,6 +80,8 @@ def test_sqlite_discovery(sqlite_sample_tap: SQLTap): assert stream.metadata.root.table_key_properties == ["c1"] assert stream.primary_keys == ["c1"] + assert stream.schema["properties"]["c1"] == {"type": ["integer"]} + assert stream.schema["required"] == ["c1"] def test_sqlite_input_catalog(sqlite_sample_tap: SQLTap): @@ -90,7 +92,7 @@ def test_sqlite_input_catalog(sqlite_sample_tap: SQLTap): for schema in [stream.schema, stream.stream_maps[0].transformed_schema]: assert len(schema["properties"]) == 2 - assert schema["properties"]["c1"] == {"type": ["integer", "null"]} + assert schema["properties"]["c1"] == {"type": ["integer"]} assert schema["properties"]["c2"] == {"type": ["string", "null"]} assert stream.name == stream.tap_stream_id == "main-t1"