diff --git a/pyproject.toml b/pyproject.toml index 7558d9e18..b1dde771a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,6 +134,26 @@ pytest-codspeed = ">=2.2.0" [tool.pytest.ini_options] addopts = '--ignore=singer_sdk/helpers/_simpleeval.py -m "not external"' +filterwarnings = [ + "error", + "ignore:Could not configure external gitlab tests:UserWarning", + "ignore:No records were available to test:UserWarning", + # https://github.com/meltano/sdk/issues/1354 + "ignore:The function singer_sdk.testing.get_standard_tap_tests is deprecated:DeprecationWarning", + # https://docs.python.org/3.12/whatsnew/3.12.html#deprecated + # https://github.com/danthedeckie/simpleeval/blob/master/simpleeval.py + "ignore:ast.NameConstant is deprecated:DeprecationWarning:simpleeval", + # https://docs.python.org/3.12/whatsnew/3.12.html#deprecated + # https://github.com/danthedeckie/simpleeval/blob/master/simpleeval.py + "ignore:ast.Num is deprecated:DeprecationWarning:simpleeval", + # https://github.com/joblib/joblib/pull/1518 + "ignore:ast.Num is deprecated:DeprecationWarning:joblib._utils", + # https://docs.python.org/3.12/whatsnew/3.12.html#deprecated + # https://github.com/danthedeckie/simpleeval/blob/master/simpleeval.py + "ignore:ast.Str is deprecated:DeprecationWarning:simpleeval", + # https://github.com/joblib/joblib/pull/1518 + "ignore:Attribute n is deprecated:DeprecationWarning:joblib._utils", +] markers = [ "external: Tests relying on external resources", "windows: Tests that only run on Windows", diff --git a/samples/sample_tap_gitlab/gitlab_rest_streams.py b/samples/sample_tap_gitlab/gitlab_rest_streams.py index 0d5968132..53e0c8fa1 100644 --- a/samples/sample_tap_gitlab/gitlab_rest_streams.py +++ b/samples/sample_tap_gitlab/gitlab_rest_streams.py @@ -13,6 +13,7 @@ DateTimeType, DateType, IntegerType, + ObjectType, PropertiesList, Property, StringType, @@ -158,7 +159,6 @@ class EpicsStream(ProjectBasedStream): Property("title", StringType), Property("description", StringType), Property("state", StringType), - Property("author_id", IntegerType), Property("start_date", DateType), Property("end_date", DateType), Property("due_date", DateType), @@ -167,6 +167,17 @@ class EpicsStream(ProjectBasedStream): Property("labels", ArrayType(StringType)), Property("upvotes", IntegerType), Property("downvotes", IntegerType), + Property( + "author", + ObjectType( + Property("id", IntegerType), + Property("name", StringType), + Property("username", StringType), + Property("state", StringType), + Property("avatar_url", StringType), + Property("web_url", StringType), + ), + ), ).to_dict() def get_child_context( diff --git a/samples/sample_tap_gitlab/schemas/issues.json b/samples/sample_tap_gitlab/schemas/issues.json index 98119c7f8..3bdd3cf7b 100644 --- a/samples/sample_tap_gitlab/schemas/issues.json +++ b/samples/sample_tap_gitlab/schemas/issues.json @@ -1,223 +1,365 @@ { - "type": "object", - "properties": { + "type": "object", + "properties": { + "project_id": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "iid": { + "type": "integer" + }, + "milestone": { + "type": [ + "null", + "object" + ], + "properties": { "project_id": { - "type": "integer" + "type": "integer" }, - "id": { - "type": "integer" + "description": { + "type": [ + "null", + "string" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "due_date": { + "type": [ + "null", + "string" + ], + "format": "date" }, "iid": { - "type": "integer" + "type": "integer" }, - "milestone_id": { - "type": [ - "null", - "integer" - ] + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" }, - "author_id": { - "type": [ - "null", - "integer" - ] + "title": { + "type": [ + "null", + "string" + ] }, - "assignee_id": { - "type": [ - "null", - "integer" - ] + "id": { + "type": "integer" }, - "assignees": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "username": { - "type": "string" - }, - "state": { - "type": "string" - }, - "avatar_url": { - "type": [ - "null", - "string" - ] - }, - "web_url": { - "type": [ - "null", - "string" - ] - } - } - } - }, - "closed_by_id": { - "type": [ - "null", - "integer" - ] + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + }, + "example": { + "project_id" : 1, + "description" : "Ducimus nam enim ex consequatur cumque ratione.", + "state" : "closed", + "due_date" : null, + "iid" : 2, + "created_at" : "2016-01-04T15:31:39.996Z", + "title" : "v4.0", + "id" : 17, + "updated_at" : "2016-01-04T15:31:39.996Z" + } + }, + "author": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": "integer" }, - "title": { - "type": [ - "null", - "string" - ] + "name": { + "type": "string" }, - "description": { - "type": [ - "null", - "string" - ] + "username": { + "type": "string" }, "state": { - "type": [ - "null", - "string" - ] + "type": "string" }, - "labels": { - "type": "array", - "items": { - "type": "string" - } + "avatar_url": { + "type": [ + "null", + "string" + ] }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] + "web_url": { + "type": [ + "null", + "string" + ] + } + } + }, + "assignee": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": "string" }, - "updated_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] + "id": { + "type": "integer" }, - "closed_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] + "name": { + "type": "string" }, - "subscribed": { - "type": [ - "null", - "boolean" - ] - }, - "upvotes": { - "type": [ - "null", - "integer" - ] + "web_url": { + "type": [ + "null", + "string" + ] }, - "downvotes": { - "type": [ - "null", - "integer" - ] + "avatar_url": { + "type": [ + "null", + "string" + ] }, - "merge_requests_count": { + "username": { + "type": "string" + } + } + }, + "assignees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "username": { + "type": "string" + }, + "state": { + "type": "string" + }, + "avatar_url": { "type": [ - "null", - "integer" + "null", + "string" ] - }, - "user_notes_count": { + }, + "web_url": { "type": [ - "null", - "integer" + "null", + "string" ] + } + } + } + }, + "closed_by": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": "string" }, - "due_date": { - "type": [ - "null", - "string" - ] + "id": { + "type": "integer" }, - "weight": { - "type": [ - "null", - "integer" - ] + "name": { + "type": "string" }, "web_url": { - "type": [ - "null", - "string" - ] + "type": [ + "null", + "string" + ] }, - "confidential": { - "type": [ - "null", - "boolean" - ] + "avatar_url": { + "type": [ + "null", + "string" + ] }, - "discussion_locked": { - "type": [ - "null", - "boolean" - ] + "username": { + "type": "string" + } + } + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "created_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" }, - "has_tasks": { - "type": [ - "null", - "boolean" - ] + { + "type": "null" + } + ] + }, + "updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" }, - "task_status": { - "type": [ - "null", - "string" - ] + { + "type": "null" + } + ] + }, + "closed_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" }, + { + "type": "null" + } + ] + }, + "upvotes": { + "type": [ + "null", + "integer" + ] + }, + "downvotes": { + "type": [ + "null", + "integer" + ] + }, + "merge_requests_count": { + "type": [ + "null", + "integer" + ] + }, + "user_notes_count": { + "type": [ + "null", + "integer" + ] + }, + "due_date": { + "type": [ + "null", + "string" + ], + "format": "date" + }, + "weight": { + "type": [ + "null", + "integer" + ] + }, + "web_url": { + "type": [ + "null", + "string" + ] + }, + "confidential": { + "type": [ + "null", + "boolean" + ] + }, + "discussion_locked": { + "type": [ + "null", + "boolean" + ] + }, + "has_tasks": { + "type": [ + "null", + "boolean" + ] + }, + "task_status": { + "type": [ + "null", + "string" + ] + }, + "time_stats": { + "type": [ + "null", + "object" + ], + "properties": { "time_estimate": { - "type": [ - "null", - "integer" - ] + "type": [ + "null", + "integer" + ] }, "total_time_spent": { - "type": [ - "null", - "integer" - ] + "type": [ + "null", + "integer" + ] }, "human_time_estimate": { - "type": [ - "null", - "string" - ] + "type": [ + "null", + "string" + ] }, "human_total_time_spent": { - "type": [ - "null", - "string" - ] + "type": [ + "null", + "string" + ] } + } } -} \ No newline at end of file + } +} diff --git a/samples/sample_tap_gitlab/schemas/projects.json b/samples/sample_tap_gitlab/schemas/projects.json index d02858e5f..2f9ececc4 100644 --- a/samples/sample_tap_gitlab/schemas/projects.json +++ b/samples/sample_tap_gitlab/schemas/projects.json @@ -10,8 +10,9 @@ "avatar_url": { "type": ["null", "string"] }, - "builds_enabled": { - "type": ["null", "boolean"] + "builds_access_level": { + "type": ["null", "string"], + "enum": ["disabled", "private", "enabled"] }, "container_registry_enabled": { "type": ["null", "boolean"] @@ -86,15 +87,12 @@ "only_allow_merge_if_all_discussions_are_resolved": { "type": ["null", "boolean"] }, - "only_allow_merge_if_build_succeeds": { + "only_allow_merge_if_pipeline_succeeds": { "type": ["null", "boolean"] }, "open_issues_count": { "type": ["null", "integer"] }, - "owner_id": { - "type": ["null", "integer"] - }, "path": { "type": ["null", "string"] }, @@ -128,10 +126,7 @@ } } }, - "public": { - "type": ["null", "boolean"] - }, - "public_builds": { + "public_jobs": { "type": ["null", "boolean"] }, "request_access_enabled": { @@ -166,37 +161,15 @@ "star_count": { "type": ["null", "integer"] }, - "statistics": { - "type": ["object", "null"], - "properties": { - "commit_count": { - "type": ["null", "integer"] - }, - "storage_size": { - "type": ["null", "integer"] - }, - "repository_size": { - "type": ["null", "integer"] - }, - "lfs_objects_size": { - "type": ["null", "integer"] - }, - "job_artifacts_size": { - "type": ["null", "integer"] - } - } - }, "tag_list": { "type": ["array", "null"], "items": { "type": "string" } }, - "visibility_level": { - "type": ["null", "integer"] - }, "visibility": { - "type": ["null", "string"] + "type": ["null", "string"], + "enum": ["private", "internal", "public"] }, "web_url": { "type": ["null", "string"] diff --git a/samples/sample_tap_gitlab/schemas/releases.json b/samples/sample_tap_gitlab/schemas/releases.json index 9165c802d..972a1f182 100644 --- a/samples/sample_tap_gitlab/schemas/releases.json +++ b/samples/sample_tap_gitlab/schemas/releases.json @@ -4,20 +4,102 @@ "tag_name": { "type": "string" }, - "author_id": { - "type": ["null", "integer"] + "author": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "username": { + "type": "string" + }, + "state": { + "type": "string" + }, + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "web_url": { + "type": [ + "null", + "string" + ] + } + } }, - "commit_id": { - "type": ["string", "null"] + "commit": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": "string" + }, + "short_id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "parent_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "message": { + "type": "string" + }, + "author_name": { + "type": "string" + }, + "author_email": { + "type": "string" + }, + "authored_date": { + "type": "string", + "format": "date-time" + }, + "committer_name": { + "type": "string" + }, + "committer_email": { + "type": "string" + }, + "committed_date": { + "type": "string", + "format": "date-time" + } + } }, "project_id": { "type": "string" }, "description": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "name": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "created_at": { "anyOf": [ diff --git a/samples/sample_tap_google_analytics/schemas/simple-sample.json b/samples/sample_tap_google_analytics/schemas/simple-sample.json index 794869260..0a5f97f4d 100644 --- a/samples/sample_tap_google_analytics/schemas/simple-sample.json +++ b/samples/sample_tap_google_analytics/schemas/simple-sample.json @@ -1,102 +1,11 @@ { "type": "object", "properties": { - "id": { - "type": ["string", "null"] - }, - "project_id": { - "type": ["integer", "null"] - }, - "short_id": { - "type": ["string", "null"] - }, - "title": { - "type": ["null", "string"] - }, - "author_name": { - "type": ["null", "string"] - }, - "author_email": { - "type": ["null", "string"] - }, - "authored_date": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "committer_name": { - "type": ["null", "string"] - }, - "committer_email": { - "type": ["null", "string"] - }, - "committed_date": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "message": { - "type": ["null", "string"] - }, - "allow_failure": { - "type": ["null", "boolean"] - }, - "parent_ids": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "stats": { - "anyOf": [ - { - "type": "object", - "properties": { - "additions": { - "type": "integer" - }, - "deletions": { - "type": "integer" - }, - "total": { - "type": "integer" - } - } - }, - { - "type": "null" - } - ] + "totals": { + "type": "array", + "items": { + "type": "string" + } } } } diff --git a/tests/core/test_connector_sql.py b/tests/core/test_connector_sql.py index 816086582..2a3413c01 100644 --- a/tests/core/test_connector_sql.py +++ b/tests/core/test_connector_sql.py @@ -274,7 +274,7 @@ def test_engine_json_serialization(self, connector: SQLConnector): sa.Column("attrs", sa.JSON), ) meta.create_all(engine) - with engine.connect() as conn: + with engine.connect() as conn, conn.begin(): conn.execute( table.insert(), [