From 56bf5eb7feba52fe49d586ab61af85c18e7a5da2 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Fri, 14 Aug 2020 13:28:54 -0700 Subject: [PATCH 01/35] init commit --- .../azure-schemaregistry/__init__.py | 0 .../azure/schemaregistry/__init__.py | 6 + .../azure/schemaregistry/_common/__init__.py | 0 .../azure/schemaregistry/_common/_schema.py | 18 +++ .../_generated/_azure_schema_registry.py | 4 +- .../_generated/_configuration.py | 1 + .../aio/_azure_schema_registry_async.py | 4 +- .../_generated/aio/_configuration_async.py | 1 + .../_schema_operations_async.py | 36 +++--- .../operations/_schema_operations.py | 36 +++--- .../schemaregistry/_schema_registry_client.py | 26 +++++ .../_schema_registry_serializer/__init__.py | 0 .../_schema_registry_avro_serializer.py | 14 +++ .../azure/schemaregistry/aio/__init__.py | 0 .../aio/_schema_registry_client_async.py | 26 +++++ .../_schema_registry_avro_serializer_async.py | 0 .../schemaregistry/serialization/__init__.py | 5 + .../serialization/_avro_serializer.py | 106 ++++++++++++++++++ .../serialization/_object_serializer.py | 76 +++++++++++++ .../azure-schemaregistry/dev_requirements.txt | 1 + .../azure-schemaregistry/setup.py | 3 +- 21 files changed, 322 insertions(+), 41 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/_schema_registry_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_serializer/_schema_registry_avro_serializer_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_object_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py index a933188e69d4..06fc729bbf16 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py @@ -6,3 +6,9 @@ from ._version import VERSION __version__ = VERSION + +from ._schema_registry_client import SchemaRegistryClient + +__all__ = [ + "SchemaRegistryClient" +] diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py new file mode 100644 index 000000000000..f55a16c820cf --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -0,0 +1,18 @@ +from typing import Union +from enum import Enum + + +class SerializationType(Enum): + AVRO = 0 + JSON = 1 + + +class Schema: # TODO: 1. Naming: Schema Properties/Schema Description? 2. Should it be dict like object? + def __init__(self, schema_id, serialization_type, schema_name, schema_group, schema_byte_array, parse_method): # TODO: what is schema_byte_arry and parse_method? + # type: (str, Union[str, Enum], str, str) -> None + self.schema_id = schema_id # TODO: property vs instance variable + self.serialization_type = serialization_type + self.schema_name = schema_name + self.schema_group = schema_group + self.schema = None # TODO: for Avro/Json, schema would just be a JSON string? what about protocolbuf + diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py index 7c4e41e03505..f59066a526d6 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py @@ -21,7 +21,7 @@ class AzureSchemaRegistry(object): - """AzureSchemaRegistry. + """Azure Schema Registry is as a central schema repository for enterprise-level data infrastructure, complete with support for versioning and management. :ivar schema: SchemaOperations operations :vartype schema: azure.schemaregistry._generated.operations.SchemaOperations @@ -36,7 +36,7 @@ def __init__( **kwargs # type: Any ): # type: (...) -> None - base_url = 'https://{endpoint}/$schemagroups' + base_url = 'https://{endpoint}' self._config = AzureSchemaRegistryConfiguration(endpoint, **kwargs) self._client = PipelineClient(base_url=base_url, config=self._config, **kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py index 7b9ea58dc046..a1f32c687d2b 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py @@ -39,6 +39,7 @@ def __init__( super(AzureSchemaRegistryConfiguration, self).__init__(**kwargs) self.endpoint = endpoint + self.api_version = "2018-01-01-preview" kwargs.setdefault('sdk_moniker', 'azureschemaregistry/{}'.format(VERSION)) self._configure(**kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py index 97def68d031b..a3ade7cb3f71 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py @@ -17,7 +17,7 @@ class AzureSchemaRegistry(object): - """AzureSchemaRegistry. + """Azure Schema Registry is as a central schema repository for enterprise-level data infrastructure, complete with support for versioning and management. :ivar schema: SchemaOperations operations :vartype schema: azure.schemaregistry._generated.aio.operations_async.SchemaOperations @@ -31,7 +31,7 @@ def __init__( endpoint: str, **kwargs: Any ) -> None: - base_url = 'https://{endpoint}/$schemagroups' + base_url = 'https://{endpoint}' self._config = AzureSchemaRegistryConfiguration(endpoint, **kwargs) self._client = AsyncPipelineClient(base_url=base_url, config=self._config, **kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py index c4d00a2ef815..8bf8b7084c5d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py @@ -34,6 +34,7 @@ def __init__( super(AzureSchemaRegistryConfiguration, self).__init__(**kwargs) self.endpoint = endpoint + self.api_version = "2018-01-01-preview" kwargs.setdefault('sdk_moniker', 'azureschemaregistry/{}'.format(VERSION)) self._configure(**kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py index e77a32253fcf..12e92b6fab6a 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py @@ -58,6 +58,7 @@ async def get_by_id( cls = kwargs.pop('cls', None) # type: ClsType[str] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) + api_version = "2018-01-01-preview" # Construct URL url = self.get_by_id.metadata['url'] # type: ignore @@ -69,6 +70,7 @@ async def get_by_id( # Construct parameters query_parameters = {} # type: Dict[str, Any] + query_parameters['api-version'] = self._serialize.query("api_version", api_version, 'str') # Construct headers header_parameters = {} # type: Dict[str, Any] @@ -86,7 +88,7 @@ async def get_by_id( response_headers = {} response_headers['Location']=self._deserialize('str', response.headers.get('Location')) - response_headers['X-Serialization']=self._deserialize('str', response.headers.get('X-Serialization')) + response_headers['X-Schema-Type']=self._deserialize('str', response.headers.get('X-Schema-Type')) response_headers['X-Schema-Id']=self._deserialize('str', response.headers.get('X-Schema-Id')) response_headers['X-Schema-Id-Location']=self._deserialize('str', response.headers.get('X-Schema-Id-Location')) response_headers['X-Schema-Version']=self._deserialize('int', response.headers.get('X-Schema-Version')) @@ -96,13 +98,12 @@ async def get_by_id( return cls(pipeline_response, deserialized, response_headers) return deserialized - get_by_id.metadata = {'url': '/getSchemaById/{schema-id}'} # type: ignore + get_by_id.metadata = {'url': '/$schemagroups/getSchemaById/{schema-id}'} # type: ignore - async def get_id_by_content( + async def query_id_by_content( self, group_name: str, schema_name: str, - serialization_type: str, schema_content: str, **kwargs ) -> "models.SchemaId": @@ -115,9 +116,6 @@ async def get_id_by_content( :type group_name: str :param schema_name: Name of the registered schema. :type schema_name: str - :param serialization_type: Serialization type of the registered schema. Must match - serialization type of the specified schema group. - :type serialization_type: str :param schema_content: String representation of the registered schema. :type schema_content: str :keyword callable cls: A custom type or function that will be passed the direct response @@ -128,10 +126,12 @@ async def get_id_by_content( cls = kwargs.pop('cls', None) # type: ClsType["models.SchemaId"] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) + x_schema_type = "avro" + api_version = "2018-01-01-preview" content_type = kwargs.pop("content_type", "application/json") # Construct URL - url = self.get_id_by_content.metadata['url'] # type: ignore + url = self.query_id_by_content.metadata['url'] # type: ignore path_format_arguments = { 'endpoint': self._serialize.url("self._config.endpoint", self._config.endpoint, 'str', skip_quote=True), 'group-name': self._serialize.url("group_name", group_name, 'str'), @@ -141,10 +141,11 @@ async def get_id_by_content( # Construct parameters query_parameters = {} # type: Dict[str, Any] + query_parameters['api-version'] = self._serialize.query("api_version", api_version, 'str') # Construct headers header_parameters = {} # type: Dict[str, Any] - header_parameters['serialization-type'] = self._serialize.header("serialization_type", serialization_type, 'str') + header_parameters['X-Schema-Type'] = self._serialize.header("x_schema_type", x_schema_type, 'str') header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str') header_parameters['Accept'] = 'application/json' @@ -164,7 +165,7 @@ async def get_id_by_content( response_headers = {} response_headers['Location']=self._deserialize('str', response.headers.get('Location')) - response_headers['X-Serialization']=self._deserialize('str', response.headers.get('X-Serialization')) + response_headers['X-Schema-Type']=self._deserialize('str', response.headers.get('X-Schema-Type')) response_headers['X-Schema-Id']=self._deserialize('str', response.headers.get('X-Schema-Id')) response_headers['X-Schema-Id-Location']=self._deserialize('str', response.headers.get('X-Schema-Id-Location')) response_headers['X-Schema-Version']=self._deserialize('int', response.headers.get('X-Schema-Version')) @@ -174,13 +175,12 @@ async def get_id_by_content( return cls(pipeline_response, deserialized, response_headers) return deserialized - get_id_by_content.metadata = {'url': '/{group-name}/schemas/{schema-name}'} # type: ignore + query_id_by_content.metadata = {'url': '/$schemagroups/{group-name}/schemas/{schema-name}'} # type: ignore async def register( self, group_name: str, schema_name: str, - serialization_type: str, schema_content: str, **kwargs ) -> "models.SchemaId": @@ -193,9 +193,6 @@ async def register( :type group_name: str :param schema_name: Name of schema being registered. :type schema_name: str - :param serialization_type: Serialization type for the schema being registered. Must match - serialization type of the specified schema group. - :type serialization_type: str :param schema_content: String representation of the schema being registered. :type schema_content: str :keyword callable cls: A custom type or function that will be passed the direct response @@ -206,6 +203,8 @@ async def register( cls = kwargs.pop('cls', None) # type: ClsType["models.SchemaId"] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) + x_schema_type = "avro" + api_version = "2018-01-01-preview" content_type = kwargs.pop("content_type", "application/json") # Construct URL @@ -219,10 +218,11 @@ async def register( # Construct parameters query_parameters = {} # type: Dict[str, Any] + query_parameters['api-version'] = self._serialize.query("api_version", api_version, 'str') # Construct headers header_parameters = {} # type: Dict[str, Any] - header_parameters['serialization-type'] = self._serialize.header("serialization_type", serialization_type, 'str') + header_parameters['X-Schema-Type'] = self._serialize.header("x_schema_type", x_schema_type, 'str') header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str') header_parameters['Accept'] = 'application/json' @@ -242,7 +242,7 @@ async def register( response_headers = {} response_headers['Location']=self._deserialize('str', response.headers.get('Location')) - response_headers['X-Serialization']=self._deserialize('str', response.headers.get('X-Serialization')) + response_headers['X-Schema-Type']=self._deserialize('str', response.headers.get('X-Schema-Type')) response_headers['X-Schema-Id']=self._deserialize('str', response.headers.get('X-Schema-Id')) response_headers['X-Schema-Id-Location']=self._deserialize('str', response.headers.get('X-Schema-Id-Location')) response_headers['X-Schema-Version']=self._deserialize('int', response.headers.get('X-Schema-Version')) @@ -252,4 +252,4 @@ async def register( return cls(pipeline_response, deserialized, response_headers) return deserialized - register.metadata = {'url': '/{group-name}/schemas/{schema-name}'} # type: ignore + register.metadata = {'url': '/$schemagroups/{group-name}/schemas/{schema-name}'} # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py index c2eb7573a328..2a5130cce974 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py @@ -63,6 +63,7 @@ def get_by_id( cls = kwargs.pop('cls', None) # type: ClsType[str] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) + api_version = "2018-01-01-preview" # Construct URL url = self.get_by_id.metadata['url'] # type: ignore @@ -74,6 +75,7 @@ def get_by_id( # Construct parameters query_parameters = {} # type: Dict[str, Any] + query_parameters['api-version'] = self._serialize.query("api_version", api_version, 'str') # Construct headers header_parameters = {} # type: Dict[str, Any] @@ -91,7 +93,7 @@ def get_by_id( response_headers = {} response_headers['Location']=self._deserialize('str', response.headers.get('Location')) - response_headers['X-Serialization']=self._deserialize('str', response.headers.get('X-Serialization')) + response_headers['X-Schema-Type']=self._deserialize('str', response.headers.get('X-Schema-Type')) response_headers['X-Schema-Id']=self._deserialize('str', response.headers.get('X-Schema-Id')) response_headers['X-Schema-Id-Location']=self._deserialize('str', response.headers.get('X-Schema-Id-Location')) response_headers['X-Schema-Version']=self._deserialize('int', response.headers.get('X-Schema-Version')) @@ -101,13 +103,12 @@ def get_by_id( return cls(pipeline_response, deserialized, response_headers) return deserialized - get_by_id.metadata = {'url': '/getSchemaById/{schema-id}'} # type: ignore + get_by_id.metadata = {'url': '/$schemagroups/getSchemaById/{schema-id}'} # type: ignore - def get_id_by_content( + def query_id_by_content( self, group_name, # type: str schema_name, # type: str - serialization_type, # type: str schema_content, # type: str **kwargs # type: Any ): @@ -121,9 +122,6 @@ def get_id_by_content( :type group_name: str :param schema_name: Name of the registered schema. :type schema_name: str - :param serialization_type: Serialization type of the registered schema. Must match - serialization type of the specified schema group. - :type serialization_type: str :param schema_content: String representation of the registered schema. :type schema_content: str :keyword callable cls: A custom type or function that will be passed the direct response @@ -134,10 +132,12 @@ def get_id_by_content( cls = kwargs.pop('cls', None) # type: ClsType["models.SchemaId"] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) + x_schema_type = "avro" + api_version = "2018-01-01-preview" content_type = kwargs.pop("content_type", "application/json") # Construct URL - url = self.get_id_by_content.metadata['url'] # type: ignore + url = self.query_id_by_content.metadata['url'] # type: ignore path_format_arguments = { 'endpoint': self._serialize.url("self._config.endpoint", self._config.endpoint, 'str', skip_quote=True), 'group-name': self._serialize.url("group_name", group_name, 'str'), @@ -147,10 +147,11 @@ def get_id_by_content( # Construct parameters query_parameters = {} # type: Dict[str, Any] + query_parameters['api-version'] = self._serialize.query("api_version", api_version, 'str') # Construct headers header_parameters = {} # type: Dict[str, Any] - header_parameters['serialization-type'] = self._serialize.header("serialization_type", serialization_type, 'str') + header_parameters['X-Schema-Type'] = self._serialize.header("x_schema_type", x_schema_type, 'str') header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str') header_parameters['Accept'] = 'application/json' @@ -170,7 +171,7 @@ def get_id_by_content( response_headers = {} response_headers['Location']=self._deserialize('str', response.headers.get('Location')) - response_headers['X-Serialization']=self._deserialize('str', response.headers.get('X-Serialization')) + response_headers['X-Schema-Type']=self._deserialize('str', response.headers.get('X-Schema-Type')) response_headers['X-Schema-Id']=self._deserialize('str', response.headers.get('X-Schema-Id')) response_headers['X-Schema-Id-Location']=self._deserialize('str', response.headers.get('X-Schema-Id-Location')) response_headers['X-Schema-Version']=self._deserialize('int', response.headers.get('X-Schema-Version')) @@ -180,13 +181,12 @@ def get_id_by_content( return cls(pipeline_response, deserialized, response_headers) return deserialized - get_id_by_content.metadata = {'url': '/{group-name}/schemas/{schema-name}'} # type: ignore + query_id_by_content.metadata = {'url': '/$schemagroups/{group-name}/schemas/{schema-name}'} # type: ignore def register( self, group_name, # type: str schema_name, # type: str - serialization_type, # type: str schema_content, # type: str **kwargs # type: Any ): @@ -200,9 +200,6 @@ def register( :type group_name: str :param schema_name: Name of schema being registered. :type schema_name: str - :param serialization_type: Serialization type for the schema being registered. Must match - serialization type of the specified schema group. - :type serialization_type: str :param schema_content: String representation of the schema being registered. :type schema_content: str :keyword callable cls: A custom type or function that will be passed the direct response @@ -213,6 +210,8 @@ def register( cls = kwargs.pop('cls', None) # type: ClsType["models.SchemaId"] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) + x_schema_type = "avro" + api_version = "2018-01-01-preview" content_type = kwargs.pop("content_type", "application/json") # Construct URL @@ -226,10 +225,11 @@ def register( # Construct parameters query_parameters = {} # type: Dict[str, Any] + query_parameters['api-version'] = self._serialize.query("api_version", api_version, 'str') # Construct headers header_parameters = {} # type: Dict[str, Any] - header_parameters['serialization-type'] = self._serialize.header("serialization_type", serialization_type, 'str') + header_parameters['X-Schema-Type'] = self._serialize.header("x_schema_type", x_schema_type, 'str') header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str') header_parameters['Accept'] = 'application/json' @@ -249,7 +249,7 @@ def register( response_headers = {} response_headers['Location']=self._deserialize('str', response.headers.get('Location')) - response_headers['X-Serialization']=self._deserialize('str', response.headers.get('X-Serialization')) + response_headers['X-Schema-Type']=self._deserialize('str', response.headers.get('X-Schema-Type')) response_headers['X-Schema-Id']=self._deserialize('str', response.headers.get('X-Schema-Id')) response_headers['X-Schema-Id-Location']=self._deserialize('str', response.headers.get('X-Schema-Id-Location')) response_headers['X-Schema-Version']=self._deserialize('int', response.headers.get('X-Schema-Version')) @@ -259,4 +259,4 @@ def register( return cls(pipeline_response, deserialized, response_headers) return deserialized - register.metadata = {'url': '/{group-name}/schemas/{schema-name}'} # type: ignore + register.metadata = {'url': '/$schemagroups/{group-name}/schemas/{schema-name}'} # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py new file mode 100644 index 000000000000..2f5ea5786124 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -0,0 +1,26 @@ +from ._generated._azure_schema_registry import AzureSchemaRegistry +from ._common._schema import Schema + + +class SchemaRegistryClient: + def __init__( + self, + endpoint, + credential, # TODO P0: how to set credential/build pipeline? + max_caches_size=0, # TODO: cache for schemas? + **kwargs + ): + self._client = AzureSchemaRegistry(endpoint=endpoint) + + # TODO: all http request needs to be parsed? + def register_schema(self, schema_group, schema_name, serialization_type, schema_string): + # type: (str, str, str, str) -> Schema + return self._client.schema.register(schema_group, schema_name, serialization_type, schema_string) + + def get_schema(self, schema_id): + # type: (str) -> Schema + return self._client.schema.get_by_id(schema_id) + + def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string): # TODO: all should match? + # type: (str, str, str, Union[str, Enum]) -> str + return self._client.schema.get_id_by_content(schema_group, schema_name, serialization_type, schema_string) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/_schema_registry_avro_serializer.py new file mode 100644 index 000000000000..f94f60795636 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/_schema_registry_avro_serializer.py @@ -0,0 +1,14 @@ +from .._schema_registry_client import SchemaRegistryClient +from .._common._serializer._avro_serializer import AvroObjectSerializer + + +class SchemaRegistryAvroSerializer: + def __init__(self, endpoint, credential, max_cache_size): # TODO: or just use_cache flag? + self._schema_registry_client = SchemaRegistryClient(endpoint, credential) + self._serializer = AvroObjectSerializer() + + def serialize(self, data, schema): + pass + + def deserialize(self, data): + pass diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py new file mode 100644 index 000000000000..308ccf97af7b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -0,0 +1,26 @@ +from .._schema import Schema + + +class SchemaRegistryClient: + def __init__( + self, + endpoint, + credential, + max_caches_size, # TODO: cache for schemas? + + **kwargs + ): + self._generated_client = None + pass + + async def register_schema(self, schema_group, schema_name, schema_string, schema_type): + # type: (str, str, str, str) -> Schema + pass + + async def get_schema(self, schema_id): + # type: (str) -> Schema + pass + + async def get_schema_id(self, schema_group, schema_name, schema_string, serialization_type): # TODO: all should match? + # type: (str, str, str, Union[str, Enum]) -> str + pass diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_serializer/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_serializer/_schema_registry_avro_serializer_async.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/__init__.py new file mode 100644 index 000000000000..686ddfdaad73 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/__init__.py @@ -0,0 +1,5 @@ +from ._avro_serializer import AvroObjectSerializer + +__all__ = [ + "AvroObjectSerializer" +] \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_avro_serializer.py new file mode 100644 index 000000000000..0d913db4418d --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_avro_serializer.py @@ -0,0 +1,106 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from typing import BinaryIO, Optional, Any, Union, Type +from avro.datafile import DataFileWriter, DataFileReader +from avro.io import DatumWriter, DatumReader + +from ._object_serializer import ObjectSerializer + + +class AvroObjectSerializer(ObjectSerializer): + + def __init__(self, codec=None): + """A Avro serializer using avro lib from Apache. + :param str codec: The writer codec. If None, let the avro library decides. + """ + try: + import avro # pylint: disable=unused-import + except ImportError: + raise ImportError("In order to create a AvroObjectSerializer you need to install the 'avro' library") + + self._writer_codec = codec + + def serialize( + self, + stream, # type: BinaryIO + value, # type: ObjectType + schema=None, # type: Optional[Any] + ): + # type: (...) -> None + """Convert the provided value to it's binary representation and write it to the stream. + Schema must be a Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + :param stream: A stream of bytes or bytes directly + :type stream: BinaryIO + :param value: An object to serialize + :param schema: A Avro RecordSchema + """ + if not schema: + raise ValueError("Schema is required in Avro serializer.") + + kwargs = {} + if self._writer_codec: + kwargs['codec'] = self._writer_codec + + writer = DataFileWriter( + stream, + DatumWriter(), + schema, + **kwargs + ) + writer.append(value) + writer.flush() + + def deserialize( + self, + data, # type: Union[bytes, BinaryIO] + return_type=None, # type: Optional[Type[ObjectType]] + ): + # type: (...) -> ObjectType + """Read the binary representation into a specific type. + Return type will be ignored, since the schema is deduced from the provided bytes. + :param data: A stream of bytes or bytes directly + :type data: BinaryIO or bytes + :param return_type: Return type is not supported in the Avro serializer. + :returns: An instanciated object + :rtype: ObjectType + """ + if not hasattr(data, 'read'): + from io import BytesIO + data = BytesIO(data) + + reader = DataFileReader( + data, + DatumReader() + ) + obj = next(reader) + try: + next(reader) + raise ValueError("This avro stream is multiple object stream") + except StopIteration: + pass + reader.close() + return obj \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_object_serializer.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_object_serializer.py new file mode 100644 index 000000000000..fac99d185f35 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_object_serializer.py @@ -0,0 +1,76 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import abc +from typing import BinaryIO, Union, Type, TypeVar, Optional + + +try: + ABC = abc.ABC +except AttributeError: # Python 2.7, abc exists, but not ABC + ABC = abc.ABCMeta("ABC", (object,), {"__slots__": ()}) # type: ignore + +ObjectType = TypeVar("ObjectType") + + +class ObjectSerializer(ABC): + + @abc.abstractmethod + def serialize( + self, + stream, # type: BinaryIO + value, # type: ObjectType + schema=None, # type: Optional[Any] + ): + # type: (...) -> None + """Convert the provided value to it's binary representation and write it to the stream. + Schema can be used to pass additional information on how to serialize the value. Schema + might be ignored, and usage is dependent of the serializer chose. Please refer to the documentation + of the serializer implementation for details of schema structure. + :param stream: A stream of bytes or bytes directly + :type stream: BinaryIO + :param value: An object to serialize + :param schema: A schema used validate or help serialization. Usage is dependent of the serializer. + """ + raise NotImplementedError() + + @abc.abstractmethod + def deserialize( + self, + data, # type: Union[bytes, BinaryIO] + return_type=None, # type: Optional[Type[ObjectType]] + ): + # type: (...) -> ObjectType + """Read the binary representation into a specific type. + Return type can be used to express the expected return type if the serializer is able to instiate classes. + If not, the return type will be decided by the deserializer implementation (likely dict). + Please refer to the documentation of the serializer implementation for details. + :param data: A stream of bytes or bytes directly + :type data: BinaryIO or bytes + :param return_type: The type of the object to convert to and return, if supported. + :returns: An instanciated object + :rtype: ObjectType + """ + raise NotImplementedError() \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt index e69de29bb2d1..07dd69cdc21b 100644 --- a/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt +++ b/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt @@ -0,0 +1 @@ +-e ../../identity/azure-identity \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/setup.py b/sdk/schemaregistry/azure-schemaregistry/setup.py index 8f8ca94f9e59..739998589981 100644 --- a/sdk/schemaregistry/azure-schemaregistry/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry/setup.py @@ -79,7 +79,8 @@ ]), install_requires=[ 'msrest>=0.5.0', - 'azure-core<2.0.0,>=1.2.2' + 'azure-core<2.0.0,>=1.2.2', + 'avro>=1.0.0,<2.0.0' ], extras_require={ ":python_version<'3.0'": ['azure-nspkg', 'futures'], From d1c96f90c426fe399870d3b90b93656fa81345b5 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Mon, 17 Aug 2020 17:31:40 -0700 Subject: [PATCH 02/35] avro serializer structure --- .../CHANGELOG.md | 3 + .../MANIFEST.in | 4 + .../README.md | 30 +++++ .../azure/__init__.py | 1 + .../azure/schemaregistry/__init__.py | 1 + .../schemaregistry/serializer/__init__.py | 1 + .../serializer/_avro_serializer/__init__.py | 1 + .../_avro_serializer/_avro_serializer.py} | 0 .../serializer/_avro_serializer/_version.py | 6 + .../dev_requirements.txt | 2 + .../mypy.ini | 6 + .../samples/sample_avro_serializer.py} | 0 .../sdk_packaging.toml | 2 + .../setup.cfg | 2 + .../setup.py | 88 +++++++++++++++ .../tests/test_avro_serializer.py | 0 .../azure/schemaregistry/__init__.py | 4 +- .../schemaregistry/_common/_constants.py} | 13 ++- .../_generated/_azure_schema_registry.py | 7 +- .../_generated/_configuration.py | 12 ++ .../aio/_azure_schema_registry_async.py | 11 +- .../_generated/aio/_configuration_async.py | 16 ++- .../schemaregistry/_schema_registry_client.py | 49 ++++++-- .../_schema_registry_avro_serializer.py | 14 --- .../schemaregistry/serialization/__init__.py | 5 - .../serialization/_avro_serializer.py | 106 ------------------ .../serialization/_object_serializer.py | 76 ------------- .../samples/schema_registry.py | 53 +++++++++ .../azure-schemaregistry/setup.py | 3 +- .../azure-schemaregistry/swagger/README.md | 2 + 30 files changed, 297 insertions(+), 221 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/MANIFEST.in create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/__init__.py rename sdk/schemaregistry/{azure-schemaregistry/__init__.py => azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_avro_serializer.py} (100%) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_version.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/mypy.ini rename sdk/schemaregistry/{azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/__init__.py => azure-schemaregistry-avro-serializer/samples/sample_avro_serializer.py} (100%) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/sdk_packaging.toml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.cfg create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py rename sdk/schemaregistry/azure-schemaregistry/{samples/sample_placeholder.py => azure/schemaregistry/_common/_constants.py} (58%) delete mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/_schema_registry_avro_serializer.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_avro_serializer.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_object_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/samples/schema_registry.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md new file mode 100644 index 000000000000..332564950c28 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md @@ -0,0 +1,3 @@ +# Release History + +## 1.0.0b1 (Unreleased) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/MANIFEST.in b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/MANIFEST.in new file mode 100644 index 000000000000..3ba8ed1d9a45 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/MANIFEST.in @@ -0,0 +1,4 @@ +include *.md +include azure/__init__.py +recursive-include tests *.py +recursive-include samples *.py \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md new file mode 100644 index 000000000000..b96064bbd540 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md @@ -0,0 +1,30 @@ +# Azure Schema Registry Avro Serializer library for Python + +Azure Schema Registry Avro Serializer is a ... +<> + +## Getting started + +## Key concepts + +## Examples + +## Troubleshooting + +## Next steps + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + + diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py new file mode 100644 index 000000000000..0d1f7edf5dc6 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py new file mode 100644 index 000000000000..0d1f7edf5dc6 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py new file mode 100644 index 000000000000..0d1f7edf5dc6 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/__init__.py new file mode 100644 index 000000000000..0d1f7edf5dc6 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_avro_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_version.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_version.py new file mode 100644 index 000000000000..ac9f392f513e --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_version.py @@ -0,0 +1,6 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +VERSION = "1.0.0b1" diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt new file mode 100644 index 000000000000..394099b2743a --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt @@ -0,0 +1,2 @@ +-e ../../identity/azure-identity +-e ../azure-schemaregistry \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/mypy.ini b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/mypy.ini new file mode 100644 index 000000000000..c5adef3f8a6d --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +python_version = 3.7 +warn_unused_configs = True +ignore_missing_imports = True + +# Per-module options: diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sample_avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sample_avro_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/sdk_packaging.toml b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/sdk_packaging.toml new file mode 100644 index 000000000000..e7687fdae93b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/sdk_packaging.toml @@ -0,0 +1,2 @@ +[packaging] +auto_update = false \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.cfg b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.cfg new file mode 100644 index 000000000000..3c6e79cf31da --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py new file mode 100644 index 000000000000..56b0db61c88b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +import re +import os.path +from io import open +from setuptools import find_packages, setup + +# Change the PACKAGE_NAME only to change folder and different name +PACKAGE_NAME = "azure-schemaregistry-avro-serializer" +PACKAGE_PPRINT_NAME = "Schema Registry Avro Serializer" + +# a-b-c => a/b/c +package_folder_path = PACKAGE_NAME.replace('-', '/') +# a-b-c => a.b.c +namespace_name = PACKAGE_NAME.replace('-', '.') + +# azure v0.x is not compatible with this package +# azure v0.x used to have a __version__ attribute (newer versions don't) +try: + import azure + try: + ver = azure.__version__ + raise Exception( + 'This package is incompatible with azure=={}. '.format(ver) + + 'Uninstall it with "pip uninstall azure".' + ) + except AttributeError: + pass +except ImportError: + pass + +# Version extraction inspired from 'requests' +with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', + fd.read(), re.MULTILINE).group(1) + +if not version: + raise RuntimeError('Cannot find version information') + +with open('README.md', encoding='utf-8') as f: + readme = f.read() +with open('CHANGELOG.md', encoding='utf-8') as f: + changelog = f.read() + +setup( + name=PACKAGE_NAME, + version=version, + description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME), + long_description=readme + '\n\n' + changelog, + long_description_content_type='text/markdown', + license='MIT License', + author='Microsoft Corporation', + author_email='azpysdkhelp@microsoft.com', + url='https://github.com/Azure/azure-sdk-for-python', + classifiers=[ + "Development Status :: 4 - Beta", + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'License :: OSI Approved :: MIT License', + ], + zip_safe=False, + packages=find_packages(exclude=[ + 'tests', + 'samples', + # Exclude packages that will be covered by PEP420 or nspkg + 'azure', + ]), + install_requires=[ + 'msrest>=0.5.0', + 'azure-core<2.0.0,>=1.2.2' + 'azure-schemaregistry==1.0.0b1' + ], + extras_require={ + ":python_version<'3.0'": ['azure-nspkg', 'futures'], + } +) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py index 06fc729bbf16..4ab85d35853d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py @@ -8,7 +8,9 @@ __version__ = VERSION from ._schema_registry_client import SchemaRegistryClient +from ._common._constants import SerializationType __all__ = [ - "SchemaRegistryClient" + "SchemaRegistryClient", + "SerializationType" ] diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sample_placeholder.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py similarity index 58% rename from sdk/schemaregistry/azure-schemaregistry/samples/sample_placeholder.py rename to sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py index e779593d1124..d203c0b95879 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sample_placeholder.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py @@ -1,7 +1,10 @@ -# ------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from enum import Enum -# Samples will be added later + +class SerializationType(Enum): + AVRO = "AVRO" + JSON = "JSON" diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py index f59066a526d6..02bfd51add11 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py @@ -15,6 +15,8 @@ # pylint: disable=unused-import,ungrouped-imports from typing import Any + from azure.core.credentials import TokenCredential + from ._configuration import AzureSchemaRegistryConfiguration from .operations import SchemaOperations from . import models @@ -25,6 +27,8 @@ class AzureSchemaRegistry(object): :ivar schema: SchemaOperations operations :vartype schema: azure.schemaregistry._generated.operations.SchemaOperations + :param credential: Credential needed for the client to connect to Azure. + :type credential: ~azure.core.credentials.TokenCredential :param endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :type endpoint: str :keyword int polling_interval: Default waiting time between two polls for LRO operations if no Retry-After header is present. @@ -32,12 +36,13 @@ class AzureSchemaRegistry(object): def __init__( self, + credential, # type: "TokenCredential" endpoint, # type: str **kwargs # type: Any ): # type: (...) -> None base_url = 'https://{endpoint}' - self._config = AzureSchemaRegistryConfiguration(endpoint, **kwargs) + self._config = AzureSchemaRegistryConfiguration(credential, endpoint, **kwargs) self._client = PipelineClient(base_url=base_url, config=self._config, **kwargs) client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)} diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py index a1f32c687d2b..072156a901e2 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py @@ -17,6 +17,8 @@ # pylint: disable=unused-import,ungrouped-imports from typing import Any + from azure.core.credentials import TokenCredential + class AzureSchemaRegistryConfiguration(Configuration): """Configuration for AzureSchemaRegistry. @@ -24,22 +26,30 @@ class AzureSchemaRegistryConfiguration(Configuration): Note that all parameters used to create this instance are saved as instance attributes. + :param credential: Credential needed for the client to connect to Azure. + :type credential: ~azure.core.credentials.TokenCredential :param endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :type endpoint: str """ def __init__( self, + credential, # type: "TokenCredential" endpoint, # type: str **kwargs # type: Any ): # type: (...) -> None + if credential is None: + raise ValueError("Parameter 'credential' must not be None.") if endpoint is None: raise ValueError("Parameter 'endpoint' must not be None.") super(AzureSchemaRegistryConfiguration, self).__init__(**kwargs) + self.credential = credential self.endpoint = endpoint self.api_version = "2018-01-01-preview" + self.credential_scopes = ['https://eventhubs.azure.net/.default'] + self.credential_scopes.extend(kwargs.pop('credential_scopes', [])) kwargs.setdefault('sdk_moniker', 'azureschemaregistry/{}'.format(VERSION)) self._configure(**kwargs) @@ -56,3 +66,5 @@ def _configure( self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs) self.redirect_policy = kwargs.get('redirect_policy') or policies.RedirectPolicy(**kwargs) self.authentication_policy = kwargs.get('authentication_policy') + if self.credential and not self.authentication_policy: + self.authentication_policy = policies.BearerTokenCredentialPolicy(self.credential, *self.credential_scopes, **kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py index a3ade7cb3f71..7bc485467f31 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py @@ -6,11 +6,15 @@ # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -from typing import Any +from typing import Any, TYPE_CHECKING from azure.core import AsyncPipelineClient from msrest import Deserializer, Serializer +if TYPE_CHECKING: + # pylint: disable=unused-import,ungrouped-imports + from azure.core.credentials_async import AsyncTokenCredential + from ._configuration_async import AzureSchemaRegistryConfiguration from .operations_async import SchemaOperations from .. import models @@ -21,6 +25,8 @@ class AzureSchemaRegistry(object): :ivar schema: SchemaOperations operations :vartype schema: azure.schemaregistry._generated.aio.operations_async.SchemaOperations + :param credential: Credential needed for the client to connect to Azure. + :type credential: ~azure.core.credentials_async.AsyncTokenCredential :param endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :type endpoint: str :keyword int polling_interval: Default waiting time between two polls for LRO operations if no Retry-After header is present. @@ -28,11 +34,12 @@ class AzureSchemaRegistry(object): def __init__( self, + credential: "AsyncTokenCredential", endpoint: str, **kwargs: Any ) -> None: base_url = 'https://{endpoint}' - self._config = AzureSchemaRegistryConfiguration(endpoint, **kwargs) + self._config = AzureSchemaRegistryConfiguration(credential, endpoint, **kwargs) self._client = AsyncPipelineClient(base_url=base_url, config=self._config, **kwargs) client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)} diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py index 8bf8b7084c5d..71d0942e1f3e 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py @@ -6,13 +6,17 @@ # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -from typing import Any +from typing import Any, TYPE_CHECKING from azure.core.configuration import Configuration from azure.core.pipeline import policies from .._version import VERSION +if TYPE_CHECKING: + # pylint: disable=unused-import,ungrouped-imports + from azure.core.credentials_async import AsyncTokenCredential + class AzureSchemaRegistryConfiguration(Configuration): """Configuration for AzureSchemaRegistry. @@ -20,21 +24,29 @@ class AzureSchemaRegistryConfiguration(Configuration): Note that all parameters used to create this instance are saved as instance attributes. + :param credential: Credential needed for the client to connect to Azure. + :type credential: ~azure.core.credentials_async.AsyncTokenCredential :param endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :type endpoint: str """ def __init__( self, + credential: "AsyncTokenCredential", endpoint: str, **kwargs: Any ) -> None: + if credential is None: + raise ValueError("Parameter 'credential' must not be None.") if endpoint is None: raise ValueError("Parameter 'endpoint' must not be None.") super(AzureSchemaRegistryConfiguration, self).__init__(**kwargs) + self.credential = credential self.endpoint = endpoint self.api_version = "2018-01-01-preview" + self.credential_scopes = ['https://eventhubs.azure.net/.default'] + self.credential_scopes.extend(kwargs.pop('credential_scopes', [])) kwargs.setdefault('sdk_moniker', 'azureschemaregistry/{}'.format(VERSION)) self._configure(**kwargs) @@ -50,3 +62,5 @@ def _configure( self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs) self.redirect_policy = kwargs.get('redirect_policy') or policies.AsyncRedirectPolicy(**kwargs) self.authentication_policy = kwargs.get('authentication_policy') + if self.credential and not self.authentication_policy: + self.authentication_policy = policies.AsyncBearerTokenCredentialPolicy(self.credential, *self.credential_scopes, **kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index 2f5ea5786124..da0649eb4d7e 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -1,26 +1,59 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from typing import Any, TYPE_CHECKING + from ._generated._azure_schema_registry import AzureSchemaRegistry from ._common._schema import Schema +if TYPE_CHECKING: + from azure.core.credentials import TokenCredential + class SchemaRegistryClient: + """ + + """ def __init__( - self, - endpoint, - credential, # TODO P0: how to set credential/build pipeline? - max_caches_size=0, # TODO: cache for schemas? - **kwargs + self, + endpoint, + credential, + **kwargs ): - self._client = AzureSchemaRegistry(endpoint=endpoint) + # type: (str, TokenCredential, Any) -> None + self._client = AzureSchemaRegistry(credential=credential, endpoint=endpoint) - # TODO: all http request needs to be parsed? def register_schema(self, schema_group, schema_name, serialization_type, schema_string): # type: (str, str, str, str) -> Schema - return self._client.schema.register(schema_group, schema_name, serialization_type, schema_string) + """ + + :param schema_group: + :param schema_name: + :param serialization_type: + :param schema_string: + :return: + """ + res = self._client.schema.register(schema_group, schema_name, schema_string) + return res def get_schema(self, schema_id): # type: (str) -> Schema + """ + + :param schema_id: + :return: + """ return self._client.schema.get_by_id(schema_id) def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string): # TODO: all should match? # type: (str, str, str, Union[str, Enum]) -> str + """ + + :param schema_group: + :param schema_name: + :param serialization_type: + :param schema_string: + :return: + """ return self._client.schema.get_id_by_content(schema_group, schema_name, serialization_type, schema_string) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/_schema_registry_avro_serializer.py deleted file mode 100644 index f94f60795636..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_serializer/_schema_registry_avro_serializer.py +++ /dev/null @@ -1,14 +0,0 @@ -from .._schema_registry_client import SchemaRegistryClient -from .._common._serializer._avro_serializer import AvroObjectSerializer - - -class SchemaRegistryAvroSerializer: - def __init__(self, endpoint, credential, max_cache_size): # TODO: or just use_cache flag? - self._schema_registry_client = SchemaRegistryClient(endpoint, credential) - self._serializer = AvroObjectSerializer() - - def serialize(self, data, schema): - pass - - def deserialize(self, data): - pass diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/__init__.py deleted file mode 100644 index 686ddfdaad73..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from ._avro_serializer import AvroObjectSerializer - -__all__ = [ - "AvroObjectSerializer" -] \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_avro_serializer.py deleted file mode 100644 index 0d913db4418d..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_avro_serializer.py +++ /dev/null @@ -1,106 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -from typing import BinaryIO, Optional, Any, Union, Type -from avro.datafile import DataFileWriter, DataFileReader -from avro.io import DatumWriter, DatumReader - -from ._object_serializer import ObjectSerializer - - -class AvroObjectSerializer(ObjectSerializer): - - def __init__(self, codec=None): - """A Avro serializer using avro lib from Apache. - :param str codec: The writer codec. If None, let the avro library decides. - """ - try: - import avro # pylint: disable=unused-import - except ImportError: - raise ImportError("In order to create a AvroObjectSerializer you need to install the 'avro' library") - - self._writer_codec = codec - - def serialize( - self, - stream, # type: BinaryIO - value, # type: ObjectType - schema=None, # type: Optional[Any] - ): - # type: (...) -> None - """Convert the provided value to it's binary representation and write it to the stream. - Schema must be a Avro RecordSchema: - https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - :param stream: A stream of bytes or bytes directly - :type stream: BinaryIO - :param value: An object to serialize - :param schema: A Avro RecordSchema - """ - if not schema: - raise ValueError("Schema is required in Avro serializer.") - - kwargs = {} - if self._writer_codec: - kwargs['codec'] = self._writer_codec - - writer = DataFileWriter( - stream, - DatumWriter(), - schema, - **kwargs - ) - writer.append(value) - writer.flush() - - def deserialize( - self, - data, # type: Union[bytes, BinaryIO] - return_type=None, # type: Optional[Type[ObjectType]] - ): - # type: (...) -> ObjectType - """Read the binary representation into a specific type. - Return type will be ignored, since the schema is deduced from the provided bytes. - :param data: A stream of bytes or bytes directly - :type data: BinaryIO or bytes - :param return_type: Return type is not supported in the Avro serializer. - :returns: An instanciated object - :rtype: ObjectType - """ - if not hasattr(data, 'read'): - from io import BytesIO - data = BytesIO(data) - - reader = DataFileReader( - data, - DatumReader() - ) - obj = next(reader) - try: - next(reader) - raise ValueError("This avro stream is multiple object stream") - except StopIteration: - pass - reader.close() - return obj \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_object_serializer.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_object_serializer.py deleted file mode 100644 index fac99d185f35..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serialization/_object_serializer.py +++ /dev/null @@ -1,76 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import abc -from typing import BinaryIO, Union, Type, TypeVar, Optional - - -try: - ABC = abc.ABC -except AttributeError: # Python 2.7, abc exists, but not ABC - ABC = abc.ABCMeta("ABC", (object,), {"__slots__": ()}) # type: ignore - -ObjectType = TypeVar("ObjectType") - - -class ObjectSerializer(ABC): - - @abc.abstractmethod - def serialize( - self, - stream, # type: BinaryIO - value, # type: ObjectType - schema=None, # type: Optional[Any] - ): - # type: (...) -> None - """Convert the provided value to it's binary representation and write it to the stream. - Schema can be used to pass additional information on how to serialize the value. Schema - might be ignored, and usage is dependent of the serializer chose. Please refer to the documentation - of the serializer implementation for details of schema structure. - :param stream: A stream of bytes or bytes directly - :type stream: BinaryIO - :param value: An object to serialize - :param schema: A schema used validate or help serialization. Usage is dependent of the serializer. - """ - raise NotImplementedError() - - @abc.abstractmethod - def deserialize( - self, - data, # type: Union[bytes, BinaryIO] - return_type=None, # type: Optional[Type[ObjectType]] - ): - # type: (...) -> ObjectType - """Read the binary representation into a specific type. - Return type can be used to express the expected return type if the serializer is able to instiate classes. - If not, the return type will be decided by the deserializer implementation (likely dict). - Please refer to the documentation of the serializer implementation for details. - :param data: A stream of bytes or bytes directly - :type data: BinaryIO or bytes - :param return_type: The type of the object to convert to and return, if supported. - :returns: An instanciated object - :rtype: ObjectType - """ - raise NotImplementedError() \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/schema_registry.py new file mode 100644 index 000000000000..81d7686b0578 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/samples/schema_registry.py @@ -0,0 +1,53 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import os + +from azure.identity import ClientSecretCredential +from azure.schemaregistry import SchemaRegistryClient, SerializationType + +TENANT_ID = os.environ['AZURE_TENANT_ID'] +CLIENT_ID = os.environ['AZURE_CLIENT_ID'] +CLIENT_SECRET = os.environ['AZURE_CLIENT_SECRET'] + +SCHEMA_REGISTRY_ENDPOINT = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_GROUP = os.environ['SCHEMA_GROUP'] +SCHEMA_NAME = os.environ['SCHEMA_NAME'] +SERIALIZATION_TYPE = SerializationType.AVRO +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET +) + + +def register_schema(client, schema_group, schema_name, serialization_type, schema_string): + res = client.register_schema(schema_group, schema_name, "AVRO", schema_string) + print(res) + pass + + +def get_schema_by_id(): + pass + + +def get_schema(): + pass + + +schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) +register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/setup.py b/sdk/schemaregistry/azure-schemaregistry/setup.py index 739998589981..8f8ca94f9e59 100644 --- a/sdk/schemaregistry/azure-schemaregistry/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry/setup.py @@ -79,8 +79,7 @@ ]), install_requires=[ 'msrest>=0.5.0', - 'azure-core<2.0.0,>=1.2.2', - 'avro>=1.0.0,<2.0.0' + 'azure-core<2.0.0,>=1.2.2' ], extras_require={ ":python_version<'3.0'": ['azure-nspkg', 'futures'], diff --git a/sdk/schemaregistry/azure-schemaregistry/swagger/README.md b/sdk/schemaregistry/azure-schemaregistry/swagger/README.md index 6acc525b597f..d74afae801ca 100644 --- a/sdk/schemaregistry/azure-schemaregistry/swagger/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/swagger/README.md @@ -16,5 +16,7 @@ no-namespace-folders: true license-header: MICROSOFT_MIT_NO_VERSION clear-output-folder: true python: true +add-credential: true +credential-scopes: "https://eventhubs.azure.net/.default" package-version: "1.0.0b1" ``` From 6311e92fa99b102e785b06ca0f0563e9767951c7 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 20 Aug 2020 13:23:05 -0700 Subject: [PATCH 03/35] adding avro serializer --- .../azure/__init__.py | 6 +- .../azure/schemaregistry/__init__.py | 6 +- .../serializer/avro_serializer/__init__.py | 9 ++ .../avro_serializer/_avro_serializer.py | 12 ++ .../schemaregistry/serializer/__init__.py | 5 +- .../serializer/_avro_serializer/__init__.py | 1 - .../_avro_serializer/_avro_serializer.py | 0 .../serializer/avro_serializer/__init__.py | 9 ++ .../avro_serializer/_avro_serializer.py | 113 ++++++++++++++++++ .../_schema_registry_avro_serializer.py | 38 ++++++ .../_version.py | 0 .../azure-schemaregistry/azure/__init__.py | 6 +- .../azure/schemaregistry/__init__.py | 1 - .../azure/schemaregistry/_common/__init__.py | 4 + .../azure/schemaregistry/_common/_schema.py | 18 --- .../schemaregistry/_schema_registry_client.py | 27 +++-- .../azure/schemaregistry/aio/__init__.py | 9 ++ .../aio/_schema_registry_client_async.py | 97 +++++++++++---- .../_schema_registry_avro_serializer_async.py | 0 .../async_samples/schema_registry_async.py | 49 ++++++++ .../samples/schema_registry.py | 53 -------- .../samples/sync_samples/schema_registry.py | 47 ++++++++ 22 files changed, 402 insertions(+), 108 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_avro_serializer.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py rename sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/{_avro_serializer => avro_serializer}/_version.py (100%) delete mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_serializer/_schema_registry_avro_serializer_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry/samples/schema_registry.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py index 0d1f7edf5dc6..f70db896c615 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py @@ -1 +1,5 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py index 0d1f7edf5dc6..f70db896c615 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py @@ -1 +1,5 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py new file mode 100644 index 000000000000..3e67c6f26bdc --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py @@ -0,0 +1,9 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from ._avro_serializer import SchemaRegistryAvroSerializer + +__all__ = [ + "SchemaRegistryAvroSerializer" +] \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_avro_serializer.py new file mode 100644 index 000000000000..3d611146d991 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_avro_serializer.py @@ -0,0 +1,12 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +import avro + +from azure.schemaregistry.aio import SchemaRegistryClient + + +class SchemaRegistryAvroSerializer: + def __init__(self): + self._schema_dict = {} diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py index 0d1f7edf5dc6..34913fb394d7 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py @@ -1 +1,4 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/__init__.py deleted file mode 100644 index 0d1f7edf5dc6..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_avro_serializer.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py new file mode 100644 index 000000000000..5bad038b4890 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py @@ -0,0 +1,9 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from ._schema_registry_avro_serializer import SchemaRegistryAvroSerializer + +__all__ = [ + "SchemaRegistryAvroSerializer" +] \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py new file mode 100644 index 000000000000..b298581ba8eb --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -0,0 +1,113 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import abc +from typing import BinaryIO, Union, Type, TypeVar, Optional, Any + +from avro.datafile import DataFileWriter, DataFileReader +from avro.io import DatumWriter, DatumReader + +try: + ABC = abc.ABC +except AttributeError: # Python 2.7, abc exists, but not ABC + ABC = abc.ABCMeta("ABC", (object,), {"__slots__": ()}) # type: ignore + +ObjectType = TypeVar("ObjectType") + + +class AvroObjectSerializer(): + + def __init__(self, codec=None): + """A Avro serializer using avro lib from Apache. + :param str codec: The writer codec. If None, let the avro library decides. + """ + try: + import avro # pylint: disable=unused-import + except ImportError: + raise ImportError("In order to create a AvroObjectSerializer you need to install the 'avro' library") + + self._writer_codec = codec + + def serialize( + self, + stream, # type: BinaryIO + value, # type: ObjectType + schema=None, # type: Optional[Any] + ): + # type: (...) -> None + """Convert the provided value to it's binary representation and write it to the stream. + Schema must be a Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + :param stream: A stream of bytes or bytes directly + :type stream: BinaryIO + :param value: An object to serialize + :param schema: A Avro RecordSchema + """ + if not schema: + raise ValueError("Schema is required in Avro serializer.") + + kwargs = {} + if self._writer_codec: + kwargs['codec'] = self._writer_codec + + writer = DataFileWriter( + stream, + DatumWriter(), + schema, + **kwargs + ) + writer.append(value) + writer.flush() + + def deserialize( + self, + data, # type: Union[bytes, BinaryIO] + return_type=None, # type: Optional[Type[ObjectType]] + ): + # type: (...) -> ObjectType + """Read the binary representation into a specific type. + Return type will be ignored, since the schema is deduced from the provided bytes. + :param data: A stream of bytes or bytes directly + :type data: BinaryIO or bytes + :param return_type: Return type is not supported in the Avro serializer. + :returns: An instanciated object + :rtype: ObjectType + """ + if not hasattr(data, 'read'): + from io import BytesIO + data = BytesIO(data) + + reader = DataFileReader( + data, + DatumReader() + ) + obj = next(reader) + try: + next(reader) + raise ValueError("This avro stream is multiple object stream") + except StopIteration: + pass + reader.close() + return obj diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py new file mode 100644 index 000000000000..8f2f5ac97d18 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -0,0 +1,38 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from io import BytesIO + +from azure.schemaregistry import SchemaRegistryClient + +from ._avro_serializer import AvroObjectSerializer + + +class SchemaRegistryAvroSerializer: + def __init__(self, credential, endpoint, schema_group, **kwargs): + self._schema_group = schema_group + self._avro_serializer = AvroObjectSerializer() + self._scheme_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) + + self._schema_dict = {} + + def _get_schema_id(self, schema): + if schema in self._schema_dict: + return self._schema_dict[schema] + else: + self._scheme_registry_client.register_schema() + + def serialize(self, data, schema_name, schema): + """ + + :param schema: + :param data: + :return: + """ + stream = BytesIO() + data_payload = self._avro_serializer.serialize(stream, data, schema) + + + def deserialize(self): + pass diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_version.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_version.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/_avro_serializer/_version.py rename to sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_version.py diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/__init__.py index e604558a8bed..215195d57db0 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/__init__.py @@ -1 +1,5 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py index 4ab85d35853d..3926cce5980d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # ------------------------------------------------------------------------- - from ._version import VERSION __version__ = VERSION diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py index e69de29bb2d1..66794229797f 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py @@ -0,0 +1,4 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py deleted file mode 100644 index f55a16c820cf..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Union -from enum import Enum - - -class SerializationType(Enum): - AVRO = 0 - JSON = 1 - - -class Schema: # TODO: 1. Naming: Schema Properties/Schema Description? 2. Should it be dict like object? - def __init__(self, schema_id, serialization_type, schema_name, schema_group, schema_byte_array, parse_method): # TODO: what is schema_byte_arry and parse_method? - # type: (str, Union[str, Enum], str, str) -> None - self.schema_id = schema_id # TODO: property vs instance variable - self.serialization_type = serialization_type - self.schema_name = schema_name - self.schema_group = schema_group - self.schema = None # TODO: for Avro/Json, schema would just be a JSON string? what about protocolbuf - diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index da0649eb4d7e..0331711a834f 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -2,10 +2,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from typing import Any, TYPE_CHECKING +from typing import Any, TYPE_CHECKING, Union +from enum import Enum from ._generated._azure_schema_registry import AzureSchemaRegistry -from ._common._schema import Schema if TYPE_CHECKING: from azure.core.credentials import TokenCredential @@ -22,10 +22,10 @@ def __init__( **kwargs ): # type: (str, TokenCredential, Any) -> None - self._client = AzureSchemaRegistry(credential=credential, endpoint=endpoint) + self._generated_client = AzureSchemaRegistry(credential=credential, endpoint=endpoint, **kwargs) def register_schema(self, schema_group, schema_name, serialization_type, schema_string): - # type: (str, str, str, str) -> Schema + # type: (str, str, str, str) -> str """ :param schema_group: @@ -34,11 +34,15 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ :param schema_string: :return: """ - res = self._client.schema.register(schema_group, schema_name, schema_string) - return res + schema_id_model = self._generated_client.schema.register( + group_name=schema_group, + schema_name=schema_name, + schema_content=schema_string + ) + return schema_id_model.id def get_schema(self, schema_id): - # type: (str) -> Schema + # type: (str) -> str """ :param schema_id: @@ -46,7 +50,7 @@ def get_schema(self, schema_id): """ return self._client.schema.get_by_id(schema_id) - def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string): # TODO: all should match? + def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string): # type: (str, str, str, Union[str, Enum]) -> str """ @@ -56,4 +60,9 @@ def get_schema_id(self, schema_group, schema_name, serialization_type, schema_st :param schema_string: :return: """ - return self._client.schema.get_id_by_content(schema_group, schema_name, serialization_type, schema_string) + schema_id_model = self._generated_client.schema.query_id_by_content( + group_name=schema_group, + schema_name=schema_name, + schema_content=schema_string + ) + return schema_id_model.id diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py index e69de29bb2d1..229d30c6bd00 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py @@ -0,0 +1,9 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from ._schema_registry_client_async import SchemaRegistryClient + +__all__ = [ + "SchemaRegistryClient" +] \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 308ccf97af7b..5a66f7c20283 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -1,26 +1,79 @@ -from .._schema import Schema +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from typing import Any, TYPE_CHECKING, Union +from enum import Enum + +from .._generated.aio._azure_schema_registry_async import AzureSchemaRegistry + +if TYPE_CHECKING: + from azure.core.credentials_async import AsyncTokenCredential class SchemaRegistryClient: + """ + + """ def __init__( - self, - endpoint, - credential, - max_caches_size, # TODO: cache for schemas? - - **kwargs - ): - self._generated_client = None - pass - - async def register_schema(self, schema_group, schema_name, schema_string, schema_type): - # type: (str, str, str, str) -> Schema - pass - - async def get_schema(self, schema_id): - # type: (str) -> Schema - pass - - async def get_schema_id(self, schema_group, schema_name, schema_string, serialization_type): # TODO: all should match? - # type: (str, str, str, Union[str, Enum]) -> str - pass + self, + endpoint: str, + credential: AsyncTokenCredential, + **kwargs: Any + ) -> None: + self._generated_client = AzureSchemaRegistry(credential, endpoint, **kwargs) + + async def register_schema( + self, + schema_group: str, + schema_name: str, + serialization_type: str, + schema_string: str + ) -> str: + """ + + :param schema_group: + :param schema_name: + :param serialization_type: + :param schema_string: + :return: + """ + schema_id_model = await self._generated_client.schema.register( + group_name=schema_group, + schema_name=schema_name, + schema_content=schema_string + ) + return schema_id_model.id + + async def get_schema( + self, + schema_id: str + ) -> str: + """ + + :param schema_id: + :return: + """ + return await self._generated_client.schema.get_by_id(schema_id=schema_id) + + async def get_schema_id( + self, + schema_group: str, + schema_name: str, + serialization_type: Union[str, Enum], + schema_string: str + ) -> str: + """ + + :param schema_group: + :param schema_name: + :param serialization_type: + :param schema_string: + :return: + """ + schema_id_model = await self._generated_client.schema.query_id_by_content( + group_name=schema_group, + schema_name=schema_name, + schema_content=schema_string + ) + return schema_id_model.id diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_serializer/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_serializer/_schema_registry_avro_serializer_async.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py new file mode 100644 index 000000000000..5e3cba460f87 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py @@ -0,0 +1,49 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import os + +from azure.identity import ClientSecretCredential +from azure.schemaregistry import SchemaRegistryClient, SerializationType + +TENANT_ID=os.environ['AZURE_TENANT_ID'] +CLIENT_ID=os.environ['AZURE_CLIENT_ID'] +CLIENT_SECRET=os.environ['AZURE_CLIENT_SECRET'] + +SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_GROUP=os.environ['SCHEMA_GROUP'] +SCHEMA_NAME=os.environ['SCHEMA_NAME'] +SERIALIZATION_TYPE=SerializationType.AVRO +SCHEMA_STRING="""{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET +) + + +def register_schema(client, schema_group, schema_name, serialization_type, schema_string): + res = client.register_schema(schema_group, schema_name, "AVRO", schema_string) + print(type(res)) + print(res) + + +def get_schema_by_id(client, schema_id): + res = client.get_schema(schema_id) + print(type(res)) + print(res) + + +def get_schema_id(client, schema_group, schema_name, serialization_type, schema_string): + res = client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + print(type(res)) + print(res) + + +schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) +# schema_id = register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) +# schema_str = get_schema_by_id(schema_registry_client, schema_id=schema_id) +schema_id = get_schema_id(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, "dsaijdia") diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/schema_registry.py deleted file mode 100644 index 81d7686b0578..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/samples/schema_registry.py +++ /dev/null @@ -1,53 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import os - -from azure.identity import ClientSecretCredential -from azure.schemaregistry import SchemaRegistryClient, SerializationType - -TENANT_ID = os.environ['AZURE_TENANT_ID'] -CLIENT_ID = os.environ['AZURE_CLIENT_ID'] -CLIENT_SECRET = os.environ['AZURE_CLIENT_SECRET'] - -SCHEMA_REGISTRY_ENDPOINT = os.environ['SCHEMA_REGISTRY_ENDPOINT'] -SCHEMA_GROUP = os.environ['SCHEMA_GROUP'] -SCHEMA_NAME = os.environ['SCHEMA_NAME'] -SERIALIZATION_TYPE = SerializationType.AVRO -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET -) - - -def register_schema(client, schema_group, schema_name, serialization_type, schema_string): - res = client.register_schema(schema_group, schema_name, "AVRO", schema_string) - print(res) - pass - - -def get_schema_by_id(): - pass - - -def get_schema(): - pass - - -schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) -register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py new file mode 100644 index 000000000000..700ccf167a28 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py @@ -0,0 +1,47 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import os + +from azure.identity.aio import ClientSecretCredential +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry import SerializationType + +TENANT_ID=os.environ['AZURE_TENANT_ID'] +CLIENT_ID=os.environ['AZURE_CLIENT_ID'] +CLIENT_SECRET=os.environ['AZURE_CLIENT_SECRET'] + +SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_GROUP=os.environ['SCHEMA_GROUP'] +SCHEMA_NAME=os.environ['SCHEMA_NAME'] +SERIALIZATION_TYPE=SerializationType.AVRO +SCHEMA_STRING="""{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET +) + + +def register_schema(client, schema_group, schema_name, serialization_type, schema_string): + schema_id = client.register_schema(schema_group, schema_name, serialization_type, schema_string) + return schema_id + + +def get_schema_by_id(client, schema_id): + schema_str = client.get_schema(schema_id) + return schema_str + + +def get_schema_id(client, schema_group, schema_name, serialization_type, schema_string): + schema_id = client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + return schema_id + + +schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) +schema_id = register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) +schema_str = get_schema_by_id(schema_registry_client, schema_id=schema_id) +schema_id = get_schema_id(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, "dsaijdia") From 0096ef24c3ff7769797055822b7fe3cfa3dd7fe6 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 20 Aug 2020 13:41:43 -0700 Subject: [PATCH 04/35] tweak api version and fix a typo --- .../aio/operations_async/_schema_operations_async.py | 6 +++--- .../_generated/operations/_schema_operations.py | 6 +++--- .../azure/schemaregistry/_schema_registry_client.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py index 12e92b6fab6a..28d4fe3914cc 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py @@ -58,7 +58,7 @@ async def get_by_id( cls = kwargs.pop('cls', None) # type: ClsType[str] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) - api_version = "2018-01-01-preview" + api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" # Construct URL url = self.get_by_id.metadata['url'] # type: ignore @@ -127,7 +127,7 @@ async def query_id_by_content( error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) x_schema_type = "avro" - api_version = "2018-01-01-preview" + api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" content_type = kwargs.pop("content_type", "application/json") # Construct URL @@ -204,7 +204,7 @@ async def register( error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) x_schema_type = "avro" - api_version = "2018-01-01-preview" + api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" content_type = kwargs.pop("content_type", "application/json") # Construct URL diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py index 2a5130cce974..a32f58dabb93 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py @@ -63,7 +63,7 @@ def get_by_id( cls = kwargs.pop('cls', None) # type: ClsType[str] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) - api_version = "2018-01-01-preview" + api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" # Construct URL url = self.get_by_id.metadata['url'] # type: ignore @@ -133,7 +133,7 @@ def query_id_by_content( error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) x_schema_type = "avro" - api_version = "2018-01-01-preview" + api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" content_type = kwargs.pop("content_type", "application/json") # Construct URL @@ -211,7 +211,7 @@ def register( error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) x_schema_type = "avro" - api_version = "2018-01-01-preview" + api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" content_type = kwargs.pop("content_type", "application/json") # Construct URL diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index 0331711a834f..c19f88a16d58 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -48,7 +48,7 @@ def get_schema(self, schema_id): :param schema_id: :return: """ - return self._client.schema.get_by_id(schema_id) + return self._generated_client.schema.get_by_id(schema_id) def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string): # type: (str, str, str, Union[str, Enum]) -> str From 2e950014ffe95a2ee1a063c66d73e07d457f2123 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 20 Aug 2020 20:32:03 -0700 Subject: [PATCH 05/35] test template --- .../_schema_registry_avro_serializer.py | 61 ++++++++++++--- .../sample_avro_serializer.py | 0 .../schemaregistry/_schema_registry_client.py | 10 +-- .../azure-schemaregistry/dev_requirements.txt | 4 +- .../samples/sync_samples/schema_registry.py | 8 +- .../tests/schemaregistry_preparer.py | 75 +++++++++++++++++++ .../tests/test_placeholder.py | 2 - .../tests/test_schema_registry.py | 48 ++++++++++++ sdk/schemaregistry/tests.yml | 4 + 9 files changed, 189 insertions(+), 23 deletions(-) rename sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/{ => sync_samples}/sample_avro_serializer.py (100%) create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/test_placeholder.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index 8f2f5ac97d18..16e0d5c2c863 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -10,29 +10,68 @@ class SchemaRegistryAvroSerializer: + """ + + """ def __init__(self, credential, endpoint, schema_group, **kwargs): self._schema_group = schema_group self._avro_serializer = AvroObjectSerializer() self._scheme_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) - self._schema_dict = {} + self._id_to_schema_dict = {} + self._schema_to_id_dict = {} def _get_schema_id(self, schema): - if schema in self._schema_dict: - return self._schema_dict[schema] - else: - self._scheme_registry_client.register_schema() + """ + + :param str schema: + :return: + """ + try: + return self._schema_to_id_dict[str(schema)] # TODO: if schema is str/bytes, may need to convert it to Schema object and back to str form the Schema object to normalize. + except KeyError: + schema_id = self._scheme_registry_client.register_schema(self._schema_group, "name", "avro", schema) + self._schema_to_id_dict[schema] = schema_id + self._id_to_schema_dict[schema_id] = str(schema) + return schema_id + + def _get_schema(self, schema_id): + """ - def serialize(self, data, schema_name, schema): + :param str schema_id: + :return: + """ + try: + return self._id_to_schema_dict[schema_id] + except KeyError: + schema = self._scheme_registry_client.get_schema(schema_id) + self._id_to_schema_dict[schema_id] = schema + self._schema_to_id_dict[schema] = schema_id + return schema + + def serialize(self, data, schema): + # avro space name + """ - :param schema: - :param data: + :param str schema: # TODO: support schema object/str/bytes? + :param dict data: :return: """ + # TODO: schema name generated through schema namespace + name + schema_id = self._get_schema_id(schema) stream = BytesIO() - data_payload = self._avro_serializer.serialize(stream, data, schema) + self._avro_serializer.serialize(stream, data, schema) + res = schema_id.encode('utf-8').join(stream.getvalue()) + return res + def deserialize(self, data): + """ + + :param dict data: + :return: + """ + schema_id = data[0:32] + schema_str = self._get_schema(schema_id) + dict_data = self._avro_serializer.deserialize(data[32:]) # TODO: do validation + return dict_data - def deserialize(self): - pass diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sample_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sample_avro_serializer.py rename to sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index c19f88a16d58..a4bb2d60a048 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -28,10 +28,10 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ # type: (str, str, str, str) -> str """ - :param schema_group: - :param schema_name: - :param serialization_type: - :param schema_string: + :param str schema_group: + :param str schema_name: + :param str serialization_type: + :param str schema_string: :return: """ schema_id_model = self._generated_client.schema.register( @@ -45,7 +45,7 @@ def get_schema(self, schema_id): # type: (str) -> str """ - :param schema_id: + :param str schema_id: :return: """ return self._generated_client.schema.get_by_id(schema_id) diff --git a/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt index 07dd69cdc21b..e79e74219e31 100644 --- a/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt +++ b/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt @@ -1 +1,3 @@ --e ../../identity/azure-identity \ No newline at end of file +-e ../../identity/azure-identity +-e ../../../tools/azure-devtools +-e ../../../tools/azure-sdk-tools \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py index 700ccf167a28..4fb4a2ce82fb 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py @@ -9,12 +9,12 @@ from azure.schemaregistry.aio import SchemaRegistryClient from azure.schemaregistry import SerializationType -TENANT_ID=os.environ['AZURE_TENANT_ID'] -CLIENT_ID=os.environ['AZURE_CLIENT_ID'] -CLIENT_SECRET=os.environ['AZURE_CLIENT_SECRET'] +TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] +CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] +CLIENT_SECRET=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] -SCHEMA_GROUP=os.environ['SCHEMA_GROUP'] +SCHEMA_GROUP=os.environ['SCHEMA_REGISTRY_GROUP'] SCHEMA_NAME=os.environ['SCHEMA_NAME'] SERIALIZATION_TYPE=SerializationType.AVRO SCHEMA_STRING="""{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py new file mode 100644 index 000000000000..bf65f4fa1127 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py @@ -0,0 +1,75 @@ +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +import functools +import hashlib +import os +from collections import namedtuple + +from azure.identity import ClientSecretCredential +from azure_devtools.scenario_tests.exceptions import AzureTestError +from devtools_testutils import ( + ResourceGroupPreparer, AzureMgmtPreparer, FakeResource +) + + +SCHEMA_REGISTRY_ENDPOINT_PARAM = "schemaregistry_endpoint" +SCHEMA_REGISTRY_GROUP_PARAM = "schemaregistry_group" +SCHEMA_REGISTRY_CREDENTIAL_PARAM = "schemaregistry_credential" +SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME = 'SCHEMA_REGISTRY_ENDPOINT' +SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME = 'SCHEMA_REGISTRY_GROUP' +AZURE_TENANT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_TENANT_ID' +AZURE_CLIENT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_ID' +AZURE_CLIENT_SECRET_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_SECRET' + + +class SchemaRegistryNamespacePreparer(AzureMgmtPreparer): + # TODO: SR doesn't have mgmt package + def __init__(self): + pass + + def create_resource(self, name, **kwargs): + pass + + def remove_resource(self, name, **kwargs): + pass + + +class SchemaRegistryPreparer(AzureMgmtPreparer): + def __init__( + self, + name_prefix='' + ): + super(SchemaRegistryPreparer, self).__init__(name_prefix, 24) + + def create_resource(self, name, **kwargs): + # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using api, in the future we should be able to dynamically create and remove resources + return { + SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], + SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME], + SCHEMA_REGISTRY_CREDENTIAL_PARAM: ClientSecretCredential( + tenant_id=os.environ[AZURE_TENANT_ID_ENV_KEY_NAME], + client_id=os.environ[AZURE_CLIENT_ID_ENV_KEY_NAME], + client_secret=os.environ[AZURE_CLIENT_SECRET_ENV_KEY_NAME] + ) + } + + def remove_resource(self, name, **kwargs): + pass diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_placeholder.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_placeholder.py deleted file mode 100644 index ace0d42403eb..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_placeholder.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_placeholder(): - assert 1 == 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py new file mode 100644 index 000000000000..5248a4403fce --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -0,0 +1,48 @@ +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import pytest +import uuid +import os + +from azure.schemaregistry import SchemaRegistryClient + +from schemaregistry_preparer import SchemaRegistryPreparer +from devtools_testutils import AzureMgmtTestCase + + +class SchemaRegistryTests(AzureMgmtTestCase): + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + def test_register_schema(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=schemaregistry_credential) + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + serialization_type = "avro" + schema_id = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + + assert id is not None + + returned_schema_str = client.get_schema(schema_id=schema_id) + returned_schema_id = client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) + + assert returned_schema_str == schema_str + assert returned_schema_id == schema_id diff --git a/sdk/schemaregistry/tests.yml b/sdk/schemaregistry/tests.yml index a800f04f3613..350037172da8 100644 --- a/sdk/schemaregistry/tests.yml +++ b/sdk/schemaregistry/tests.yml @@ -11,6 +11,10 @@ jobs: AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id) AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id) AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) + SCHEMA_REGISTRY_AZURE_TENANT_ID: $(schema-registry-sdk-test-tenant-id) + SCHEMA_REGISTRY_AZURE_CLIENT_ID: $(schema-registry-sdk-test-client-id) + SCHEMA_REGISTRY_AZURE_CLIENT_SECRET: $(schema-registry-sdk-test-client-secret) + SCHEMA_REGISTRY_ENDPOINT: $(schema-registry-sdk-test-endpoint) AZURE_TEST_RUN_LIVE: 'true' Matrix: Linux_Python35: From 2e8bcc726f66bfe28ef122b8460ceea172951198 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Fri, 21 Aug 2020 20:29:08 -0700 Subject: [PATCH 06/35] avro serializer sync draft --- .../schemaregistry/serializer/__init__.py | 1 + .../serializer/avro_serializer/__init__.py | 4 ++ .../_schema_registry_avro_serializer.py | 41 +++++++++++------- .../sync_samples/sample_avro_serializer.py | 43 +++++++++++++++++++ .../setup.py | 24 ++--------- .../azure/schemaregistry/__init__.py | 2 + .../schemaregistry/_schema_registry_client.py | 2 +- .../schemaregistry/serializer/__init__.py | 5 +++ .../async_samples/schema_registry_async.py | 5 ++- .../samples/sync_samples/schema_registry.py | 15 ++++--- 10 files changed, 97 insertions(+), 45 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py index 34913fb394d7..446811a8e263 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py @@ -2,3 +2,4 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py index 5bad038b4890..ffd2b0188dd4 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py @@ -2,6 +2,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from ._version import VERSION + +__version__ = VERSION + from ._schema_registry_avro_serializer import SchemaRegistryAvroSerializer __all__ = [ diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index 16e0d5c2c863..6cbb4dada325 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -3,8 +3,9 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- from io import BytesIO +import avro -from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry import SchemaRegistryClient, SerializationType from ._avro_serializer import AvroObjectSerializer @@ -21,18 +22,24 @@ def __init__(self, credential, endpoint, schema_group, **kwargs): self._id_to_schema_dict = {} self._schema_to_id_dict = {} - def _get_schema_id(self, schema): + def _get_schema_id(self, schema_name, schema_str): """ - :param str schema: + :param schema_name: + :param schema_str: :return: """ try: - return self._schema_to_id_dict[str(schema)] # TODO: if schema is str/bytes, may need to convert it to Schema object and back to str form the Schema object to normalize. + return self._schema_to_id_dict[schema_str] except KeyError: - schema_id = self._scheme_registry_client.register_schema(self._schema_group, "name", "avro", schema) - self._schema_to_id_dict[schema] = schema_id - self._id_to_schema_dict[schema_id] = str(schema) + schema_id = self._scheme_registry_client.register_schema( + self._schema_group, + schema_name, + SerializationType.AVRO, + schema_str + ) + self._schema_to_id_dict[schema_str] = schema_id + self._id_to_schema_dict[schema_id] = str(schema_str) return schema_id def _get_schema(self, schema_id): @@ -44,24 +51,26 @@ def _get_schema(self, schema_id): try: return self._id_to_schema_dict[schema_id] except KeyError: - schema = self._scheme_registry_client.get_schema(schema_id) - self._id_to_schema_dict[schema_id] = schema - self._schema_to_id_dict[schema] = schema_id - return schema + schema_str = self._scheme_registry_client.get_schema(schema_id) + self._id_to_schema_dict[schema_id] = schema_str + self._schema_to_id_dict[schema_str] = schema_id + return schema_str def serialize(self, data, schema): - # avro space name + """ - :param str schema: # TODO: support schema object/str/bytes? + :param schema: # TODO: support schema object/str/bytes? :param dict data: :return: """ # TODO: schema name generated through schema namespace + name - schema_id = self._get_schema_id(schema) + if not isinstance(schema, avro.schema.Schema): + schema = avro.schema.parse(schema) + + schema_id = self._get_schema_id(schema.fullname, str(schema)) stream = BytesIO() self._avro_serializer.serialize(stream, data, schema) - res = schema_id.encode('utf-8').join(stream.getvalue()) + res = schema_id.encode('utf-8') + stream.getvalue() return res def deserialize(self, data): @@ -70,7 +79,7 @@ def deserialize(self, data): :param dict data: :return: """ - schema_id = data[0:32] + schema_id = data[0:32].decode('utf-8') schema_str = self._get_schema(schema_id) dict_data = self._avro_serializer.deserialize(data[32:]) # TODO: do validation return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py index e69de29bb2d1..98dd804ade69 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py @@ -0,0 +1,43 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import os + +from azure.identity import ClientSecretCredential +from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer + +TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] +CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] +CLIENT_SECRET=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] + +SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_GROUP=os.environ['SCHEMA_REGISTRY_GROUP'] +SCHEMA_STRING=""" +{"namespace":"example.avro","type":"record","name": +"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]} +""" + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET +) + + +def serialize(serializer, schema): + dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + bytes = serializer.serialize(dict_data, schema) + return bytes + + +def deserialize(serializer, dict_data): + return serializer.deserialize(dict_data) + +serializer = SchemaRegistryAvroSerializer(token_credential, SCHEMA_REGISTRY_ENDPOINT, SCHEMA_GROUP) +payload_bytes = serialize(serializer, SCHEMA_STRING) +print(payload_bytes) +dict_data = deserialize(serializer, payload_bytes) +print(dict_data) + diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py index 56b0db61c88b..42b63e82dd56 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py @@ -15,25 +15,8 @@ PACKAGE_NAME = "azure-schemaregistry-avro-serializer" PACKAGE_PPRINT_NAME = "Schema Registry Avro Serializer" -# a-b-c => a/b/c -package_folder_path = PACKAGE_NAME.replace('-', '/') -# a-b-c => a.b.c -namespace_name = PACKAGE_NAME.replace('-', '.') - -# azure v0.x is not compatible with this package -# azure v0.x used to have a __version__ attribute (newer versions don't) -try: - import azure - try: - ver = azure.__version__ - raise Exception( - 'This package is incompatible with azure=={}. '.format(ver) + - 'Uninstall it with "pip uninstall azure".' - ) - except AttributeError: - pass -except ImportError: - pass +package_folder_path = "azure/schemaregistry/serializer/avro_serializer" +namespace_name = "azure.schemaregistry.serializer.avro_serializer" # Version extraction inspired from 'requests' with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: @@ -78,9 +61,8 @@ 'azure', ]), install_requires=[ - 'msrest>=0.5.0', 'azure-core<2.0.0,>=1.2.2' - 'azure-schemaregistry==1.0.0b1' + 'azure-schemaregistry<2.0.0,>=1.0.0b1' ], extras_require={ ":python_version<'3.0'": ['azure-nspkg', 'futures'], diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py index 3926cce5980d..4f6c36ad5f3d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py @@ -3,6 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # ------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore + from ._version import VERSION __version__ = VERSION diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index a4bb2d60a048..b5440bb1a4a3 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -25,7 +25,7 @@ def __init__( self._generated_client = AzureSchemaRegistry(credential=credential, endpoint=endpoint, **kwargs) def register_schema(self, schema_group, schema_name, serialization_type, schema_string): - # type: (str, str, str, str) -> str + # type: (str, str, Union[str, Enum], str) -> str """ :param str schema_group: diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py new file mode 100644 index 000000000000..446811a8e263 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py @@ -0,0 +1,5 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py index 5e3cba460f87..89a71f93a9a9 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py @@ -5,8 +5,9 @@ # -------------------------------------------------------------------------- import os -from azure.identity import ClientSecretCredential -from azure.schemaregistry import SchemaRegistryClient, SerializationType +from azure.identity.aio import ClientSecretCredential +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry import SerializationType TENANT_ID=os.environ['AZURE_TENANT_ID'] CLIENT_ID=os.environ['AZURE_CLIENT_ID'] diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py index 4fb4a2ce82fb..241c36a40084 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py @@ -5,9 +5,8 @@ # -------------------------------------------------------------------------- import os -from azure.identity.aio import ClientSecretCredential -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry import SerializationType +from azure.identity import ClientSecretCredential +from azure.schemaregistry import SchemaRegistryClient, SerializationType TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] @@ -17,7 +16,10 @@ SCHEMA_GROUP=os.environ['SCHEMA_REGISTRY_GROUP'] SCHEMA_NAME=os.environ['SCHEMA_NAME'] SERIALIZATION_TYPE=SerializationType.AVRO -SCHEMA_STRING="""{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" +SCHEMA_STRING=""" +{"namespace":"example.avro","type":"record","name": +"User","fields":[{"name":"namee","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]} +""" token_credential = ClientSecretCredential( tenant_id=TENANT_ID, @@ -43,5 +45,8 @@ def get_schema_id(client, schema_group, schema_name, serialization_type, schema_ schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) schema_id = register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) +print(schema_id) schema_str = get_schema_by_id(schema_registry_client, schema_id=schema_id) -schema_id = get_schema_id(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, "dsaijdia") +print(schema_str) +schema_id = get_schema_id(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) +print(schema_id) From 6248ff1c7db7607f8566e1d88db4a3fdbf5e5661 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Mon, 24 Aug 2020 21:11:49 -0700 Subject: [PATCH 07/35] major azure sr client work done --- .../azure/__init__.py | 27 ++- .../azure/schemaregistry/__init__.py | 27 ++- .../schemaregistry/serializer/__init__.py | 27 ++- .../serializer/avro_serializer/__init__.py | 27 ++- .../avro_serializer/_avro_serializer.py | 2 +- .../_schema_registry_avro_serializer.py | 52 ++++-- .../serializer/avro_serializer/_version.py | 29 +++- .../azure-schemaregistry/azure/__init__.py | 27 ++- .../azure/schemaregistry/__init__.py | 33 +++- .../azure/schemaregistry/_common/__init__.py | 27 ++- .../schemaregistry/_common/_constants.py | 31 +++- .../_common/_response_handlers.py | 55 +++++++ .../azure/schemaregistry/_common/_schema.py | 155 ++++++++++++++++++ .../schemaregistry/_schema_registry_client.py | 111 ++++++++++--- .../azure/schemaregistry/_version.py | 29 +++- .../aio/_schema_registry_client_async.py | 81 ++++++--- .../schemaregistry/serializer/__init__.py | 5 - .../azure-schemaregistry/samples/README.md | 49 ++++++ .../async_samples/schema_registry_async.py | 87 ++++++---- .../samples/sync_samples/schema_registry.py | 79 +++++---- .../async_tests/test_schema_registry_async.py | 142 ++++++++++++++++ .../tests/schemaregistry_preparer.py | 12 +- .../tests/test_schema_registry.py | 98 ++++++++++- 23 files changed, 1035 insertions(+), 177 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py index f70db896c615..80f86cb969ec 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py @@ -1,5 +1,26 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py index f70db896c615..80f86cb969ec 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py @@ -1,5 +1,26 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py index 446811a8e263..e7ad8fc407f9 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py @@ -1,5 +1,26 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py index ffd2b0188dd4..0edc418a9f22 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py @@ -1,7 +1,28 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- from ._version import VERSION __version__ = VERSION diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py index b298581ba8eb..27f712eed55f 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -37,7 +37,7 @@ ObjectType = TypeVar("ObjectType") -class AvroObjectSerializer(): +class AvroObjectSerializer(ABC): def __init__(self, codec=None): """A Avro serializer using avro lib from Apache. diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index 6cbb4dada325..ebf31d111fb9 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -1,7 +1,28 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- from io import BytesIO import avro @@ -10,17 +31,16 @@ from ._avro_serializer import AvroObjectSerializer -class SchemaRegistryAvroSerializer: +class SchemaRegistryAvroSerializer(object): """ """ def __init__(self, credential, endpoint, schema_group, **kwargs): self._schema_group = schema_group self._avro_serializer = AvroObjectSerializer() - self._scheme_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) - - self._id_to_schema_dict = {} - self._schema_to_id_dict = {} + self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) + self._id_to_schema = {} + self._schema_to_id = {} def _get_schema_id(self, schema_name, schema_str): """ @@ -30,16 +50,16 @@ def _get_schema_id(self, schema_name, schema_str): :return: """ try: - return self._schema_to_id_dict[schema_str] + return self._schema_to_id[schema_str] except KeyError: - schema_id = self._scheme_registry_client.register_schema( + schema_id = self._schema_registry_client.register_schema( self._schema_group, schema_name, SerializationType.AVRO, schema_str ) - self._schema_to_id_dict[schema_str] = schema_id - self._id_to_schema_dict[schema_id] = str(schema_str) + self._schema_to_id[schema_str] = schema_id + self._id_to_schema[schema_id] = str(schema_str) return schema_id def _get_schema(self, schema_id): @@ -49,11 +69,11 @@ def _get_schema(self, schema_id): :return: """ try: - return self._id_to_schema_dict[schema_id] + return self._id_to_schema[schema_id] except KeyError: - schema_str = self._scheme_registry_client.get_schema(schema_id) - self._id_to_schema_dict[schema_id] = schema_str - self._schema_to_id_dict[schema_str] = schema_id + schema_str = self._schema_registry_client.get_schema(schema_id) + self._id_to_schema[schema_id] = schema_str + self._schema_to_id[schema_str] = schema_id return schema_str def serialize(self, data, schema): diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_version.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_version.py index ac9f392f513e..d316addb36cb 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_version.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_version.py @@ -1,6 +1,27 @@ -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- VERSION = "1.0.0b1" diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/__init__.py index 215195d57db0..6f1f89cde8df 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/__init__.py @@ -1,5 +1,26 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- __path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py index 4f6c36ad5f3d..28fc5d4961cf 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py @@ -1,8 +1,28 @@ -# ------------------------------------------------------------------------ +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore from ._version import VERSION @@ -10,8 +30,11 @@ from ._schema_registry_client import SchemaRegistryClient from ._common._constants import SerializationType +from ._common._schema import Schema, SchemaId __all__ = [ "SchemaRegistryClient", - "SerializationType" + "SerializationType", + "Schema", + "SchemaId" ] diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py index 66794229797f..9ecdd4ff2c41 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py @@ -1,4 +1,25 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- \ No newline at end of file +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py index d203c0b95879..cd5424dc42c4 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py @@ -1,10 +1,31 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- from enum import Enum class SerializationType(Enum): - AVRO = "AVRO" - JSON = "JSON" + AVRO = "Avro" + JSON = "Json" diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py new file mode 100644 index 000000000000..fc135e907b9f --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py @@ -0,0 +1,55 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from typing import Any, Dict + +from azure.core.pipeline import PipelineResponse + +from .._generated.models import SchemaId as InternalSchemaId +from ._schema import SchemaId, Schema + + +def _parse_response_schema_id(pipeline_response, deserialized, response_headers): + # type: (PipelineResponse, InternalSchemaId, Dict[str, Any]) -> SchemaId + """ + + :param pipeline_response: + :param deserialized: + :param response_headers: + :return: + """ + return SchemaId(schema_id=deserialized.id, **response_headers) + + +def _parse_response_schema(pipeline_response, deserialized, response_headers): + # type: (PipelineResponse, str, Dict[str, Any]) -> Schema + """ + + :param pipeline_response: + :param deserialized: + :param response_headers: + :return: + """ + return Schema(schema_str=deserialized, **response_headers) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py new file mode 100644 index 000000000000..e70ca12c9437 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -0,0 +1,155 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from typing import Any, Dict + + +class DictMixin(object): + + def __setitem__(self, key, item): + self.__dict__[key] = item + + def __getitem__(self, key): + return self.__dict__[key] + + def __repr__(self): + return str(self) + + def __len__(self): + return len(self.keys()) + + def __delitem__(self, key): + self.__dict__[key] = None + + def __eq__(self, other): + """Compare objects by comparing all attributes.""" + if isinstance(other, self.__class__): + return self.__dict__ == other.__dict__ + return False + + def __ne__(self, other): + """Compare objects by comparing all attributes.""" + return not self.__eq__(other) + + def __str__(self): + return str({k: v for k, v in self.__dict__.items() if not k.startswith('_')}) + + def has_key(self, k): + return k in self.__dict__ + + def update(self, *args, **kwargs): + return self.__dict__.update(*args, **kwargs) + + def keys(self): + return [k for k in self.__dict__ if not k.startswith('_')] + + def values(self): + return [v for k, v in self.__dict__.items() if not k.startswith('_')] + + def items(self): + return [(k, v) for k, v in self.__dict__.items() if not k.startswith('_')] + + def get(self, key, default=None): + if key in self.__dict__: + return self.__dict__[key] + return default + + +class SchemaMeta(DictMixin): + """ + Id and meta properties of a schema. + + :ivar id: References specific schema in registry namespace. + :type id: str + :ivar location: URL location of schema, identified by schema group, schema name, and version. + :type location: str + :ivar id_location: URL location of schema, identified by schema ID. # TODO: JS name is LocationById + :type id_location: str + :ivar type: Serialization type for the schema being stored. + :type type: str + :ivar version: Version of the returned schema. + :type version: int + """ + def __init__( + self, + **kwargs + ): + self.location = kwargs.get('Location') + self.id = kwargs.get("X-Schema-Id") + self.id_location = kwargs.get('X-Schema-Id-Location') + self.type = kwargs.get('X-Schema-Type') + self.version = kwargs.get('X-Schema-Version') + + +class SchemaId(SchemaMeta): # TODO: need a better name here? + """ + Id and meta properties of a schema. + + :ivar id: References specific schema in registry namespace. + :type id: str + :ivar location: URL location of schema, identified by schema group, schema name, and version. + :type location: str + :ivar location_by_id: URL location of schema, identified by schema ID. + :type location_by_id: str + :ivar type: Serialization type for the schema being stored. + :type type: str + :ivar version: Version of the returned schema. + :type version: int + """ + def __init__( + self, + schema_id, + **kwargs + ): + # type: (str, Dict[str, Any]) -> None + super(SchemaId, self).__init__(**kwargs) + self.id = schema_id + + +class Schema(SchemaMeta): + """ + Id, meta properties and schema content of a schema. + + :ivar content: The content of the schema. + :type content: str + :ivar id: References specific schema in registry namespace. + :type id: str + :ivar location: URL location of schema, identified by schema group, schema name, and version. + :type location: str + :ivar location_by_id: URL location of schema, identified by schema ID. + :type location_by_id: str + :ivar type: Serialization type for the schema being stored. + :type type: str + :ivar version: Version of the returned schema. + :type version: int + """ + def __init__( + self, + schema_str, + **kwargs + ): + # type: (str, Dict[str, Any]) -> None + super(Schema, self).__init__(**kwargs) + self.content = schema_str diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index b5440bb1a4a3..a30aaa80268b 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -1,10 +1,33 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- from typing import Any, TYPE_CHECKING, Union from enum import Enum +from ._common._schema import Schema, SchemaId +from ._common._response_handlers import _parse_response_schema, _parse_response_schema_id from ._generated._azure_schema_registry import AzureSchemaRegistry if TYPE_CHECKING: @@ -13,6 +36,12 @@ class SchemaRegistryClient: """ + SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, + complete with support for versioning and management. + + :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. + :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. + :type credential: TokenCredential """ def __init__( @@ -24,45 +53,75 @@ def __init__( # type: (str, TokenCredential, Any) -> None self._generated_client = AzureSchemaRegistry(credential=credential, endpoint=endpoint, **kwargs) - def register_schema(self, schema_group, schema_name, serialization_type, schema_string): - # type: (str, str, Union[str, Enum], str) -> str + def __enter__(self): + # type: () -> SchemaRegistryClient + self._generated_client.__enter__() + return self + + def __exit__(self, *exc_details): + # type: (Any) -> None + self._generated_client.__exit__(*exc_details) + + def close(self): + # type: () -> None + """ This method is to close the sockets opened by the client. + It need not be used when using with a context manager. """ + self._generated_client.close() - :param str schema_group: - :param str schema_name: - :param str serialization_type: - :param str schema_string: - :return: + def register_schema(self, schema_group, schema_name, serialization_type, schema_string): # TODO: serialization_type Enum + string? + # type: (str, str, Union[str, Enum], str) -> SchemaId """ - schema_id_model = self._generated_client.schema.register( + Register new schema. If schema of specified name does not exist in specified group, + schema is created at version 1. If schema of specified name exists already in specified group, + schema is created at latest version + 1. + + :param str schema_group: Schema group under which schema should be registered. + :param str schema_name: Name of schema being registered. + :param serialization_type: Serialization type for the schema being registered. + :type serialization_type: Union[str, Enum] + :param str schema_string: String representation of the schema being registered. + :rtype: SchemaId + """ + return self._generated_client.schema.register( group_name=schema_group, schema_name=schema_name, - schema_content=schema_string + schema_content=schema_string, + #serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter + cls=_parse_response_schema_id ) - return schema_id_model.id def get_schema(self, schema_id): - # type: (str) -> str + # type: (str) -> Schema """ + Gets a registered schema by its unique ID. + Azure Schema Registry guarantees that ID is unique within a namespace. - :param str schema_id: - :return: + :param str schema_id: References specific schema in registry namespace. + :rtype: Schema """ - return self._generated_client.schema.get_by_id(schema_id) + return self._generated_client.schema.get_by_id( + schema_id, + cls=_parse_response_schema + ) def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string): - # type: (str, str, str, Union[str, Enum]) -> str + # type: (str, str, str, Union[str, Enum]) -> SchemaId """ + Gets the ID referencing an existing schema within the specified schema group, + as matched by schema content comparison. - :param schema_group: - :param schema_name: - :param serialization_type: - :param schema_string: - :return: + :param str schema_group: Schema group under which schema should be registered. + :param str schema_name: Name of schema being registered. + :param serialization_type: Serialization type for the schema being registered. + :type serialization_type: Union[str, Enum] + :param str schema_string: String representation of the schema being registered. + :rtype: SchemaId """ - schema_id_model = self._generated_client.schema.query_id_by_content( + return self._generated_client.schema.query_id_by_content( group_name=schema_group, schema_name=schema_name, - schema_content=schema_string + schema_content=schema_string, + #serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter + cls=_parse_response_schema_id ) - return schema_id_model.id diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_version.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_version.py index ac9f392f513e..d316addb36cb 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_version.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_version.py @@ -1,6 +1,27 @@ -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- VERSION = "1.0.0b1" diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 5a66f7c20283..55e4b5dca16f 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -5,6 +5,8 @@ from typing import Any, TYPE_CHECKING, Union from enum import Enum +from .._common._response_handlers import _parse_response_schema_id, _parse_response_schema +from .._common._schema import SchemaId, Schema from .._generated.aio._azure_schema_registry_async import AzureSchemaRegistry if TYPE_CHECKING: @@ -13,48 +15,77 @@ class SchemaRegistryClient: """ + SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, + complete with support for versioning and management. + + :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. + :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. + :type credential: TokenCredential """ def __init__( self, endpoint: str, - credential: AsyncTokenCredential, + credential: "AsyncTokenCredential", **kwargs: Any ) -> None: self._generated_client = AzureSchemaRegistry(credential, endpoint, **kwargs) + async def __aenter__(self): + await self._generated_client.__aenter__() + return self + + async def __aexit__(self, *args): + await self._generated_client.__aexit__(*args) + + async def close(self): + """ This method is to close the sockets opened by the client. + It need not be used when using with a context manager. + """ + await self._generated_client.close() + async def register_schema( self, schema_group: str, schema_name: str, - serialization_type: str, + serialization_type: Union[str, Enum], schema_string: str - ) -> str: + ) -> SchemaId: """ + Register new schema. If schema of specified name does not exist in specified group, + schema is created at version 1. If schema of specified name exists already in specified group, + schema is created at latest version + 1. - :param schema_group: - :param schema_name: - :param serialization_type: - :param schema_string: - :return: + :param str schema_group: Schema group under which schema should be registered. + :param str schema_name: Name of schema being registered. + :param serialization_type: Serialization type for the schema being registered. + :type serialization_type: Union[str, Enum] + :param str schema_string: String representation of the schema being registered. + :rtype: SchemaId """ - schema_id_model = await self._generated_client.schema.register( + return await self._generated_client.schema.register( group_name=schema_group, schema_name=schema_name, - schema_content=schema_string + schema_content=schema_string, + # serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter + cls=_parse_response_schema_id, ) - return schema_id_model.id async def get_schema( self, schema_id: str - ) -> str: + ) -> Schema: """ + Gets a registered schema by its unique ID. + Azure Schema Registry guarantees that ID is unique within a namespace. - :param schema_id: - :return: + :param str schema_id: References specific schema in registry namespace. + :rtype: Schema """ - return await self._generated_client.schema.get_by_id(schema_id=schema_id) + return await self._generated_client.schema.get_by_id( + schema_id=schema_id, + cls=_parse_response_schema + ) async def get_schema_id( self, @@ -64,16 +95,20 @@ async def get_schema_id( schema_string: str ) -> str: """ + Gets the ID referencing an existing schema within the specified schema group, + as matched by schema content comparison. - :param schema_group: - :param schema_name: - :param serialization_type: - :param schema_string: - :return: + :param str schema_group: Schema group under which schema should be registered. + :param str schema_name: Name of schema being registered. + :param serialization_type: Serialization type for the schema being registered. + :type serialization_type: Union[str, Enum] + :param str schema_string: String representation of the schema being registered. + :rtype: SchemaId """ - schema_id_model = await self._generated_client.schema.query_id_by_content( + return await self._generated_client.schema.query_id_by_content( group_name=schema_group, schema_name=schema_name, - schema_content=schema_string + schema_content=schema_string, + # serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter + cls=_parse_response_schema_id ) - return schema_id_model.id diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py deleted file mode 100644 index 446811a8e263..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/README.md b/sdk/schemaregistry/azure-schemaregistry/samples/README.md index e69de29bb2d1..1aaa7736383d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/samples/README.md @@ -0,0 +1,49 @@ +--- +page_type: sample +languages: + - python +products: + - azure + - azure-schema-registry +urlFragment: schemaregistry-samples +--- + +# Azure Schema Registry client library for Python Samples + +These are code samples that show common scenario operations with the Schema Registry client library. +The async versions of the samples (the python sample files appended with `_async`) show asynchronous operations, +and require Python 3.5 or later. + +Several Schema Registry Python SDK samples are available to you in the SDK's GitHub repository. These samples provide example code for additional scenarios commonly encountered while working with Schema Registry: + +* [schema_registry.py](./sync_samples/schema_registry.py) ([async version](./async_samples/schema_registry_async.py)) - Examples for common Schema Registry tasks: + * Register a schema + * Get schema by id + * Get schema id + +## Prerequisites +- Python 2.7, 3.5 or later. +- **Microsoft Azure Subscription:** To use Azure services, including Azure Schema Registry, you'll need a subscription. +If you do not have an existing Azure account, you may sign up for a free trial or use your MSDN subscriber benefits when you [create an account](https://account.windowsazure.com/Home/Index). + +## Setup + +1. Install the Azure Schema Registry client library for Python with [pip](https://pypi.org/project/pip/): + +```bash +pip install azure-schemaregistry +``` + +2. Clone or download this sample repository +3. Open the sample folder in Visual Studio Code or your IDE of choice. + +## Running the samples + +1. Open a terminal window and `cd` to the directory that the samples are saved in. +2. Set the environment variables specified in the sample file you wish to run. +3. Follow the usage described in the file, e.g. `python schema_registry.py` + +## Next steps + +Check out the [API reference documentation](https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry/latest/index.html) to learn more about +what you can do with the Azure Schema Registry client library. diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py index 89a71f93a9a9..0df168f0ed80 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py @@ -3,48 +3,77 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- + +""" +Example to show basic usage of schema registry asynchronously: + - register a schema + - get schema by id + - get schema id +""" + +import asyncio import os from azure.identity.aio import ClientSecretCredential from azure.schemaregistry.aio import SchemaRegistryClient from azure.schemaregistry import SerializationType -TENANT_ID=os.environ['AZURE_TENANT_ID'] -CLIENT_ID=os.environ['AZURE_CLIENT_ID'] -CLIENT_SECRET=os.environ['AZURE_CLIENT_SECRET'] +TENANT_ID = os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] +CLIENT_ID = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] +CLIENT_SECRET = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] + +SCHEMA_REGISTRY_ENDPOINT = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_GROUP = os.environ['SCHEMA_REGISTRY_GROUP'] +SCHEMA_NAME = 'your-schema-name' +SERIALIZATION_TYPE = SerializationType.AVRO +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" -SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] -SCHEMA_GROUP=os.environ['SCHEMA_GROUP'] -SCHEMA_NAME=os.environ['SCHEMA_NAME'] -SERIALIZATION_TYPE=SerializationType.AVRO -SCHEMA_STRING="""{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET -) +async def register_schema(client, schema_group, schema_name, serialization_type, schema_string): + print("Registering schema...") + schema_id = await client.register_schema(schema_group, schema_name, serialization_type, schema_string) + print("Schema registered, returned schema id is {}".format(schema_id.id)) + print("Schema meta properties are {}".format(schema_id)) + return schema_id.id -def register_schema(client, schema_group, schema_name, serialization_type, schema_string): - res = client.register_schema(schema_group, schema_name, "AVRO", schema_string) - print(type(res)) - print(res) +async def get_schema_by_id(client, schema_id): + print("Getting schema by id...") + schema = await client.get_schema(schema_id) + print("The schema string of schema id: {} string is {}".format(schema_id, schema.content)) + print("Schema meta properties are {}".format(schema_id)) + return schema.content -def get_schema_by_id(client, schema_id): - res = client.get_schema(schema_id) - print(type(res)) - print(res) +async def get_schema_id(client, schema_group, schema_name, serialization_type, schema_string): + print("Getting schema id...") + schema_id = await client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + print("The schema id is: {}".format(schema_id.id)) + print("Schema meta properties are {}".format(schema_id)) + return schema_id.id -def get_schema_id(client, schema_group, schema_name, serialization_type, schema_string): - res = client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) - print(type(res)) - print(res) +async def main(): + token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET + ) + schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) + async with token_credential, schema_registry_client: + schema_id = await register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) + schema_str = await get_schema_by_id(schema_registry_client, schema_id) + schema_id = await get_schema_id(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) -schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) -# schema_id = register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) -# schema_str = get_schema_by_id(schema_registry_client, schema_id=schema_id) -schema_id = get_schema_id(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, "dsaijdia") +loop = asyncio.get_event_loop() +loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py index 241c36a40084..3d96b52edda5 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py @@ -3,50 +3,73 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- + +""" +Example to show basic usage of schema registry: + - register a schema + - get schema by id + - get schema id +""" + + import os from azure.identity import ClientSecretCredential from azure.schemaregistry import SchemaRegistryClient, SerializationType -TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] -CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] -CLIENT_SECRET=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] - -SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] -SCHEMA_GROUP=os.environ['SCHEMA_REGISTRY_GROUP'] -SCHEMA_NAME=os.environ['SCHEMA_NAME'] -SERIALIZATION_TYPE=SerializationType.AVRO -SCHEMA_STRING=""" -{"namespace":"example.avro","type":"record","name": -"User","fields":[{"name":"namee","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]} -""" +TENANT_ID = os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] +CLIENT_ID = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] +CLIENT_SECRET = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET -) +SCHEMA_REGISTRY_ENDPOINT = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_GROUP = os.environ['SCHEMA_REGISTRY_GROUP'] +SCHEMA_NAME = 'your-schema-name' +SERIALIZATION_TYPE = SerializationType.AVRO +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" def register_schema(client, schema_group, schema_name, serialization_type, schema_string): + print("Registering schema...") schema_id = client.register_schema(schema_group, schema_name, serialization_type, schema_string) - return schema_id + print("Schema registered, returned schema id is {}".format(schema_id.id)) + print("Schema meta properties are {}".format(schema_id)) + return schema_id.id def get_schema_by_id(client, schema_id): - schema_str = client.get_schema(schema_id) - return schema_str + print("Getting schema by id...") + schema = client.get_schema(schema_id) + print("The schema string of schema id: {} string is {}".format(schema_id, schema.content)) + print("Schema meta properties are {}".format(schema_id)) + return schema.content def get_schema_id(client, schema_group, schema_name, serialization_type, schema_string): + print("Getting schema id...") schema_id = client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) - return schema_id + print("The schema id is: {}".format(schema_id.id)) + print("Schema meta properties are {}".format(schema_id)) + return schema_id.id + +if __name__ == '__main__': + token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET + ) + schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) + with schema_registry_client: + schema_id = register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) + schema_str = get_schema_by_id(schema_registry_client, schema_id=schema_id) + schema_id = get_schema_id(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) -schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) -schema_id = register_schema(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) -print(schema_id) -schema_str = get_schema_by_id(schema_registry_client, schema_id=schema_id) -print(schema_str) -schema_id = get_schema_id(schema_registry_client, SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) -print(schema_id) diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py new file mode 100644 index 000000000000..0ee5326c1f88 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py @@ -0,0 +1,142 @@ +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import pytest +import uuid + +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.identity.aio import ClientSecretCredential +from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError + +from schemaregistry_preparer import SchemaRegistryPreparer +from devtools_testutils import AzureMgmtTestCase + + +class SchemaRegistryAsyncTests(AzureMgmtTestCase): + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) + async with client, credential: + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + serialization_type = "Avro" + schema_id = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + + assert schema_id.id is not None + assert schema_id.location is not None + assert schema_id.id_location is not None + assert schema_id.version is 1 + assert schema_id.type == "Avro" + + returned_schema = await client.get_schema(schema_id=schema_id.id) + + assert returned_schema.id == schema_id.id + assert returned_schema.location is not None + assert returned_schema.id_location is not None + assert returned_schema.version == 1 + assert returned_schema.type == "Avro" + assert returned_schema.content == schema_str + + returned_schema_id = await client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) + + assert returned_schema_id.id == schema_id.id + assert returned_schema_id.location is not None + assert returned_schema_id.id_location is not None + assert returned_schema_id.version == 1 + assert returned_schema_id.type == "Avro" + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) + async with client, credential: + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + serialization_type = "Avro" + schema_id = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + + assert schema_id.id is not None + assert schema_id.location is not None + assert schema_id.id_location is not None + assert schema_id.version == 1 + assert schema_id.type == "Avro" + + schema_str_new = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_food","type":["string","null"]}]}""" + new_schema_id = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str_new) + + assert new_schema_id.id is not None + assert new_schema_id.location is not None + assert new_schema_id.id_location is not None + assert new_schema_id.version == 2 + assert new_schema_id.type == "Avro" + + new_schema = await client.get_schema(schema_id=new_schema_id.id) + + assert new_schema_id.id != schema_id.id + assert new_schema.id == new_schema_id.id + assert new_schema.location is not None + assert new_schema.id_location is not None + assert new_schema.content == schema_str_new + assert new_schema.version == 2 + assert new_schema_id.type == "Avro" + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) + async with client, credential: + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + serialization_type = "Avro" + with pytest.raises(ClientAuthenticationError): + await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + async def test_schema_negative_wrong_endpoint_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=str(uuid.uuid4()) + ".servicebus.windows.net", credential=credential) + async with client, credential: + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + serialization_type = "Avro" + with pytest.raises(ServiceRequestError): + await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + async def test_schema_negative_no_schema_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) + async with client, credential: + with pytest.raises(HttpResponseError): + await client.get_schema('a') + + with pytest.raises(HttpResponseError): + await client.get_schema('a' * 32) diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py index bf65f4fa1127..7f33dd0c50e7 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py @@ -32,7 +32,9 @@ SCHEMA_REGISTRY_ENDPOINT_PARAM = "schemaregistry_endpoint" SCHEMA_REGISTRY_GROUP_PARAM = "schemaregistry_group" -SCHEMA_REGISTRY_CREDENTIAL_PARAM = "schemaregistry_credential" +SCHEMA_REGISTRY_TENANT_ID_PARAM = "schemaregistry_tenant_id" +SCHEMA_REGISTRY_CLIENT_ID_PARAM = "schemaregistry_client_id" +SCHEMA_REGISTRY_CLIENT_SECRET_PARAM = "schemaregistry_client_secret" SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME = 'SCHEMA_REGISTRY_ENDPOINT' SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME = 'SCHEMA_REGISTRY_GROUP' AZURE_TENANT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_TENANT_ID' @@ -64,11 +66,9 @@ def create_resource(self, name, **kwargs): return { SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME], - SCHEMA_REGISTRY_CREDENTIAL_PARAM: ClientSecretCredential( - tenant_id=os.environ[AZURE_TENANT_ID_ENV_KEY_NAME], - client_id=os.environ[AZURE_CLIENT_ID_ENV_KEY_NAME], - client_secret=os.environ[AZURE_CLIENT_SECRET_ENV_KEY_NAME] - ) + SCHEMA_REGISTRY_TENANT_ID_PARAM: os.environ[AZURE_TENANT_ID_ENV_KEY_NAME], + SCHEMA_REGISTRY_CLIENT_ID_PARAM: os.environ[AZURE_CLIENT_ID_ENV_KEY_NAME], + SCHEMA_REGISTRY_CLIENT_SECRET_PARAM: os.environ[AZURE_CLIENT_SECRET_ENV_KEY_NAME] } def remove_resource(self, name, **kwargs): diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index 5248a4403fce..5d8885bf7da2 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -19,9 +19,10 @@ # -------------------------------------------------------------------------- import pytest import uuid -import os from azure.schemaregistry import SchemaRegistryClient +from azure.identity import ClientSecretCredential +from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError from schemaregistry_preparer import SchemaRegistryPreparer from devtools_testutils import AzureMgmtTestCase @@ -32,17 +33,100 @@ class SchemaRegistryTests(AzureMgmtTestCase): @pytest.mark.liveTest @pytest.mark.live_test_only @SchemaRegistryPreparer() - def test_register_schema(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): + def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=schemaregistry_credential) schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - serialization_type = "avro" + serialization_type = "Avro" schema_id = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - assert id is not None + assert schema_id.id is not None + assert schema_id.location is not None + assert schema_id.id_location is not None + assert schema_id.version is 1 + assert schema_id.type == "Avro" + + returned_schema = client.get_schema(schema_id=schema_id.id) + + assert returned_schema.id == schema_id.id + assert returned_schema.location is not None + assert returned_schema.id_location is not None + assert returned_schema.version == 1 + assert returned_schema.type == "Avro" + assert returned_schema.content == schema_str - returned_schema_str = client.get_schema(schema_id=schema_id) returned_schema_id = client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) - assert returned_schema_str == schema_str - assert returned_schema_id == schema_id + assert returned_schema_id.id == schema_id.id + assert returned_schema_id.location is not None + assert returned_schema_id.id_location is not None + assert returned_schema_id.version == 1 + assert returned_schema_id.type == "Avro" + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=schemaregistry_credential) + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + serialization_type = "Avro" + schema_id = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + + assert schema_id.id is not None + assert schema_id.location is not None + assert schema_id.id_location is not None + assert schema_id.version == 1 + assert schema_id.type == "Avro" + + schema_str_new = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_food","type":["string","null"]}]}""" + new_schema_id = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str_new) + + assert new_schema_id.id is not None + assert new_schema_id.location is not None + assert new_schema_id.id_location is not None + assert new_schema_id.version == 2 + assert new_schema_id.type == "Avro" + + new_schema = client.get_schema(schema_id=new_schema_id.id) + + assert new_schema_id.id != schema_id.id + assert new_schema.id == new_schema_id.id + assert new_schema.location is not None + assert new_schema.id_location is not None + assert new_schema.content == schema_str_new + assert new_schema.version == 2 + assert new_schema_id.type == "Avro" + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=ClientSecretCredential("fake", "fake", "fake")) + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + serialization_type = "Avro" + with pytest.raises(ClientAuthenticationError): + client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): + client = SchemaRegistryClient(endpoint=str(uuid.uuid4())+".servicebus.windows.net", credential=schemaregistry_credential) + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + serialization_type = "Avro" + with pytest.raises(ServiceRequestError): + client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + def test_schema_negative_no_schema(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=schemaregistry_credential) + with pytest.raises(HttpResponseError): + client.get_schema('a') + + with pytest.raises(HttpResponseError): + client.get_schema('a' * 32) From f97478e5e8774f9de39b01a9b1bfa2bb97e89015 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 25 Aug 2020 14:31:54 -0700 Subject: [PATCH 08/35] add sample docstring for sr --- .../azure/schemaregistry/_common/_schema.py | 20 ++++ .../schemaregistry/_schema_registry_client.py | 39 ++++++++ .../aio/_schema_registry_client_async.py | 39 ++++++++ .../sample_schema_registry_code_async.py | 78 +++++++++++++++ .../sample_schema_registry_code.py | 96 +++++++++++++++++++ .../tests/test_schema_registry.py | 25 +++-- 6 files changed, 287 insertions(+), 10 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index e70ca12c9437..4d79179f8438 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -117,6 +117,16 @@ class SchemaId(SchemaMeta): # TODO: need a better name here? :type type: str :ivar version: Version of the returned schema. :type version: int + + .. admonition:: Example: + + .. literalinclude:: ../samples/sync_samples/sample_code_schemaregistry.py + :start-after: [START print_schema_id] + :end-before: [END print_schema_id] + :language: python + :dedent: 4 + :caption: SchemaId object. + """ def __init__( self, @@ -144,6 +154,16 @@ class Schema(SchemaMeta): :type type: str :ivar version: Version of the returned schema. :type version: int + + .. admonition:: Example: + + .. literalinclude:: ../samples/sync_samples/sample_code_schemaregistry.py + :start-after: [START print_schema] + :end-before: [END print_schema] + :language: python + :dedent: 4 + :caption: Schema object. + """ def __init__( self, diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index a30aaa80268b..4735ab3fc460 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -43,6 +43,15 @@ class SchemaRegistryClient: :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. :type credential: TokenCredential + .. admonition:: Example: + + .. literalinclude:: ../samples/sync_samples/sample_code_schemaregistry.py + :start-after: [START create_sr_client_sync] + :end-before: [END create_sr_client_sync] + :language: python + :dedent: 4 + :caption: Create a new instance of the SchemaRegistryClient. + """ def __init__( self, @@ -82,6 +91,16 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ :type serialization_type: Union[str, Enum] :param str schema_string: String representation of the schema being registered. :rtype: SchemaId + + .. admonition:: Example: + + .. literalinclude:: ../samples/sync_samples/sample_code_schemaregistry.py + :start-after: [START register_schema_sync] + :end-before: [END register_schema_sync] + :language: python + :dedent: 4 + :caption: Register a new schema. + """ return self._generated_client.schema.register( group_name=schema_group, @@ -99,6 +118,16 @@ def get_schema(self, schema_id): :param str schema_id: References specific schema in registry namespace. :rtype: Schema + + .. admonition:: Example: + + .. literalinclude:: ../samples/sync_samples/sample_code_schemaregistry.py + :start-after: [START get_schema_sync] + :end-before: [END get_schema_sync] + :language: python + :dedent: 4 + :caption: Get schema by id. + """ return self._generated_client.schema.get_by_id( schema_id, @@ -117,6 +146,16 @@ def get_schema_id(self, schema_group, schema_name, serialization_type, schema_st :type serialization_type: Union[str, Enum] :param str schema_string: String representation of the schema being registered. :rtype: SchemaId + + .. admonition:: Example: + + .. literalinclude:: ../samples/sync_samples/sample_code_schemaregistry.py + :start-after: [START get_schema_id_sync] + :end-before: [END get_schema_id_sync] + :language: python + :dedent: 4 + :caption: Get schema id. + """ return self._generated_client.schema.query_id_by_content( group_name=schema_group, diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 55e4b5dca16f..2a0023b35cb8 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -22,6 +22,15 @@ class SchemaRegistryClient: :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. :type credential: TokenCredential + .. admonition:: Example: + + .. literalinclude:: ../samples/async_samples/sample_code_schemaregistry_async.py + :start-after: [START create_sr_client_async] + :end-before: [END create_sr_client_async] + :language: python + :dedent: 4 + :caption: Create a new instance of the SchemaRegistryClient. + """ def __init__( self, @@ -62,6 +71,16 @@ async def register_schema( :type serialization_type: Union[str, Enum] :param str schema_string: String representation of the schema being registered. :rtype: SchemaId + + .. admonition:: Example: + + .. literalinclude:: ../samples/async_samples/sample_code_schemaregistry.py + :start-after: [START register_schema_async] + :end-before: [END register_schema_async] + :language: python + :dedent: 4 + :caption: Register a new schema. + """ return await self._generated_client.schema.register( group_name=schema_group, @@ -81,6 +100,16 @@ async def get_schema( :param str schema_id: References specific schema in registry namespace. :rtype: Schema + + .. admonition:: Example: + + .. literalinclude:: ../samples/async_samples/sample_code_schemaregistry.py + :start-after: [START get_schema_async] + :end-before: [END get_schema_async] + :language: python + :dedent: 4 + :caption: Get schema by id. + """ return await self._generated_client.schema.get_by_id( schema_id=schema_id, @@ -104,6 +133,16 @@ async def get_schema_id( :type serialization_type: Union[str, Enum] :param str schema_string: String representation of the schema being registered. :rtype: SchemaId + + .. admonition:: Example: + + .. literalinclude:: ../samples/async_samples/sample_code_schemaregistry_async.py + :start-after: [START get_schema_id_async] + :end-before: [END get_schema_id_async] + :language: python + :dedent: 4 + :caption:Get schema id. + """ return await self._generated_client.schema.query_id_by_content( group_name=schema_group, diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py new file mode 100644 index 000000000000..8b4592ebe9f3 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py @@ -0,0 +1,78 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os + +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry import SerializationType +from azure.identity.aio import ClientSecretCredential, DefaultAzureCredential + + +def create_client(): + # [START create_sr_client_async] + SCHEMA_REGISTRY_ENDPOINT = os.environ['SCHEMA_REGISTRY_ENDPOINT'] + token_credential = DefaultAzureCredential() + schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) + # [END create_sr_client_async] + TENANT_ID = os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] + CLIENT_ID = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] + CLIENT_SECRET = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] + token_credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) + return schema_registry_client + + +async def register_scehma(schema_registry_client): + # [START register_schema_async] + SCHEMA_GROUP = os.environ['SCHEMA_REGISTRY_GROUP'] + SCHEMA_NAME = 'your-schema-name' + SERIALIZATION_TYPE = SerializationType.AVRO + SCHEMA_STRING = """ + {"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] + }""" + schema_id_response = await schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) + schem_id = schema_id_response.id + # [END register_schema_async] + + +async def get_schema(schema_registry_client, schema_id): + # [START get_schema_async] + schema_response = await schema_registry_client.get_schema(schema_id) + schema_content = schema_response.content + # [END get_schema_async] + + +async def get_schema_id(schema_registry_client, schema_group, schema_name, serialization_type, schema_string): + # [START get_schema_id_async] + schema_id_response = await schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + schem_id = schema_id_response.id + # [END get_schema_id_async] diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py new file mode 100644 index 000000000000..4a1c4387f4ed --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py @@ -0,0 +1,96 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os + +from azure.schemaregistry import SchemaRegistryClient, SerializationType +from azure.identity import ClientSecretCredential, DefaultAzureCredential + + +def create_client(): + # [START create_sr_client_sync] + SCHEMA_REGISTRY_ENDPOINT = os.environ['SCHEMA_REGISTRY_ENDPOINT'] + token_credential = DefaultAzureCredential() + schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) + # [END create_sr_client_sync] + TENANT_ID = os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] + CLIENT_ID = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] + CLIENT_SECRET = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] + token_credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) + return schema_registry_client + + +def register_scehma(schema_registry_client): + # [START register_schema_sync] + SCHEMA_GROUP = os.environ['SCHEMA_REGISTRY_GROUP'] + SCHEMA_NAME = 'your-schema-name' + SERIALIZATION_TYPE = SerializationType.AVRO + SCHEMA_STRING = """ + {"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] + }""" + schema_id_response = schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) + schem_id = schema_id_response.id + # [END register_schema_sync] + + # [START print_schema_id] + print(schema_id_response.id) + print(schema_id_response.location) + print(schema_id_response.id_location) + print(schema_id_response.type) + print(schema_id_response.version) + # [END print_schema_id] + + return schem_id + + +def get_schema(schema_registry_client, schema_id): + # [START get_schema_sync] + schema_response = schema_registry_client.get_schema(schema_id) + schema_content = schema_response.content + # [END get_schema_sync] + + # [START print_schema] + print(schema_response.id) + print(schema_response.location) + print(schema_response.id_location) + print(schema_response.type) + print(schema_response.version) + print(schema_response.content) + # [END print_schema] + + +def get_schema_id(schema_registry_client, schema_group, schema_name, serialization_type, schema_string): + # [START get_schema_id_sync] + schema_id_response = schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + schem_id = schema_id_response.id + # [END get_schema_id_sync] diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index 5d8885bf7da2..90b0a56f2df7 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -33,8 +33,9 @@ class SchemaRegistryTests(AzureMgmtTestCase): @pytest.mark.liveTest @pytest.mark.live_test_only @SchemaRegistryPreparer() - def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=schemaregistry_credential) + def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" @@ -66,8 +67,9 @@ def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schem @pytest.mark.liveTest @pytest.mark.live_test_only @SchemaRegistryPreparer() - def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=schemaregistry_credential) + def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" @@ -101,8 +103,9 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, sche @pytest.mark.liveTest @pytest.mark.live_test_only @SchemaRegistryPreparer() - def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=ClientSecretCredential("fake", "fake", "fake")) + def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" @@ -112,8 +115,9 @@ def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemar @pytest.mark.liveTest @pytest.mark.live_test_only @SchemaRegistryPreparer() - def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): - client = SchemaRegistryClient(endpoint=str(uuid.uuid4())+".servicebus.windows.net", credential=schemaregistry_credential) + def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=str(uuid.uuid4()) + ".servicebus.windows.net", credential=credential) schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" @@ -123,8 +127,9 @@ def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemareg @pytest.mark.liveTest @pytest.mark.live_test_only @SchemaRegistryPreparer() - def test_schema_negative_no_schema(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_credential, **kwargs): - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=schemaregistry_credential) + def test_schema_negative_no_schema(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) with pytest.raises(HttpResponseError): client.get_schema('a') From 3cf645987f6f0f75b2de5d314e59f6e102054182 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 25 Aug 2020 21:07:52 -0700 Subject: [PATCH 09/35] avro serializer async impl --- .../azure/schemaregistry/aio/__init__.py | 0 .../schemaregistry/aio/serializer/__init__.py | 0 .../serializer/avro_serializer/__init__.py | 29 +++- .../avro_serializer/_avro_serializer.py | 12 -- .../_schema_registry_avro_serializer_async.py | 137 ++++++++++++++++++ .../schemaregistry/serializer/__init__.py | 1 - .../_schema_registry_avro_serializer.py | 53 +++++-- .../async_samples/avro_serializer_async.py | 78 ++++++++++ .../samples/sync_samples/avro_serializer.py | 72 +++++++++ .../sync_samples/sample_avro_serializer.py | 43 ------ .../async_tests/test_avro_serializer_async.py | 0 .../tests/schemaregistry_preparer.py | 75 ++++++++++ .../schemaregistry/_schema_registry_client.py | 1 + .../azure/schemaregistry/aio/__init__.py | 27 +++- .../aio/_schema_registry_client_async.py | 28 +++- .../schemaregistry/aio/serializer/__init__.py | 26 ++++ .../schemaregistry/serializer/__init__.py | 26 ++++ .../samples/sync_samples/schema_registry.py | 26 +++- sdk/schemaregistry/tests.yml | 1 + 19 files changed, 556 insertions(+), 79 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/schemaregistry_preparer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/serializer/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py index 3e67c6f26bdc..3af91d793ced 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py @@ -1,8 +1,29 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -from ._avro_serializer import SchemaRegistryAvroSerializer +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from ._schema_registry_avro_serializer_async import SchemaRegistryAvroSerializer __all__ = [ "SchemaRegistryAvroSerializer" diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_avro_serializer.py deleted file mode 100644 index 3d611146d991..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_avro_serializer.py +++ /dev/null @@ -1,12 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -import avro - -from azure.schemaregistry.aio import SchemaRegistryClient - - -class SchemaRegistryAvroSerializer: - def __init__(self): - self._schema_dict = {} diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py new file mode 100644 index 000000000000..2d08adcb8c65 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py @@ -0,0 +1,137 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from io import BytesIO +from typing import Any, Dict, Union +import avro + +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry import SerializationType + +from ....serializer.avro_serializer._avro_serializer import AvroObjectSerializer + + +class SchemaRegistryAvroSerializer: + """ + + """ + def __init__(self, credential, endpoint, schema_group, **kwargs): + self._schema_group = schema_group + self._avro_serializer = AvroObjectSerializer() + self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) + self._id_to_schema = {} + self._schema_to_id = {} + + async def __aenter__(self): + await self._schema_registry_client.__aenter__() + return self + + async def __aexit__(self, *args): + await self._schema_registry_client.__aexit__(*args) + + async def close(self): + """ This method is to close the sockets opened by the client. + It need not be used when using with a context manager. + """ + await self._schema_registry_client.close() + + async def _get_schema_id(self, schema_name, schema_str): + # type: (str, str) -> str + """ + + :param schema_name: + :param schema_str: + :return: + """ + try: + return self._schema_to_id[schema_str] + except KeyError: + schema_id = (await self._schema_registry_client.register_schema( + self._schema_group, + schema_name, + SerializationType.AVRO, + schema_str + )).id + self._schema_to_id[schema_str] = schema_id + self._id_to_schema[schema_id] = str(schema_str) + return schema_id + + async def _get_schema(self, schema_id): + # type: (str) -> str + """ + + :param str schema_id: + :return: + """ + try: + return self._id_to_schema[schema_id] + except KeyError: + schema_str = (await self._schema_registry_client.get_schema(schema_id)).content + self._id_to_schema[schema_id] = schema_str + self._schema_to_id[schema_str] = schema_id + return schema_str + + async def serialize(self, data, schema): + # type: (Dict[str, Any], Union[str, bytes, avro.schema.Schema]) -> bytes + """ + + :param schema: # TODO: support schema object/str/bytes? + :param dict data: + :return: + """ + if not isinstance(schema, avro.schema.Schema): + schema = avro.schema.parse(schema) + + schema_id = await self._get_schema_id(schema.fullname, str(schema)) + stream = BytesIO() + self._avro_serializer.serialize(stream, data, schema) + # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. + # This is intended to become a record format identifier in the future. + # Right now, you can just put \x00\x00\x00\x00. + record_format_identifier = b'\0\0\0\0' + res = record_format_identifier + schema_id.encode('utf-8') + stream.getvalue() + return res + + async def deserialize(self, data): + # type: (bytes) -> Dict[str, Any] + """ + + :param bytes data: + :rtype: Dict[str, Any] + """ + # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. + # This is intended to become a record format identifier in the future. + # Right now, you can just put \x00\x00\x00\x00. + record_format_identifier = data[0:4] + schema_id = data[4:36].decode('utf-8') + schema_content = await self._get_schema(schema_id) + dict_data = self._avro_serializer.deserialize(data[36:]) + try: # TODO: this part is a draft validation process, but I have the concern that the performance is poor + schema = avro.schema.parse(schema_content) + stream = BytesIO() + self._avro_serializer.serialize(stream, dict_data, schema) + except avro.schema.AvroException: + raise + return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py index e7ad8fc407f9..c36aaed14908 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py @@ -23,4 +23,3 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index ebf31d111fb9..ffa0a668b458 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -24,6 +24,7 @@ # # -------------------------------------------------------------------------- from io import BytesIO +from typing import Any, Dict, Union import avro from azure.schemaregistry import SchemaRegistryClient, SerializationType @@ -42,7 +43,24 @@ def __init__(self, credential, endpoint, schema_group, **kwargs): self._id_to_schema = {} self._schema_to_id = {} + def __enter__(self): + # type: () -> SchemaRegistryClient + self._schema_registry_client.__enter__() + return self + + def __exit__(self, *exc_details): + # type: (Any) -> None + self._schema_registry_client.__exit__(*exc_details) + + def close(self): + # type: () -> None + """ This method is to close the sockets opened by the client. + It need not be used when using with a context manager. + """ + self._schema_registry_client.close() + def _get_schema_id(self, schema_name, schema_str): + # type: (str, str) -> str """ :param schema_name: @@ -57,12 +75,13 @@ def _get_schema_id(self, schema_name, schema_str): schema_name, SerializationType.AVRO, schema_str - ) + ).id self._schema_to_id[schema_str] = schema_id self._id_to_schema[schema_id] = str(schema_str) return schema_id def _get_schema(self, schema_id): + # type: (str) -> str """ :param str schema_id: @@ -71,36 +90,50 @@ def _get_schema(self, schema_id): try: return self._id_to_schema[schema_id] except KeyError: - schema_str = self._schema_registry_client.get_schema(schema_id) + schema_str = self._schema_registry_client.get_schema(schema_id).content self._id_to_schema[schema_id] = schema_str self._schema_to_id[schema_str] = schema_id return schema_str def serialize(self, data, schema): + # type: (Dict[str, Any], Union[str, bytes, avro.schema.Schema]) -> bytes """ :param schema: # TODO: support schema object/str/bytes? :param dict data: :return: """ - # TODO: schema name generated through schema namespace + name if not isinstance(schema, avro.schema.Schema): schema = avro.schema.parse(schema) schema_id = self._get_schema_id(schema.fullname, str(schema)) stream = BytesIO() self._avro_serializer.serialize(stream, data, schema) - res = schema_id.encode('utf-8') + stream.getvalue() + # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. + # This is intended to become a record format identifier in the future. + # Right now, you can just put \x00\x00\x00\x00. + record_format_identifier = b'\0\0\0\0' + res = record_format_identifier + schema_id.encode('utf-8') + stream.getvalue() return res def deserialize(self, data): + # type: (bytes) -> Dict[str, Any] """ - :param dict data: - :return: + :param bytes data: + :rtype: Dict[str, Any] """ - schema_id = data[0:32].decode('utf-8') - schema_str = self._get_schema(schema_id) - dict_data = self._avro_serializer.deserialize(data[32:]) # TODO: do validation + # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. + # This is intended to become a record format identifier in the future. + # Right now, you can just put \x00\x00\x00\x00. + record_format_identifier = data[0:4] + schema_id = data[4:36].decode('utf-8') + schema_content = self._get_schema(schema_id) + dict_data = self._avro_serializer.deserialize(data[36:]) + try: # TODO: this part is a draft validation process, but I have the concern that the performance is poor + schema = avro.schema.parse(schema_content) + stream = BytesIO() + self._avro_serializer.serialize(stream, dict_data, schema) + except avro.schema.AvroException: + raise return dict_data - diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py new file mode 100644 index 000000000000..214e85917419 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py @@ -0,0 +1,78 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import asyncio +import os + +from azure.identity.aio import ClientSecretCredential +from azure.schemaregistry.aio.serializer.avro_serializer import SchemaRegistryAvroSerializer + +TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] +CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] +CLIENT_SECRET=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] + +SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_GROUP=os.environ['SCHEMA_REGISTRY_GROUP'] +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +async def serialize(serializer, schema, dict_data): + bytes = await serializer.serialize(dict_data, schema) + print('Encoded bytes are: ', bytes) + return bytes + + +async def deserialize(serializer, bytes): + dict_data = await serializer.deserialize(bytes) + print('Deserialized data is: ', dict_data) + return dict_data + + +async def main(): + token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET + ) + + serializer = SchemaRegistryAvroSerializer(token_credential, SCHEMA_REGISTRY_ENDPOINT, SCHEMA_GROUP) + async with serializer: + dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + payload_bytes = await serialize(serializer, SCHEMA_STRING, dict_data) + dict_data = await deserialize(serializer, payload_bytes) + + await token_credential.close() + +loop = asyncio.get_event_loop() +loop.run_until_complete(main()) \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py new file mode 100644 index 000000000000..231246f2d75b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py @@ -0,0 +1,72 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os + +from azure.identity import ClientSecretCredential +from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer + +TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] +CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] +CLIENT_SECRET=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] + +SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_GROUP=os.environ['SCHEMA_REGISTRY_GROUP'] +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET +) + + +def serialize(serializer, schema, dict_data): + bytes = serializer.serialize(dict_data, schema) + print('Encoded bytes are: ', bytes) + return bytes + + +def deserialize(serializer, bytes): + dict_data = serializer.deserialize(bytes) + print('Deserialized data is: ', dict_data) + return dict_data + + +if __name__ == '__main__': + serializer = SchemaRegistryAvroSerializer(token_credential, SCHEMA_REGISTRY_ENDPOINT, SCHEMA_GROUP) + dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + payload_bytes = serialize(serializer, SCHEMA_STRING, dict_data) + dict_data = deserialize(serializer, payload_bytes) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py deleted file mode 100644 index 98dd804ade69..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/sample_avro_serializer.py +++ /dev/null @@ -1,43 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import os - -from azure.identity import ClientSecretCredential -from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer - -TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] -CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] -CLIENT_SECRET=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] - -SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] -SCHEMA_GROUP=os.environ['SCHEMA_REGISTRY_GROUP'] -SCHEMA_STRING=""" -{"namespace":"example.avro","type":"record","name": -"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]} -""" - -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET -) - - -def serialize(serializer, schema): - dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - bytes = serializer.serialize(dict_data, schema) - return bytes - - -def deserialize(serializer, dict_data): - return serializer.deserialize(dict_data) - -serializer = SchemaRegistryAvroSerializer(token_credential, SCHEMA_REGISTRY_ENDPOINT, SCHEMA_GROUP) -payload_bytes = serialize(serializer, SCHEMA_STRING) -print(payload_bytes) -dict_data = deserialize(serializer, payload_bytes) -print(dict_data) - diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/schemaregistry_preparer.py new file mode 100644 index 000000000000..7f33dd0c50e7 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/schemaregistry_preparer.py @@ -0,0 +1,75 @@ +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +import functools +import hashlib +import os +from collections import namedtuple + +from azure.identity import ClientSecretCredential +from azure_devtools.scenario_tests.exceptions import AzureTestError +from devtools_testutils import ( + ResourceGroupPreparer, AzureMgmtPreparer, FakeResource +) + + +SCHEMA_REGISTRY_ENDPOINT_PARAM = "schemaregistry_endpoint" +SCHEMA_REGISTRY_GROUP_PARAM = "schemaregistry_group" +SCHEMA_REGISTRY_TENANT_ID_PARAM = "schemaregistry_tenant_id" +SCHEMA_REGISTRY_CLIENT_ID_PARAM = "schemaregistry_client_id" +SCHEMA_REGISTRY_CLIENT_SECRET_PARAM = "schemaregistry_client_secret" +SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME = 'SCHEMA_REGISTRY_ENDPOINT' +SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME = 'SCHEMA_REGISTRY_GROUP' +AZURE_TENANT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_TENANT_ID' +AZURE_CLIENT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_ID' +AZURE_CLIENT_SECRET_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_SECRET' + + +class SchemaRegistryNamespacePreparer(AzureMgmtPreparer): + # TODO: SR doesn't have mgmt package + def __init__(self): + pass + + def create_resource(self, name, **kwargs): + pass + + def remove_resource(self, name, **kwargs): + pass + + +class SchemaRegistryPreparer(AzureMgmtPreparer): + def __init__( + self, + name_prefix='' + ): + super(SchemaRegistryPreparer, self).__init__(name_prefix, 24) + + def create_resource(self, name, **kwargs): + # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using api, in the future we should be able to dynamically create and remove resources + return { + SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], + SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME], + SCHEMA_REGISTRY_TENANT_ID_PARAM: os.environ[AZURE_TENANT_ID_ENV_KEY_NAME], + SCHEMA_REGISTRY_CLIENT_ID_PARAM: os.environ[AZURE_CLIENT_ID_ENV_KEY_NAME], + SCHEMA_REGISTRY_CLIENT_SECRET_PARAM: os.environ[AZURE_CLIENT_SECRET_ENV_KEY_NAME] + } + + def remove_resource(self, name, **kwargs): + pass diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index 4735ab3fc460..da0073ee89ba 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -33,6 +33,7 @@ if TYPE_CHECKING: from azure.core.credentials import TokenCredential +# TODO: Arthur: schema id will be the type of long instead of string class SchemaRegistryClient: """ diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py index 229d30c6bd00..e1ff81128426 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py @@ -1,7 +1,28 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- from ._schema_registry_client_async import SchemaRegistryClient __all__ = [ diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 2a0023b35cb8..741c29ebf289 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -1,7 +1,28 @@ -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- from typing import Any, TYPE_CHECKING, Union from enum import Enum @@ -12,6 +33,7 @@ if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential +# TODO: Arthur: schema id will be the type of long instead of string class SchemaRegistryClient: """ diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/serializer/__init__.py new file mode 100644 index 000000000000..dba3fde4964b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/serializer/__init__.py @@ -0,0 +1,26 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +# __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py new file mode 100644 index 000000000000..dba3fde4964b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py @@ -0,0 +1,26 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +# __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py index 3d96b52edda5..626139d3dae9 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py @@ -1,7 +1,27 @@ -# ------------------------------------------------------------------------- +# -------------------------------------------------------------------------- +# # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# # -------------------------------------------------------------------------- """ diff --git a/sdk/schemaregistry/tests.yml b/sdk/schemaregistry/tests.yml index 350037172da8..9e4c4ca2b84c 100644 --- a/sdk/schemaregistry/tests.yml +++ b/sdk/schemaregistry/tests.yml @@ -15,6 +15,7 @@ jobs: SCHEMA_REGISTRY_AZURE_CLIENT_ID: $(schema-registry-sdk-test-client-id) SCHEMA_REGISTRY_AZURE_CLIENT_SECRET: $(schema-registry-sdk-test-client-secret) SCHEMA_REGISTRY_ENDPOINT: $(schema-registry-sdk-test-endpoint) + SCHEMA_REGISTRY_GROUP： $(schema-registry-sdk-test-group) AZURE_TEST_RUN_LIVE: 'true' Matrix: Linux_Python35: From 58fb59fc4df04495af18958dab9b5133b4b193bc Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Wed, 26 Aug 2020 14:49:31 -0700 Subject: [PATCH 10/35] close the writer --- .../serializer/avro_serializer/_avro_serializer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py index 27f712eed55f..05458cb3c321 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -80,6 +80,7 @@ def serialize( ) writer.append(value) writer.flush() + writer.close() def deserialize( self, From 02c60ecbe0970b069b463f8c73e35417d6d99329 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Wed, 26 Aug 2020 17:48:37 -0700 Subject: [PATCH 11/35] update avro se/de impl --- .../avro_serializer/_avro_serializer.py | 31 +++----- .../_schema_registry_avro_serializer.py | 1 + .../tests/test_avro_serializer.py | 60 +++++++++++++++ .../async_tests/schemaregistry_preparer.py | 75 +++++++++++++++++++ 4 files changed, 145 insertions(+), 22 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/schemaregistry_preparer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py index 05458cb3c321..57f63953484b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -26,8 +26,7 @@ import abc from typing import BinaryIO, Union, Type, TypeVar, Optional, Any -from avro.datafile import DataFileWriter, DataFileReader -from avro.io import DatumWriter, DatumReader +from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder try: ABC = abc.ABC @@ -72,18 +71,12 @@ def serialize( if self._writer_codec: kwargs['codec'] = self._writer_codec - writer = DataFileWriter( - stream, - DatumWriter(), - schema, - **kwargs - ) - writer.append(value) - writer.flush() - writer.close() + writer = DatumWriter(schema) # TODO: cache it + writer.write(value, BinaryEncoder(stream)) def deserialize( self, + schema, # type: data, # type: Union[bytes, BinaryIO] return_type=None, # type: Optional[Type[ObjectType]] ): @@ -92,6 +85,7 @@ def deserialize( Return type will be ignored, since the schema is deduced from the provided bytes. :param data: A stream of bytes or bytes directly :type data: BinaryIO or bytes + :param schema: A Avro RecordSchema :param return_type: Return type is not supported in the Avro serializer. :returns: An instanciated object :rtype: ObjectType @@ -100,15 +94,8 @@ def deserialize( from io import BytesIO data = BytesIO(data) - reader = DataFileReader( - data, - DatumReader() - ) - obj = next(reader) - try: - next(reader) - raise ValueError("This avro stream is multiple object stream") - except StopIteration: - pass - reader.close() + avro_reader = DatumReader(writers_schema=schema) + bin_decoder = BinaryDecoder(data) + obj = avro_reader.read(bin_decoder) + data.close() return obj diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index ffa0a668b458..9660e443e3e9 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -114,6 +114,7 @@ def serialize(self, data, schema): # Right now, you can just put \x00\x00\x00\x00. record_format_identifier = b'\0\0\0\0' res = record_format_identifier + schema_id.encode('utf-8') + stream.getvalue() + stream.close() return res def deserialize(self, data): diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py index e69de29bb2d1..22774d268b77 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py @@ -0,0 +1,60 @@ +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import pytest +import uuid +import avro +import avro.io +from io import BytesIO + +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.identity import ClientSecretCredential +from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError + +from schemaregistry_preparer import SchemaRegistryPreparer +from devtools_testutils import AzureMgmtTestCase + + +class SchemaRegistryTests(AzureMgmtTestCase): + + @pytest.mark.liveTest + @pytest.mark.live_test_only + def test_basic_avro_serializer(self): + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + + # encoding part + encode_stream = BytesIO() + writer = avro.io.DatumWriter(schema) + writer.write(dict_data, avro.io.BinaryEncoder(encode_stream)) + encoded_payload = encode_stream.getvalue() + encode_stream.close() + + # decoding part + decode_stream = BytesIO(encoded_payload) + avro_reader = avro.io.DatumReader(writers_schema=schema) + bin_decoder = avro.io.BinaryDecoder(decode_stream) + decoded_data = avro_reader.read(bin_decoder) + decode_stream.close() + + assert decoded_data['name'] == "Ben" + assert decoded_data['favorite_number'] == 7 + assert decoded_data['favorite_color'] == 'red' diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/schemaregistry_preparer.py new file mode 100644 index 000000000000..7f33dd0c50e7 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/schemaregistry_preparer.py @@ -0,0 +1,75 @@ +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +import functools +import hashlib +import os +from collections import namedtuple + +from azure.identity import ClientSecretCredential +from azure_devtools.scenario_tests.exceptions import AzureTestError +from devtools_testutils import ( + ResourceGroupPreparer, AzureMgmtPreparer, FakeResource +) + + +SCHEMA_REGISTRY_ENDPOINT_PARAM = "schemaregistry_endpoint" +SCHEMA_REGISTRY_GROUP_PARAM = "schemaregistry_group" +SCHEMA_REGISTRY_TENANT_ID_PARAM = "schemaregistry_tenant_id" +SCHEMA_REGISTRY_CLIENT_ID_PARAM = "schemaregistry_client_id" +SCHEMA_REGISTRY_CLIENT_SECRET_PARAM = "schemaregistry_client_secret" +SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME = 'SCHEMA_REGISTRY_ENDPOINT' +SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME = 'SCHEMA_REGISTRY_GROUP' +AZURE_TENANT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_TENANT_ID' +AZURE_CLIENT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_ID' +AZURE_CLIENT_SECRET_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_SECRET' + + +class SchemaRegistryNamespacePreparer(AzureMgmtPreparer): + # TODO: SR doesn't have mgmt package + def __init__(self): + pass + + def create_resource(self, name, **kwargs): + pass + + def remove_resource(self, name, **kwargs): + pass + + +class SchemaRegistryPreparer(AzureMgmtPreparer): + def __init__( + self, + name_prefix='' + ): + super(SchemaRegistryPreparer, self).__init__(name_prefix, 24) + + def create_resource(self, name, **kwargs): + # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using api, in the future we should be able to dynamically create and remove resources + return { + SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], + SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME], + SCHEMA_REGISTRY_TENANT_ID_PARAM: os.environ[AZURE_TENANT_ID_ENV_KEY_NAME], + SCHEMA_REGISTRY_CLIENT_ID_PARAM: os.environ[AZURE_CLIENT_ID_ENV_KEY_NAME], + SCHEMA_REGISTRY_CLIENT_SECRET_PARAM: os.environ[AZURE_CLIENT_SECRET_ENV_KEY_NAME] + } + + def remove_resource(self, name, **kwargs): + pass From 85d6766b05811eed408f8513250d072052a48c95 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 27 Aug 2020 13:45:29 -0700 Subject: [PATCH 12/35] update avro serializer impl --- .../_schema_registry_avro_serializer_async.py | 12 +++++------- .../avro_serializer/_avro_serializer.py | 12 ++++++------ .../_schema_registry_avro_serializer.py | 15 ++++++--------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py index 2d08adcb8c65..2ba119a04c41 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py @@ -112,6 +112,7 @@ async def serialize(self, data, schema): # Right now, you can just put \x00\x00\x00\x00. record_format_identifier = b'\0\0\0\0' res = record_format_identifier + schema_id.encode('utf-8') + stream.getvalue() + stream.close() return res async def deserialize(self, data): @@ -127,11 +128,8 @@ async def deserialize(self, data): record_format_identifier = data[0:4] schema_id = data[4:36].decode('utf-8') schema_content = await self._get_schema(schema_id) - dict_data = self._avro_serializer.deserialize(data[36:]) - try: # TODO: this part is a draft validation process, but I have the concern that the performance is poor - schema = avro.schema.parse(schema_content) - stream = BytesIO() - self._avro_serializer.serialize(stream, dict_data, schema) - except avro.schema.AvroException: - raise + + # TODO: schema_id to datumreader cache + schema = avro.schema.parse(schema_content) + dict_data = self._avro_serializer.deserialize(schema, data[36:]) return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py index 57f63953484b..8469a65f6047 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -25,7 +25,7 @@ # -------------------------------------------------------------------------- import abc from typing import BinaryIO, Union, Type, TypeVar, Optional, Any - +from io import BytesIO from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder try: @@ -53,7 +53,7 @@ def serialize( self, stream, # type: BinaryIO value, # type: ObjectType - schema=None, # type: Optional[Any] + schema, # type: Optional[Any] ): # type: (...) -> None """Convert the provided value to it's binary representation and write it to the stream. @@ -91,11 +91,11 @@ def deserialize( :rtype: ObjectType """ if not hasattr(data, 'read'): - from io import BytesIO data = BytesIO(data) - avro_reader = DatumReader(writers_schema=schema) + avro_reader = DatumReader(writers_schema=schema) # TODO: cache it bin_decoder = BinaryDecoder(data) - obj = avro_reader.read(bin_decoder) + decoded_data = avro_reader.read(bin_decoder) data.close() - return obj + + return decoded_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index 9660e443e3e9..ed7e23f983b8 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -105,7 +105,7 @@ def serialize(self, data, schema): """ if not isinstance(schema, avro.schema.Schema): schema = avro.schema.parse(schema) - + # TODO: schema_id to datumwrtier cache schema_id = self._get_schema_id(schema.fullname, str(schema)) stream = BytesIO() self._avro_serializer.serialize(stream, data, schema) @@ -113,7 +113,7 @@ def serialize(self, data, schema): # This is intended to become a record format identifier in the future. # Right now, you can just put \x00\x00\x00\x00. record_format_identifier = b'\0\0\0\0' - res = record_format_identifier + schema_id.encode('utf-8') + stream.getvalue() + res = record_format_identifier + schema_id.encode('utf-8') + stream.getvalue() # TODO: should we use struck.pack and unpack for interoperability, could be a cross-language problem stream.close() return res @@ -130,11 +130,8 @@ def deserialize(self, data): record_format_identifier = data[0:4] schema_id = data[4:36].decode('utf-8') schema_content = self._get_schema(schema_id) - dict_data = self._avro_serializer.deserialize(data[36:]) - try: # TODO: this part is a draft validation process, but I have the concern that the performance is poor - schema = avro.schema.parse(schema_content) - stream = BytesIO() - self._avro_serializer.serialize(stream, dict_data, schema) - except avro.schema.AvroException: - raise + + # TODO: schema_id to datumreader cache + schema = avro.schema.parse(schema_content) + dict_data = self._avro_serializer.deserialize(schema, data[36:]) return dict_data From 5fa4b43576927235298652aec65c3de64cd90ded Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 27 Aug 2020 14:35:31 -0700 Subject: [PATCH 13/35] fix apireview reported error in sr --- .../azure/schemaregistry/_common/_schema.py | 8 +++-- .../schemaregistry/_schema_registry_client.py | 29 +++++++++++----- .../aio/_schema_registry_client_async.py | 34 +++++++++++++------ 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index 4d79179f8438..9186942bcb8b 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -26,6 +26,9 @@ from typing import Any, Dict +# TODO: Arthur: schema id will be the type of long instead of string + + class DictMixin(object): def __setitem__(self, key, item): @@ -96,6 +99,7 @@ def __init__( self, **kwargs ): + # type: (Any) -> None self.location = kwargs.get('Location') self.id = kwargs.get("X-Schema-Id") self.id_location = kwargs.get('X-Schema-Id-Location') @@ -133,7 +137,7 @@ def __init__( schema_id, **kwargs ): - # type: (str, Dict[str, Any]) -> None + # type: (str, Any) -> None super(SchemaId, self).__init__(**kwargs) self.id = schema_id @@ -170,6 +174,6 @@ def __init__( schema_str, **kwargs ): - # type: (str, Dict[str, Any]) -> None + # type: (str, Any) -> None super(Schema, self).__init__(**kwargs) self.content = schema_str diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index da0073ee89ba..839f3448234b 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -26,6 +26,7 @@ from typing import Any, TYPE_CHECKING, Union from enum import Enum +from ._common._constants import SerializationType from ._common._schema import Schema, SchemaId from ._common._response_handlers import _parse_response_schema, _parse_response_schema_id from ._generated._azure_schema_registry import AzureSchemaRegistry @@ -35,6 +36,7 @@ # TODO: Arthur: schema id will be the type of long instead of string + class SchemaRegistryClient: """ SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, @@ -79,8 +81,8 @@ def close(self): """ self._generated_client.close() - def register_schema(self, schema_group, schema_name, serialization_type, schema_string): # TODO: serialization_type Enum + string? - # type: (str, str, Union[str, Enum], str) -> SchemaId + def register_schema(self, schema_group, schema_name, serialization_type, schema_string, **kwargs): # TODO: serialization_type Enum + string? + # type: (str, str, Union[str, SerializationType], str, Any) -> SchemaId """ Register new schema. If schema of specified name does not exist in specified group, schema is created at version 1. If schema of specified name exists already in specified group, @@ -89,7 +91,7 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ :param str schema_group: Schema group under which schema should be registered. :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. - :type serialization_type: Union[str, Enum] + :type serialization_type: Union[str, SerializationType] :param str schema_string: String representation of the schema being registered. :rtype: SchemaId @@ -103,6 +105,11 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ :caption: Register a new schema. """ + try: + serialization_type = serialization_type.value + except AttributeError: + pass + return self._generated_client.schema.register( group_name=schema_group, schema_name=schema_name, @@ -111,8 +118,8 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ cls=_parse_response_schema_id ) - def get_schema(self, schema_id): - # type: (str) -> Schema + def get_schema(self, schema_id, **kwargs): + # type: (str, Any) -> Schema """ Gets a registered schema by its unique ID. Azure Schema Registry guarantees that ID is unique within a namespace. @@ -135,8 +142,8 @@ def get_schema(self, schema_id): cls=_parse_response_schema ) - def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string): - # type: (str, str, str, Union[str, Enum]) -> SchemaId + def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string, **kwargs): + # type: (str, str, str, Union[str, SerializationType], Any) -> SchemaId """ Gets the ID referencing an existing schema within the specified schema group, as matched by schema content comparison. @@ -144,7 +151,8 @@ def get_schema_id(self, schema_group, schema_name, serialization_type, schema_st :param str schema_group: Schema group under which schema should be registered. :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. - :type serialization_type: Union[str, Enum] + The + :type serialization_type: Union[str, SerializationType] :param str schema_string: String representation of the schema being registered. :rtype: SchemaId @@ -158,6 +166,11 @@ def get_schema_id(self, schema_group, schema_name, serialization_type, schema_st :caption: Get schema id. """ + try: + serialization_type = serialization_type.value + except AttributeError: + pass + return self._generated_client.schema.query_id_by_content( group_name=schema_group, schema_name=schema_name, diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 741c29ebf289..a9c0ae5454cc 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -24,8 +24,8 @@ # # -------------------------------------------------------------------------- from typing import Any, TYPE_CHECKING, Union -from enum import Enum +from .._common._constants import SerializationType from .._common._response_handlers import _parse_response_schema_id, _parse_response_schema from .._common._schema import SchemaId, Schema from .._generated.aio._azure_schema_registry_async import AzureSchemaRegistry @@ -35,6 +35,7 @@ # TODO: Arthur: schema id will be the type of long instead of string + class SchemaRegistryClient: """ SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, @@ -42,7 +43,7 @@ class SchemaRegistryClient: :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. - :type credential: TokenCredential + :type credential: AsyncTokenCredential .. admonition:: Example: @@ -79,8 +80,9 @@ async def register_schema( self, schema_group: str, schema_name: str, - serialization_type: Union[str, Enum], - schema_string: str + serialization_type: Union[str, SerializationType], + schema_string: str, + **kwargs: Any ) -> SchemaId: """ Register new schema. If schema of specified name does not exist in specified group, @@ -90,7 +92,7 @@ async def register_schema( :param str schema_group: Schema group under which schema should be registered. :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. - :type serialization_type: Union[str, Enum] + :type serialization_type: Union[str, SerializationType] :param str schema_string: String representation of the schema being registered. :rtype: SchemaId @@ -104,6 +106,11 @@ async def register_schema( :caption: Register a new schema. """ + try: + serialization_type = serialization_type.value + except AttributeError: + pass + return await self._generated_client.schema.register( group_name=schema_group, schema_name=schema_name, @@ -114,7 +121,8 @@ async def register_schema( async def get_schema( self, - schema_id: str + schema_id: str, + **kwargs: Any ) -> Schema: """ Gets a registered schema by its unique ID. @@ -142,9 +150,10 @@ async def get_schema_id( self, schema_group: str, schema_name: str, - serialization_type: Union[str, Enum], - schema_string: str - ) -> str: + serialization_type: Union[str, SerializationType], + schema_string: str, + **kwargs: Any + ) -> SchemaId: """ Gets the ID referencing an existing schema within the specified schema group, as matched by schema content comparison. @@ -152,7 +161,7 @@ async def get_schema_id( :param str schema_group: Schema group under which schema should be registered. :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. - :type serialization_type: Union[str, Enum] + :type serialization_type: Union[str, SerializationType] :param str schema_string: String representation of the schema being registered. :rtype: SchemaId @@ -166,6 +175,11 @@ async def get_schema_id( :caption:Get schema id. """ + try: + serialization_type = serialization_type.value + except AttributeError: + pass + return await self._generated_client.schema.query_id_by_content( group_name=schema_group, schema_name=schema_name, From b910027c8d04713e6d531574faa62053c0ed28d6 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 27 Aug 2020 16:07:39 -0700 Subject: [PATCH 14/35] srav namespace, setup update --- .../schemaregistry/aio/serializer/__init__.py | 0 .../schemaregistry/serializer/__init__.py | 1 + .../avro_serializer/_avro_serializer.py | 2 +- .../_schema_registry_avro_serializer.py | 20 +++++++--- .../avro_serializer/aio}/__init__.py | 0 .../_schema_registry_avro_serializer_async.py | 38 +++++++++++-------- .../dev_requirements.txt | 2 +- .../aio/__init__.py => samples/README.md} | 0 .../async_samples/avro_serializer_async.py | 2 +- .../setup.py | 21 ++++++---- .../tests/test_avro_serializer.py | 18 +++++---- .../schemaregistry/aio/serializer/__init__.py | 26 ------------- .../schemaregistry/serializer/__init__.py | 2 +- .../azure-schemaregistry/setup.py | 14 ++++--- 14 files changed, 73 insertions(+), 73 deletions(-) delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/__init__.py rename sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/{aio/serializer/avro_serializer => serializer/avro_serializer/aio}/__init__.py (100%) rename sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/{aio/serializer/avro_serializer => serializer/avro_serializer/aio}/_schema_registry_avro_serializer_async.py (76%) rename sdk/schemaregistry/azure-schemaregistry-avro-serializer/{azure/schemaregistry/aio/__init__.py => samples/README.md} (100%) delete mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/serializer/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py index c36aaed14908..80f86cb969ec 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py @@ -23,3 +23,4 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py index 8469a65f6047..e4b801cec618 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -76,8 +76,8 @@ def serialize( def deserialize( self, - schema, # type: data, # type: Union[bytes, BinaryIO] + schema, # type: return_type=None, # type: Optional[Type[ObjectType]] ): # type: (...) -> ObjectType diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index ed7e23f983b8..967998e926db 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -34,6 +34,13 @@ class SchemaRegistryAvroSerializer(object): """ + SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, + complete with support for versioning and management. + + :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. + :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. + :type credential: TokenCredential + :param str schema_group """ def __init__(self, credential, endpoint, schema_group, **kwargs): @@ -44,7 +51,7 @@ def __init__(self, credential, endpoint, schema_group, **kwargs): self._schema_to_id = {} def __enter__(self): - # type: () -> SchemaRegistryClient + # type: () -> SchemaRegistryAvroSerializer self._schema_registry_client.__enter__() return self @@ -98,9 +105,11 @@ def _get_schema(self, schema_id): def serialize(self, data, schema): # type: (Dict[str, Any], Union[str, bytes, avro.schema.Schema]) -> bytes """ + Encode dict data with the given schema. - :param schema: # TODO: support schema object/str/bytes? - :param dict data: + :param data: The dict data to be encoded. + :param schema: The schema used to encode the data. # TODO: support schema object/str/bytes? + :type schema: Union[str, bytes, avro.schema.Schema]) :return: """ if not isinstance(schema, avro.schema.Schema): @@ -120,8 +129,9 @@ def serialize(self, data, schema): def deserialize(self, data): # type: (bytes) -> Dict[str, Any] """ + Decode bytes data. - :param bytes data: + :param bytes data: The bytes data needs to be decoded. :rtype: Dict[str, Any] """ # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. @@ -133,5 +143,5 @@ def deserialize(self, data): # TODO: schema_id to datumreader cache schema = avro.schema.parse(schema_content) - dict_data = self._avro_serializer.deserialize(schema, data[36:]) + dict_data = self._avro_serializer.deserialize(data[36:], schema) return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/__init__.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py similarity index 76% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py rename to sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py index 2ba119a04c41..54bd1653d264 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/serializer/avro_serializer/_schema_registry_avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py @@ -30,11 +30,18 @@ from azure.schemaregistry.aio import SchemaRegistryClient from azure.schemaregistry import SerializationType -from ....serializer.avro_serializer._avro_serializer import AvroObjectSerializer +from .._avro_serializer import AvroObjectSerializer class SchemaRegistryAvroSerializer: """ + SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, + complete with support for versioning and management. + + :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. + :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. + :type credential: TokenCredential + :param str schema_group """ def __init__(self, credential, endpoint, schema_group, **kwargs): @@ -44,21 +51,20 @@ def __init__(self, credential, endpoint, schema_group, **kwargs): self._id_to_schema = {} self._schema_to_id = {} - async def __aenter__(self): + async def __aenter__(self) -> "SchemaRegistryAvroSerializer": await self._schema_registry_client.__aenter__() return self - async def __aexit__(self, *args): + async def __aexit__(self, *args: Any) -> None: await self._schema_registry_client.__aexit__(*args) - async def close(self): + async def close(self) -> None: """ This method is to close the sockets opened by the client. It need not be used when using with a context manager. """ await self._schema_registry_client.close() - async def _get_schema_id(self, schema_name, schema_str): - # type: (str, str) -> str + async def _get_schema_id(self, schema_name: str, schema_str: str) -> str: """ :param schema_name: @@ -78,8 +84,7 @@ async def _get_schema_id(self, schema_name, schema_str): self._id_to_schema[schema_id] = str(schema_str) return schema_id - async def _get_schema(self, schema_id): - # type: (str) -> str + async def _get_schema(self, schema_id: str) -> str: """ :param str schema_id: @@ -93,12 +98,13 @@ async def _get_schema(self, schema_id): self._schema_to_id[schema_str] = schema_id return schema_str - async def serialize(self, data, schema): - # type: (Dict[str, Any], Union[str, bytes, avro.schema.Schema]) -> bytes + async def serialize(self, data: Dict[str, Any], schema: Union[str, bytes, avro.schema.Schema]) -> bytes: """ + Encode dict data with the given schema. - :param schema: # TODO: support schema object/str/bytes? - :param dict data: + :param data: The dict data to be encoded. + :param schema: The schema used to encode the data. # TODO: support schema object/str/bytes? + :type schema: Union[str, bytes, avro.schema.Schema]) :return: """ if not isinstance(schema, avro.schema.Schema): @@ -115,11 +121,11 @@ async def serialize(self, data, schema): stream.close() return res - async def deserialize(self, data): - # type: (bytes) -> Dict[str, Any] + async def deserialize(self, data: bytes) -> Dict[str, Any]: """ + Decode bytes data. - :param bytes data: + :param bytes data: The bytes data needs to be decoded. :rtype: Dict[str, Any] """ # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. @@ -131,5 +137,5 @@ async def deserialize(self, data): # TODO: schema_id to datumreader cache schema = avro.schema.parse(schema_content) - dict_data = self._avro_serializer.deserialize(schema, data[36:]) + dict_data = self._avro_serializer.deserialize(data[36:], schema) return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt index 394099b2743a..2225991bf987 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt @@ -1,2 +1,2 @@ -e ../../identity/azure-identity --e ../azure-schemaregistry \ No newline at end of file +../azure-schemaregistry \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/aio/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py index 214e85917419..10571530ba2b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py @@ -27,7 +27,7 @@ import os from azure.identity.aio import ClientSecretCredential -from azure.schemaregistry.aio.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avro_serializer.aio import SchemaRegistryAvroSerializer TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py index 42b63e82dd56..77b047a54c6b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py @@ -31,6 +31,15 @@ with open('CHANGELOG.md', encoding='utf-8') as f: changelog = f.read() +exclude_packages = [ + 'tests', + 'samples', + # Exclude packages that will be covered by PEP420 or nspkg + 'azure', + 'azure.schemaregistry', + 'azure.schemaregistry.serializer' + ] + setup( name=PACKAGE_NAME, version=version, @@ -54,15 +63,11 @@ 'License :: OSI Approved :: MIT License', ], zip_safe=False, - packages=find_packages(exclude=[ - 'tests', - 'samples', - # Exclude packages that will be covered by PEP420 or nspkg - 'azure', - ]), + packages=find_packages(exclude=exclude_packages), install_requires=[ - 'azure-core<2.0.0,>=1.2.2' - 'azure-schemaregistry<2.0.0,>=1.0.0b1' + 'azure-core<2.0.0,>=1.2.2', + 'azure-schemaregistry<2.0.0,>=1.0.0b1', + 'avro>=1.0.0' ], extras_require={ ":python_version<'3.0'": ['azure-nspkg', 'futures'], diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py index 22774d268b77..045faf227e08 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py @@ -25,6 +25,7 @@ from azure.schemaregistry import SchemaRegistryClient from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avro_serializer._avro_serializer import AvroObjectSerializer from azure.identity import ClientSecretCredential from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError @@ -32,7 +33,7 @@ from devtools_testutils import AzureMgmtTestCase -class SchemaRegistryTests(AzureMgmtTestCase): +class SchemaRegistryAvroSerializerTests(AzureMgmtTestCase): @pytest.mark.liveTest @pytest.mark.live_test_only @@ -41,20 +42,21 @@ def test_basic_avro_serializer(self): schema = avro.schema.parse(schema_str) dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + raw_avro_object_serailizer = AvroObjectSerializer() + # encoding part encode_stream = BytesIO() - writer = avro.io.DatumWriter(schema) - writer.write(dict_data, avro.io.BinaryEncoder(encode_stream)) + raw_avro_object_serailizer.serialize(encode_stream, dict_data, schema) encoded_payload = encode_stream.getvalue() encode_stream.close() # decoding part - decode_stream = BytesIO(encoded_payload) - avro_reader = avro.io.DatumReader(writers_schema=schema) - bin_decoder = avro.io.BinaryDecoder(decode_stream) - decoded_data = avro_reader.read(bin_decoder) - decode_stream.close() + decoded_data = raw_avro_object_serailizer.deserialize(encoded_payload, schema) assert decoded_data['name'] == "Ben" assert decoded_data['favorite_number'] == 7 assert decoded_data['favorite_color'] == 'red' + + + + diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/serializer/__init__.py deleted file mode 100644 index dba3fde4964b..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/serializer/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -# __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py index dba3fde4964b..80f86cb969ec 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py @@ -23,4 +23,4 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -# __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/setup.py b/sdk/schemaregistry/azure-schemaregistry/setup.py index 8f8ca94f9e59..54de509f9a5d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry/setup.py @@ -48,6 +48,13 @@ with open('CHANGELOG.md', encoding='utf-8') as f: changelog = f.read() +exclude_packages = [ + 'tests', + 'samples', + # Exclude packages that will be covered by PEP420 or nspkg + 'azure', + ] + setup( name=PACKAGE_NAME, version=version, @@ -71,12 +78,7 @@ 'License :: OSI Approved :: MIT License', ], zip_safe=False, - packages=find_packages(exclude=[ - 'tests', - 'samples', - # Exclude packages that will be covered by PEP420 or nspkg - 'azure', - ]), + packages=find_packages(exclude=exclude_packages), install_requires=[ 'msrest>=0.5.0', 'azure-core<2.0.0,>=1.2.2' From 6b6c8b2328b187dedf0916df790e26e672ca64da Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Fri, 28 Aug 2020 10:48:57 -0700 Subject: [PATCH 15/35] doc update --- .../_schema_registry_avro_serializer.py | 2 -- .../azure-schemaregistry-avro-serializer/setup.py | 2 +- .../tests/test_avro_serializer.py | 11 +++++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index 967998e926db..cecda4ff7365 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -34,8 +34,6 @@ class SchemaRegistryAvroSerializer(object): """ - SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, - complete with support for versioning and management. :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py index 77b047a54c6b..e1746a434d21 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py @@ -67,7 +67,7 @@ install_requires=[ 'azure-core<2.0.0,>=1.2.2', 'azure-schemaregistry<2.0.0,>=1.0.0b1', - 'avro>=1.0.0' + 'avro<2.0.0,>=1.0.0' ], extras_require={ ":python_version<'3.0'": ['azure-nspkg', 'futures'], diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py index 045faf227e08..19dca0a0cc1d 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py @@ -37,7 +37,7 @@ class SchemaRegistryAvroSerializerTests(AzureMgmtTestCase): @pytest.mark.liveTest @pytest.mark.live_test_only - def test_basic_avro_serializer(self): + def test_raw_avro_serializer(self): schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} @@ -57,6 +57,9 @@ def test_basic_avro_serializer(self): assert decoded_data['favorite_number'] == 7 assert decoded_data['favorite_color'] == 'red' - - - + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, + schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + pass From c465be18c443f50e5ce8d32afec5a01c9bb77432 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Sat, 29 Aug 2020 17:08:25 -0700 Subject: [PATCH 16/35] update doc and api --- .../azure-schemaregistry/CHANGELOG.md | 2 + .../azure-schemaregistry/README.md | 119 +++++++++++++++++- .../azure/schemaregistry/__init__.py | 4 +- .../_common/_response_handlers.py | 6 +- .../azure/schemaregistry/_common/_schema.py | 6 +- .../schemaregistry/_schema_registry_client.py | 22 ++-- .../aio/_schema_registry_client_async.py | 22 ++-- .../async_tests/schemaregistry_preparer.py | 75 ----------- .../tests/schemaregistry_preparer.py | 2 +- 9 files changed, 149 insertions(+), 109 deletions(-) delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/schemaregistry_preparer.py diff --git a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md index 332564950c28..f3535254bb01 100644 --- a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md @@ -1,3 +1,5 @@ # Release History ## 1.0.0b1 (Unreleased) + +Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry. \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/README.md b/sdk/schemaregistry/azure-schemaregistry/README.md index 412d4a03b90e..3340aeb9e5fb 100644 --- a/sdk/schemaregistry/azure-schemaregistry/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/README.md @@ -1,18 +1,128 @@ # Azure Schema Registry client library for Python -Azure Schema Registry is a ... -<> +Azure Schema Registry is a service that provides the ability to store and retrieve different types of schemas such as +Avro, Json, etc. ## Getting started +### Install the package + +Install the Azure Service Bus client library for Python with [pip][pip]: + +```Bash +pip install azure-schemaregistry +``` + +### Prerequisites: +To use this package, you must have: +* Azure subscription - [Create a free account][azure_sub] +* Azure Schema Registry +* Python 2.7, 3.5 or later - [Install Python][python] + ## Key concepts +- Schema: Schema is the organization or structure for data. + ## Examples +The following sections provide several code snippets covering some of the most common Schema Registry tasks, including: + +- [Register a schema](register-a-schema) +- [Get the schema by id](get-the-schema-by-id) +- [Get the id of a schema](get-the-id-of-a-schema) + +### Register a schema + +```python +import os + +from azure.identity import DefaultAzureCredential +from azure.schemaregistry import SchemaRegistryClient + +token_credential = DefaultAzureCredential() +endpoint = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +schema_group = "" +schema_name = "" +serialization_type = "Avro" +schema_content = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +} +""" + +schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) +with schema_registry_client, token_credential: + schema_properties = schema_registry_client.register_schema(schema_group, schema_name, serilization_type, schema_string) + schema_id = schema_properties.id +``` + +### Get the schema by id + +```python +import os + +from azure.identity import DefaultAzureCredential +from azure.schemaregistry import SchemaRegistryClient + +token_credential = DefaultAzureCredential() +endpoint = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +schema_id = '' + +schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) +with schema_registry_client, token_credential: + schema = schema_registry_client.get_schema(schema_id) + schema_content = schema.content +``` + +### Get the id of a schema + +```python +import os + +from azure.identity import DefaultAzureCredential +from azure.schemaregistry import SchemaRegistryClient + +token_credential = DefaultAzureCredential() +endpoint = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +schema_group = "" +schema_name = "" +serialization_type = "Avro" +schema_content = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +} +""" + +schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) +with schema_registry_client, token_credential: + schema_properties = schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_content) + schema_id = schema_properties.id +``` + ## Troubleshooting +### General + +Schema Registry clients raise exceptions defined in [Azure Core][azure_core]. + +### Logging + ## Next steps +### More sample code + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a @@ -27,4 +137,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - +[pip]: https://pypi.org/project/pip/ +[python]: https://www.python.org/downloads/ +[azure_core]: https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/README.md +[azure_sub]: https://azure.microsoft.com/free/ \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py index 28fc5d4961cf..6c68f01b1092 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py @@ -30,11 +30,11 @@ from ._schema_registry_client import SchemaRegistryClient from ._common._constants import SerializationType -from ._common._schema import Schema, SchemaId +from ._common._schema import Schema, SchemaProperties __all__ = [ "SchemaRegistryClient", "SerializationType", "Schema", - "SchemaId" + "SchemaProperties" ] diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py index fc135e907b9f..def5143bfc77 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py @@ -28,11 +28,11 @@ from azure.core.pipeline import PipelineResponse from .._generated.models import SchemaId as InternalSchemaId -from ._schema import SchemaId, Schema +from ._schema import SchemaProperties, Schema def _parse_response_schema_id(pipeline_response, deserialized, response_headers): - # type: (PipelineResponse, InternalSchemaId, Dict[str, Any]) -> SchemaId + # type: (PipelineResponse, InternalSchemaId, Dict[str, Any]) -> SchemaProperties """ :param pipeline_response: @@ -40,7 +40,7 @@ def _parse_response_schema_id(pipeline_response, deserialized, response_headers) :param response_headers: :return: """ - return SchemaId(schema_id=deserialized.id, **response_headers) + return SchemaProperties(schema_id=deserialized.id, **response_headers) def _parse_response_schema(pipeline_response, deserialized, response_headers): diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index 9186942bcb8b..51bedf838529 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -107,7 +107,7 @@ def __init__( self.version = kwargs.get('X-Schema-Version') -class SchemaId(SchemaMeta): # TODO: need a better name here? +class SchemaProperties: """ Id and meta properties of a schema. @@ -138,13 +138,13 @@ def __init__( **kwargs ): # type: (str, Any) -> None - super(SchemaId, self).__init__(**kwargs) + super(SchemaProperties, self).__init__(**kwargs) self.id = schema_id class Schema(SchemaMeta): """ - Id, meta properties and schema content of a schema. + The schema content of a schema, along with id and meta properties. :ivar content: The content of the schema. :type content: str diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index 839f3448234b..3170e444260f 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -27,7 +27,7 @@ from enum import Enum from ._common._constants import SerializationType -from ._common._schema import Schema, SchemaId +from ._common._schema import Schema, SchemaProperties from ._common._response_handlers import _parse_response_schema, _parse_response_schema_id from ._generated._azure_schema_registry import AzureSchemaRegistry @@ -81,8 +81,8 @@ def close(self): """ self._generated_client.close() - def register_schema(self, schema_group, schema_name, serialization_type, schema_string, **kwargs): # TODO: serialization_type Enum + string? - # type: (str, str, Union[str, SerializationType], str, Any) -> SchemaId + def register_schema(self, schema_group, schema_name, serialization_type, schema_content, **kwargs): # TODO: serialization_type Enum + string? + # type: (str, str, Union[str, SerializationType], str, Any) -> SchemaProperties """ Register new schema. If schema of specified name does not exist in specified group, schema is created at version 1. If schema of specified name exists already in specified group, @@ -92,8 +92,8 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. :type serialization_type: Union[str, SerializationType] - :param str schema_string: String representation of the schema being registered. - :rtype: SchemaId + :param str schema_content: String representation of the schema being registered. + :rtype: SchemaProperties .. admonition:: Example: @@ -113,7 +113,7 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ return self._generated_client.schema.register( group_name=schema_group, schema_name=schema_name, - schema_content=schema_string, + schema_content=schema_content, #serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter cls=_parse_response_schema_id ) @@ -142,8 +142,8 @@ def get_schema(self, schema_id, **kwargs): cls=_parse_response_schema ) - def get_schema_id(self, schema_group, schema_name, serialization_type, schema_string, **kwargs): - # type: (str, str, str, Union[str, SerializationType], Any) -> SchemaId + def get_schema_id(self, schema_group, schema_name, serialization_type, schema_content, **kwargs): + # type: (str, str, str, Union[str, SerializationType], Any) -> SchemaProperties """ Gets the ID referencing an existing schema within the specified schema group, as matched by schema content comparison. @@ -153,8 +153,8 @@ def get_schema_id(self, schema_group, schema_name, serialization_type, schema_st :param serialization_type: Serialization type for the schema being registered. The :type serialization_type: Union[str, SerializationType] - :param str schema_string: String representation of the schema being registered. - :rtype: SchemaId + :param str schema_content: String representation of the schema being registered. + :rtype: SchemaProperties .. admonition:: Example: @@ -174,7 +174,7 @@ def get_schema_id(self, schema_group, schema_name, serialization_type, schema_st return self._generated_client.schema.query_id_by_content( group_name=schema_group, schema_name=schema_name, - schema_content=schema_string, + schema_content=schema_content, #serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter cls=_parse_response_schema_id ) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index a9c0ae5454cc..36c69eb0ca61 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -27,7 +27,7 @@ from .._common._constants import SerializationType from .._common._response_handlers import _parse_response_schema_id, _parse_response_schema -from .._common._schema import SchemaId, Schema +from .._common._schema import SchemaProperties, Schema from .._generated.aio._azure_schema_registry_async import AzureSchemaRegistry if TYPE_CHECKING: @@ -81,9 +81,9 @@ async def register_schema( schema_group: str, schema_name: str, serialization_type: Union[str, SerializationType], - schema_string: str, + schema_content: str, **kwargs: Any - ) -> SchemaId: + ) -> SchemaProperties: """ Register new schema. If schema of specified name does not exist in specified group, schema is created at version 1. If schema of specified name exists already in specified group, @@ -93,8 +93,8 @@ async def register_schema( :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. :type serialization_type: Union[str, SerializationType] - :param str schema_string: String representation of the schema being registered. - :rtype: SchemaId + :param str schema_content: String representation of the schema being registered. + :rtype: SchemaProperties .. admonition:: Example: @@ -114,7 +114,7 @@ async def register_schema( return await self._generated_client.schema.register( group_name=schema_group, schema_name=schema_name, - schema_content=schema_string, + schema_content=schema_content, # serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter cls=_parse_response_schema_id, ) @@ -151,9 +151,9 @@ async def get_schema_id( schema_group: str, schema_name: str, serialization_type: Union[str, SerializationType], - schema_string: str, + schema_content: str, **kwargs: Any - ) -> SchemaId: + ) -> SchemaProperties: """ Gets the ID referencing an existing schema within the specified schema group, as matched by schema content comparison. @@ -162,8 +162,8 @@ async def get_schema_id( :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. :type serialization_type: Union[str, SerializationType] - :param str schema_string: String representation of the schema being registered. - :rtype: SchemaId + :param str schema_content: String representation of the schema being registered. + :rtype: SchemaProperties .. admonition:: Example: @@ -183,7 +183,7 @@ async def get_schema_id( return await self._generated_client.schema.query_id_by_content( group_name=schema_group, schema_name=schema_name, - schema_content=schema_string, + schema_content=schema_content, # serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter cls=_parse_response_schema_id ) diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/schemaregistry_preparer.py deleted file mode 100644 index 7f33dd0c50e7..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/schemaregistry_preparer.py +++ /dev/null @@ -1,75 +0,0 @@ -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- - -import functools -import hashlib -import os -from collections import namedtuple - -from azure.identity import ClientSecretCredential -from azure_devtools.scenario_tests.exceptions import AzureTestError -from devtools_testutils import ( - ResourceGroupPreparer, AzureMgmtPreparer, FakeResource -) - - -SCHEMA_REGISTRY_ENDPOINT_PARAM = "schemaregistry_endpoint" -SCHEMA_REGISTRY_GROUP_PARAM = "schemaregistry_group" -SCHEMA_REGISTRY_TENANT_ID_PARAM = "schemaregistry_tenant_id" -SCHEMA_REGISTRY_CLIENT_ID_PARAM = "schemaregistry_client_id" -SCHEMA_REGISTRY_CLIENT_SECRET_PARAM = "schemaregistry_client_secret" -SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME = 'SCHEMA_REGISTRY_ENDPOINT' -SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME = 'SCHEMA_REGISTRY_GROUP' -AZURE_TENANT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_TENANT_ID' -AZURE_CLIENT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_ID' -AZURE_CLIENT_SECRET_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_SECRET' - - -class SchemaRegistryNamespacePreparer(AzureMgmtPreparer): - # TODO: SR doesn't have mgmt package - def __init__(self): - pass - - def create_resource(self, name, **kwargs): - pass - - def remove_resource(self, name, **kwargs): - pass - - -class SchemaRegistryPreparer(AzureMgmtPreparer): - def __init__( - self, - name_prefix='' - ): - super(SchemaRegistryPreparer, self).__init__(name_prefix, 24) - - def create_resource(self, name, **kwargs): - # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using api, in the future we should be able to dynamically create and remove resources - return { - SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], - SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME], - SCHEMA_REGISTRY_TENANT_ID_PARAM: os.environ[AZURE_TENANT_ID_ENV_KEY_NAME], - SCHEMA_REGISTRY_CLIENT_ID_PARAM: os.environ[AZURE_CLIENT_ID_ENV_KEY_NAME], - SCHEMA_REGISTRY_CLIENT_SECRET_PARAM: os.environ[AZURE_CLIENT_SECRET_ENV_KEY_NAME] - } - - def remove_resource(self, name, **kwargs): - pass diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py index 7f33dd0c50e7..70c6ab9aaf06 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py @@ -62,7 +62,7 @@ def __init__( super(SchemaRegistryPreparer, self).__init__(name_prefix, 24) def create_resource(self, name, **kwargs): - # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using api, in the future we should be able to dynamically create and remove resources + # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using mgmt api, in the future we should be able to dynamically create and remove resources return { SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME], From 63a278c5b522c945e3b8dee50d1991cd974273d7 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Sun, 30 Aug 2020 19:24:57 -0700 Subject: [PATCH 17/35] impl, doc update --- .../CHANGELOG.md | 2 + .../README.md | 83 ++++++++++++++++++- .../avro_serializer/_avro_serializer.py | 53 +++++++----- .../_schema_registry_avro_serializer.py | 24 +++--- .../_schema_registry_avro_serializer_async.py | 21 ++--- .../dev_requirements.txt | 2 + .../samples/README.md | 48 +++++++++++ .../async_samples/avro_serializer_async.py | 4 +- .../samples/sync_samples/avro_serializer.py | 2 +- .../async_tests/test_avro_serializer_async.py | 66 +++++++++++++++ .../tests/test_avro_serializer.py | 54 ++++++++++-- .../azure-schemaregistry/README.md | 4 +- .../schemaregistry/_schema_registry_client.py | 2 +- 13 files changed, 306 insertions(+), 59 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md index 332564950c28..582d5d5bdb16 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md @@ -1,3 +1,5 @@ # Release History ## 1.0.0b1 (Unreleased) + +Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry Avro Serializer. \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md index b96064bbd540..27a9a5281672 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md @@ -1,18 +1,92 @@ # Azure Schema Registry Avro Serializer library for Python -Azure Schema Registry Avro Serializer is a ... -<> +Azure Schema Registry Avro Serializer provides the ability to serialize and deserialize data according +to the given avro schema. It would automatically register, get and cache the schema. ## Getting started +### Install the package + +Install the Azure Service Bus client library for Python with [pip][pip]: + +```Bash +pip install azure-schemaregistry-avro-serializer +``` + +### Prerequisites: +To use this package, you must have: +* Azure subscription - [Create a free account][azure_sub] +* Azure Schema Registry +* Python 2.7, 3.5 or later - [Install Python][python] + ## Key concepts +- Avro: Apache Avro™ is a data serialization system. + ## Examples +The following sections provide several code snippets covering some of the most common Schema Registry tasks, including: + +- [Serialization](serialization) +- [Deserialization](deserialization) + +### Serialization + +```python +import os +from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.identity import DefaultAzureCredential + +token_credential = DefaultAzureCredential() +endpoint = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +schema_group = "" + +serializer = SchemaRegistryAvroSerializer(endpoint, token_credential, schema_group) + +schema_string = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + +with serializer: + dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + encoded_bytes = serializer.serialize(dict_data, schema_string) +``` + +### Deserialization + +```python +import os +from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.identity import DefaultAzureCredential + +token_credential = DefaultAzureCredential() +endpoint = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +schema_group = "" + +serializer = SchemaRegistryAvroSerializer(endpoint, token_credential, schema_group) + +with serializer: + encoded_bytes = b'' + decoded_data = serializer.deserialize(encoded_bytes) +``` + ## Troubleshooting +Azure Schema Registry Avro Serializer raise exceptions defined in [Azure Core][azure_core]. + ## Next steps +### More sample code + +Please find further examples in the [samples](./samples) directory demonstrating common Azure Schema Registry Avro Serializer scenarios. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a @@ -27,4 +101,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - +[pip]: https://pypi.org/project/pip/ +[python]: https://www.python.org/downloads/ +[azure_core]: https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/README.md +[azure_sub]: https://azure.microsoft.com/free/ diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py index e4b801cec618..808879f6c37f 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -24,8 +24,9 @@ # # -------------------------------------------------------------------------- import abc -from typing import BinaryIO, Union, Type, TypeVar, Optional, Any +from typing import BinaryIO, Union, Type, TypeVar, Optional, Any, Dict from io import BytesIO +import avro from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder try: @@ -42,37 +43,41 @@ def __init__(self, codec=None): """A Avro serializer using avro lib from Apache. :param str codec: The writer codec. If None, let the avro library decides. """ - try: - import avro # pylint: disable=unused-import - except ImportError: - raise ImportError("In order to create a AvroObjectSerializer you need to install the 'avro' library") - self._writer_codec = codec + self._schema_writer_cache = {} # type: Dict[str, DatumWriter] + self._schema_reader_cache = {} # type: Dict[str, DatumReader] def serialize( self, - stream, # type: BinaryIO - value, # type: ObjectType + data, # type: ObjectType schema, # type: Optional[Any] ): - # type: (...) -> None + # type: (...) -> bytes """Convert the provided value to it's binary representation and write it to the stream. Schema must be a Avro RecordSchema: https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - :param stream: A stream of bytes or bytes directly - :type stream: BinaryIO - :param value: An object to serialize + :param data: An object to serialize :param schema: A Avro RecordSchema """ if not schema: raise ValueError("Schema is required in Avro serializer.") - kwargs = {} - if self._writer_codec: - kwargs['codec'] = self._writer_codec + if not isinstance(schema, avro.schema.Schema): + schema = avro.schema.parse(schema) + + try: + writer = self._schema_writer_cache[str(schema)] + except KeyError: + writer = DatumWriter(schema) + self._schema_writer_cache[str(schema)] = writer + + stream = BytesIO() - writer = DatumWriter(schema) # TODO: cache it - writer.write(value, BinaryEncoder(stream)) + writer.write(data, BinaryEncoder(stream)) + encoded_data = stream.getvalue() + + stream.close() + return encoded_data def deserialize( self, @@ -87,15 +92,23 @@ def deserialize( :type data: BinaryIO or bytes :param schema: A Avro RecordSchema :param return_type: Return type is not supported in the Avro serializer. - :returns: An instanciated object + :returns: An instantiated object :rtype: ObjectType """ if not hasattr(data, 'read'): data = BytesIO(data) - avro_reader = DatumReader(writers_schema=schema) # TODO: cache it + if not isinstance(schema, avro.schema.Schema): + schema = avro.schema.parse(schema) + + try: + reader = self._schema_reader_cache[str(schema)] + except KeyError: + reader = DatumReader(writers_schema=schema) + self._schema_reader_cache[str(schema)] = reader + bin_decoder = BinaryDecoder(data) - decoded_data = avro_reader.read(bin_decoder) + decoded_data = reader.read(bin_decoder) data.close() return decoded_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index cecda4ff7365..b79bb8b2a6ae 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -34,14 +34,16 @@ class SchemaRegistryAvroSerializer(object): """ + SchemaRegistryAvroSerializer provides the ability to serialize and deserialize data according + to the given avro schema. It would automatically register, get and cache the schema. :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. :type credential: TokenCredential - :param str schema_group + :param str schema_group: Schema group under which schema should be registered. """ - def __init__(self, credential, endpoint, schema_group, **kwargs): + def __init__(self, endpoint, credential, schema_group, **kwargs): self._schema_group = schema_group self._avro_serializer = AvroObjectSerializer() self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) @@ -101,28 +103,26 @@ def _get_schema(self, schema_id): return schema_str def serialize(self, data, schema): - # type: (Dict[str, Any], Union[str, bytes, avro.schema.Schema]) -> bytes + # type: (Dict[str, Any], Union[str, bytes]) -> bytes """ Encode dict data with the given schema. :param data: The dict data to be encoded. :param schema: The schema used to encode the data. # TODO: support schema object/str/bytes? - :type schema: Union[str, bytes, avro.schema.Schema]) + :type schema: Union[str, bytes] :return: """ if not isinstance(schema, avro.schema.Schema): schema = avro.schema.parse(schema) - # TODO: schema_id to datumwrtier cache + schema_id = self._get_schema_id(schema.fullname, str(schema)) - stream = BytesIO() - self._avro_serializer.serialize(stream, data, schema) + data_bytes = self._avro_serializer.serialize(data, schema) # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. # This is intended to become a record format identifier in the future. # Right now, you can just put \x00\x00\x00\x00. record_format_identifier = b'\0\0\0\0' - res = record_format_identifier + schema_id.encode('utf-8') + stream.getvalue() # TODO: should we use struck.pack and unpack for interoperability, could be a cross-language problem - stream.close() - return res + payload = record_format_identifier + schema_id.encode('utf-8') + data_bytes # TODO: should we use struck.pack and unpack for interoperability, could be a cross-language problem + return payload def deserialize(self, data): # type: (bytes) -> Dict[str, Any] @@ -139,7 +139,5 @@ def deserialize(self, data): schema_id = data[4:36].decode('utf-8') schema_content = self._get_schema(schema_id) - # TODO: schema_id to datumreader cache - schema = avro.schema.parse(schema_content) - dict_data = self._avro_serializer.deserialize(data[36:], schema) + dict_data = self._avro_serializer.deserialize(data[36:], schema_content) return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py index 54bd1653d264..4e2815e9d12b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py @@ -35,8 +35,8 @@ class SchemaRegistryAvroSerializer: """ - SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, - complete with support for versioning and management. + SchemaRegistryAvroSerializer provides the ability to serialize and deserialize data according + to the given avro schema. It would automatically register, get and cache the schema. :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. @@ -44,10 +44,10 @@ class SchemaRegistryAvroSerializer: :param str schema_group """ - def __init__(self, credential, endpoint, schema_group, **kwargs): + def __init__(self, endpoint, credential, schema_group, **kwargs): self._schema_group = schema_group self._avro_serializer = AvroObjectSerializer() - self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) + self._schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=credential) self._id_to_schema = {} self._schema_to_id = {} @@ -98,28 +98,26 @@ async def _get_schema(self, schema_id: str) -> str: self._schema_to_id[schema_str] = schema_id return schema_str - async def serialize(self, data: Dict[str, Any], schema: Union[str, bytes, avro.schema.Schema]) -> bytes: + async def serialize(self, data: Dict[str, Any], schema: Union[str, bytes]) -> bytes: """ Encode dict data with the given schema. :param data: The dict data to be encoded. :param schema: The schema used to encode the data. # TODO: support schema object/str/bytes? - :type schema: Union[str, bytes, avro.schema.Schema]) + :type schema: Union[str, bytes] :return: """ if not isinstance(schema, avro.schema.Schema): schema = avro.schema.parse(schema) schema_id = await self._get_schema_id(schema.fullname, str(schema)) - stream = BytesIO() - self._avro_serializer.serialize(stream, data, schema) + data_bytes = self._avro_serializer.serialize(data, schema) # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. # This is intended to become a record format identifier in the future. # Right now, you can just put \x00\x00\x00\x00. record_format_identifier = b'\0\0\0\0' - res = record_format_identifier + schema_id.encode('utf-8') + stream.getvalue() - stream.close() - return res + payload = record_format_identifier + schema_id.encode('utf-8') + data_bytes + return payload async def deserialize(self, data: bytes) -> Dict[str, Any]: """ @@ -135,7 +133,6 @@ async def deserialize(self, data: bytes) -> Dict[str, Any]: schema_id = data[4:36].decode('utf-8') schema_content = await self._get_schema(schema_id) - # TODO: schema_id to datumreader cache schema = avro.schema.parse(schema_content) dict_data = self._avro_serializer.deserialize(data[36:], schema) return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt index 2225991bf987..fd489752ebe0 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt @@ -1,2 +1,4 @@ -e ../../identity/azure-identity +-e ../../../tools/azure-devtools +-e ../../../tools/azure-sdk-tools ../azure-schemaregistry \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md index e69de29bb2d1..00cff3f1b745 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md @@ -0,0 +1,48 @@ +--- +page_type: sample +languages: + - python +products: + - azure + - azure-schema-registry-avro-serializer +urlFragment: schemaregistry-avro-serializer-samples +--- + +# Azure Schema Registry Avro Serializer library for Python Samples + +These are code samples that show common scenario operations with the Schema Registry Avro Serializer library. +The async versions of the samples (the python sample files appended with `_async`) show asynchronous operations, +and require Python 3.5 or later. + +Several Schema Registry Avro Serializer Python SDK samples are available to you in the SDK's GitHub repository. These samples provide example code for additional scenarios commonly encountered while working with Schema Registry Avro Serializer: + +* [avro_serializer.py](./sync_samples/avro_serializer.py) ([async version](./async_samples/avro_serializer_async.py)) - Examples for common Schema Registry Avro Serializer tasks: + * Serialize data according to the given schema + * Deserialize data + +## Prerequisites +- Python 2.7, 3.5 or later. +- **Microsoft Azure Subscription:** To use Azure services, including Azure Schema Registry, you'll need a subscription. +If you do not have an existing Azure account, you may sign up for a free trial or use your MSDN subscriber benefits when you [create an account](https://account.windowsazure.com/Home/Index). + +## Setup + +1. Install the Azure Schema Registry client library for Python with [pip](https://pypi.org/project/pip/): + +```bash +pip install azure-schemaregistry-avro-serializer +``` + +2. Clone or download this sample repository +3. Open the sample folder in Visual Studio Code or your IDE of choice. + +## Running the samples + +1. Open a terminal window and `cd` to the directory that the samples are saved in. +2. Set the environment variables specified in the sample file you wish to run. +3. Follow the usage described in the file, e.g. `python avro_serializer.py` + +## Next steps + +Check out the [API reference documentation](https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry-avro-serializer/latest/index.html) to learn more about +what you can do with the Azure Schema Registry Avro Serializer library. diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py index 10571530ba2b..f97be292328f 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py @@ -66,11 +66,11 @@ async def main(): client_secret=CLIENT_SECRET ) - serializer = SchemaRegistryAvroSerializer(token_credential, SCHEMA_REGISTRY_ENDPOINT, SCHEMA_GROUP) + serializer = SchemaRegistryAvroSerializer(SCHEMA_REGISTRY_ENDPOINT, token_credential, SCHEMA_GROUP) async with serializer: dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} payload_bytes = await serialize(serializer, SCHEMA_STRING, dict_data) - dict_data = await deserialize(serializer, payload_bytes) + decoded_data = await deserialize(serializer, payload_bytes) await token_credential.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py index 231246f2d75b..1c3c1eaff909 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py @@ -66,7 +66,7 @@ def deserialize(serializer, bytes): if __name__ == '__main__': - serializer = SchemaRegistryAvroSerializer(token_credential, SCHEMA_REGISTRY_ENDPOINT, SCHEMA_GROUP) + serializer = SchemaRegistryAvroSerializer(SCHEMA_REGISTRY_ENDPOINT, token_credential, SCHEMA_GROUP) dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} payload_bytes = serialize(serializer, SCHEMA_STRING, dict_data) dict_data = deserialize(serializer, payload_bytes) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py index e69de29bb2d1..a945052c67f5 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py @@ -0,0 +1,66 @@ +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import pytest +import uuid +import avro +import avro.io +from io import BytesIO + +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry.serializer.avro_serializer.aio import SchemaRegistryAvroSerializer +from azure.identity.aio import ClientSecretCredential +from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError + +from schemaregistry_preparer import SchemaRegistryPreparer +from devtools_testutils import AzureMgmtTestCase + + +class SchemaRegistryAvroSerializerAsyncTests(AzureMgmtTestCase): + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + async def test_basic_sr_avro_serializer_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, + schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, + client_secret=schemaregistry_client_secret) + + sr_avro_serializer = SchemaRegistryAvroSerializer(schemaregistry_endpoint, credential, schemaregistry_group) + sr_client = SchemaRegistryClient(schemaregistry_endpoint, credential) + + random_schema_namespace = str(uuid.uuid4())[0:8] + schema_str = "{\"namespace\":\"" + random_schema_namespace + "\"" +\ + ""","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + + dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + encoded_data = await sr_avro_serializer.serialize(dict_data, schema_str) + + assert encoded_data[0:4] == b'\0\0\0\0' + schema_id = (await sr_client.get_schema_id(schemaregistry_group, schema.fullname, "Avro", str(schema))).id + assert encoded_data[4:36] == schema_id.encode('utf-8') + + decoded_data = await sr_avro_serializer.deserialize(encoded_data) + assert decoded_data['name'] == "Ben" + assert decoded_data['favorite_number'] == 7 + assert decoded_data['favorite_color'] == 'red' + + await sr_avro_serializer.close() + await sr_client.close() + await credential.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py index 19dca0a0cc1d..aeb390b04cf1 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py @@ -45,10 +45,7 @@ def test_raw_avro_serializer(self): raw_avro_object_serailizer = AvroObjectSerializer() # encoding part - encode_stream = BytesIO() - raw_avro_object_serailizer.serialize(encode_stream, dict_data, schema) - encoded_payload = encode_stream.getvalue() - encode_stream.close() + encoded_payload = raw_avro_object_serailizer.serialize(dict_data, schema) # decoding part decoded_data = raw_avro_object_serailizer.deserialize(encoded_payload, schema) @@ -57,9 +54,56 @@ def test_raw_avro_serializer(self): assert decoded_data['favorite_number'] == 7 assert decoded_data['favorite_color'] == 'red' + dict_data_missing_optional_fields = {"name": "Alice"} + encoded_payload = raw_avro_object_serailizer.serialize(dict_data_missing_optional_fields, schema) + decoded_data = raw_avro_object_serailizer.deserialize(encoded_payload, schema) + + assert decoded_data['name'] == "Alice" + assert not decoded_data['favorite_number'] + assert not decoded_data['favorite_color'] + + @pytest.mark.liveTest + @pytest.mark.live_test_only + def test_raw_avro_serializer_negative(self): + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + + raw_avro_object_serailizer = AvroObjectSerializer() + dict_data_wrong_type = {"name": "Ben", "favorite_number": "something", "favorite_color": "red"} + with pytest.raises(avro.io.AvroTypeException): + raw_avro_object_serailizer.serialize(dict_data_wrong_type, schema) + + dict_data_missing_required_field = {"favorite_number": 7, "favorite_color": "red"} + with pytest.raises(avro.io.AvroTypeException): + raw_avro_object_serailizer.serialize(dict_data_missing_required_field, schema) + @pytest.mark.liveTest @pytest.mark.live_test_only @SchemaRegistryPreparer() def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - pass + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, + client_secret=schemaregistry_client_secret) + + sr_avro_serializer = SchemaRegistryAvroSerializer(schemaregistry_endpoint, credential, schemaregistry_group) + sr_client = SchemaRegistryClient(schemaregistry_endpoint, credential) + + random_schema_namespace = str(uuid.uuid4())[0:8] + schema_str = "{\"namespace\":\"" + random_schema_namespace + "\"" +\ + ""","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + + dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + encoded_data = sr_avro_serializer.serialize(dict_data, schema_str) + + assert encoded_data[0:4] == b'\0\0\0\0' + schema_id = sr_client.get_schema_id(schemaregistry_group, schema.fullname, "Avro", str(schema)).id + assert encoded_data[4:36] == schema_id.encode('utf-8') + + decoded_data = sr_avro_serializer.deserialize(encoded_data) + assert decoded_data['name'] == "Ben" + assert decoded_data['favorite_number'] == 7 + assert decoded_data['favorite_color'] == 'red' + + sr_avro_serializer.close() + sr_client.close() diff --git a/sdk/schemaregistry/azure-schemaregistry/README.md b/sdk/schemaregistry/azure-schemaregistry/README.md index 3340aeb9e5fb..5c093d7cb9f0 100644 --- a/sdk/schemaregistry/azure-schemaregistry/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/README.md @@ -75,7 +75,7 @@ endpoint = os.environ['SCHEMA_REGISTRY_ENDPOINT'] schema_id = '' schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) -with schema_registry_client, token_credential: +with schema_registry_client: schema = schema_registry_client.get_schema(schema_id) schema_content = schema.content ``` @@ -106,7 +106,7 @@ schema_content = """ """ schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) -with schema_registry_client, token_credential: +with schema_registry_client: schema_properties = schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_content) schema_id = schema_properties.id ``` diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index 3170e444260f..f824b75c7fc9 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -143,7 +143,7 @@ def get_schema(self, schema_id, **kwargs): ) def get_schema_id(self, schema_group, schema_name, serialization_type, schema_content, **kwargs): - # type: (str, str, str, Union[str, SerializationType], Any) -> SchemaProperties + # type: (str, str, Union[str, SerializationType], str, Any) -> SchemaProperties """ Gets the ID referencing an existing schema within the specified schema group, as matched by schema content comparison. From dc363f43c504dd6b53e5ff993ec71189531d2b4d Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Mon, 31 Aug 2020 20:52:25 -0700 Subject: [PATCH 18/35] partial update according to laruent's feedback --- .../README.md | 15 ++ .../_schema_registry_avro_serializer.py | 47 +++--- .../avro_serializer/aio/__init__.py | 30 ---- .../_schema_registry_avro_serializer_async.py | 138 ------------------ .../samples/README.md | 2 +- .../async_samples/avro_serializer_async.py | 78 ---------- .../setup.py | 6 +- .../async_tests/test_avro_serializer_async.py | 66 --------- .../azure-schemaregistry/README.md | 20 ++- .../azure/schemaregistry/_common/__init__.py | 2 +- .../schemaregistry/_common/_constants.py | 5 +- .../_common/_response_handlers.py | 3 +- .../azure/schemaregistry/_common/_schema.py | 79 +++------- .../schemaregistry/_schema_registry_client.py | 7 +- .../aio/_schema_registry_client_async.py | 4 +- .../sample_schema_registry_code_async.py | 12 +- .../async_samples/schema_registry_async.py | 20 +-- .../sample_schema_registry_code.py | 30 ++-- .../samples/sync_samples/schema_registry.py | 20 +-- .../azure-schemaregistry/setup.py | 16 +- .../async_tests/test_schema_registry_async.py | 56 +++---- .../tests/test_schema_registry.py | 83 ++++++----- 22 files changed, 210 insertions(+), 529 deletions(-) delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md index 27a9a5281672..22559618ef1e 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md @@ -19,6 +19,21 @@ To use this package, you must have: * Azure Schema Registry * Python 2.7, 3.5 or later - [Install Python][python] +### Authenticate the client +Interaction with Schema Registry Avro Serializer starts with an instance of SchemaRegistryAvroSerializer class. You need the endpoint, AAD credential and schema group name to instantiate the client object. + +**Create client using the azure-identity library:** + +```python +from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.identity import DefaultAzureCredential + +credential = DefaultAzureCredential() +endpoint = '<< ENDPOINT OF THE SCHEMA REGISTRY >>' +schema_group = '<< GROUP NAME OF THE SCHEMA >>' +schema_registry_client = SchemaRegistryAvroSerializer(endpoint, credential, schema_group) +``` + ## Key concepts - Avro: Apache Avro™ is a data serialization system. diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index b79bb8b2a6ae..c43ac2980fda 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -31,8 +31,8 @@ from ._avro_serializer import AvroObjectSerializer - -class SchemaRegistryAvroSerializer(object): +# TODO: inherit from context manager +class SchemaRegistryAvroSerializer(): # suffix Client? """ SchemaRegistryAvroSerializer provides the ability to serialize and deserialize data according to the given avro schema. It would automatically register, get and cache the schema. @@ -49,6 +49,7 @@ def __init__(self, endpoint, credential, schema_group, **kwargs): self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) self._id_to_schema = {} self._schema_to_id = {} + self._user_input_schema_cache = {} def __enter__(self): # type: () -> SchemaRegistryAvroSerializer @@ -66,14 +67,15 @@ def close(self): """ self._schema_registry_client.close() - def _get_schema_id(self, schema_name, schema_str): - # type: (str, str) -> str + def _get_schema_id(self, schema_name, schema): + # type: (str, avro.schema.Schema) -> str """ :param schema_name: - :param schema_str: + :param schema: :return: """ + schema_str = str(schema) try: return self._schema_to_id[schema_str] except KeyError: @@ -84,7 +86,7 @@ def _get_schema_id(self, schema_name, schema_str): schema_str ).id self._schema_to_id[schema_str] = schema_id - self._id_to_schema[schema_id] = str(schema_str) + self._id_to_schema[schema_id] = schema_str return schema_id def _get_schema(self, schema_id): @@ -108,33 +110,40 @@ def serialize(self, data, schema): Encode dict data with the given schema. :param data: The dict data to be encoded. - :param schema: The schema used to encode the data. # TODO: support schema object/str/bytes? + :param schema: The schema used to encode the data. :type schema: Union[str, bytes] :return: """ - if not isinstance(schema, avro.schema.Schema): - schema = avro.schema.parse(schema) + try: + schema = self._user_input_schema_cache[schema] + except KeyError: + if not isinstance(schema, avro.schema.Schema): + schema = avro.schema.parse(schema) + self._user_input_schema_cache[schema] = schema - schema_id = self._get_schema_id(schema.fullname, str(schema)) + record_format_identifier = b'\0\0\0\0' # TODO: BytesIO to append bytes zero + schema_id = self._get_schema_id(schema.fullname, schema) data_bytes = self._avro_serializer.serialize(data, schema) - # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. - # This is intended to become a record format identifier in the future. - # Right now, you can just put \x00\x00\x00\x00. - record_format_identifier = b'\0\0\0\0' - payload = record_format_identifier + schema_id.encode('utf-8') + data_bytes # TODO: should we use struck.pack and unpack for interoperability, could be a cross-language problem + + stream = BytesIO() + + stream.write(record_format_identifier) + stream.write(schema_id.encode('utf-8')) + stream.write(data_bytes) + stream.flush() + + payload = stream.getvalue() + stream.close() return payload def deserialize(self, data): - # type: (bytes) -> Dict[str, Any] + # type: (bytes) -> Dict[str, Any] # TODO: add Io/generator support in the future """ Decode bytes data. :param bytes data: The bytes data needs to be decoded. :rtype: Dict[str, Any] """ - # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. - # This is intended to become a record format identifier in the future. - # Right now, you can just put \x00\x00\x00\x00. record_format_identifier = data[0:4] schema_id = data[4:36].decode('utf-8') schema_content = self._get_schema(schema_id) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/__init__.py deleted file mode 100644 index 3af91d793ced..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -from ._schema_registry_avro_serializer_async import SchemaRegistryAvroSerializer - -__all__ = [ - "SchemaRegistryAvroSerializer" -] \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py deleted file mode 100644 index 4e2815e9d12b..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/aio/_schema_registry_avro_serializer_async.py +++ /dev/null @@ -1,138 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -from io import BytesIO -from typing import Any, Dict, Union -import avro - -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry import SerializationType - -from .._avro_serializer import AvroObjectSerializer - - -class SchemaRegistryAvroSerializer: - """ - SchemaRegistryAvroSerializer provides the ability to serialize and deserialize data according - to the given avro schema. It would automatically register, get and cache the schema. - - :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. - :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. - :type credential: TokenCredential - :param str schema_group - - """ - def __init__(self, endpoint, credential, schema_group, **kwargs): - self._schema_group = schema_group - self._avro_serializer = AvroObjectSerializer() - self._schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=credential) - self._id_to_schema = {} - self._schema_to_id = {} - - async def __aenter__(self) -> "SchemaRegistryAvroSerializer": - await self._schema_registry_client.__aenter__() - return self - - async def __aexit__(self, *args: Any) -> None: - await self._schema_registry_client.__aexit__(*args) - - async def close(self) -> None: - """ This method is to close the sockets opened by the client. - It need not be used when using with a context manager. - """ - await self._schema_registry_client.close() - - async def _get_schema_id(self, schema_name: str, schema_str: str) -> str: - """ - - :param schema_name: - :param schema_str: - :return: - """ - try: - return self._schema_to_id[schema_str] - except KeyError: - schema_id = (await self._schema_registry_client.register_schema( - self._schema_group, - schema_name, - SerializationType.AVRO, - schema_str - )).id - self._schema_to_id[schema_str] = schema_id - self._id_to_schema[schema_id] = str(schema_str) - return schema_id - - async def _get_schema(self, schema_id: str) -> str: - """ - - :param str schema_id: - :return: - """ - try: - return self._id_to_schema[schema_id] - except KeyError: - schema_str = (await self._schema_registry_client.get_schema(schema_id)).content - self._id_to_schema[schema_id] = schema_str - self._schema_to_id[schema_str] = schema_id - return schema_str - - async def serialize(self, data: Dict[str, Any], schema: Union[str, bytes]) -> bytes: - """ - Encode dict data with the given schema. - - :param data: The dict data to be encoded. - :param schema: The schema used to encode the data. # TODO: support schema object/str/bytes? - :type schema: Union[str, bytes] - :return: - """ - if not isinstance(schema, avro.schema.Schema): - schema = avro.schema.parse(schema) - - schema_id = await self._get_schema_id(schema.fullname, str(schema)) - data_bytes = self._avro_serializer.serialize(data, schema) - # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. - # This is intended to become a record format identifier in the future. - # Right now, you can just put \x00\x00\x00\x00. - record_format_identifier = b'\0\0\0\0' - payload = record_format_identifier + schema_id.encode('utf-8') + data_bytes - return payload - - async def deserialize(self, data: bytes) -> Dict[str, Any]: - """ - Decode bytes data. - - :param bytes data: The bytes data needs to be decoded. - :rtype: Dict[str, Any] - """ - # TODO: Arthur: We are adding 4 bytes to the beginning of each SR payload. - # This is intended to become a record format identifier in the future. - # Right now, you can just put \x00\x00\x00\x00. - record_format_identifier = data[0:4] - schema_id = data[4:36].decode('utf-8') - schema_content = await self._get_schema(schema_id) - - schema = avro.schema.parse(schema_content) - dict_data = self._avro_serializer.deserialize(data[36:], schema) - return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md index 00cff3f1b745..8cefc321f7e8 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md @@ -16,7 +16,7 @@ and require Python 3.5 or later. Several Schema Registry Avro Serializer Python SDK samples are available to you in the SDK's GitHub repository. These samples provide example code for additional scenarios commonly encountered while working with Schema Registry Avro Serializer: -* [avro_serializer.py](./sync_samples/avro_serializer.py) ([async version](./async_samples/avro_serializer_async.py)) - Examples for common Schema Registry Avro Serializer tasks: +* [avro_serializer.py](./sync_samples/avro_serializer.py) - Examples for common Schema Registry Avro Serializer tasks: * Serialize data according to the given schema * Deserialize data diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py deleted file mode 100644 index f97be292328f..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/async_samples/avro_serializer_async.py +++ /dev/null @@ -1,78 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import asyncio -import os - -from azure.identity.aio import ClientSecretCredential -from azure.schemaregistry.serializer.avro_serializer.aio import SchemaRegistryAvroSerializer - -TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] -CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] -CLIENT_SECRET=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] - -SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] -SCHEMA_GROUP=os.environ['SCHEMA_REGISTRY_GROUP'] -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -async def serialize(serializer, schema, dict_data): - bytes = await serializer.serialize(dict_data, schema) - print('Encoded bytes are: ', bytes) - return bytes - - -async def deserialize(serializer, bytes): - dict_data = await serializer.deserialize(bytes) - print('Deserialized data is: ', dict_data) - return dict_data - - -async def main(): - token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET - ) - - serializer = SchemaRegistryAvroSerializer(SCHEMA_REGISTRY_ENDPOINT, token_credential, SCHEMA_GROUP) - async with serializer: - dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - payload_bytes = await serialize(serializer, SCHEMA_STRING, dict_data) - decoded_data = await deserialize(serializer, payload_bytes) - - await token_credential.close() - -loop = asyncio.get_event_loop() -loop.run_until_complete(main()) \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py index e1746a434d21..1b4bd6f8b09c 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py @@ -65,11 +65,7 @@ zip_safe=False, packages=find_packages(exclude=exclude_packages), install_requires=[ - 'azure-core<2.0.0,>=1.2.2', 'azure-schemaregistry<2.0.0,>=1.0.0b1', 'avro<2.0.0,>=1.0.0' - ], - extras_require={ - ":python_version<'3.0'": ['azure-nspkg', 'futures'], - } + ] ) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py deleted file mode 100644 index a945052c67f5..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/async_tests/test_avro_serializer_async.py +++ /dev/null @@ -1,66 +0,0 @@ -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import pytest -import uuid -import avro -import avro.io -from io import BytesIO - -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry.serializer.avro_serializer.aio import SchemaRegistryAvroSerializer -from azure.identity.aio import ClientSecretCredential -from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError - -from schemaregistry_preparer import SchemaRegistryPreparer -from devtools_testutils import AzureMgmtTestCase - - -class SchemaRegistryAvroSerializerAsyncTests(AzureMgmtTestCase): - @pytest.mark.liveTest - @pytest.mark.live_test_only - @SchemaRegistryPreparer() - async def test_basic_sr_avro_serializer_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, - schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, - client_secret=schemaregistry_client_secret) - - sr_avro_serializer = SchemaRegistryAvroSerializer(schemaregistry_endpoint, credential, schemaregistry_group) - sr_client = SchemaRegistryClient(schemaregistry_endpoint, credential) - - random_schema_namespace = str(uuid.uuid4())[0:8] - schema_str = "{\"namespace\":\"" + random_schema_namespace + "\"" +\ - ""","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - schema = avro.schema.parse(schema_str) - - dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - encoded_data = await sr_avro_serializer.serialize(dict_data, schema_str) - - assert encoded_data[0:4] == b'\0\0\0\0' - schema_id = (await sr_client.get_schema_id(schemaregistry_group, schema.fullname, "Avro", str(schema))).id - assert encoded_data[4:36] == schema_id.encode('utf-8') - - decoded_data = await sr_avro_serializer.deserialize(encoded_data) - assert decoded_data['name'] == "Ben" - assert decoded_data['favorite_number'] == 7 - assert decoded_data['favorite_color'] == 'red' - - await sr_avro_serializer.close() - await sr_client.close() - await credential.close() diff --git a/sdk/schemaregistry/azure-schemaregistry/README.md b/sdk/schemaregistry/azure-schemaregistry/README.md index 5c093d7cb9f0..28145c90afb2 100644 --- a/sdk/schemaregistry/azure-schemaregistry/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/README.md @@ -19,6 +19,20 @@ To use this package, you must have: * Azure Schema Registry * Python 2.7, 3.5 or later - [Install Python][python] +### Authenticate the client +Interaction with Schema Registry starts with an instance of SchemaRegistryClient class. You need the endpoint and AAD credential to instantiate the client object. + +**Create client using the azure-identity library:** + +```python +from azure.schemaregistry import SchemaRegistryClient +from azure.identity import DefaultAzureCredential + +credential = DefaultAzureCredential() +endpoint = '<< ENDPOINT OF THE SCHEMA REGISTRY >>' +schema_registry_client = SchemaRegistryClient(endpoint, credential) +``` + ## Key concepts - Schema: Schema is the organization or structure for data. @@ -58,8 +72,8 @@ schema_content = """ schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) with schema_registry_client, token_credential: - schema_properties = schema_registry_client.register_schema(schema_group, schema_name, serilization_type, schema_string) - schema_id = schema_properties.id + schema_properties = schema_registry_client.register_schema(schema_group, schema_name, serialization_type, schema_content) + schema_id = schema_properties.schema_id ``` ### Get the schema by id @@ -108,7 +122,7 @@ schema_content = """ schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) with schema_registry_client: schema_properties = schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_content) - schema_id = schema_properties.id + schema_id = schema_properties.schema_id ``` ## Troubleshooting diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py index 9ecdd4ff2c41..c36aaed14908 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/__init__.py @@ -22,4 +22,4 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # -# -------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------- diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py index cd5424dc42c4..9761f2ba601f 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py @@ -26,6 +26,9 @@ from enum import Enum -class SerializationType(Enum): +class SerializationType(str, Enum): AVRO = "Avro" JSON = "Json" + + def __str__(self): + return self.value diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py index def5143bfc77..e24d7e04a052 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py @@ -52,4 +52,5 @@ def _parse_response_schema(pipeline_response, deserialized, response_headers): :param response_headers: :return: """ - return Schema(schema_str=deserialized, **response_headers) + + return Schema(schema_content=deserialized, schema_properties=SchemaProperties(**response_headers)) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index 51bedf838529..9f8e3fedf60d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -23,10 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import Any, Dict - - -# TODO: Arthur: schema id will be the type of long instead of string +from typing import Any, Optional class DictMixin(object): @@ -80,36 +77,9 @@ def get(self, key, default=None): return default -class SchemaMeta(DictMixin): - """ - Id and meta properties of a schema. - - :ivar id: References specific schema in registry namespace. - :type id: str - :ivar location: URL location of schema, identified by schema group, schema name, and version. - :type location: str - :ivar id_location: URL location of schema, identified by schema ID. # TODO: JS name is LocationById - :type id_location: str - :ivar type: Serialization type for the schema being stored. - :type type: str - :ivar version: Version of the returned schema. - :type version: int - """ - def __init__( - self, - **kwargs - ): - # type: (Any) -> None - self.location = kwargs.get('Location') - self.id = kwargs.get("X-Schema-Id") - self.id_location = kwargs.get('X-Schema-Id-Location') - self.type = kwargs.get('X-Schema-Type') - self.version = kwargs.get('X-Schema-Version') - - -class SchemaProperties: +class SchemaProperties(DictMixin): """ - Id and meta properties of a schema. + Meta properties of a schema. :ivar id: References specific schema in registry namespace. :type id: str @@ -117,8 +87,8 @@ class SchemaProperties: :type location: str :ivar location_by_id: URL location of schema, identified by schema ID. :type location_by_id: str - :ivar type: Serialization type for the schema being stored. - :type type: str + :ivar serialization_type: Serialization type for the schema being stored. + :type serialization_type: str :ivar version: Version of the returned schema. :type version: int @@ -134,30 +104,25 @@ class SchemaProperties: """ def __init__( self, - schema_id, + schema_id=None, **kwargs ): - # type: (str, Any) -> None - super(SchemaProperties, self).__init__(**kwargs) - self.id = schema_id + # type: (Optional[str], Any) -> None + self.location = kwargs.get('Location') + self.schema_id = schema_id or kwargs.get("X-Schema-Id") + self.location_by_id = kwargs.get('X-Schema-Id-Location') + self.serialization_type = kwargs.get('X-Schema-Type') + self.version = kwargs.get('X-Schema-Version') -class Schema(SchemaMeta): +class Schema(DictMixin): """ The schema content of a schema, along with id and meta properties. - :ivar content: The content of the schema. - :type content: str - :ivar id: References specific schema in registry namespace. - :type id: str - :ivar location: URL location of schema, identified by schema group, schema name, and version. - :type location: str - :ivar location_by_id: URL location of schema, identified by schema ID. - :type location_by_id: str - :ivar type: Serialization type for the schema being stored. - :type type: str - :ivar version: Version of the returned schema. - :type version: int + :ivar schema_content: The content of the schema. + :type schema_content: str + :ivar schema_properties: The properties of the schema. + :type schema_properties: SchemaProperties .. admonition:: Example: @@ -171,9 +136,9 @@ class Schema(SchemaMeta): """ def __init__( self, - schema_str, - **kwargs + schema_content, + schema_properties, ): - # type: (str, Any) -> None - super(Schema, self).__init__(**kwargs) - self.content = schema_str + # type: (str, SchemaProperties) -> None + self.schema_content = schema_content + self.schema_properties = schema_properties diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index f824b75c7fc9..3f62047391d2 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -24,7 +24,6 @@ # # -------------------------------------------------------------------------- from typing import Any, TYPE_CHECKING, Union -from enum import Enum from ._common._constants import SerializationType from ._common._schema import Schema, SchemaProperties @@ -34,10 +33,8 @@ if TYPE_CHECKING: from azure.core.credentials import TokenCredential -# TODO: Arthur: schema id will be the type of long instead of string - -class SchemaRegistryClient: +class SchemaRegistryClient(object): """ SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, complete with support for versioning and management. @@ -81,7 +78,7 @@ def close(self): """ self._generated_client.close() - def register_schema(self, schema_group, schema_name, serialization_type, schema_content, **kwargs): # TODO: serialization_type Enum + string? + def register_schema(self, schema_group, schema_name, serialization_type, schema_content, **kwargs): # type: (str, str, Union[str, SerializationType], str, Any) -> SchemaProperties """ Register new schema. If schema of specified name does not exist in specified group, diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 36c69eb0ca61..c328d3e51487 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -33,10 +33,8 @@ if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential -# TODO: Arthur: schema id will be the type of long instead of string - -class SchemaRegistryClient: +class SchemaRegistryClient(object): """ SchemaRegistryClient is as a central schema repository for enterprise-level data infrastructure, complete with support for versioning and management. diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py index 8b4592ebe9f3..6cecc9c6188a 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py @@ -59,20 +59,20 @@ async def register_scehma(schema_registry_client): {"name": "favorite_color", "type": ["string", "null"]} ] }""" - schema_id_response = await schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) - schem_id = schema_id_response.id + schema_properties = await schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) + schem_id = schema_properties.schema_id # [END register_schema_async] async def get_schema(schema_registry_client, schema_id): # [START get_schema_async] - schema_response = await schema_registry_client.get_schema(schema_id) - schema_content = schema_response.content + schema = await schema_registry_client.get_schema(schema_id) + schema_content = schema.schema_content # [END get_schema_async] async def get_schema_id(schema_registry_client, schema_group, schema_name, serialization_type, schema_string): # [START get_schema_id_async] - schema_id_response = await schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) - schem_id = schema_id_response.id + schema_properties = await schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + schem_id = schema_properties.schema_id # [END get_schema_id_async] diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py index 0df168f0ed80..84e8cca571eb 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py @@ -40,26 +40,26 @@ async def register_schema(client, schema_group, schema_name, serialization_type, schema_string): print("Registering schema...") - schema_id = await client.register_schema(schema_group, schema_name, serialization_type, schema_string) - print("Schema registered, returned schema id is {}".format(schema_id.id)) - print("Schema meta properties are {}".format(schema_id)) - return schema_id.id + schema_properties = await client.register_schema(schema_group, schema_name, serialization_type, schema_string) + print("Schema registered, returned schema id is {}".format(schema_properties.schema_id)) + print("Schema meta properties are {}".format(schema_properties)) + return schema_properties.schema_id async def get_schema_by_id(client, schema_id): print("Getting schema by id...") schema = await client.get_schema(schema_id) - print("The schema string of schema id: {} string is {}".format(schema_id, schema.content)) + print("The schema string of schema id: {} string is {}".format(schema_id, schema.schema_content)) print("Schema meta properties are {}".format(schema_id)) - return schema.content + return schema.schema_content async def get_schema_id(client, schema_group, schema_name, serialization_type, schema_string): print("Getting schema id...") - schema_id = await client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) - print("The schema id is: {}".format(schema_id.id)) - print("Schema meta properties are {}".format(schema_id)) - return schema_id.id + schema_properties = await client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + print("The schema id is: {}".format(schema_properties.schema_id)) + print("Schema meta properties are {}".format(schema_properties)) + return schema_properties.schema_id async def main(): diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py index 4a1c4387f4ed..eb7f54dfc29d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py @@ -58,16 +58,16 @@ def register_scehma(schema_registry_client): {"name": "favorite_color", "type": ["string", "null"]} ] }""" - schema_id_response = schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) - schem_id = schema_id_response.id + schema_properties = schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) + schem_id = schema_properties.schema_id # [END register_schema_sync] # [START print_schema_id] - print(schema_id_response.id) - print(schema_id_response.location) - print(schema_id_response.id_location) - print(schema_id_response.type) - print(schema_id_response.version) + print(schema_properties.schema_id) + print(schema_properties.location) + print(schema_properties.location_by_id) + print(schema_properties.type) + print(schema_properties.version) # [END print_schema_id] return schem_id @@ -75,22 +75,18 @@ def register_scehma(schema_registry_client): def get_schema(schema_registry_client, schema_id): # [START get_schema_sync] - schema_response = schema_registry_client.get_schema(schema_id) - schema_content = schema_response.content + schema = schema_registry_client.get_schema(schema_id) + schema_content = schema.schema_content # [END get_schema_sync] # [START print_schema] - print(schema_response.id) - print(schema_response.location) - print(schema_response.id_location) - print(schema_response.type) - print(schema_response.version) - print(schema_response.content) + print(schema.schema_content) + print(schema.schema_properties) # [END print_schema] def get_schema_id(schema_registry_client, schema_group, schema_name, serialization_type, schema_string): # [START get_schema_id_sync] - schema_id_response = schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) - schem_id = schema_id_response.id + schema_properties = schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + schem_id = schema_properties.schema_id # [END get_schema_id_sync] diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py index 626139d3dae9..a94f6e0586e6 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py @@ -59,26 +59,26 @@ def register_schema(client, schema_group, schema_name, serialization_type, schema_string): print("Registering schema...") - schema_id = client.register_schema(schema_group, schema_name, serialization_type, schema_string) - print("Schema registered, returned schema id is {}".format(schema_id.id)) - print("Schema meta properties are {}".format(schema_id)) - return schema_id.id + schema_properties = client.register_schema(schema_group, schema_name, serialization_type, schema_string) + print("Schema registered, returned schema id is {}".format(schema_properties.schema_id)) + print("Schema meta properties are {}".format(schema_properties)) + return schema_properties.schema_id def get_schema_by_id(client, schema_id): print("Getting schema by id...") schema = client.get_schema(schema_id) - print("The schema string of schema id: {} string is {}".format(schema_id, schema.content)) + print("The schema string of schema id: {} string is {}".format(schema_id, schema.schema_content)) print("Schema meta properties are {}".format(schema_id)) - return schema.content + return schema.schema_content def get_schema_id(client, schema_group, schema_name, serialization_type, schema_string): print("Getting schema id...") - schema_id = client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) - print("The schema id is: {}".format(schema_id.id)) - print("Schema meta properties are {}".format(schema_id)) - return schema_id.id + schema_properties = client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + print("The schema id is: {}".format(schema_properties.schema_id)) + print("Schema meta properties are {}".format(schema_properties)) + return schema_properties.schema_id if __name__ == '__main__': diff --git a/sdk/schemaregistry/azure-schemaregistry/setup.py b/sdk/schemaregistry/azure-schemaregistry/setup.py index 54de509f9a5d..118bd0775d15 100644 --- a/sdk/schemaregistry/azure-schemaregistry/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry/setup.py @@ -20,20 +20,6 @@ # a-b-c => a.b.c namespace_name = PACKAGE_NAME.replace('-', '.') -# azure v0.x is not compatible with this package -# azure v0.x used to have a __version__ attribute (newer versions don't) -try: - import azure - try: - ver = azure.__version__ - raise Exception( - 'This package is incompatible with azure=={}. '.format(ver) + - 'Uninstall it with "pip uninstall azure".' - ) - except AttributeError: - pass -except ImportError: - pass # Version extraction inspired from 'requests' with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: @@ -84,6 +70,6 @@ 'azure-core<2.0.0,>=1.2.2' ], extras_require={ - ":python_version<'3.0'": ['azure-nspkg', 'futures'], + ":python_version<'3.0'": ['azure-nspkg'] } ) diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py index 0ee5326c1f88..8528a4f25b52 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py @@ -40,28 +40,28 @@ async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_ schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" - schema_id = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - assert schema_id.id is not None - assert schema_id.location is not None - assert schema_id.id_location is not None - assert schema_id.version is 1 - assert schema_id.type == "Avro" + assert schema_properties.schema_id is not None + assert schema_properties.location is not None + assert schema_properties.location_by_id is not None + assert schema_properties.version is 1 + assert schema_properties.type == "Avro" - returned_schema = await client.get_schema(schema_id=schema_id.id) + returned_schema = await client.get_schema(schema_id=schema_properties.schema_id) - assert returned_schema.id == schema_id.id + assert returned_schema.schema_id == schema_properties.schema_id assert returned_schema.location is not None - assert returned_schema.id_location is not None + assert returned_schema.location_by_id is not None assert returned_schema.version == 1 assert returned_schema.type == "Avro" assert returned_schema.content == schema_str returned_schema_id = await client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) - assert returned_schema_id.id == schema_id.id + assert returned_schema_id.schema_id == schema_properties.schema_id assert returned_schema_id.location is not None - assert returned_schema_id.id_location is not None + assert returned_schema_id.location_by_id is not None assert returned_schema_id.version == 1 assert returned_schema_id.type == "Avro" @@ -75,32 +75,32 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" - schema_id = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - assert schema_id.id is not None - assert schema_id.location is not None - assert schema_id.id_location is not None - assert schema_id.version == 1 - assert schema_id.type == "Avro" + assert schema_properties.schema_id is not None + assert schema_properties.location is not None + assert schema_properties.location_by_id is not None + assert schema_properties.version == 1 + assert schema_properties.type == "Avro" schema_str_new = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_food","type":["string","null"]}]}""" - new_schema_id = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str_new) + new_schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str_new) - assert new_schema_id.id is not None - assert new_schema_id.location is not None - assert new_schema_id.id_location is not None - assert new_schema_id.version == 2 - assert new_schema_id.type == "Avro" + assert new_schema_properties.schema_id is not None + assert new_schema_properties.location is not None + assert new_schema_properties.location_by_id is not None + assert new_schema_properties.version == 2 + assert new_schema_properties.type == "Avro" - new_schema = await client.get_schema(schema_id=new_schema_id.id) + new_schema = await client.get_schema(schema_id=new_schema_properties.schema_id) - assert new_schema_id.id != schema_id.id - assert new_schema.id == new_schema_id.id + assert new_schema.schema_id != schema_properties.schema_id + assert new_schema.schema_id == new_schema_properties.schema_id assert new_schema.location is not None - assert new_schema.id_location is not None + assert new_schema.location_by_id is not None assert new_schema.content == schema_str_new assert new_schema.version == 2 - assert new_schema_id.type == "Avro" + assert new_schema.type == "Avro" @pytest.mark.liveTest @pytest.mark.live_test_only diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index 90b0a56f2df7..3b4a5bc66f0b 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -39,30 +39,30 @@ def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schem schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" - schema_id = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - assert schema_id.id is not None - assert schema_id.location is not None - assert schema_id.id_location is not None - assert schema_id.version is 1 - assert schema_id.type == "Avro" + assert schema_properties.schema_id is not None + assert schema_properties.location is not None + assert schema_properties.location_by_id is not None + assert schema_properties.version is 1 + assert schema_properties.serialization_type == "Avro" - returned_schema = client.get_schema(schema_id=schema_id.id) + returned_schema = client.get_schema(schema_id=schema_properties.schema_id) - assert returned_schema.id == schema_id.id + assert returned_schema.schema_id == schema_properties.schema_id assert returned_schema.location is not None - assert returned_schema.id_location is not None + assert returned_schema.location_by_id is not None assert returned_schema.version == 1 - assert returned_schema.type == "Avro" - assert returned_schema.content == schema_str + assert returned_schema.serialization_type == "Avro" + assert returned_schema.schema_content == schema_str - returned_schema_id = client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) + returned_schema_properties = client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) - assert returned_schema_id.id == schema_id.id - assert returned_schema_id.location is not None - assert returned_schema_id.id_location is not None - assert returned_schema_id.version == 1 - assert returned_schema_id.type == "Avro" + assert returned_schema_properties.schema_id == schema_properties.schema_id + assert returned_schema_properties.location is not None + assert returned_schema_properties.location_by_id is not None + assert returned_schema_properties.version == 1 + assert returned_schema_properties.serialization_type == "Avro" @pytest.mark.liveTest @pytest.mark.live_test_only @@ -73,32 +73,45 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, sche schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" - schema_id = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - assert schema_id.id is not None - assert schema_id.location is not None - assert schema_id.id_location is not None - assert schema_id.version == 1 - assert schema_id.type == "Avro" + assert schema_properties.schema_id is not None + assert schema_properties.location is not None + assert schema_properties.location_by_id is not None + assert schema_properties.version == 1 + assert schema_properties.serialization_type == "Avro" schema_str_new = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_food","type":["string","null"]}]}""" - new_schema_id = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str_new) + new_schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str_new) - assert new_schema_id.id is not None - assert new_schema_id.location is not None - assert new_schema_id.id_location is not None - assert new_schema_id.version == 2 - assert new_schema_id.type == "Avro" + assert new_schema_properties.schema_id is not None + assert new_schema_properties.location is not None + assert new_schema_properties.location_by_id is not None + assert new_schema_properties.version == 2 + assert new_schema_properties.serialization_type == "Avro" - new_schema = client.get_schema(schema_id=new_schema_id.id) + new_schema = client.get_schema(schema_id=new_schema_properties.schema_id) - assert new_schema_id.id != schema_id.id - assert new_schema.id == new_schema_id.id + assert new_schema_properties.schema_id != schema_properties.schema_id + assert new_schema.schema_id == new_schema_properties.schema_id assert new_schema.location is not None - assert new_schema.id_location is not None - assert new_schema.content == schema_str_new + assert new_schema.location_by_id is not None + assert new_schema.schema_content == schema_str_new assert new_schema.version == 2 - assert new_schema_id.type == "Avro" + assert new_schema.serialization_type == "Avro" + + @pytest.mark.liveTest + @pytest.mark.live_test_only + @SchemaRegistryPreparer() + def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) + client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) + schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":["int","null"]},{"name":"city","type":["string","null"]}]}""" + serialization_type = "Avro" + schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + schema_properties_second = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + assert schema_properties.schema_id == schema_properties_second.schema_id @pytest.mark.liveTest @pytest.mark.live_test_only From 740de0e0d83fd77f0b35e40b433deecbc836430a Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 1 Sep 2020 13:44:08 -0700 Subject: [PATCH 19/35] be consistent with eh extension structure --- .../azure/schemaregistry/__init__.py | 1 - .../azure/schemaregistry/serializer/__init__.py | 1 - .../azure/schemaregistry/serializer/__init__.py | 1 - 3 files changed, 3 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py index 80f86cb969ec..c36aaed14908 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py @@ -23,4 +23,3 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py index 80f86cb969ec..c36aaed14908 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py @@ -23,4 +23,3 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py index 80f86cb969ec..c36aaed14908 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/serializer/__init__.py @@ -23,4 +23,3 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore From 7734c42d3763e6576443dfaf1cb76d58423eb269 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 1 Sep 2020 15:35:52 -0700 Subject: [PATCH 20/35] more update code according to feedback --- .../README.md | 8 +- .../_schema_registry_avro_serializer.py | 29 ++-- .../setup.py | 2 +- ...ializer.test_basic_sr_avro_serializer.yaml | 104 +++++++++++++ .../tests/test_avro_serializer.py | 9 +- .../schemaregistry/_common/_constants.py | 3 - .../azure/schemaregistry/_common/_schema.py | 4 +- .../azure-schemaregistry/dev_requirements.txt | 4 +- .../sample_schema_registry_code_async.py | 40 ++++- .../async_samples/schema_registry_async.py | 6 +- .../sample_schema_registry_code.py | 35 ++++- .../samples/sync_samples/schema_registry.py | 6 +- ...hema_registry_async.test_schema_basic.yaml | 98 ++++++++++++ ...egistry_async.test_schema_basic_async.yaml | 98 ++++++++++++ ....test_schema_negative_no_schema_async.yaml | 51 +++++++ ...ema_registry_async.test_schema_update.yaml | 98 ++++++++++++ ...gistry_async.test_schema_update_async.yaml | 98 ++++++++++++ .../async_tests/test_schema_registry_async.py | 44 +++--- ..._schema_registry.test_register_schema.yaml | 140 ++++++++++++++++++ ...est_schema_registry.test_schema_basic.yaml | 140 ++++++++++++++++++ ...gistry.test_schema_negative_no_schema.yaml | 67 +++++++++ ...chema_registry.test_schema_same_twice.yaml | 98 ++++++++++++ ...st_schema_registry.test_schema_update.yaml | 140 ++++++++++++++++++ .../tests/test_schema_registry.py | 22 +-- 24 files changed, 1266 insertions(+), 78 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_register_schema.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md index 22559618ef1e..1c4dbb4da456 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md @@ -1,16 +1,16 @@ -# Azure Schema Registry Avro Serializer library for Python +# Azure Schema Registry Avro Serializer client library for Python Azure Schema Registry Avro Serializer provides the ability to serialize and deserialize data according -to the given avro schema. It would automatically register, get and cache the schema. +to the given avro schema. It is integrated with Azure Schema Registry SDK and will automatically register and get schema. ## Getting started ### Install the package -Install the Azure Service Bus client library for Python with [pip][pip]: +Install the Azure Schema Registry Avro Serializer client library for Python with [pip][pip]: ```Bash -pip install azure-schemaregistry-avro-serializer +pip install azure-schemaregistry-avroserializer ``` ### Prerequisites: diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index c43ac2980fda..acacd23981e3 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -31,8 +31,8 @@ from ._avro_serializer import AvroObjectSerializer -# TODO: inherit from context manager -class SchemaRegistryAvroSerializer(): # suffix Client? + +class SchemaRegistryAvroSerializer(object): """ SchemaRegistryAvroSerializer provides the ability to serialize and deserialize data according to the given avro schema. It would automatically register, get and cache the schema. @@ -84,7 +84,7 @@ def _get_schema_id(self, schema_name, schema): schema_name, SerializationType.AVRO, schema_str - ).id + ).schema_id self._schema_to_id[schema_str] = schema_id self._id_to_schema[schema_id] = schema_str return schema_id @@ -104,8 +104,8 @@ def _get_schema(self, schema_id): self._schema_to_id[schema_str] = schema_id return schema_str - def serialize(self, data, schema): - # type: (Dict[str, Any], Union[str, bytes]) -> bytes + def serialize(self, data, schema, **kwargs): + # type: (Dict[str, Any], Union[str, bytes], Any) -> bytes """ Encode dict data with the given schema. @@ -114,16 +114,17 @@ def serialize(self, data, schema): :type schema: Union[str, bytes] :return: """ + raw_input_schema = schema try: - schema = self._user_input_schema_cache[schema] + cached_schema = self._user_input_schema_cache[raw_input_schema] except KeyError: - if not isinstance(schema, avro.schema.Schema): - schema = avro.schema.parse(schema) - self._user_input_schema_cache[schema] = schema + parsed_schema = avro.schema.parse(raw_input_schema) + self._user_input_schema_cache[raw_input_schema] = parsed_schema + cached_schema = parsed_schema - record_format_identifier = b'\0\0\0\0' # TODO: BytesIO to append bytes zero - schema_id = self._get_schema_id(schema.fullname, schema) - data_bytes = self._avro_serializer.serialize(data, schema) + record_format_identifier = b'\0\0\0\0' + schema_id = self._get_schema_id(cached_schema.fullname, cached_schema) + data_bytes = self._avro_serializer.serialize(data, cached_schema) stream = BytesIO() @@ -136,8 +137,8 @@ def serialize(self, data, schema): stream.close() return payload - def deserialize(self, data): - # type: (bytes) -> Dict[str, Any] # TODO: add Io/generator support in the future + def deserialize(self, data, **kwargs): + # type: (bytes, Any) -> Dict[str, Any] """ Decode bytes data. diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py index 1b4bd6f8b09c..c166eb266d22 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py @@ -12,7 +12,7 @@ from setuptools import find_packages, setup # Change the PACKAGE_NAME only to change folder and different name -PACKAGE_NAME = "azure-schemaregistry-avro-serializer" +PACKAGE_NAME = "azure-schemaregistry-avroserializer" PACKAGE_PPRINT_NAME = "Schema Registry Avro Serializer" package_folder_path = "azure/schemaregistry/serializer/avro_serializer" diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml new file mode 100644 index 000000000000..b87eebc8e8ed --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml @@ -0,0 +1,104 @@ +interactions: +- request: + body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"testschemafc48df19\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", + \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], + \"name\": \"favorite_color\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '271' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User?api-version=2017-04 + response: + body: + string: '{"id":"07809b86375a42ae8463e77de063ab0b"}' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 22:34:11 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 07809b86375a42ae8463e77de063ab0b + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/07809b86375a42ae8463e77de063ab0b?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"testschemafc48df19\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", + \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], + \"name\": \"favorite_color\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '271' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: POST + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User?api-version=2017-04 + response: + body: + string: '{"id":"07809b86375a42ae8463e77de063ab0b"}' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 22:34:12 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 07809b86375a42ae8463e77de063ab0b + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/07809b86375a42ae8463e77de063ab0b?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py index aeb390b04cf1..df7ed8b29b14 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py @@ -88,7 +88,7 @@ def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_ sr_avro_serializer = SchemaRegistryAvroSerializer(schemaregistry_endpoint, credential, schemaregistry_group) sr_client = SchemaRegistryClient(schemaregistry_endpoint, credential) - random_schema_namespace = str(uuid.uuid4())[0:8] + random_schema_namespace = 'testschema' + str(uuid.uuid4())[0:8] schema_str = "{\"namespace\":\"" + random_schema_namespace + "\"" +\ ""","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) @@ -96,10 +96,15 @@ def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_ dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} encoded_data = sr_avro_serializer.serialize(dict_data, schema_str) + assert schema_str in sr_avro_serializer._user_input_schema_cache + assert str(avro.schema.parse(schema_str)) in sr_avro_serializer._schema_to_id + assert encoded_data[0:4] == b'\0\0\0\0' - schema_id = sr_client.get_schema_id(schemaregistry_group, schema.fullname, "Avro", str(schema)).id + schema_id = sr_client.get_schema_id(schemaregistry_group, schema.fullname, "Avro", str(schema)).schema_id assert encoded_data[4:36] == schema_id.encode('utf-8') + assert schema_id in sr_avro_serializer._id_to_schema + decoded_data = sr_avro_serializer.deserialize(encoded_data) assert decoded_data['name'] == "Ben" assert decoded_data['favorite_number'] == 7 diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py index 9761f2ba601f..bb4a49db913d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py @@ -29,6 +29,3 @@ class SerializationType(str, Enum): AVRO = "Avro" JSON = "Json" - - def __str__(self): - return self.value diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index 9f8e3fedf60d..45677a014958 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -95,8 +95,8 @@ class SchemaProperties(DictMixin): .. admonition:: Example: .. literalinclude:: ../samples/sync_samples/sample_code_schemaregistry.py - :start-after: [START print_schema_id] - :end-before: [END print_schema_id] + :start-after: [START print_schema_properties] + :end-before: [END print_schema_properties] :language: python :dedent: 4 :caption: SchemaId object. diff --git a/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt index e79e74219e31..eeda56db3177 100644 --- a/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt +++ b/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt @@ -1,3 +1,5 @@ +-e ../../core/azure-core -e ../../identity/azure-identity -e ../../../tools/azure-devtools --e ../../../tools/azure-sdk-tools \ No newline at end of file +-e ../../../tools/azure-sdk-tools +aiohttp>=3.0; python_version >= '3.5' \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py index 6cecc9c6188a..75788da911b7 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py @@ -24,6 +24,7 @@ # # -------------------------------------------------------------------------- import os +import asyncio from azure.schemaregistry.aio import SchemaRegistryClient from azure.schemaregistry import SerializationType @@ -41,7 +42,7 @@ def create_client(): CLIENT_SECRET = os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] token_credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) schema_registry_client = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) - return schema_registry_client + return schema_registry_client, token_credential async def register_scehma(schema_registry_client): @@ -49,7 +50,7 @@ async def register_scehma(schema_registry_client): SCHEMA_GROUP = os.environ['SCHEMA_REGISTRY_GROUP'] SCHEMA_NAME = 'your-schema-name' SERIALIZATION_TYPE = SerializationType.AVRO - SCHEMA_STRING = """ + SCHEMA_CONTENT = """ {"namespace": "example.avro", "type": "record", "name": "User", @@ -59,9 +60,10 @@ async def register_scehma(schema_registry_client): {"name": "favorite_color", "type": ["string", "null"]} ] }""" - schema_properties = await schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) + schema_properties = await schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_CONTENT) schem_id = schema_properties.schema_id # [END register_schema_async] + return schem_id async def get_schema(schema_registry_client, schema_id): @@ -71,8 +73,36 @@ async def get_schema(schema_registry_client, schema_id): # [END get_schema_async] -async def get_schema_id(schema_registry_client, schema_group, schema_name, serialization_type, schema_string): +async def get_schema_id(schema_registry_client): + schema_group = os.environ['SCHEMA_REGISTRY_GROUP'] + schema_name = 'your-schema-name' + serialization_type = SerializationType.AVRO + schema_content = """ + {"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] + }""" + # [START get_schema_id_async] - schema_properties = await schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + schema_properties = await schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_content) schem_id = schema_properties.schema_id # [END get_schema_id_async] + + +async def main(): + client, credential = create_client() + async with client, credential: + id = await register_scehma(client) + schema = await get_schema(client, id) + id = await get_schema_id(client) + + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py index 84e8cca571eb..2efadb028dca 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py @@ -42,7 +42,7 @@ async def register_schema(client, schema_group, schema_name, serialization_type, print("Registering schema...") schema_properties = await client.register_schema(schema_group, schema_name, serialization_type, schema_string) print("Schema registered, returned schema id is {}".format(schema_properties.schema_id)) - print("Schema meta properties are {}".format(schema_properties)) + print("Schema properties are {}".format(schema_properties)) return schema_properties.schema_id @@ -50,7 +50,7 @@ async def get_schema_by_id(client, schema_id): print("Getting schema by id...") schema = await client.get_schema(schema_id) print("The schema string of schema id: {} string is {}".format(schema_id, schema.schema_content)) - print("Schema meta properties are {}".format(schema_id)) + print("Schema properties are {}".format(schema_id)) return schema.schema_content @@ -58,7 +58,7 @@ async def get_schema_id(client, schema_group, schema_name, serialization_type, s print("Getting schema id...") schema_properties = await client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) print("The schema id is: {}".format(schema_properties.schema_id)) - print("Schema meta properties are {}".format(schema_properties)) + print("Schema properties are {}".format(schema_properties)) return schema_properties.schema_id diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py index eb7f54dfc29d..30c7781f889d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py @@ -48,7 +48,7 @@ def register_scehma(schema_registry_client): SCHEMA_GROUP = os.environ['SCHEMA_REGISTRY_GROUP'] SCHEMA_NAME = 'your-schema-name' SERIALIZATION_TYPE = SerializationType.AVRO - SCHEMA_STRING = """ + SCHEMA_CONTENT = """ {"namespace": "example.avro", "type": "record", "name": "User", @@ -58,17 +58,17 @@ def register_scehma(schema_registry_client): {"name": "favorite_color", "type": ["string", "null"]} ] }""" - schema_properties = schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_STRING) + schema_properties = schema_registry_client.register_schema(SCHEMA_GROUP, SCHEMA_NAME, SERIALIZATION_TYPE, SCHEMA_CONTENT) schem_id = schema_properties.schema_id # [END register_schema_sync] - # [START print_schema_id] + # [START print_schema_properties] print(schema_properties.schema_id) print(schema_properties.location) print(schema_properties.location_by_id) - print(schema_properties.type) + print(schema_properties.serialization_type) print(schema_properties.version) - # [END print_schema_id] + # [END print_schema_properties] return schem_id @@ -85,8 +85,29 @@ def get_schema(schema_registry_client, schema_id): # [END print_schema] -def get_schema_id(schema_registry_client, schema_group, schema_name, serialization_type, schema_string): +def get_schema_id(schema_registry_client): + schema_group = os.environ['SCHEMA_REGISTRY_GROUP'] + schema_name = 'your-schema-name' + serialization_type = SerializationType.AVRO + schema_content = """ + {"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] + }""" # [START get_schema_id_sync] - schema_properties = schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) + schema_properties = schema_registry_client.get_schema_id(schema_group, schema_name, serialization_type, schema_content) schem_id = schema_properties.schema_id # [END get_schema_id_sync] + + +if __name__ == '__main__': + client = create_client() + with client: + id = register_scehma(client) + schema = get_schema(client, id) + id = get_schema_id(client) diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py index a94f6e0586e6..e1c1d80e6641 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py @@ -61,7 +61,7 @@ def register_schema(client, schema_group, schema_name, serialization_type, schem print("Registering schema...") schema_properties = client.register_schema(schema_group, schema_name, serialization_type, schema_string) print("Schema registered, returned schema id is {}".format(schema_properties.schema_id)) - print("Schema meta properties are {}".format(schema_properties)) + print("Schema properties are {}".format(schema_properties)) return schema_properties.schema_id @@ -69,7 +69,7 @@ def get_schema_by_id(client, schema_id): print("Getting schema by id...") schema = client.get_schema(schema_id) print("The schema string of schema id: {} string is {}".format(schema_id, schema.schema_content)) - print("Schema meta properties are {}".format(schema_id)) + print("Schema properties are {}".format(schema_id)) return schema.schema_content @@ -77,7 +77,7 @@ def get_schema_id(client, schema_group, schema_name, serialization_type, schema_ print("Getting schema id...") schema_properties = client.get_schema_id(schema_group, schema_name, serialization_type, schema_string) print("The schema id is: {}".format(schema_properties.schema_id)) - print("Schema meta properties are {}".format(schema_properties)) + print("Schema properties are {}".format(schema_properties)) return schema_properties.schema_id diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic.yaml new file mode 100644 index 000000000000..7f21ba6b7f6e --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4?api-version=2017-04 + response: + body: + string: '{"id":"42278eb1818b4a16bc2580ed5f70e417"}' + headers: + content-type: application/json + date: Tue, 25 Aug 2020 03:07:09 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 42278eb1818b4a16bc2580ed5f70e417 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4?api-version=2017-04 +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: application/json + date: Tue, 25 Aug 2020 03:07:09 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 42278eb1818b4a16bc2580ed5f70e417 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: POST + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4?api-version=2017-04 + response: + body: + string: '{"id":"42278eb1818b4a16bc2580ed5f70e417"}' + headers: + content-type: application/json + date: Tue, 25 Aug 2020 03:07:09 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 42278eb1818b4a16bc2580ed5f70e417 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml new file mode 100644 index 000000000000..bb6efc9c06e7 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b?api-version=2017-04 + response: + body: + string: '{"id":"25e3ce5c21844faba13d9cb8a76aa0de"}' + headers: + content-type: application/json + date: Tue, 01 Sep 2020 21:36:47 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 25e3ce5c21844faba13d9cb8a76aa0de + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b?api-version=2017-04 +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: application/json + date: Tue, 01 Sep 2020 21:36:47 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 25e3ce5c21844faba13d9cb8a76aa0de + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: POST + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b?api-version=2017-04 + response: + body: + string: '{"id":"25e3ce5c21844faba13d9cb8a76aa0de"}' + headers: + content-type: application/json + date: Tue, 01 Sep 2020 21:36:48 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 25e3ce5c21844faba13d9cb8a76aa0de + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml new file mode 100644 index 000000000000..3dd2693c06ca --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml @@ -0,0 +1,51 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 + response: + body: + string: 400SubCode=40000, UnknownType:The request + is invalid. [MGResponseHttpError=BadRequest]. TrackingId:6451c6b4-cd1d-4154-9144-1d8198dbe79c_G4, + SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/a, + Timestamp:2020-09-01T21:36:49 + headers: + content-type: application/xml; charset=utf-8 + date: Tue, 01 Sep 2020 21:36:49 GMT + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 400 + message: Bad Request + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 + response: + body: + string: 404Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + does not exist. TrackingId:8ba23261-9feb-4628-89d5-61946780df35_G4, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, + Timestamp:2020-09-01T21:36:50 + headers: + content-type: application/xml; charset=utf-8 + date: Tue, 01 Sep 2020 21:36:49 GMT + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 404 + message: Not Found + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update.yaml new file mode 100644 index 000000000000..cfae03285dbc --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d?api-version=2017-04 + response: + body: + string: '{"id":"fd3af450331b427a90260487b6a593f1"}' + headers: + content-type: application/json + date: Tue, 25 Aug 2020 03:11:20 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: fd3af450331b427a90260487b6a593f1 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/fd3af450331b427a90260487b6a593f1?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d?api-version=2017-04 +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '244' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d?api-version=2017-04 + response: + body: + string: '{"id":"5a04b4ccfd9e4dc6bc98adcd04785031"}' + headers: + content-type: application/json + date: Tue, 25 Aug 2020 03:11:20 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions/2?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 5a04b4ccfd9e4dc6bc98adcd04785031 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5a04b4ccfd9e4dc6bc98adcd04785031?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '2' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d?api-version=2017-04 +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/5a04b4ccfd9e4dc6bc98adcd04785031?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: application/json + date: Tue, 25 Aug 2020 03:11:21 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions/2?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 5a04b4ccfd9e4dc6bc98adcd04785031 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5a04b4ccfd9e4dc6bc98adcd04785031?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '2' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/5a04b4ccfd9e4dc6bc98adcd04785031?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml new file mode 100644 index 000000000000..185e5087a705 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc?api-version=2017-04 + response: + body: + string: '{"id":"67ab61f42b964a42820ed8b6e8cec8e7"}' + headers: + content-type: application/json + date: Tue, 01 Sep 2020 21:36:56 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 67ab61f42b964a42820ed8b6e8cec8e7 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/67ab61f42b964a42820ed8b6e8cec8e7?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc?api-version=2017-04 +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '244' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc?api-version=2017-04 + response: + body: + string: '{"id":"163cc506a02c4b31b4b068b347bfae4c"}' + headers: + content-type: application/json + date: Tue, 01 Sep 2020 21:36:56 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions/2?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 163cc506a02c4b31b4b068b347bfae4c + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/163cc506a02c4b31b4b068b347bfae4c?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '2' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc?api-version=2017-04 +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/163cc506a02c4b31b4b068b347bfae4c?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: application/json + date: Tue, 01 Sep 2020 21:36:57 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions/2?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 163cc506a02c4b31b4b068b347bfae4c + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/163cc506a02c4b31b4b068b347bfae4c?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '2' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/163cc506a02c4b31b4b068b347bfae4c?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py index 8528a4f25b52..e370466dfa65 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py @@ -46,24 +46,24 @@ async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_ assert schema_properties.location is not None assert schema_properties.location_by_id is not None assert schema_properties.version is 1 - assert schema_properties.type == "Avro" + assert schema_properties.serialization_type == "Avro" returned_schema = await client.get_schema(schema_id=schema_properties.schema_id) - assert returned_schema.schema_id == schema_properties.schema_id - assert returned_schema.location is not None - assert returned_schema.location_by_id is not None - assert returned_schema.version == 1 - assert returned_schema.type == "Avro" - assert returned_schema.content == schema_str + assert returned_schema.schema_properties.schema_id == schema_properties.schema_id + assert returned_schema.schema_properties.location is not None + assert returned_schema.schema_properties.location_by_id is not None + assert returned_schema.schema_properties.version == 1 + assert returned_schema.schema_properties.serialization_type == "Avro" + assert returned_schema.schema_content == schema_str - returned_schema_id = await client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) + returned_schema_properties = await client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) - assert returned_schema_id.schema_id == schema_properties.schema_id - assert returned_schema_id.location is not None - assert returned_schema_id.location_by_id is not None - assert returned_schema_id.version == 1 - assert returned_schema_id.type == "Avro" + assert returned_schema_properties.schema_id == schema_properties.schema_id + assert returned_schema_properties.location is not None + assert returned_schema_properties.location_by_id is not None + assert returned_schema_properties.version == 1 + assert returned_schema_properties.serialization_type == "Avro" @pytest.mark.liveTest @pytest.mark.live_test_only @@ -81,7 +81,7 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry assert schema_properties.location is not None assert schema_properties.location_by_id is not None assert schema_properties.version == 1 - assert schema_properties.type == "Avro" + assert schema_properties.serialization_type == "Avro" schema_str_new = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_food","type":["string","null"]}]}""" new_schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str_new) @@ -90,17 +90,17 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry assert new_schema_properties.location is not None assert new_schema_properties.location_by_id is not None assert new_schema_properties.version == 2 - assert new_schema_properties.type == "Avro" + assert new_schema_properties.serialization_type == "Avro" new_schema = await client.get_schema(schema_id=new_schema_properties.schema_id) - assert new_schema.schema_id != schema_properties.schema_id - assert new_schema.schema_id == new_schema_properties.schema_id - assert new_schema.location is not None - assert new_schema.location_by_id is not None - assert new_schema.content == schema_str_new - assert new_schema.version == 2 - assert new_schema.type == "Avro" + assert new_schema.schema_properties.schema_id != schema_properties.schema_id + assert new_schema.schema_properties.schema_id == new_schema_properties.schema_id + assert new_schema.schema_properties.location is not None + assert new_schema.schema_properties.location_by_id is not None + assert new_schema.schema_content == schema_str_new + assert new_schema.schema_properties.version == 2 + assert new_schema.schema_properties.serialization_type == "Avro" @pytest.mark.liveTest @pytest.mark.live_test_only diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_register_schema.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_register_schema.yaml new file mode 100644 index 000000000000..25f94760b004 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_register_schema.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: '"somestr"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '9' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43?api-version=2017-04 + response: + body: + string: '{"id":"2cc9a5eb78484e1cac2e5b547b0374ef"}' + headers: + content-type: + - application/json + date: + - Tue, 25 Aug 2020 00:43:02 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 2cc9a5eb78484e1cac2e5b547b0374ef + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2cc9a5eb78484e1cac2e5b547b0374ef?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/2cc9a5eb78484e1cac2e5b547b0374ef?api-version=2017-04 + response: + body: + string: '"somestr"' + headers: + content-type: + - application/json + date: + - Tue, 25 Aug 2020 00:43:02 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 2cc9a5eb78484e1cac2e5b547b0374ef + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2cc9a5eb78484e1cac2e5b547b0374ef?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"somestr"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '9' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: POST + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43?api-version=2017-04 + response: + body: + string: '{"id":"2cc9a5eb78484e1cac2e5b547b0374ef"}' + headers: + content-type: + - application/json + date: + - Tue, 25 Aug 2020 00:43:04 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 2cc9a5eb78484e1cac2e5b547b0374ef + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2cc9a5eb78484e1cac2e5b547b0374ef?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml new file mode 100644 index 000000000000..a23e6fd49851 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61?api-version=2017-04 + response: + body: + string: '{"id":"b7dda07056e6400aaa547a86d181e7bc"}' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 21:34:59 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - b7dda07056e6400aaa547a86d181e7bc + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b7dda07056e6400aaa547a86d181e7bc?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/b7dda07056e6400aaa547a86d181e7bc?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 21:34:59 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - b7dda07056e6400aaa547a86d181e7bc + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b7dda07056e6400aaa547a86d181e7bc?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: POST + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61?api-version=2017-04 + response: + body: + string: '{"id":"b7dda07056e6400aaa547a86d181e7bc"}' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 21:34:59 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - b7dda07056e6400aaa547a86d181e7bc + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b7dda07056e6400aaa547a86d181e7bc?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml new file mode 100644 index 000000000000..5bfe01241dcc --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 + response: + body: + string: 400SubCode=40000, UnknownType:The request + is invalid. [MGResponseHttpError=BadRequest]. TrackingId:d63523f2-3c03-4d54-afab-fc9386bd81fb_G0, + SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/a, + Timestamp:2020-09-01T21:35:01 + headers: + content-type: + - application/xml; charset=utf-8 + date: + - Tue, 01 Sep 2020 21:35:01 GMT + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 400 + message: Bad Request +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 + response: + body: + string: 404Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + does not exist. TrackingId:a7b590bf-6862-43cd-b9ce-c6ed5d6d5017_G0, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, + Timestamp:2020-09-01T21:35:01 + headers: + content-type: + - application/xml; charset=utf-8 + date: + - Tue, 01 Sep 2020 21:35:01 GMT + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml new file mode 100644 index 000000000000..2d7a1411ad3f --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":[\"int\",\"null\"]},{\"name\":\"city\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '223' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5?api-version=2017-04 + response: + body: + string: '{"id":"c02dde8d030743b2bd76781606ed5b61"}' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 21:35:08 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - c02dde8d030743b2bd76781606ed5b61 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c02dde8d030743b2bd76781606ed5b61?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":[\"int\",\"null\"]},{\"name\":\"city\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '223' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5?api-version=2017-04 + response: + body: + string: '{"id":"c02dde8d030743b2bd76781606ed5b61"}' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 21:35:08 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - c02dde8d030743b2bd76781606ed5b61 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c02dde8d030743b2bd76781606ed5b61?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml new file mode 100644 index 000000000000..b1f25e1e1957 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b?api-version=2017-04 + response: + body: + string: '{"id":"e7f36b1ffc6142c2a9fe713d02a61448"}' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 21:35:09 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - e7f36b1ffc6142c2a9fe713d02a61448 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/e7f36b1ffc6142c2a9fe713d02a61448?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '244' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b?api-version=2017-04 + response: + body: + string: '{"id":"b6caf7d1944a4f8e8247cf2f2bb9c15d"}' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 21:35:09 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions/2?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - b6caf7d1944a4f8e8247cf2f2bb9c15d + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b6caf7d1944a4f8e8247cf2f2bb9c15d?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '2' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/b6caf7d1944a4f8e8247cf2f2bb9c15d?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: + - application/json + date: + - Tue, 01 Sep 2020 21:35:11 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions/2?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - b6caf7d1944a4f8e8247cf2f2bb9c15d + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b6caf7d1944a4f8e8247cf2f2bb9c15d?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '2' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index 3b4a5bc66f0b..fd78e1bd2d83 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -49,11 +49,11 @@ def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schem returned_schema = client.get_schema(schema_id=schema_properties.schema_id) - assert returned_schema.schema_id == schema_properties.schema_id - assert returned_schema.location is not None - assert returned_schema.location_by_id is not None - assert returned_schema.version == 1 - assert returned_schema.serialization_type == "Avro" + assert returned_schema.schema_properties.schema_id == schema_properties.schema_id + assert returned_schema.schema_properties.location is not None + assert returned_schema.schema_properties.location_by_id is not None + assert returned_schema.schema_properties.version == 1 + assert returned_schema.schema_properties.serialization_type == "Avro" assert returned_schema.schema_content == schema_str returned_schema_properties = client.get_schema_id(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -92,13 +92,13 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, sche new_schema = client.get_schema(schema_id=new_schema_properties.schema_id) - assert new_schema_properties.schema_id != schema_properties.schema_id - assert new_schema.schema_id == new_schema_properties.schema_id - assert new_schema.location is not None - assert new_schema.location_by_id is not None + assert new_schema.schema_properties.schema_id != schema_properties.schema_id + assert new_schema.schema_properties.schema_id == new_schema_properties.schema_id + assert new_schema.schema_properties.location is not None + assert new_schema.schema_properties.location_by_id is not None assert new_schema.schema_content == schema_str_new - assert new_schema.version == 2 - assert new_schema.serialization_type == "Avro" + assert new_schema.schema_properties.version == 2 + assert new_schema.schema_properties.serialization_type == "Avro" @pytest.mark.liveTest @pytest.mark.live_test_only From 92cd38501d9bbae6422ef364894749255bf86ef1 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 1 Sep 2020 15:39:36 -0700 Subject: [PATCH 21/35] update credential config --- sdk/schemaregistry/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/schemaregistry/tests.yml b/sdk/schemaregistry/tests.yml index 9e4c4ca2b84c..a3a5d667351a 100644 --- a/sdk/schemaregistry/tests.yml +++ b/sdk/schemaregistry/tests.yml @@ -11,11 +11,11 @@ jobs: AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id) AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id) AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) - SCHEMA_REGISTRY_AZURE_TENANT_ID: $(schema-registry-sdk-test-tenant-id) - SCHEMA_REGISTRY_AZURE_CLIENT_ID: $(schema-registry-sdk-test-client-id) - SCHEMA_REGISTRY_AZURE_CLIENT_SECRET: $(schema-registry-sdk-test-client-secret) - SCHEMA_REGISTRY_ENDPOINT: $(schema-registry-sdk-test-endpoint) - SCHEMA_REGISTRY_GROUP： $(schema-registry-sdk-test-group) + SCHEMA_REGISTRY_AZURE_TENANT_ID: $(python-schema-registry-sdk-test-tenant-id) + SCHEMA_REGISTRY_AZURE_CLIENT_ID: $(python-schema-registry-sdk-test-client-id) + SCHEMA_REGISTRY_AZURE_CLIENT_SECRET: $(python-schema-registry-sdk-test-client-secret) + SCHEMA_REGISTRY_ENDPOINT: $(python-schema-registry-sdk-test-endpoint) + SCHEMA_REGISTRY_GROUP: $(python-schema-registry-sdk-test-group) AZURE_TEST_RUN_LIVE: 'true' Matrix: Linux_Python35: From 1c6067669532d3f54711fc02f5ea5216c6902a95 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 1 Sep 2020 15:42:58 -0700 Subject: [PATCH 22/35] rename package name to azure-schemaregistry-avroserializer --- .../CHANGELOG.md | 0 .../MANIFEST.in | 0 .../README.md | 0 .../azure/__init__.py | 0 .../azure/schemaregistry/__init__.py | 0 .../azure/schemaregistry/serializer/__init__.py | 0 .../azure/schemaregistry/serializer/avro_serializer/__init__.py | 0 .../schemaregistry/serializer/avro_serializer/_avro_serializer.py | 0 .../avro_serializer/_schema_registry_avro_serializer.py | 0 .../azure/schemaregistry/serializer/avro_serializer/_version.py | 0 .../dev_requirements.txt | 0 .../mypy.ini | 0 .../samples/README.md | 0 .../samples/sync_samples/avro_serializer.py | 0 .../sdk_packaging.toml | 0 .../setup.cfg | 0 .../setup.py | 0 .../test_avro_serializer.test_basic_sr_avro_serializer.yaml | 0 .../tests/schemaregistry_preparer.py | 0 .../tests/test_avro_serializer.py | 0 20 files changed, 0 insertions(+), 0 deletions(-) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/CHANGELOG.md (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/MANIFEST.in (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/README.md (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/azure/__init__.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/azure/schemaregistry/__init__.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/azure/schemaregistry/serializer/__init__.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/azure/schemaregistry/serializer/avro_serializer/__init__.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/azure/schemaregistry/serializer/avro_serializer/_version.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/dev_requirements.txt (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/mypy.ini (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/samples/README.md (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/samples/sync_samples/avro_serializer.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/sdk_packaging.toml (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/setup.cfg (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/setup.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/tests/schemaregistry_preparer.py (100%) rename sdk/schemaregistry/{azure-schemaregistry-avro-serializer => azure-schemaregistry-avroserializer}/tests/test_avro_serializer.py (100%) diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/CHANGELOG.md rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/MANIFEST.in b/sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/MANIFEST.in rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/README.md rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/__init__.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/__init__.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_version.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_version.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/azure/schemaregistry/serializer/avro_serializer/_version.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_version.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/dev_requirements.txt rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/mypy.ini b/sdk/schemaregistry/azure-schemaregistry-avroserializer/mypy.ini similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/mypy.ini rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/mypy.ini diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/README.md rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/samples/sync_samples/avro_serializer.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/avro_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/sdk_packaging.toml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/sdk_packaging.toml similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/sdk_packaging.toml rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/sdk_packaging.toml diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.cfg b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.cfg similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.cfg rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.cfg diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/setup.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/schemaregistry_preparer.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avro-serializer/tests/test_avro_serializer.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py From f20bba0a93b9eca0571ebce35d140275327efd8f Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 1 Sep 2020 16:04:08 -0700 Subject: [PATCH 23/35] fix pylint --- .../serializer/avro_serializer/__init__.py | 2 +- .../avro_serializer/_avro_serializer.py | 2 +- .../_schema_registry_avro_serializer.py | 21 ++++++++++--------- .../_common/_response_handlers.py | 4 ++-- .../schemaregistry/_schema_registry_client.py | 9 +++++--- .../azure/schemaregistry/aio/__init__.py | 2 +- .../aio/_schema_registry_client_async.py | 9 +++++--- 7 files changed, 28 insertions(+), 21 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/__init__.py index 0edc418a9f22..fe999769d03e 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/__init__.py @@ -31,4 +31,4 @@ __all__ = [ "SchemaRegistryAvroSerializer" -] \ No newline at end of file +] diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py index 808879f6c37f..76ff8c105083 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -83,7 +83,7 @@ def deserialize( self, data, # type: Union[bytes, BinaryIO] schema, # type: - return_type=None, # type: Optional[Type[ObjectType]] + return_type=None, # type: Optional[Type[ObjectType]] # pylint: disable=unused-argument ): # type: (...) -> ObjectType """Read the binary representation into a specific type. diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index acacd23981e3..44e3c95d480b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -46,7 +46,7 @@ class SchemaRegistryAvroSerializer(object): def __init__(self, endpoint, credential, schema_group, **kwargs): self._schema_group = schema_group self._avro_serializer = AvroObjectSerializer() - self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint) + self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint, **kwargs) self._id_to_schema = {} self._schema_to_id = {} self._user_input_schema_cache = {} @@ -67,8 +67,8 @@ def close(self): """ self._schema_registry_client.close() - def _get_schema_id(self, schema_name, schema): - # type: (str, avro.schema.Schema) -> str + def _get_schema_id(self, schema_name, schema, **kwargs): + # type: (str, avro.schema.Schema, Any) -> str """ :param schema_name: @@ -83,14 +83,15 @@ def _get_schema_id(self, schema_name, schema): self._schema_group, schema_name, SerializationType.AVRO, - schema_str + schema_str, + **kwargs ).schema_id self._schema_to_id[schema_str] = schema_id self._id_to_schema[schema_id] = schema_str return schema_id - def _get_schema(self, schema_id): - # type: (str) -> str + def _get_schema(self, schema_id, **kwargs): + # type: (str, Any) -> str """ :param str schema_id: @@ -99,7 +100,7 @@ def _get_schema(self, schema_id): try: return self._id_to_schema[schema_id] except KeyError: - schema_str = self._schema_registry_client.get_schema(schema_id).content + schema_str = self._schema_registry_client.get_schema(schema_id, **kwargs).schema_content self._id_to_schema[schema_id] = schema_str self._schema_to_id[schema_str] = schema_id return schema_str @@ -123,7 +124,7 @@ def serialize(self, data, schema, **kwargs): cached_schema = parsed_schema record_format_identifier = b'\0\0\0\0' - schema_id = self._get_schema_id(cached_schema.fullname, cached_schema) + schema_id = self._get_schema_id(cached_schema.fullname, cached_schema, **kwargs) data_bytes = self._avro_serializer.serialize(data, cached_schema) stream = BytesIO() @@ -145,9 +146,9 @@ def deserialize(self, data, **kwargs): :param bytes data: The bytes data needs to be decoded. :rtype: Dict[str, Any] """ - record_format_identifier = data[0:4] + record_format_identifier = data[0:4] # pylint: disable=unused-variable schema_id = data[4:36].decode('utf-8') - schema_content = self._get_schema(schema_id) + schema_content = self._get_schema(schema_id, **kwargs) dict_data = self._avro_serializer.deserialize(data[36:], schema_content) return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py index e24d7e04a052..e1ce9f4424c6 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_response_handlers.py @@ -31,7 +31,7 @@ from ._schema import SchemaProperties, Schema -def _parse_response_schema_id(pipeline_response, deserialized, response_headers): +def _parse_response_schema_id(pipeline_response, deserialized, response_headers): # pylint: disable=unused-argument # type: (PipelineResponse, InternalSchemaId, Dict[str, Any]) -> SchemaProperties """ @@ -43,7 +43,7 @@ def _parse_response_schema_id(pipeline_response, deserialized, response_headers) return SchemaProperties(schema_id=deserialized.id, **response_headers) -def _parse_response_schema(pipeline_response, deserialized, response_headers): +def _parse_response_schema(pipeline_response, deserialized, response_headers): # pylint: disable=unused-argument # type: (PipelineResponse, str, Dict[str, Any]) -> Schema """ diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index 3f62047391d2..b9d384ca33cd 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -112,7 +112,8 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ schema_name=schema_name, schema_content=schema_content, #serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter - cls=_parse_response_schema_id + cls=_parse_response_schema_id, + **kwargs ) def get_schema(self, schema_id, **kwargs): @@ -136,7 +137,8 @@ def get_schema(self, schema_id, **kwargs): """ return self._generated_client.schema.get_by_id( schema_id, - cls=_parse_response_schema + cls=_parse_response_schema, + **kwargs ) def get_schema_id(self, schema_group, schema_name, serialization_type, schema_content, **kwargs): @@ -173,5 +175,6 @@ def get_schema_id(self, schema_group, schema_name, serialization_type, schema_co schema_name=schema_name, schema_content=schema_content, #serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter - cls=_parse_response_schema_id + cls=_parse_response_schema_id, + **kwargs ) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py index e1ff81128426..07b86b655bcd 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/__init__.py @@ -27,4 +27,4 @@ __all__ = [ "SchemaRegistryClient" -] \ No newline at end of file +] diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index c328d3e51487..2eca2c1579f5 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -68,7 +68,7 @@ async def __aenter__(self): async def __aexit__(self, *args): await self._generated_client.__aexit__(*args) - async def close(self): + async def close(self) -> None: """ This method is to close the sockets opened by the client. It need not be used when using with a context manager. """ @@ -115,6 +115,7 @@ async def register_schema( schema_content=schema_content, # serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter cls=_parse_response_schema_id, + **kwargs ) async def get_schema( @@ -141,7 +142,8 @@ async def get_schema( """ return await self._generated_client.schema.get_by_id( schema_id=schema_id, - cls=_parse_response_schema + cls=_parse_response_schema, + **kwargs ) async def get_schema_id( @@ -183,5 +185,6 @@ async def get_schema_id( schema_name=schema_name, schema_content=schema_content, # serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter - cls=_parse_response_schema_id + cls=_parse_response_schema_id, + **kwargs ) From c81f16bd4c569953c39219dcd4562c8854377e21 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Wed, 2 Sep 2020 11:40:50 -0700 Subject: [PATCH 24/35] try ci fix --- .../azure-schemaregistry-avroserializer/setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py index c166eb266d22..92705a2877cf 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py @@ -67,5 +67,10 @@ install_requires=[ 'azure-schemaregistry<2.0.0,>=1.0.0b1', 'avro<2.0.0,>=1.0.0' - ] + ], + extras_require={ + ":python_version<'3.0'": ['futures'], + ":python_version<'3.4'": ['enum34>=1.0.4'], + ":python_version<'3.5'": ["typing"] + } ) From 41ee64b908b3273ebe3a8a98130ed56529e86f78 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Wed, 2 Sep 2020 17:40:49 -0700 Subject: [PATCH 25/35] fix test for py27 as avro only accept unicode --- .../samples/sync_samples/avro_serializer.py | 2 +- ...ializer.test_basic_sr_avro_serializer.yaml | 48 +++++++++---------- .../tests/test_avro_serializer.py | 48 +++++++++---------- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/avro_serializer.py index 1c3c1eaff909..2bf1894abec8 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/avro_serializer.py @@ -67,6 +67,6 @@ def deserialize(serializer, bytes): if __name__ == '__main__': serializer = SchemaRegistryAvroSerializer(SCHEMA_REGISTRY_ENDPOINT, token_credential, SCHEMA_GROUP) - dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} payload_bytes = serialize(serializer, SCHEMA_STRING, dict_data) dict_data = deserialize(serializer, payload_bytes) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml index b87eebc8e8ed..911c35c88fc7 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml @@ -1,9 +1,9 @@ interactions: - request: - body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"testschemafc48df19\", - \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", - \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], - \"name\": \"favorite_color\"}]}"' + body: !!python/unicode '"{\"type\": \"record\", \"namespace\": \"testschema09bb017e\", + \"name\": \"User\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, + {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", + \"null\"], \"name\": \"favorite_color\"}]}"' headers: Accept: - application/json @@ -16,21 +16,21 @@ interactions: Content-Type: - application/json User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + - azsdk-python-azureschemaregistry/1.0.0b1 Python/2.7.18 (Windows-10-10.0.19041) X-Schema-Type: - avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User?api-version=2017-04 response: body: - string: '{"id":"07809b86375a42ae8463e77de063ab0b"}' + string: !!python/unicode '{"id":"035f2334e3af4417a8011cb0f895088d"}' headers: content-type: - application/json date: - - Tue, 01 Sep 2020 22:34:11 GMT + - Thu, 03 Sep 2020 00:39:52 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -38,23 +38,23 @@ interactions: transfer-encoding: - chunked x-schema-id: - - 07809b86375a42ae8463e77de063ab0b + - 035f2334e3af4417a8011cb0f895088d x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/07809b86375a42ae8463e77de063ab0b?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/035f2334e3af4417a8011cb0f895088d?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User/versions?api-version=2017-04 status: code: 200 message: OK - request: - body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"testschemafc48df19\", - \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", - \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], - \"name\": \"favorite_color\"}]}"' + body: !!python/unicode '"{\"type\": \"record\", \"namespace\": \"testschema09bb017e\", + \"name\": \"User\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, + {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", + \"null\"], \"name\": \"favorite_color\"}]}"' headers: Accept: - application/json @@ -67,21 +67,21 @@ interactions: Content-Type: - application/json User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + - azsdk-python-azureschemaregistry/1.0.0b1 Python/2.7.18 (Windows-10-10.0.19041) X-Schema-Type: - avro method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User?api-version=2017-04 response: body: - string: '{"id":"07809b86375a42ae8463e77de063ab0b"}' + string: !!python/unicode '{"id":"035f2334e3af4417a8011cb0f895088d"}' headers: content-type: - application/json date: - - Tue, 01 Sep 2020 22:34:12 GMT + - Thu, 03 Sep 2020 00:39:53 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -89,15 +89,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - 07809b86375a42ae8463e77de063ab0b + - 035f2334e3af4417a8011cb0f895088d x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/07809b86375a42ae8463e77de063ab0b?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/035f2334e3af4417a8011cb0f895088d?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemafc48df19.User/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User/versions?api-version=2017-04 status: code: 200 message: OK diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py index df7ed8b29b14..42bc09eb4fe0 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -40,27 +40,27 @@ class SchemaRegistryAvroSerializerTests(AzureMgmtTestCase): def test_raw_avro_serializer(self): schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) - dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - raw_avro_object_serailizer = AvroObjectSerializer() + raw_avro_object_serializer = AvroObjectSerializer() # encoding part - encoded_payload = raw_avro_object_serailizer.serialize(dict_data, schema) + encoded_payload = raw_avro_object_serializer.serialize(dict_data, schema) # decoding part - decoded_data = raw_avro_object_serailizer.deserialize(encoded_payload, schema) + decoded_data = raw_avro_object_serializer.deserialize(encoded_payload, schema) - assert decoded_data['name'] == "Ben" - assert decoded_data['favorite_number'] == 7 - assert decoded_data['favorite_color'] == 'red' + assert decoded_data["name"] == u"Ben" + assert decoded_data["favorite_number"] == 7 + assert decoded_data["favorite_color"] == u"red" - dict_data_missing_optional_fields = {"name": "Alice"} - encoded_payload = raw_avro_object_serailizer.serialize(dict_data_missing_optional_fields, schema) - decoded_data = raw_avro_object_serailizer.deserialize(encoded_payload, schema) + dict_data_missing_optional_fields = {"name": u"Alice"} + encoded_payload = raw_avro_object_serializer.serialize(dict_data_missing_optional_fields, schema) + decoded_data = raw_avro_object_serializer.deserialize(encoded_payload, schema) - assert decoded_data['name'] == "Alice" - assert not decoded_data['favorite_number'] - assert not decoded_data['favorite_color'] + assert decoded_data["name"] == u"Alice" + assert not decoded_data["favorite_number"] + assert not decoded_data["favorite_color"] @pytest.mark.liveTest @pytest.mark.live_test_only @@ -68,14 +68,14 @@ def test_raw_avro_serializer_negative(self): schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) - raw_avro_object_serailizer = AvroObjectSerializer() - dict_data_wrong_type = {"name": "Ben", "favorite_number": "something", "favorite_color": "red"} + raw_avro_object_serializer = AvroObjectSerializer() + dict_data_wrong_type = {"name": u"Ben", "favorite_number": u"something", "favorite_color": u"red"} with pytest.raises(avro.io.AvroTypeException): - raw_avro_object_serailizer.serialize(dict_data_wrong_type, schema) + raw_avro_object_serializer.serialize(dict_data_wrong_type, schema) - dict_data_missing_required_field = {"favorite_number": 7, "favorite_color": "red"} + dict_data_missing_required_field = {"favorite_number": 7, "favorite_color": u"red"} with pytest.raises(avro.io.AvroTypeException): - raw_avro_object_serailizer.serialize(dict_data_missing_required_field, schema) + raw_avro_object_serializer.serialize(dict_data_missing_required_field, schema) @pytest.mark.liveTest @pytest.mark.live_test_only @@ -88,12 +88,12 @@ def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_ sr_avro_serializer = SchemaRegistryAvroSerializer(schemaregistry_endpoint, credential, schemaregistry_group) sr_client = SchemaRegistryClient(schemaregistry_endpoint, credential) - random_schema_namespace = 'testschema' + str(uuid.uuid4())[0:8] + random_schema_namespace = "testschema" + str(uuid.uuid4())[0:8] schema_str = "{\"namespace\":\"" + random_schema_namespace + "\"" +\ ""","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) - dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} encoded_data = sr_avro_serializer.serialize(dict_data, schema_str) assert schema_str in sr_avro_serializer._user_input_schema_cache @@ -101,14 +101,14 @@ def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_ assert encoded_data[0:4] == b'\0\0\0\0' schema_id = sr_client.get_schema_id(schemaregistry_group, schema.fullname, "Avro", str(schema)).schema_id - assert encoded_data[4:36] == schema_id.encode('utf-8') + assert encoded_data[4:36] == schema_id.encode("utf-8") assert schema_id in sr_avro_serializer._id_to_schema decoded_data = sr_avro_serializer.deserialize(encoded_data) - assert decoded_data['name'] == "Ben" - assert decoded_data['favorite_number'] == 7 - assert decoded_data['favorite_color'] == 'red' + assert decoded_data["name"] == u"Ben" + assert decoded_data["favorite_number"] == 7 + assert decoded_data["favorite_color"] == u"red" sr_avro_serializer.close() sr_client.close() From 26753310697ea0caba32f37f40767e8739d35814 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Wed, 2 Sep 2020 20:52:29 -0700 Subject: [PATCH 26/35] first round of review feedback --- .../CHANGELOG.md | 10 ++++- .../MANIFEST.in | 2 + .../README.md | 36 +++++++++++++++++ .../avro_serializer/_avro_serializer.py | 23 +++++------ .../_schema_registry_avro_serializer.py | 21 ++++++---- .../dev_requirements.txt | 2 +- .../samples/README.md | 8 ++-- .../{sync_samples => }/avro_serializer.py | 0 .../tests/test_avro_serializer.py | 4 -- .../azure-schemaregistry/CHANGELOG.md | 11 ++++- .../azure-schemaregistry/README.md | 40 +++++++++++++++++-- .../azure/schemaregistry/_common/_schema.py | 4 +- .../azure-schemaregistry/dev_requirements.txt | 4 +- 13 files changed, 123 insertions(+), 42 deletions(-) rename sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/{sync_samples => }/avro_serializer.py (100%) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md index 582d5d5bdb16..d132099f47a4 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History -## 1.0.0b1 (Unreleased) +## 1.0.0b1 (2020-09-08) -Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry Avro Serializer. \ No newline at end of file +Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry Avro Serializer. + +**New features** + +- `SchemaRegistryAvroSerializer` is the top-level client class that provides the functionality to encode and decode avro data utilizing the avro library. It will automatically register schema and retrieve schema from Azure Schema Registry Service. It provides two methods: + - `serialize`: Serialize dict data into bytes according to the given schema and register schema if needed. + - `deserialize`: Deserialize bytes data into dict data by automatically retrieving schema from the service. diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in b/sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in index 3ba8ed1d9a45..90e5d6eb2bd7 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in @@ -1,4 +1,6 @@ include *.md include azure/__init__.py +include azure/schemaregistry/__init__.py +include azure/schemaregistry/serializer/__init__.py recursive-include tests *.py recursive-include samples *.py \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md index 1c4dbb4da456..9f300f3962e2 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md @@ -94,8 +94,43 @@ with serializer: ## Troubleshooting +### General + Azure Schema Registry Avro Serializer raise exceptions defined in [Azure Core][azure_core]. +### Logging +This library uses the standard +[logging][python_logging] library for logging. +Basic information about HTTP sessions (URLs, headers, etc.) is logged at INFO +level. + +Detailed DEBUG level logging, including request/response bodies and unredacted +headers, can be enabled on a client with the `logging_enable` argument: +```python +import sys +import logging +from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.identity import DefaultAzureCredential + +# Create a logger for the SDK +logger = logging.getLogger('azure.schemaregistry') +logger.setLevel(logging.DEBUG) + +# Configure a console output +handler = logging.StreamHandler(stream=sys.stdout) +logger.addHandler(handler) + +credential = DefaultAzureCredential() +# This client will log detailed information about its HTTP sessions, at DEBUG level +avro_serializer = SchemaRegistryAvroSerializer("you_end_point", credential, "your_group_name", logging_enable=True) +``` + +Similarly, `logging_enable` can enable detailed logging for a single operation, +even when it isn't enabled for the client: +```py +avro_serializer.serialie(dict_data, schema_content, logging_enable=True) +``` + ## Next steps ### More sample code @@ -120,3 +155,4 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio [python]: https://www.python.org/downloads/ [azure_core]: https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/README.md [azure_sub]: https://azure.microsoft.com/free/ +[python_logging]: https://docs.python.org/3/library/logging.html diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py index 76ff8c105083..a3fc8d3ad6ba 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py @@ -23,21 +23,15 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -import abc -from typing import BinaryIO, Union, Type, TypeVar, Optional, Any, Dict +from typing import BinaryIO, Union, TypeVar, Dict from io import BytesIO import avro from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder -try: - ABC = abc.ABC -except AttributeError: # Python 2.7, abc exists, but not ABC - ABC = abc.ABCMeta("ABC", (object,), {"__slots__": ()}) # type: ignore - ObjectType = TypeVar("ObjectType") -class AvroObjectSerializer(ABC): +class AvroObjectSerializer(object): def __init__(self, codec=None): """A Avro serializer using avro lib from Apache. @@ -50,14 +44,16 @@ def __init__(self, codec=None): def serialize( self, data, # type: ObjectType - schema, # type: Optional[Any] + schema, # type: Union[str, bytes, avro.schema.Schema] ): - # type: (...) -> bytes + # type: (ObjectType, Union[str, bytes, avro.schema.Schema]) -> bytes """Convert the provided value to it's binary representation and write it to the stream. Schema must be a Avro RecordSchema: https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema :param data: An object to serialize + :type data: ObjectType :param schema: A Avro RecordSchema + :type schema: Union[str, bytes, avro.schema.Schema] """ if not schema: raise ValueError("Schema is required in Avro serializer.") @@ -82,16 +78,15 @@ def serialize( def deserialize( self, data, # type: Union[bytes, BinaryIO] - schema, # type: - return_type=None, # type: Optional[Type[ObjectType]] # pylint: disable=unused-argument + schema, # type: Union[str, bytes, avro.schema.Schema] ): - # type: (...) -> ObjectType + # type: (Union[bytes, BinaryIO], Union[str, bytes, avro.schema.Schema]) -> ObjectType """Read the binary representation into a specific type. Return type will be ignored, since the schema is deduced from the provided bytes. :param data: A stream of bytes or bytes directly :type data: BinaryIO or bytes :param schema: A Avro RecordSchema - :param return_type: Return type is not supported in the Avro serializer. + :type schema: Union[str, bytes, avro.schema.Schema] :returns: An instantiated object :rtype: ObjectType """ diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py index 44e3c95d480b..ebe6e8da1b74 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py @@ -70,10 +70,15 @@ def close(self): def _get_schema_id(self, schema_name, schema, **kwargs): # type: (str, avro.schema.Schema, Any) -> str """ - - :param schema_name: - :param schema: - :return: + Get schema id from local cache with the given schema. + If there is no item in the local cache, get schema id from the service and cache it. + + :param schema_name: Name of the schema + :type schema_name: str + :param schema: Schema object + :type schema: avro.schema.Schema + :return: Schema Id + :rtype: str """ schema_str = str(schema) try: @@ -93,9 +98,11 @@ def _get_schema_id(self, schema_name, schema, **kwargs): def _get_schema(self, schema_id, **kwargs): # type: (str, Any) -> str """ + Get schema content from local cache with the given schema id. + If there is no item in the local cache, get schema from the service and cache it. - :param str schema_id: - :return: + :param str schema_id: Schema id + :return: Schema content """ try: return self._id_to_schema[schema_id] @@ -146,7 +153,7 @@ def deserialize(self, data, **kwargs): :param bytes data: The bytes data needs to be decoded. :rtype: Dict[str, Any] """ - record_format_identifier = data[0:4] # pylint: disable=unused-variable + # record_format_identifier = data[0:4] # The first 4 bytes are retained for future record format identifier. schema_id = data[4:36].decode('utf-8') schema_content = self._get_schema(schema_id, **kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt index fd489752ebe0..c2d5098aa720 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt @@ -1,4 +1,4 @@ --e ../../identity/azure-identity -e ../../../tools/azure-devtools -e ../../../tools/azure-sdk-tools +-e ../../identity/azure-identity ../azure-schemaregistry \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md index 8cefc321f7e8..f3689b6f6a7b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md @@ -11,12 +11,10 @@ urlFragment: schemaregistry-avro-serializer-samples # Azure Schema Registry Avro Serializer library for Python Samples These are code samples that show common scenario operations with the Schema Registry Avro Serializer library. -The async versions of the samples (the python sample files appended with `_async`) show asynchronous operations, -and require Python 3.5 or later. Several Schema Registry Avro Serializer Python SDK samples are available to you in the SDK's GitHub repository. These samples provide example code for additional scenarios commonly encountered while working with Schema Registry Avro Serializer: -* [avro_serializer.py](./sync_samples/avro_serializer.py) - Examples for common Schema Registry Avro Serializer tasks: +* [avro_serializer.py](./avro_serializer.py) - Examples for common Schema Registry Avro Serializer tasks: * Serialize data according to the given schema * Deserialize data @@ -27,10 +25,10 @@ If you do not have an existing Azure account, you may sign up for a free trial o ## Setup -1. Install the Azure Schema Registry client library for Python with [pip](https://pypi.org/project/pip/): +1. Install the Azure Schema Registry Avro Serializer client library for Python with [pip](https://pypi.org/project/pip/): ```bash -pip install azure-schemaregistry-avro-serializer +pip install azure-schemaregistry-avroserializer ``` 2. Clone or download this sample repository diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/avro_serializer.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py index 42bc09eb4fe0..6319f4660815 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -35,8 +35,6 @@ class SchemaRegistryAvroSerializerTests(AzureMgmtTestCase): - @pytest.mark.liveTest - @pytest.mark.live_test_only def test_raw_avro_serializer(self): schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) @@ -62,8 +60,6 @@ def test_raw_avro_serializer(self): assert not decoded_data["favorite_number"] assert not decoded_data["favorite_color"] - @pytest.mark.liveTest - @pytest.mark.live_test_only def test_raw_avro_serializer_negative(self): schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) diff --git a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md index f3535254bb01..22deb5fe313e 100644 --- a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md @@ -1,5 +1,12 @@ # Release History -## 1.0.0b1 (Unreleased) +## 1.0.0b1 (2020-09-08) -Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry. \ No newline at end of file +Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry. + +**New features** + +- `SchemaRegistryClient` is the top-level client class interacting with the Azure Schema Registry Service. It provides three methods: + - `register_schema`: Store schema into the service. + - `get_schema`: Get schema content and its properties by schema id. + - `get_schema_id`: Get schema id and its properties by schema group, schema name, serialization type and schema content. diff --git a/sdk/schemaregistry/azure-schemaregistry/README.md b/sdk/schemaregistry/azure-schemaregistry/README.md index 28145c90afb2..342ff33cbc47 100644 --- a/sdk/schemaregistry/azure-schemaregistry/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/README.md @@ -7,7 +7,7 @@ Avro, Json, etc. ### Install the package -Install the Azure Service Bus client library for Python with [pip][pip]: +Install the Azure Schema Registry client library for Python with [pip][pip]: ```Bash pip install azure-schemaregistry @@ -71,7 +71,7 @@ schema_content = """ """ schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) -with schema_registry_client, token_credential: +with schema_registry_client: schema_properties = schema_registry_client.register_schema(schema_group, schema_name, serialization_type, schema_content) schema_id = schema_properties.schema_id ``` @@ -132,11 +132,44 @@ with schema_registry_client: Schema Registry clients raise exceptions defined in [Azure Core][azure_core]. ### Logging +This library uses the standard +[logging][python_logging] library for logging. +Basic information about HTTP sessions (URLs, headers, etc.) is logged at INFO +level. + +Detailed DEBUG level logging, including request/response bodies and unredacted +headers, can be enabled on a client with the `logging_enable` argument: +```python +import sys +import logging +from azure.schemaregistry import SchemaRegistryClient +from azure.identity import DefaultAzureCredential + +# Create a logger for the SDK +logger = logging.getLogger('azure.schemaregistry') +logger.setLevel(logging.DEBUG) + +# Configure a console output +handler = logging.StreamHandler(stream=sys.stdout) +logger.addHandler(handler) + +credential = DefaultAzureCredential() +# This client will log detailed information about its HTTP sessions, at DEBUG level +schema_registry_client = SchemaRegistryClient("you_end_point", credential, logging_enable=True) +``` + +Similarly, `logging_enable` can enable detailed logging for a single operation, +even when it isn't enabled for the client: +```py +schema_registry_client.get_schema(schema_id, logging_enable=True) +``` ## Next steps ### More sample code +Please take a look at the [samples](./samples) directory for detailed examples of how to use this library to register and retrieve schema to/from Schema Registry. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a @@ -154,4 +187,5 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio [pip]: https://pypi.org/project/pip/ [python]: https://www.python.org/downloads/ [azure_core]: https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/README.md -[azure_sub]: https://azure.microsoft.com/free/ \ No newline at end of file +[azure_sub]: https://azure.microsoft.com/free/ +[python_logging]: https://docs.python.org/3/library/logging.html \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index 45677a014958..ccd8184a61db 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -81,8 +81,8 @@ class SchemaProperties(DictMixin): """ Meta properties of a schema. - :ivar id: References specific schema in registry namespace. - :type id: str + :ivar schema_id: References specific schema in registry namespace. + :type schema_id: str :ivar location: URL location of schema, identified by schema group, schema name, and version. :type location: str :ivar location_by_id: URL location of schema, identified by schema ID. diff --git a/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt index eeda56db3177..f15ae1a63a98 100644 --- a/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt +++ b/sdk/schemaregistry/azure-schemaregistry/dev_requirements.txt @@ -1,5 +1,5 @@ --e ../../core/azure-core --e ../../identity/azure-identity -e ../../../tools/azure-devtools -e ../../../tools/azure-sdk-tools +-e ../../core/azure-core +-e ../../identity/azure-identity aiohttp>=3.0; python_version >= '3.5' \ No newline at end of file From fb0e6f9115a449710e9b9b79f0e322316a5874d3 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Wed, 2 Sep 2020 20:53:53 -0700 Subject: [PATCH 27/35] remove temp ci experiment --- .../azure-schemaregistry-avroserializer/setup.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py index 92705a2877cf..c166eb266d22 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py @@ -67,10 +67,5 @@ install_requires=[ 'azure-schemaregistry<2.0.0,>=1.0.0b1', 'avro<2.0.0,>=1.0.0' - ], - extras_require={ - ":python_version<'3.0'": ['futures'], - ":python_version<'3.4'": ['enum34>=1.0.4'], - ":python_version<'3.5'": ["typing"] - } + ] ) From 0260ea39009c788629ea4076e61e820dbe9b8e71 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Wed, 2 Sep 2020 21:14:03 -0700 Subject: [PATCH 28/35] init add conftest.py to pass py2.7 test --- .../azure-schemaregistry/conftest.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 sdk/schemaregistry/azure-schemaregistry/conftest.py diff --git a/sdk/schemaregistry/azure-schemaregistry/conftest.py b/sdk/schemaregistry/azure-schemaregistry/conftest.py new file mode 100644 index 000000000000..eb768094af4a --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/conftest.py @@ -0,0 +1,32 @@ +# ------------------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +import os +import sys +import uuid + +import pytest + +# Ignore async tests for Python < 3.5 +collect_ignore = [] +if sys.version_info < (3, 5): + collect_ignore.append("tests/async_tests") + collect_ignore.append("samples/async_samples") + + +# Note: This is duplicated between here and the basic conftest, so that it does not throw warnings if you're +# running locally to this SDK. (Everything works properly, pytest just makes a bit of noise.) +def pytest_configure(config): + # register an additional marker + config.addinivalue_line( + "markers", "liveTest: mark test to be a live test only" + ) + config.addinivalue_line( + "markers", "live_test_only: mark test to be a live test only" + ) + config.addinivalue_line( + "markers", "playback_test_only: mark test to be a playback test only" + ) From bb687cbe0be592a1bdcb30ffe3386dc789a7f459 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 3 Sep 2020 14:47:37 -0700 Subject: [PATCH 29/35] laurent feedback update --- .../README.md | 12 ++--- .../__init__.py | 0 .../_avro_serializer.py | 0 .../_schema_registry_avro_serializer.py | 9 +++- .../_version.py | 0 .../samples/README.md | 4 +- .../samples/avro_serializer.py | 2 +- .../setup.py | 4 +- ...ializer.test_basic_sr_avro_serializer.yaml | 50 +++++++++---------- .../tests/test_avro_serializer.py | 6 +-- .../azure-schemaregistry/README.md | 6 +-- .../schemaregistry/_common/_constants.py | 3 +- .../schemaregistry/_generated/__init__.py | 2 +- .../_generated/_azure_schema_registry.py | 1 - .../_generated/_configuration.py | 6 +-- .../schemaregistry/_generated/aio/__init__.py | 2 +- ...try_async.py => _azure_schema_registry.py} | 7 ++- ...nfiguration_async.py => _configuration.py} | 6 +-- .../__init__.py | 2 +- .../_schema_operations.py} | 46 +++++++++-------- .../_generated/models/__init__.py | 5 ++ .../models/_azure_schema_registry_enums.py | 31 ++++++++++++ .../operations/_schema_operations.py | 46 +++++++++-------- .../schemaregistry/_schema_registry_client.py | 5 +- .../aio/_schema_registry_client_async.py | 7 +-- .../azure-schemaregistry/conftest.py | 15 ------ .../azure-schemaregistry/samples/README.md | 4 +- .../azure-schemaregistry/swagger/README.md | 3 +- ...egistry_async.test_schema_basic_async.yaml | 50 +++++++++---------- ....test_schema_negative_no_schema_async.yaml | 12 ++--- ...gistry_async.test_schema_update_async.yaml | 50 +++++++++---------- .../async_tests/test_schema_registry_async.py | 10 ---- ...est_schema_registry.test_schema_basic.yaml | 40 +++++++-------- ...gistry.test_schema_negative_no_schema.yaml | 12 ++--- ...chema_registry.test_schema_same_twice.yaml | 32 ++++++------ ...st_schema_registry.test_schema_update.yaml | 44 ++++++++-------- .../tests/test_schema_registry.py | 12 ----- 37 files changed, 280 insertions(+), 266 deletions(-) rename sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/{avro_serializer => avroserializer}/__init__.py (100%) rename sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/{avro_serializer => avroserializer}/_avro_serializer.py (100%) rename sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/{avro_serializer => avroserializer}/_schema_registry_avro_serializer.py (94%) rename sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/{avro_serializer => avroserializer}/_version.py (100%) rename sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/{_azure_schema_registry_async.py => _azure_schema_registry.py} (89%) rename sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/{_configuration_async.py => _configuration.py} (91%) rename sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/{operations_async => operations}/__init__.py (90%) rename sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/{operations_async/_schema_operations_async.py => operations/_schema_operations.py} (88%) create mode 100644 sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/models/_azure_schema_registry_enums.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md index 9f300f3962e2..df33e31bcfa4 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md @@ -7,10 +7,10 @@ to the given avro schema. It is integrated with Azure Schema Registry SDK and wi ### Install the package -Install the Azure Schema Registry Avro Serializer client library for Python with [pip][pip]: +Install the Azure Schema Registry Avro Serializer client library and Azure Identity client library for Python with [pip][pip]: ```Bash -pip install azure-schemaregistry-avroserializer +pip install azure-schemaregistry-avroserializer azure-identity ``` ### Prerequisites: @@ -25,7 +25,7 @@ Interaction with Schema Registry Avro Serializer starts with an instance of Sche **Create client using the azure-identity library:** ```python -from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer from azure.identity import DefaultAzureCredential credential = DefaultAzureCredential() @@ -49,7 +49,7 @@ The following sections provide several code snippets covering some of the most c ```python import os -from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer from azure.identity import DefaultAzureCredential token_credential = DefaultAzureCredential() @@ -78,7 +78,7 @@ with serializer: ```python import os -from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer from azure.identity import DefaultAzureCredential token_credential = DefaultAzureCredential() @@ -109,7 +109,7 @@ headers, can be enabled on a client with the `logging_enable` argument: ```python import sys import logging -from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer from azure.identity import DefaultAzureCredential # Create a logger for the SDK diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/__init__.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/__init__.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/__init__.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_avro_serializer.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py similarity index 94% rename from sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index ebe6e8da1b74..0465058c50d0 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -24,13 +24,16 @@ # # -------------------------------------------------------------------------- from io import BytesIO -from typing import Any, Dict, Union +from typing import Any, Dict, Union, TYPE_CHECKING import avro from azure.schemaregistry import SchemaRegistryClient, SerializationType from ._avro_serializer import AvroObjectSerializer +if TYPE_CHECKING: + from azure.core.credentials import TokenCredential + class SchemaRegistryAvroSerializer(object): """ @@ -41,11 +44,13 @@ class SchemaRegistryAvroSerializer(object): :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. :type credential: TokenCredential :param str schema_group: Schema group under which schema should be registered. + :keyword str codec: The writer codec. If None, let the avro library decides. """ def __init__(self, endpoint, credential, schema_group, **kwargs): + # type: (str, TokenCredential, str, Any) -> None self._schema_group = schema_group - self._avro_serializer = AvroObjectSerializer() + self._avro_serializer = AvroObjectSerializer(codec=kwargs.get("codec")) self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint, **kwargs) self._id_to_schema = {} self._schema_to_id = {} diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_version.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_version.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avro_serializer/_version.py rename to sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_version.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md index f3689b6f6a7b..972766cadf0b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md @@ -25,10 +25,10 @@ If you do not have an existing Azure account, you may sign up for a free trial o ## Setup -1. Install the Azure Schema Registry Avro Serializer client library for Python with [pip](https://pypi.org/project/pip/): +1. Install the Azure Schema Registry Avro Serializer client library and Azure Identity client library for Python with [pip](https://pypi.org/project/pip/): ```bash -pip install azure-schemaregistry-avroserializer +pip install azure-schemaregistry-avroserializer azure-identity ``` 2. Clone or download this sample repository diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py index 2bf1894abec8..769f1860dd66 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py @@ -26,7 +26,7 @@ import os from azure.identity import ClientSecretCredential -from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py index c166eb266d22..76e9259ef602 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py @@ -15,8 +15,8 @@ PACKAGE_NAME = "azure-schemaregistry-avroserializer" PACKAGE_PPRINT_NAME = "Schema Registry Avro Serializer" -package_folder_path = "azure/schemaregistry/serializer/avro_serializer" -namespace_name = "azure.schemaregistry.serializer.avro_serializer" +package_folder_path = "azure/schemaregistry/serializer/avroserializer" +namespace_name = "azure.schemaregistry.serializer.avroserializer" # Version extraction inspired from 'requests' with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml index 911c35c88fc7..e8db561effb4 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml @@ -1,9 +1,9 @@ interactions: - request: - body: !!python/unicode '"{\"type\": \"record\", \"namespace\": \"testschema09bb017e\", - \"name\": \"User\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, - {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", - \"null\"], \"name\": \"favorite_color\"}]}"' + body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"testschemabe93dcc6\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", + \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], + \"name\": \"favorite_color\"}]}"' headers: Accept: - application/json @@ -16,21 +16,21 @@ interactions: Content-Type: - application/json User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/2.7.18 (Windows-10-10.0.19041) + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User?api-version=2017-04 response: body: - string: !!python/unicode '{"id":"035f2334e3af4417a8011cb0f895088d"}' + string: '{"id":"2bd850ebc8e6415aa89684ce8fc333c3"}' headers: content-type: - application/json date: - - Thu, 03 Sep 2020 00:39:52 GMT + - Thu, 03 Sep 2020 21:39:02 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -38,23 +38,23 @@ interactions: transfer-encoding: - chunked x-schema-id: - - 035f2334e3af4417a8011cb0f895088d + - 2bd850ebc8e6415aa89684ce8fc333c3 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/035f2334e3af4417a8011cb0f895088d?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2bd850ebc8e6415aa89684ce8fc333c3?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User/versions?api-version=2017-04 status: code: 200 message: OK - request: - body: !!python/unicode '"{\"type\": \"record\", \"namespace\": \"testschema09bb017e\", - \"name\": \"User\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, - {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", - \"null\"], \"name\": \"favorite_color\"}]}"' + body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"testschemabe93dcc6\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", + \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], + \"name\": \"favorite_color\"}]}"' headers: Accept: - application/json @@ -67,21 +67,21 @@ interactions: Content-Type: - application/json User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/2.7.18 (Windows-10-10.0.19041) + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User?api-version=2017-04 response: body: - string: !!python/unicode '{"id":"035f2334e3af4417a8011cb0f895088d"}' + string: '{"id":"2bd850ebc8e6415aa89684ce8fc333c3"}' headers: content-type: - application/json date: - - Thu, 03 Sep 2020 00:39:53 GMT + - Thu, 03 Sep 2020 21:39:03 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -89,15 +89,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - 035f2334e3af4417a8011cb0f895088d + - 2bd850ebc8e6415aa89684ce8fc333c3 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/035f2334e3af4417a8011cb0f895088d?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2bd850ebc8e6415aa89684ce8fc333c3?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschema09bb017e.User/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User/versions?api-version=2017-04 status: code: 200 message: OK diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py index 6319f4660815..84521f94cb7a 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -24,8 +24,8 @@ from io import BytesIO from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.serializer.avro_serializer import SchemaRegistryAvroSerializer -from azure.schemaregistry.serializer.avro_serializer._avro_serializer import AvroObjectSerializer +from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avroserializer._avro_serializer import AvroObjectSerializer from azure.identity import ClientSecretCredential from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError @@ -73,8 +73,6 @@ def test_raw_avro_serializer_negative(self): with pytest.raises(avro.io.AvroTypeException): raw_avro_object_serializer.serialize(dict_data_missing_required_field, schema) - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): diff --git a/sdk/schemaregistry/azure-schemaregistry/README.md b/sdk/schemaregistry/azure-schemaregistry/README.md index 342ff33cbc47..61bfb236cfdd 100644 --- a/sdk/schemaregistry/azure-schemaregistry/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/README.md @@ -7,10 +7,10 @@ Avro, Json, etc. ### Install the package -Install the Azure Schema Registry client library for Python with [pip][pip]: +Install the Azure Schema Registry client library and Azure Identity client library for Python with [pip][pip]: ```Bash -pip install azure-schemaregistry +pip install azure-schemaregistry azure-identity ``` ### Prerequisites: @@ -91,7 +91,7 @@ schema_id = '' schema_registry_client = SchemaRegistryClient(endpoint=endpoint, credential=token_credential) with schema_registry_client: schema = schema_registry_client.get_schema(schema_id) - schema_content = schema.content + schema_content = schema.schema_content ``` ### Get the id of a schema diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py index bb4a49db913d..a22d118f4b84 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_constants.py @@ -27,5 +27,4 @@ class SerializationType(str, Enum): - AVRO = "Avro" - JSON = "Json" + AVRO = "avro" diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/__init__.py index 85fc8393f162..f8bdf57c1911 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/__init__.py @@ -13,7 +13,7 @@ __all__ = ['AzureSchemaRegistry'] try: - from ._patch import patch_sdk + from ._patch import patch_sdk # type: ignore patch_sdk() except ImportError: pass diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py index 02bfd51add11..c4f180986a47 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_azure_schema_registry.py @@ -31,7 +31,6 @@ class AzureSchemaRegistry(object): :type credential: ~azure.core.credentials.TokenCredential :param endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :type endpoint: str - :keyword int polling_interval: Default waiting time between two polls for LRO operations if no Retry-After header is present. """ def __init__( diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py index 072156a901e2..02e2eed29c10 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/_configuration.py @@ -47,9 +47,8 @@ def __init__( self.credential = credential self.endpoint = endpoint - self.api_version = "2018-01-01-preview" - self.credential_scopes = ['https://eventhubs.azure.net/.default'] - self.credential_scopes.extend(kwargs.pop('credential_scopes', [])) + self.api_version = "2017-04" + self.credential_scopes = kwargs.pop('credential_scopes', ['https://eventhubs.azure.net/.default']) kwargs.setdefault('sdk_moniker', 'azureschemaregistry/{}'.format(VERSION)) self._configure(**kwargs) @@ -62,6 +61,7 @@ def _configure( self.headers_policy = kwargs.get('headers_policy') or policies.HeadersPolicy(**kwargs) self.proxy_policy = kwargs.get('proxy_policy') or policies.ProxyPolicy(**kwargs) self.logging_policy = kwargs.get('logging_policy') or policies.NetworkTraceLoggingPolicy(**kwargs) + self.http_logging_policy = kwargs.get('http_logging_policy') or policies.HttpLoggingPolicy(**kwargs) self.retry_policy = kwargs.get('retry_policy') or policies.RetryPolicy(**kwargs) self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs) self.redirect_policy = kwargs.get('redirect_policy') or policies.RedirectPolicy(**kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/__init__.py index 6319c29918f1..ae36d95dbeb0 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/__init__.py @@ -6,5 +6,5 @@ # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -from ._azure_schema_registry_async import AzureSchemaRegistry +from ._azure_schema_registry import AzureSchemaRegistry __all__ = ['AzureSchemaRegistry'] diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry.py similarity index 89% rename from sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py rename to sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry.py index 7bc485467f31..dd14b4dac46a 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_azure_schema_registry.py @@ -15,8 +15,8 @@ # pylint: disable=unused-import,ungrouped-imports from azure.core.credentials_async import AsyncTokenCredential -from ._configuration_async import AzureSchemaRegistryConfiguration -from .operations_async import SchemaOperations +from ._configuration import AzureSchemaRegistryConfiguration +from .operations import SchemaOperations from .. import models @@ -24,12 +24,11 @@ class AzureSchemaRegistry(object): """Azure Schema Registry is as a central schema repository for enterprise-level data infrastructure, complete with support for versioning and management. :ivar schema: SchemaOperations operations - :vartype schema: azure.schemaregistry._generated.aio.operations_async.SchemaOperations + :vartype schema: azure.schemaregistry._generated.aio.operations.SchemaOperations :param credential: Credential needed for the client to connect to Azure. :type credential: ~azure.core.credentials_async.AsyncTokenCredential :param endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. :type endpoint: str - :keyword int polling_interval: Default waiting time between two polls for LRO operations if no Retry-After header is present. """ def __init__( diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration.py similarity index 91% rename from sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py rename to sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration.py index 71d0942e1f3e..146ad563107c 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/_configuration.py @@ -44,9 +44,8 @@ def __init__( self.credential = credential self.endpoint = endpoint - self.api_version = "2018-01-01-preview" - self.credential_scopes = ['https://eventhubs.azure.net/.default'] - self.credential_scopes.extend(kwargs.pop('credential_scopes', [])) + self.api_version = "2017-04" + self.credential_scopes = kwargs.pop('credential_scopes', ['https://eventhubs.azure.net/.default']) kwargs.setdefault('sdk_moniker', 'azureschemaregistry/{}'.format(VERSION)) self._configure(**kwargs) @@ -58,6 +57,7 @@ def _configure( self.headers_policy = kwargs.get('headers_policy') or policies.HeadersPolicy(**kwargs) self.proxy_policy = kwargs.get('proxy_policy') or policies.ProxyPolicy(**kwargs) self.logging_policy = kwargs.get('logging_policy') or policies.NetworkTraceLoggingPolicy(**kwargs) + self.http_logging_policy = kwargs.get('http_logging_policy') or policies.HttpLoggingPolicy(**kwargs) self.retry_policy = kwargs.get('retry_policy') or policies.AsyncRetryPolicy(**kwargs) self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs) self.redirect_policy = kwargs.get('redirect_policy') or policies.AsyncRedirectPolicy(**kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations/__init__.py similarity index 90% rename from sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/__init__.py rename to sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations/__init__.py index 29642724a481..61b30e467156 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations/__init__.py @@ -6,7 +6,7 @@ # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -from ._schema_operations_async import SchemaOperations +from ._schema_operations import SchemaOperations __all__ = [ 'SchemaOperations', diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations/_schema_operations.py similarity index 88% rename from sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py rename to sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations/_schema_operations.py index 28d4fe3914cc..9dd2aedd6d6d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations_async/_schema_operations_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/aio/operations/_schema_operations.py @@ -5,7 +5,7 @@ # Code generated by Microsoft (R) AutoRest Code Generator. # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -from typing import Any, Callable, Dict, Generic, Optional, TypeVar +from typing import Any, Callable, Dict, Generic, Optional, TypeVar, Union import warnings from azure.core.exceptions import HttpResponseError, ResourceExistsError, ResourceNotFoundError, map_error @@ -44,9 +44,10 @@ async def get_by_id( schema_id: str, **kwargs ) -> str: - """Gets a registered schema by its unique ID. Azure Schema Registry guarantees that ID is unique within a namespace. + """Get a registered schema by its unique ID reference. - Get a registered schema by its unique ID reference. + Gets a registered schema by its unique ID. Azure Schema Registry guarantees that ID is unique + within a namespace. :param schema_id: References specific schema in registry namespace. :type schema_id: str @@ -58,7 +59,8 @@ async def get_by_id( cls = kwargs.pop('cls', None) # type: ClsType[str] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) - api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" + api_version = "2017-04" + accept = "application/json" # Construct URL url = self.get_by_id.metadata['url'] # type: ignore @@ -74,9 +76,8 @@ async def get_by_id( # Construct headers header_parameters = {} # type: Dict[str, Any] - header_parameters['Accept'] = 'application/json' + header_parameters['Accept'] = self._serialize.header("accept", accept, 'str') - # Construct and send request request = self._client.get(url, query_parameters, header_parameters) pipeline_response = await self._client._pipeline.run(request, stream=False, **kwargs) response = pipeline_response.http_response @@ -104,18 +105,22 @@ async def query_id_by_content( self, group_name: str, schema_name: str, + x_schema_type: Union[str, "models.SerializationType"], schema_content: str, **kwargs ) -> "models.SchemaId": - """Gets the ID referencing an existing schema within the specified schema group, as matched by schema content comparison. + """Get ID for existing schema. - Get ID for existing schema. + Gets the ID referencing an existing schema within the specified schema group, as matched by + schema content comparison. :param group_name: Schema group under which schema is registered. Group's serialization type should match the serialization type specified in the request. :type group_name: str :param schema_name: Name of the registered schema. :type schema_name: str + :param x_schema_type: Serialization type for the schema being registered. + :type x_schema_type: str or ~azure.schemaregistry._generated.models.SerializationType :param schema_content: String representation of the registered schema. :type schema_content: str :keyword callable cls: A custom type or function that will be passed the direct response @@ -126,9 +131,9 @@ async def query_id_by_content( cls = kwargs.pop('cls', None) # type: ClsType["models.SchemaId"] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) - x_schema_type = "avro" - api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" + api_version = "2017-04" content_type = kwargs.pop("content_type", "application/json") + accept = "application/json" # Construct URL url = self.query_id_by_content.metadata['url'] # type: ignore @@ -147,14 +152,12 @@ async def query_id_by_content( header_parameters = {} # type: Dict[str, Any] header_parameters['X-Schema-Type'] = self._serialize.header("x_schema_type", x_schema_type, 'str') header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str') - header_parameters['Accept'] = 'application/json' + header_parameters['Accept'] = self._serialize.header("accept", accept, 'str') - # Construct and send request body_content_kwargs = {} # type: Dict[str, Any] body_content = self._serialize.body(schema_content, 'str') body_content_kwargs['content'] = body_content request = self._client.post(url, query_parameters, header_parameters, **body_content_kwargs) - pipeline_response = await self._client._pipeline.run(request, stream=False, **kwargs) response = pipeline_response.http_response @@ -181,18 +184,23 @@ async def register( self, group_name: str, schema_name: str, + x_schema_type: Union[str, "models.SerializationType"], schema_content: str, **kwargs ) -> "models.SchemaId": - """Register new schema. If schema of specified name does not exist in specified group, schema is created at version 1. If schema of specified name exists already in specified group, schema is created at latest version + 1. + """Register new schema. - Register new schema. + Register new schema. If schema of specified name does not exist in specified group, schema is + created at version 1. If schema of specified name exists already in specified group, schema is + created at latest version + 1. :param group_name: Schema group under which schema should be registered. Group's serialization type should match the serialization type specified in the request. :type group_name: str :param schema_name: Name of schema being registered. :type schema_name: str + :param x_schema_type: Serialization type for the schema being registered. + :type x_schema_type: str or ~azure.schemaregistry._generated.models.SerializationType :param schema_content: String representation of the schema being registered. :type schema_content: str :keyword callable cls: A custom type or function that will be passed the direct response @@ -203,9 +211,9 @@ async def register( cls = kwargs.pop('cls', None) # type: ClsType["models.SchemaId"] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) - x_schema_type = "avro" - api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" + api_version = "2017-04" content_type = kwargs.pop("content_type", "application/json") + accept = "application/json" # Construct URL url = self.register.metadata['url'] # type: ignore @@ -224,14 +232,12 @@ async def register( header_parameters = {} # type: Dict[str, Any] header_parameters['X-Schema-Type'] = self._serialize.header("x_schema_type", x_schema_type, 'str') header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str') - header_parameters['Accept'] = 'application/json' + header_parameters['Accept'] = self._serialize.header("accept", accept, 'str') - # Construct and send request body_content_kwargs = {} # type: Dict[str, Any] body_content = self._serialize.body(schema_content, 'str') body_content_kwargs['content'] = body_content request = self._client.put(url, query_parameters, header_parameters, **body_content_kwargs) - pipeline_response = await self._client._pipeline.run(request, stream=False, **kwargs) response = pipeline_response.http_response diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/models/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/models/__init__.py index acabe98624c0..93cd4fc50d84 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/models/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/models/__init__.py @@ -11,6 +11,11 @@ except (SyntaxError, ImportError): from ._models import SchemaId # type: ignore +from ._azure_schema_registry_enums import ( + SerializationType, +) + __all__ = [ 'SchemaId', + 'SerializationType', ] diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/models/_azure_schema_registry_enums.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/models/_azure_schema_registry_enums.py new file mode 100644 index 000000000000..a1ae7238a329 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/models/_azure_schema_registry_enums.py @@ -0,0 +1,31 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# Code generated by Microsoft (R) AutoRest Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is regenerated. +# -------------------------------------------------------------------------- + +from enum import Enum, EnumMeta +from six import with_metaclass + +class _CaseInsensitiveEnumMeta(EnumMeta): + def __getitem__(self, name): + return super().__getitem__(name.upper()) + + def __getattr__(cls, name): + """Return the enum member matching `name` + We use __getattr__ instead of descriptors or inserting into the enum + class' __dict__ in order to support `name` and `value` being both + properties for enum members (which live in the class' __dict__) and + enum members themselves. + """ + try: + return cls._member_map_[name.upper()] + except KeyError: + raise AttributeError(name) + + +class SerializationType(with_metaclass(_CaseInsensitiveEnumMeta, str, Enum)): + + AVRO = "avro" #: Avro Serialization schema type. diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py index a32f58dabb93..8472c0b9f69d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_generated/operations/_schema_operations.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: # pylint: disable=unused-import,ungrouped-imports - from typing import Any, Callable, Dict, Generic, Optional, TypeVar + from typing import Any, Callable, Dict, Generic, Optional, TypeVar, Union T = TypeVar('T') ClsType = Optional[Callable[[PipelineResponse[HttpRequest, HttpResponse], T, Dict[str, Any]], Any]] @@ -49,9 +49,10 @@ def get_by_id( **kwargs # type: Any ): # type: (...) -> str - """Gets a registered schema by its unique ID. Azure Schema Registry guarantees that ID is unique within a namespace. + """Get a registered schema by its unique ID reference. - Get a registered schema by its unique ID reference. + Gets a registered schema by its unique ID. Azure Schema Registry guarantees that ID is unique + within a namespace. :param schema_id: References specific schema in registry namespace. :type schema_id: str @@ -63,7 +64,8 @@ def get_by_id( cls = kwargs.pop('cls', None) # type: ClsType[str] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) - api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" + api_version = "2017-04" + accept = "application/json" # Construct URL url = self.get_by_id.metadata['url'] # type: ignore @@ -79,9 +81,8 @@ def get_by_id( # Construct headers header_parameters = {} # type: Dict[str, Any] - header_parameters['Accept'] = 'application/json' + header_parameters['Accept'] = self._serialize.header("accept", accept, 'str') - # Construct and send request request = self._client.get(url, query_parameters, header_parameters) pipeline_response = self._client._pipeline.run(request, stream=False, **kwargs) response = pipeline_response.http_response @@ -109,19 +110,23 @@ def query_id_by_content( self, group_name, # type: str schema_name, # type: str + x_schema_type, # type: Union[str, "models.SerializationType"] schema_content, # type: str **kwargs # type: Any ): # type: (...) -> "models.SchemaId" - """Gets the ID referencing an existing schema within the specified schema group, as matched by schema content comparison. + """Get ID for existing schema. - Get ID for existing schema. + Gets the ID referencing an existing schema within the specified schema group, as matched by + schema content comparison. :param group_name: Schema group under which schema is registered. Group's serialization type should match the serialization type specified in the request. :type group_name: str :param schema_name: Name of the registered schema. :type schema_name: str + :param x_schema_type: Serialization type for the schema being registered. + :type x_schema_type: str or ~azure.schemaregistry._generated.models.SerializationType :param schema_content: String representation of the registered schema. :type schema_content: str :keyword callable cls: A custom type or function that will be passed the direct response @@ -132,9 +137,9 @@ def query_id_by_content( cls = kwargs.pop('cls', None) # type: ClsType["models.SchemaId"] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) - x_schema_type = "avro" - api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" + api_version = "2017-04" content_type = kwargs.pop("content_type", "application/json") + accept = "application/json" # Construct URL url = self.query_id_by_content.metadata['url'] # type: ignore @@ -153,14 +158,12 @@ def query_id_by_content( header_parameters = {} # type: Dict[str, Any] header_parameters['X-Schema-Type'] = self._serialize.header("x_schema_type", x_schema_type, 'str') header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str') - header_parameters['Accept'] = 'application/json' + header_parameters['Accept'] = self._serialize.header("accept", accept, 'str') - # Construct and send request body_content_kwargs = {} # type: Dict[str, Any] body_content = self._serialize.body(schema_content, 'str') body_content_kwargs['content'] = body_content request = self._client.post(url, query_parameters, header_parameters, **body_content_kwargs) - pipeline_response = self._client._pipeline.run(request, stream=False, **kwargs) response = pipeline_response.http_response @@ -187,19 +190,24 @@ def register( self, group_name, # type: str schema_name, # type: str + x_schema_type, # type: Union[str, "models.SerializationType"] schema_content, # type: str **kwargs # type: Any ): # type: (...) -> "models.SchemaId" - """Register new schema. If schema of specified name does not exist in specified group, schema is created at version 1. If schema of specified name exists already in specified group, schema is created at latest version + 1. + """Register new schema. - Register new schema. + Register new schema. If schema of specified name does not exist in specified group, schema is + created at version 1. If schema of specified name exists already in specified group, schema is + created at latest version + 1. :param group_name: Schema group under which schema should be registered. Group's serialization type should match the serialization type specified in the request. :type group_name: str :param schema_name: Name of schema being registered. :type schema_name: str + :param x_schema_type: Serialization type for the schema being registered. + :type x_schema_type: str or ~azure.schemaregistry._generated.models.SerializationType :param schema_content: String representation of the schema being registered. :type schema_content: str :keyword callable cls: A custom type or function that will be passed the direct response @@ -210,9 +218,9 @@ def register( cls = kwargs.pop('cls', None) # type: ClsType["models.SchemaId"] error_map = {404: ResourceNotFoundError, 409: ResourceExistsError} error_map.update(kwargs.pop('error_map', {})) - x_schema_type = "avro" - api_version = "2017-04" # TODO: manually patch, the default value generated is "2018-01-01-preview" + api_version = "2017-04" content_type = kwargs.pop("content_type", "application/json") + accept = "application/json" # Construct URL url = self.register.metadata['url'] # type: ignore @@ -231,14 +239,12 @@ def register( header_parameters = {} # type: Dict[str, Any] header_parameters['X-Schema-Type'] = self._serialize.header("x_schema_type", x_schema_type, 'str') header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str') - header_parameters['Accept'] = 'application/json' + header_parameters['Accept'] = self._serialize.header("accept", accept, 'str') - # Construct and send request body_content_kwargs = {} # type: Dict[str, Any] body_content = self._serialize.body(schema_content, 'str') body_content_kwargs['content'] = body_content request = self._client.put(url, query_parameters, header_parameters, **body_content_kwargs) - pipeline_response = self._client._pipeline.run(request, stream=False, **kwargs) response = pipeline_response.http_response diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index b9d384ca33cd..524e5b81e636 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -88,6 +88,7 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ :param str schema_group: Schema group under which schema should be registered. :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. + For now Avro is the only supported serialization type by the service. :type serialization_type: Union[str, SerializationType] :param str schema_content: String representation of the schema being registered. :rtype: SchemaProperties @@ -111,7 +112,7 @@ def register_schema(self, schema_group, schema_name, serialization_type, schema_ group_name=schema_group, schema_name=schema_name, schema_content=schema_content, - #serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter + x_schema_type=serialization_type, cls=_parse_response_schema_id, **kwargs ) @@ -174,7 +175,7 @@ def get_schema_id(self, schema_group, schema_name, serialization_type, schema_co group_name=schema_group, schema_name=schema_name, schema_content=schema_content, - #serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter + x_schema_type=serialization_type, cls=_parse_response_schema_id, **kwargs ) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 2eca2c1579f5..6c3835a94d98 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -28,7 +28,7 @@ from .._common._constants import SerializationType from .._common._response_handlers import _parse_response_schema_id, _parse_response_schema from .._common._schema import SchemaProperties, Schema -from .._generated.aio._azure_schema_registry_async import AzureSchemaRegistry +from .._generated.aio._azure_schema_registry import AzureSchemaRegistry if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential @@ -90,6 +90,7 @@ async def register_schema( :param str schema_group: Schema group under which schema should be registered. :param str schema_name: Name of schema being registered. :param serialization_type: Serialization type for the schema being registered. + For now Avro is the only supported serialization type by the service. :type serialization_type: Union[str, SerializationType] :param str schema_content: String representation of the schema being registered. :rtype: SchemaProperties @@ -113,7 +114,7 @@ async def register_schema( group_name=schema_group, schema_name=schema_name, schema_content=schema_content, - # serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter + x_schema_type=serialization_type, cls=_parse_response_schema_id, **kwargs ) @@ -184,7 +185,7 @@ async def get_schema_id( group_name=schema_group, schema_name=schema_name, schema_content=schema_content, - # serialization_type=serialization_type, # TODO: current swagger doesn't support the parameter + x_schema_type=serialization_type, cls=_parse_response_schema_id, **kwargs ) diff --git a/sdk/schemaregistry/azure-schemaregistry/conftest.py b/sdk/schemaregistry/azure-schemaregistry/conftest.py index eb768094af4a..4e4d47319fb6 100644 --- a/sdk/schemaregistry/azure-schemaregistry/conftest.py +++ b/sdk/schemaregistry/azure-schemaregistry/conftest.py @@ -15,18 +15,3 @@ if sys.version_info < (3, 5): collect_ignore.append("tests/async_tests") collect_ignore.append("samples/async_samples") - - -# Note: This is duplicated between here and the basic conftest, so that it does not throw warnings if you're -# running locally to this SDK. (Everything works properly, pytest just makes a bit of noise.) -def pytest_configure(config): - # register an additional marker - config.addinivalue_line( - "markers", "liveTest: mark test to be a live test only" - ) - config.addinivalue_line( - "markers", "live_test_only: mark test to be a live test only" - ) - config.addinivalue_line( - "markers", "playback_test_only: mark test to be a playback test only" - ) diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/README.md b/sdk/schemaregistry/azure-schemaregistry/samples/README.md index 1aaa7736383d..826f4ddaadb1 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/samples/README.md @@ -28,10 +28,10 @@ If you do not have an existing Azure account, you may sign up for a free trial o ## Setup -1. Install the Azure Schema Registry client library for Python with [pip](https://pypi.org/project/pip/): +1. Install the Azure Schema Registry client library and Azure Identity client library for Python with [pip](https://pypi.org/project/pip/): ```bash -pip install azure-schemaregistry +pip install azure-schemaregistry azure-identity ``` 2. Clone or download this sample repository diff --git a/sdk/schemaregistry/azure-schemaregistry/swagger/README.md b/sdk/schemaregistry/azure-schemaregistry/swagger/README.md index d74afae801ca..be8dca675f22 100644 --- a/sdk/schemaregistry/azure-schemaregistry/swagger/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/swagger/README.md @@ -5,7 +5,8 @@ ### Generation ```ps cd C:\Work\SchemaRegistry\ -autorest --v3 --python --use=@autorest/python@5.0.0-preview.6 +autorest --reset +autorest --v3 --python ``` ### Settings ``` yaml diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml index bb6efc9c06e7..7ef9979f0e00 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml @@ -11,28 +11,28 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579?api-version=2017-04 response: body: - string: '{"id":"25e3ce5c21844faba13d9cb8a76aa0de"}' + string: '{"id":"6f12bc50b6944b8997def314d7b9a2de"}' headers: content-type: application/json - date: Tue, 01 Sep 2020 21:36:47 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions/1?api-version=2017-04 + date: Thu, 03 Sep 2020 21:36:03 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions/1?api-version=2017-04 server: Microsoft-HTTPAPI/2.0 strict-transport-security: max-age=31536000 transfer-encoding: chunked - x-schema-id: 25e3ce5c21844faba13d9cb8a76aa0de - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + x-schema-id: 6f12bc50b6944b8997def314d7b9a2de + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 x-schema-type: Avro x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions?api-version=2017-04 + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions?api-version=2017-04 status: code: 200 message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b?api-version=2017-04 + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579?api-version=2017-04 - request: body: null headers: @@ -41,26 +41,26 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 response: body: string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' headers: content-type: application/json - date: Tue, 01 Sep 2020 21:36:47 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions/1?api-version=2017-04 + date: Thu, 03 Sep 2020 21:36:03 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions/1?api-version=2017-04 server: Microsoft-HTTPAPI/2.0 strict-transport-security: max-age=31536000 transfer-encoding: chunked - x-schema-id: 25e3ce5c21844faba13d9cb8a76aa0de - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + x-schema-id: 6f12bc50b6944b8997def314d7b9a2de + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 x-schema-type: Avro x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions?api-version=2017-04 + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions?api-version=2017-04 status: code: 200 message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 - request: body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' headers: @@ -73,26 +73,26 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579?api-version=2017-04 response: body: - string: '{"id":"25e3ce5c21844faba13d9cb8a76aa0de"}' + string: '{"id":"6f12bc50b6944b8997def314d7b9a2de"}' headers: content-type: application/json - date: Tue, 01 Sep 2020 21:36:48 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions/1?api-version=2017-04 + date: Thu, 03 Sep 2020 21:36:05 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions/1?api-version=2017-04 server: Microsoft-HTTPAPI/2.0 strict-transport-security: max-age=31536000 transfer-encoding: chunked - x-schema-id: 25e3ce5c21844faba13d9cb8a76aa0de - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/25e3ce5c21844faba13d9cb8a76aa0de?api-version=2017-04 + x-schema-id: 6f12bc50b6944b8997def314d7b9a2de + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 x-schema-type: Avro x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b/versions?api-version=2017-04 + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions?api-version=2017-04 status: code: 200 message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-ab359529-4c7e-463a-a4a8-3819b6ba710b?api-version=2017-04 + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579?api-version=2017-04 version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml index 3dd2693c06ca..d6aec1d501db 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml @@ -11,12 +11,12 @@ interactions: response: body: string: 400SubCode=40000, UnknownType:The request - is invalid. [MGResponseHttpError=BadRequest]. TrackingId:6451c6b4-cd1d-4154-9144-1d8198dbe79c_G4, + is invalid. [MGResponseHttpError=BadRequest]. TrackingId:89f17396-2d5e-49d4-aaaa-402ce6f5257b_G5, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/a, - Timestamp:2020-09-01T21:36:49 + Timestamp:2020-09-03T21:36:06 headers: content-type: application/xml; charset=utf-8 - date: Tue, 01 Sep 2020 21:36:49 GMT + date: Thu, 03 Sep 2020 21:36:06 GMT server: Microsoft-HTTPAPI/2.0 strict-transport-security: max-age=31536000 transfer-encoding: chunked @@ -36,11 +36,11 @@ interactions: response: body: string: 404Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - does not exist. TrackingId:8ba23261-9feb-4628-89d5-61946780df35_G4, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, - Timestamp:2020-09-01T21:36:50 + does not exist. TrackingId:80afec87-e5b0-4843-b1d7-1e51ed76f6f5_G5, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, + Timestamp:2020-09-03T21:36:07 headers: content-type: application/xml; charset=utf-8 - date: Tue, 01 Sep 2020 21:36:49 GMT + date: Thu, 03 Sep 2020 21:36:06 GMT server: Microsoft-HTTPAPI/2.0 strict-transport-security: max-age=31536000 transfer-encoding: chunked diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml index 185e5087a705..b13e7af1bc15 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml @@ -11,28 +11,28 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91?api-version=2017-04 response: body: - string: '{"id":"67ab61f42b964a42820ed8b6e8cec8e7"}' + string: '{"id":"b03b1e31b79f4af8b285428e3ed76814"}' headers: content-type: application/json - date: Tue, 01 Sep 2020 21:36:56 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions/1?api-version=2017-04 + date: Thu, 03 Sep 2020 21:36:13 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions/1?api-version=2017-04 server: Microsoft-HTTPAPI/2.0 strict-transport-security: max-age=31536000 transfer-encoding: chunked - x-schema-id: 67ab61f42b964a42820ed8b6e8cec8e7 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/67ab61f42b964a42820ed8b6e8cec8e7?api-version=2017-04 + x-schema-id: b03b1e31b79f4af8b285428e3ed76814 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b03b1e31b79f4af8b285428e3ed76814?api-version=2017-04 x-schema-type: Avro x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions?api-version=2017-04 + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions?api-version=2017-04 status: code: 200 message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc?api-version=2017-04 + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91?api-version=2017-04 - request: body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' headers: @@ -45,28 +45,28 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91?api-version=2017-04 response: body: - string: '{"id":"163cc506a02c4b31b4b068b347bfae4c"}' + string: '{"id":"c3f0bdbe0b3d475ba7fe0bd437d9c272"}' headers: content-type: application/json - date: Tue, 01 Sep 2020 21:36:56 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions/2?api-version=2017-04 + date: Thu, 03 Sep 2020 21:36:13 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions/2?api-version=2017-04 server: Microsoft-HTTPAPI/2.0 strict-transport-security: max-age=31536000 transfer-encoding: chunked - x-schema-id: 163cc506a02c4b31b4b068b347bfae4c - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/163cc506a02c4b31b4b068b347bfae4c?api-version=2017-04 + x-schema-id: c3f0bdbe0b3d475ba7fe0bd437d9c272 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c3f0bdbe0b3d475ba7fe0bd437d9c272?api-version=2017-04 x-schema-type: Avro x-schema-version: '2' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions?api-version=2017-04 + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions?api-version=2017-04 status: code: 200 message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc?api-version=2017-04 + url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91?api-version=2017-04 - request: body: null headers: @@ -75,24 +75,24 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/163cc506a02c4b31b4b068b347bfae4c?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/c3f0bdbe0b3d475ba7fe0bd437d9c272?api-version=2017-04 response: body: string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' headers: content-type: application/json - date: Tue, 01 Sep 2020 21:36:57 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions/2?api-version=2017-04 + date: Thu, 03 Sep 2020 21:36:13 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions/2?api-version=2017-04 server: Microsoft-HTTPAPI/2.0 strict-transport-security: max-age=31536000 transfer-encoding: chunked - x-schema-id: 163cc506a02c4b31b4b068b347bfae4c - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/163cc506a02c4b31b4b068b347bfae4c?api-version=2017-04 + x-schema-id: c3f0bdbe0b3d475ba7fe0bd437d9c272 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c3f0bdbe0b3d475ba7fe0bd437d9c272?api-version=2017-04 x-schema-type: Avro x-schema-version: '2' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-b780165c-2207-42e9-ba36-ac1f3135d3cc/versions?api-version=2017-04 + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions?api-version=2017-04 status: code: 200 message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/163cc506a02c4b31b4b068b347bfae4c?api-version=2017-04 + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/c3f0bdbe0b3d475ba7fe0bd437d9c272?api-version=2017-04 version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py index e370466dfa65..b03a007243bb 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py @@ -30,8 +30,6 @@ class SchemaRegistryAsyncTests(AzureMgmtTestCase): - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) @@ -65,8 +63,6 @@ async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_ assert returned_schema_properties.version == 1 assert returned_schema_properties.serialization_type == "Avro" - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) @@ -102,8 +98,6 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry assert new_schema.schema_properties.version == 2 assert new_schema.schema_properties.serialization_type == "Avro" - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") @@ -115,8 +109,6 @@ async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpo with pytest.raises(ClientAuthenticationError): await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() async def test_schema_negative_wrong_endpoint_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) @@ -128,8 +120,6 @@ async def test_schema_negative_wrong_endpoint_async(self, schemaregistry_endpoin with pytest.raises(ServiceRequestError): await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() async def test_schema_negative_no_schema_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml index a23e6fd49851..c36bcb7474c7 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml @@ -17,17 +17,17 @@ interactions: X-Schema-Type: - avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569?api-version=2017-04 response: body: - string: '{"id":"b7dda07056e6400aaa547a86d181e7bc"}' + string: '{"id":"5b6a64e2029b43f3b212445a13ad04e1"}' headers: content-type: - application/json date: - - Tue, 01 Sep 2020 21:34:59 GMT + - Thu, 03 Sep 2020 21:34:48 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -35,15 +35,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - b7dda07056e6400aaa547a86d181e7bc + - 5b6a64e2029b43f3b212445a13ad04e1 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b7dda07056e6400aaa547a86d181e7bc?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5b6a64e2029b43f3b212445a13ad04e1?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions?api-version=2017-04 status: code: 200 message: OK @@ -59,7 +59,7 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/b7dda07056e6400aaa547a86d181e7bc?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/5b6a64e2029b43f3b212445a13ad04e1?api-version=2017-04 response: body: string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' @@ -67,9 +67,9 @@ interactions: content-type: - application/json date: - - Tue, 01 Sep 2020 21:34:59 GMT + - Thu, 03 Sep 2020 21:34:48 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -77,15 +77,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - b7dda07056e6400aaa547a86d181e7bc + - 5b6a64e2029b43f3b212445a13ad04e1 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b7dda07056e6400aaa547a86d181e7bc?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5b6a64e2029b43f3b212445a13ad04e1?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions?api-version=2017-04 status: code: 200 message: OK @@ -107,17 +107,17 @@ interactions: X-Schema-Type: - avro method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569?api-version=2017-04 response: body: - string: '{"id":"b7dda07056e6400aaa547a86d181e7bc"}' + string: '{"id":"5b6a64e2029b43f3b212445a13ad04e1"}' headers: content-type: - application/json date: - - Tue, 01 Sep 2020 21:34:59 GMT + - Thu, 03 Sep 2020 21:34:49 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -125,15 +125,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - b7dda07056e6400aaa547a86d181e7bc + - 5b6a64e2029b43f3b212445a13ad04e1 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b7dda07056e6400aaa547a86d181e7bc?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5b6a64e2029b43f3b212445a13ad04e1?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8429a139-b737-4a2a-9008-08281a4e2b61/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions?api-version=2017-04 status: code: 200 message: OK diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml index 5bfe01241dcc..777942e82949 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml @@ -15,14 +15,14 @@ interactions: response: body: string: 400SubCode=40000, UnknownType:The request - is invalid. [MGResponseHttpError=BadRequest]. TrackingId:d63523f2-3c03-4d54-afab-fc9386bd81fb_G0, + is invalid. [MGResponseHttpError=BadRequest]. TrackingId:94d3d37a-2ce8-4284-ae34-fba5253f4e8e_G2, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/a, - Timestamp:2020-09-01T21:35:01 + Timestamp:2020-09-03T21:34:50 headers: content-type: - application/xml; charset=utf-8 date: - - Tue, 01 Sep 2020 21:35:01 GMT + - Thu, 03 Sep 2020 21:34:49 GMT server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -48,13 +48,13 @@ interactions: response: body: string: 404Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - does not exist. TrackingId:a7b590bf-6862-43cd-b9ce-c6ed5d6d5017_G0, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, - Timestamp:2020-09-01T21:35:01 + does not exist. TrackingId:d5c92586-7471-41e1-8e91-7bd497a1828d_G2, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, + Timestamp:2020-09-03T21:34:51 headers: content-type: - application/xml; charset=utf-8 date: - - Tue, 01 Sep 2020 21:35:01 GMT + - Thu, 03 Sep 2020 21:34:51 GMT server: - Microsoft-HTTPAPI/2.0 strict-transport-security: diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml index 2d7a1411ad3f..bfef2ced2c46 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml @@ -15,19 +15,19 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0?api-version=2017-04 response: body: - string: '{"id":"c02dde8d030743b2bd76781606ed5b61"}' + string: '{"id":"c4ff58960b344a89a3d2a1d70388a921"}' headers: content-type: - application/json date: - - Tue, 01 Sep 2020 21:35:08 GMT + - Thu, 03 Sep 2020 21:34:57 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -35,15 +35,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - c02dde8d030743b2bd76781606ed5b61 + - c4ff58960b344a89a3d2a1d70388a921 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c02dde8d030743b2bd76781606ed5b61?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c4ff58960b344a89a3d2a1d70388a921?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0/versions?api-version=2017-04 status: code: 200 message: OK @@ -63,19 +63,19 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0?api-version=2017-04 response: body: - string: '{"id":"c02dde8d030743b2bd76781606ed5b61"}' + string: '{"id":"c4ff58960b344a89a3d2a1d70388a921"}' headers: content-type: - application/json date: - - Tue, 01 Sep 2020 21:35:08 GMT + - Thu, 03 Sep 2020 21:34:57 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -83,15 +83,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - c02dde8d030743b2bd76781606ed5b61 + - c4ff58960b344a89a3d2a1d70388a921 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c02dde8d030743b2bd76781606ed5b61?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c4ff58960b344a89a3d2a1d70388a921?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-fd831ef8-8ea1-46c9-b7ca-50f236bd63c5/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0/versions?api-version=2017-04 status: code: 200 message: OK diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml index b1f25e1e1957..c9c63163e0c0 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml @@ -15,19 +15,19 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b?api-version=2017-04 response: body: - string: '{"id":"e7f36b1ffc6142c2a9fe713d02a61448"}' + string: '{"id":"6defbafaf03c4480a9d70d5dbe78b8fb"}' headers: content-type: - application/json date: - - Tue, 01 Sep 2020 21:35:09 GMT + - Thu, 03 Sep 2020 21:35:00 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions/1?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions/1?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -35,15 +35,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - e7f36b1ffc6142c2a9fe713d02a61448 + - 6defbafaf03c4480a9d70d5dbe78b8fb x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/e7f36b1ffc6142c2a9fe713d02a61448?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/6defbafaf03c4480a9d70d5dbe78b8fb?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '1' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions?api-version=2017-04 status: code: 200 message: OK @@ -63,19 +63,19 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) X-Schema-Type: - - avro + - Avro method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b?api-version=2017-04 response: body: - string: '{"id":"b6caf7d1944a4f8e8247cf2f2bb9c15d"}' + string: '{"id":"274d0cbe53ed4002b0857d61a9139e78"}' headers: content-type: - application/json date: - - Tue, 01 Sep 2020 21:35:09 GMT + - Thu, 03 Sep 2020 21:35:00 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions/2?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions/2?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -83,15 +83,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - b6caf7d1944a4f8e8247cf2f2bb9c15d + - 274d0cbe53ed4002b0857d61a9139e78 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b6caf7d1944a4f8e8247cf2f2bb9c15d?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/274d0cbe53ed4002b0857d61a9139e78?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '2' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions?api-version=2017-04 status: code: 200 message: OK @@ -107,7 +107,7 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/b6caf7d1944a4f8e8247cf2f2bb9c15d?api-version=2017-04 + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/274d0cbe53ed4002b0857d61a9139e78?api-version=2017-04 response: body: string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' @@ -115,9 +115,9 @@ interactions: content-type: - application/json date: - - Tue, 01 Sep 2020 21:35:11 GMT + - Thu, 03 Sep 2020 21:35:01 GMT location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions/2?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions/2?api-version=2017-04 server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -125,15 +125,15 @@ interactions: transfer-encoding: - chunked x-schema-id: - - b6caf7d1944a4f8e8247cf2f2bb9c15d + - 274d0cbe53ed4002b0857d61a9139e78 x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b6caf7d1944a4f8e8247cf2f2bb9c15d?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/274d0cbe53ed4002b0857d61a9139e78?api-version=2017-04 x-schema-type: - Avro x-schema-version: - '2' x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-dcba3084-9a8d-4b82-92ea-ddd634bcc71b/versions?api-version=2017-04 + - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions?api-version=2017-04 status: code: 200 message: OK diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index fd78e1bd2d83..a419b6416037 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -30,8 +30,6 @@ class SchemaRegistryTests(AzureMgmtTestCase): - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) @@ -64,8 +62,6 @@ def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schem assert returned_schema_properties.version == 1 assert returned_schema_properties.serialization_type == "Avro" - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) @@ -100,8 +96,6 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, sche assert new_schema.schema_properties.version == 2 assert new_schema.schema_properties.serialization_type == "Avro" - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) @@ -113,8 +107,6 @@ def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, schema_properties_second = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) assert schema_properties.schema_id == schema_properties_second.schema_id - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") @@ -125,8 +117,6 @@ def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemar with pytest.raises(ClientAuthenticationError): client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) @@ -137,8 +127,6 @@ def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemareg with pytest.raises(ServiceRequestError): client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) - @pytest.mark.liveTest - @pytest.mark.live_test_only @SchemaRegistryPreparer() def test_schema_negative_no_schema(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) From d8e0986af99651e17025533a52e3c36106e5c0df Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 3 Sep 2020 15:14:20 -0700 Subject: [PATCH 30/35] remove dictmixin for b1, update comment in sample --- .../samples/avro_serializer.py | 30 +++++++--- .../azure/schemaregistry/_common/_schema.py | 55 +------------------ 2 files changed, 23 insertions(+), 62 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py index 769f1860dd66..2a32b699da07 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py @@ -53,20 +53,32 @@ ) -def serialize(serializer, schema, dict_data): - bytes = serializer.serialize(dict_data, schema) - print('Encoded bytes are: ', bytes) - return bytes +def serialize(serializer): + dict_data_ben = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + dict_data_alice = {"name": u"Alice", "favorite_number": 15, "favorite_color": u"green"} + # Schema would be automatically registered into Schema Registry and cached locally. + payload_ben = serializer.serialize(dict_data_ben, SCHEMA_STRING) + # The second call won't trigger a service call. + payload_alice = serializer.serialize(dict_data_alice, SCHEMA_STRING) + + print('Encoded bytes are: ', payload_ben) + print('Encoded bytes are: ', payload_alice) + return [payload_ben, payload_alice] + + +def deserialize(serializer, bytes_payload): + # serializer.deserialize would extract the schema id from the payload, + # retrieve schema from Schema Registry and cache the schema locally. + # If the schema id is the local cache, the call won't trigger a service call. + dict_data = serializer.deserialize(bytes_payload) -def deserialize(serializer, bytes): - dict_data = serializer.deserialize(bytes) print('Deserialized data is: ', dict_data) return dict_data if __name__ == '__main__': serializer = SchemaRegistryAvroSerializer(SCHEMA_REGISTRY_ENDPOINT, token_credential, SCHEMA_GROUP) - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - payload_bytes = serialize(serializer, SCHEMA_STRING, dict_data) - dict_data = deserialize(serializer, payload_bytes) + bytes_data_ben, bytes_data_alice = serialize(serializer) + dict_data_ben = deserialize(serializer, bytes_data_ben) + dict_data_alice = deserialize(serializer, bytes_data_alice) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index ccd8184a61db..6864db3ae5e3 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -26,58 +26,7 @@ from typing import Any, Optional -class DictMixin(object): - - def __setitem__(self, key, item): - self.__dict__[key] = item - - def __getitem__(self, key): - return self.__dict__[key] - - def __repr__(self): - return str(self) - - def __len__(self): - return len(self.keys()) - - def __delitem__(self, key): - self.__dict__[key] = None - - def __eq__(self, other): - """Compare objects by comparing all attributes.""" - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return False - - def __ne__(self, other): - """Compare objects by comparing all attributes.""" - return not self.__eq__(other) - - def __str__(self): - return str({k: v for k, v in self.__dict__.items() if not k.startswith('_')}) - - def has_key(self, k): - return k in self.__dict__ - - def update(self, *args, **kwargs): - return self.__dict__.update(*args, **kwargs) - - def keys(self): - return [k for k in self.__dict__ if not k.startswith('_')] - - def values(self): - return [v for k, v in self.__dict__.items() if not k.startswith('_')] - - def items(self): - return [(k, v) for k, v in self.__dict__.items() if not k.startswith('_')] - - def get(self, key, default=None): - if key in self.__dict__: - return self.__dict__[key] - return default - - -class SchemaProperties(DictMixin): +class SchemaProperties(object): """ Meta properties of a schema. @@ -115,7 +64,7 @@ def __init__( self.version = kwargs.get('X-Schema-Version') -class Schema(DictMixin): +class Schema(object): """ The schema content of a schema, along with id and meta properties. From b91fb4fab86964abf934870d4c4110ef696445df Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 3 Sep 2020 18:03:02 -0700 Subject: [PATCH 31/35] update api in avroserializer and update test and readme --- .../README.md | 18 +++++--- .../_schema_registry_avro_serializer.py | 15 +++---- .../samples/avro_serializer.py | 5 ++- .../tests/schemaregistry_preparer.py | 17 ++++---- .../tests/test_avro_serializer.py | 15 +++---- .../async_tests/test_schema_registry_async.py | 34 +++++++-------- .../tests/schemaregistry_preparer.py | 19 +++++---- .../tests/test_schema_registry.py | 41 ++++++++----------- sdk/schemaregistry/tests.yml | 9 ++-- 9 files changed, 85 insertions(+), 88 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md index df33e31bcfa4..055b069616ac 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md @@ -25,13 +25,15 @@ Interaction with Schema Registry Avro Serializer starts with an instance of Sche **Create client using the azure-identity library:** ```python +from azure.schemaregistry import SchemaRegistryClient from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer from azure.identity import DefaultAzureCredential credential = DefaultAzureCredential() endpoint = '<< ENDPOINT OF THE SCHEMA REGISTRY >>' schema_group = '<< GROUP NAME OF THE SCHEMA >>' -schema_registry_client = SchemaRegistryAvroSerializer(endpoint, credential, schema_group) +schema_registry_client = SchemaRegistryClient(endpoint, credential) +serializer = SchemaRegistryAvroSerializer(schema_registry_client, schema_group) ``` ## Key concepts @@ -49,6 +51,7 @@ The following sections provide several code snippets covering some of the most c ```python import os +from azure.schemaregistry import SchemaRegistryClient from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer from azure.identity import DefaultAzureCredential @@ -56,7 +59,8 @@ token_credential = DefaultAzureCredential() endpoint = os.environ['SCHEMA_REGISTRY_ENDPOINT'] schema_group = "" -serializer = SchemaRegistryAvroSerializer(endpoint, token_credential, schema_group) +schema_registry_client = SchemaRegistryClient(endpoint, token_credential) +serializer = SchemaRegistryAvroSerializer(schema_registry_client, schema_group) schema_string = """ {"namespace": "example.avro", @@ -78,6 +82,7 @@ with serializer: ```python import os +from azure.schemaregistry import SchemaRegistryClient from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer from azure.identity import DefaultAzureCredential @@ -85,7 +90,8 @@ token_credential = DefaultAzureCredential() endpoint = os.environ['SCHEMA_REGISTRY_ENDPOINT'] schema_group = "" -serializer = SchemaRegistryAvroSerializer(endpoint, token_credential, schema_group) +schema_registry_client = SchemaRegistryClient(endpoint, token_credential) +serializer = SchemaRegistryAvroSerializer(schema_registry_client, schema_group) with serializer: encoded_bytes = b'' @@ -109,6 +115,7 @@ headers, can be enabled on a client with the `logging_enable` argument: ```python import sys import logging +from azure.schemaregistry import SchemaRegistryClient from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer from azure.identity import DefaultAzureCredential @@ -121,14 +128,15 @@ handler = logging.StreamHandler(stream=sys.stdout) logger.addHandler(handler) credential = DefaultAzureCredential() +schema_registry_client = SchemaRegistryClient("", credential) # This client will log detailed information about its HTTP sessions, at DEBUG level -avro_serializer = SchemaRegistryAvroSerializer("you_end_point", credential, "your_group_name", logging_enable=True) +serializer = SchemaRegistryAvroSerializer(schema_registry_client, "", logging_enable=True) ``` Similarly, `logging_enable` can enable detailed logging for a single operation, even when it isn't enabled for the client: ```py -avro_serializer.serialie(dict_data, schema_content, logging_enable=True) +serializer.serialie(dict_data, schema_content, logging_enable=True) ``` ## Next steps diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index 0465058c50d0..001cfd2d45cc 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -31,27 +31,24 @@ from ._avro_serializer import AvroObjectSerializer -if TYPE_CHECKING: - from azure.core.credentials import TokenCredential - class SchemaRegistryAvroSerializer(object): """ SchemaRegistryAvroSerializer provides the ability to serialize and deserialize data according to the given avro schema. It would automatically register, get and cache the schema. - :param str endpoint: The Schema Registry service endpoint, for example my-namespace.servicebus.windows.net. - :param credential: To authenticate to manage the entities of the SchemaRegistry namespace. - :type credential: TokenCredential + :param schema_registry: The schema registry client + which is used to register schema and retrieve schema from the service. + :type schema_registry: SchemaRegistryClient :param str schema_group: Schema group under which schema should be registered. :keyword str codec: The writer codec. If None, let the avro library decides. """ - def __init__(self, endpoint, credential, schema_group, **kwargs): - # type: (str, TokenCredential, str, Any) -> None + def __init__(self, schema_registry, schema_group, **kwargs): + # type: (SchemaRegistryClient, str, Any) -> None self._schema_group = schema_group self._avro_serializer = AvroObjectSerializer(codec=kwargs.get("codec")) - self._schema_registry_client = SchemaRegistryClient(credential=credential, endpoint=endpoint, **kwargs) + self._schema_registry_client = schema_registry # type: SchemaRegistryClient self._id_to_schema = {} self._schema_to_id = {} self._user_input_schema_cache = {} diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py index 2a32b699da07..031ce09c7aa6 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py @@ -26,6 +26,7 @@ import os from azure.identity import ClientSecretCredential +from azure.schemaregistry import SchemaRegistryClient from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer TENANT_ID=os.environ['SCHEMA_REGISTRY_AZURE_TENANT_ID'] @@ -78,7 +79,9 @@ def deserialize(serializer, bytes_payload): if __name__ == '__main__': - serializer = SchemaRegistryAvroSerializer(SCHEMA_REGISTRY_ENDPOINT, token_credential, SCHEMA_GROUP) + schema_registry = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) + serializer = SchemaRegistryAvroSerializer(schema_registry, SCHEMA_GROUP) bytes_data_ben, bytes_data_alice = serialize(serializer) dict_data_ben = deserialize(serializer, bytes_data_ben) dict_data_alice = deserialize(serializer, bytes_data_alice) + serializer.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py index 7f33dd0c50e7..db1e468b23c2 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py @@ -63,13 +63,16 @@ def __init__( def create_resource(self, name, **kwargs): # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using api, in the future we should be able to dynamically create and remove resources - return { - SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], - SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME], - SCHEMA_REGISTRY_TENANT_ID_PARAM: os.environ[AZURE_TENANT_ID_ENV_KEY_NAME], - SCHEMA_REGISTRY_CLIENT_ID_PARAM: os.environ[AZURE_CLIENT_ID_ENV_KEY_NAME], - SCHEMA_REGISTRY_CLIENT_SECRET_PARAM: os.environ[AZURE_CLIENT_SECRET_ENV_KEY_NAME] - } + if self.is_live: + return { + SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], + SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME] + } + else: + return { + SCHEMA_REGISTRY_ENDPOINT_PARAM: "fake.servicebus.windows.net", + SCHEMA_REGISTRY_GROUP_PARAM: "fake-group" + } def remove_resource(self, name, **kwargs): pass diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py index 84521f94cb7a..31adadb1c08d 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -30,10 +30,10 @@ from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError from schemaregistry_preparer import SchemaRegistryPreparer -from devtools_testutils import AzureMgmtTestCase +from devtools_testutils import AzureTestCase -class SchemaRegistryAvroSerializerTests(AzureMgmtTestCase): +class SchemaRegistryAvroSerializerTests(AzureTestCase): def test_raw_avro_serializer(self): schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" @@ -74,13 +74,9 @@ def test_raw_avro_serializer_negative(self): raw_avro_object_serializer.serialize(dict_data_missing_required_field, schema) @SchemaRegistryPreparer() - def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, - schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, - client_secret=schemaregistry_client_secret) - - sr_avro_serializer = SchemaRegistryAvroSerializer(schemaregistry_endpoint, credential, schemaregistry_group) - sr_client = SchemaRegistryClient(schemaregistry_endpoint, credential) + def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) + sr_avro_serializer = SchemaRegistryAvroSerializer(sr_client, schemaregistry_group) random_schema_namespace = "testschema" + str(uuid.uuid4())[0:8] schema_str = "{\"namespace\":\"" + random_schema_namespace + "\"" +\ @@ -105,4 +101,3 @@ def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_ assert decoded_data["favorite_color"] == u"red" sr_avro_serializer.close() - sr_client.close() diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py index b03a007243bb..29e8df3cc26a 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py @@ -25,16 +25,15 @@ from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError from schemaregistry_preparer import SchemaRegistryPreparer -from devtools_testutils import AzureMgmtTestCase +from devtools_testutils import AzureTestCase -class SchemaRegistryAsyncTests(AzureMgmtTestCase): +class SchemaRegistryAsyncTests(AzureTestCase): @SchemaRegistryPreparer() - async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - async with client, credential: + async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) + async with client: schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" @@ -64,10 +63,9 @@ async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_ assert returned_schema_properties.serialization_type == "Avro" @SchemaRegistryPreparer() - async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - async with client, credential: + async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) + async with client: schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" @@ -99,7 +97,7 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry assert new_schema.schema_properties.serialization_type == "Avro" @SchemaRegistryPreparer() - async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) async with client, credential: @@ -110,10 +108,9 @@ async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpo await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @SchemaRegistryPreparer() - async def test_schema_negative_wrong_endpoint_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=str(uuid.uuid4()) + ".servicebus.windows.net", credential=credential) - async with client, credential: + async def test_schema_negative_wrong_endpoint_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint="nonexist.servicebus.windows.net") + async with client: schema_name = 'test-schema-' + str(uuid.uuid4()) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" @@ -121,10 +118,9 @@ async def test_schema_negative_wrong_endpoint_async(self, schemaregistry_endpoin await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @SchemaRegistryPreparer() - async def test_schema_negative_no_schema_async(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - async with client, credential: + async def test_schema_negative_no_schema_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) + async with client: with pytest.raises(HttpResponseError): await client.get_schema('a') diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py index 70c6ab9aaf06..db1e468b23c2 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py @@ -62,14 +62,17 @@ def __init__( super(SchemaRegistryPreparer, self).__init__(name_prefix, 24) def create_resource(self, name, **kwargs): - # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using mgmt api, in the future we should be able to dynamically create and remove resources - return { - SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], - SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME], - SCHEMA_REGISTRY_TENANT_ID_PARAM: os.environ[AZURE_TENANT_ID_ENV_KEY_NAME], - SCHEMA_REGISTRY_CLIENT_ID_PARAM: os.environ[AZURE_CLIENT_ID_ENV_KEY_NAME], - SCHEMA_REGISTRY_CLIENT_SECRET_PARAM: os.environ[AZURE_CLIENT_SECRET_ENV_KEY_NAME] - } + # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using api, in the future we should be able to dynamically create and remove resources + if self.is_live: + return { + SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], + SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME] + } + else: + return { + SCHEMA_REGISTRY_ENDPOINT_PARAM: "fake.servicebus.windows.net", + SCHEMA_REGISTRY_GROUP_PARAM: "fake-group" + } def remove_resource(self, name, **kwargs): pass diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index a419b6416037..9c882f12bba5 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -25,16 +25,15 @@ from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError from schemaregistry_preparer import SchemaRegistryPreparer -from devtools_testutils import AzureMgmtTestCase +from devtools_testutils import AzureTestCase -class SchemaRegistryTests(AzureMgmtTestCase): +class SchemaRegistryTests(AzureTestCase): @SchemaRegistryPreparer() - def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - schema_name = 'test-schema-' + str(uuid.uuid4()) + def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) + schema_name = 'test-schema-basic' schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -63,10 +62,9 @@ def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, schem assert returned_schema_properties.serialization_type == "Avro" @SchemaRegistryPreparer() - def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - schema_name = 'test-schema-' + str(uuid.uuid4()) + def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) + schema_name = 'test-schema-update' schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -97,10 +95,9 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, sche assert new_schema.schema_properties.serialization_type == "Avro" @SchemaRegistryPreparer() - def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - schema_name = 'test-schema-' + str(uuid.uuid4()) + def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) + schema_name = 'test-schema-twice' schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":["int","null"]},{"name":"city","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -108,29 +105,27 @@ def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, assert schema_properties.schema_id == schema_properties_second.schema_id @SchemaRegistryPreparer() - def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): + def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_name = 'test-schema-negative' schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ClientAuthenticationError): client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @SchemaRegistryPreparer() - def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=str(uuid.uuid4()) + ".servicebus.windows.net", credential=credential) - schema_name = 'test-schema-' + str(uuid.uuid4()) + def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint="nonexist.servicebus.windows.net") + schema_name = 'test-schema-nonexist' schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ServiceRequestError): client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @SchemaRegistryPreparer() - def test_schema_negative_no_schema(self, schemaregistry_endpoint, schemaregistry_group, schemaregistry_tenant_id, schemaregistry_client_id, schemaregistry_client_secret, **kwargs): - credential = ClientSecretCredential(tenant_id=schemaregistry_tenant_id, client_id=schemaregistry_client_id, client_secret=schemaregistry_client_secret) - client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) + def test_schema_negative_no_schema(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) with pytest.raises(HttpResponseError): client.get_schema('a') diff --git a/sdk/schemaregistry/tests.yml b/sdk/schemaregistry/tests.yml index a3a5d667351a..3dc85ec512a9 100644 --- a/sdk/schemaregistry/tests.yml +++ b/sdk/schemaregistry/tests.yml @@ -8,12 +8,9 @@ jobs: BuildTargetingString: azure-schemaregistry* EnvVars: AZURE_SUBSCRIPTION_ID: $(azure-subscription-id) - AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id) - AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id) - AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) - SCHEMA_REGISTRY_AZURE_TENANT_ID: $(python-schema-registry-sdk-test-tenant-id) - SCHEMA_REGISTRY_AZURE_CLIENT_ID: $(python-schema-registry-sdk-test-client-id) - SCHEMA_REGISTRY_AZURE_CLIENT_SECRET: $(python-schema-registry-sdk-test-client-secret) + AZURE_TENANT_ID: $(python-schema-registry-sdk-test-tenant-id) + AZURE_CLIENT_ID: $(python-schema-registry-sdk-test-client-id) + AZURE_CLIENT_SECRET: $(python-schema-registry-sdk-test-client-secret) SCHEMA_REGISTRY_ENDPOINT: $(python-schema-registry-sdk-test-endpoint) SCHEMA_REGISTRY_GROUP: $(python-schema-registry-sdk-test-group) AZURE_TEST_RUN_LIVE: 'true' From 929ee68b8b67b077782ce2828a40eb06413019b4 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Thu, 3 Sep 2020 22:04:06 -0700 Subject: [PATCH 32/35] update test, docs and links --- eng/ignore-links.txt | 11 +- .../README.md | 14 +- .../_schema_registry_avro_serializer.py | 11 +- .../samples/README.md | 8 +- ...ializer.test_basic_sr_avro_serializer.yaml | 104 ------------- .../tests/schemaregistry_preparer.py | 11 +- .../azure-schemaregistry/README.md | 18 ++- .../azure-schemaregistry/samples/README.md | 9 +- ...hema_registry_async.test_schema_basic.yaml | 98 ------------ ...egistry_async.test_schema_basic_async.yaml | 98 ------------ ....test_schema_negative_no_schema_async.yaml | 51 ------- ...ema_registry_async.test_schema_update.yaml | 98 ------------ ...gistry_async.test_schema_update_async.yaml | 98 ------------ .../async_tests/test_schema_registry_async.py | 93 +++++++++++- ..._schema_registry.test_register_schema.yaml | 140 ------------------ ...est_schema_registry.test_schema_basic.yaml | 140 ------------------ ...gistry.test_schema_negative_no_schema.yaml | 67 --------- ...chema_registry.test_schema_same_twice.yaml | 98 ------------ ...st_schema_registry.test_schema_update.yaml | 140 ------------------ .../tests/schemaregistry_preparer.py | 10 +- .../tests/test_schema_registry.py | 10 +- 21 files changed, 151 insertions(+), 1176 deletions(-) delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_register_schema.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml diff --git a/eng/ignore-links.txt b/eng/ignore-links.txt index bd605f16d16a..806cc64a3fe8 100644 --- a/eng/ignore-links.txt +++ b/eng/ignore-links.txt @@ -1 +1,10 @@ -https://docs.microsoft.com/python/api/overview/azure/{{package_doc_id}} \ No newline at end of file +https://docs.microsoft.com/python/api/overview/azure/{{package_doc_id}} +https://pypi.org/project/azure-schemaregistry +https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry/latest/index.html +https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py +https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py +https://pypi.org/project/azure-schemaregistry-avroserializer +https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md +https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry-avroserializer/latest/index.html +https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples +https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md index 055b069616ac..c4ef1006c343 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md @@ -3,6 +3,8 @@ Azure Schema Registry Avro Serializer provides the ability to serialize and deserialize data according to the given avro schema. It is integrated with Azure Schema Registry SDK and will automatically register and get schema. +[Source code][source_code] | [Package (PyPi)][pypi] | [API reference documentation][api_docs] | [Samples][sr_avro_samples] | [Changelog][change_log] + ## Getting started ### Install the package @@ -44,8 +46,8 @@ serializer = SchemaRegistryAvroSerializer(schema_registry_client, schema_group) The following sections provide several code snippets covering some of the most common Schema Registry tasks, including: -- [Serialization](serialization) -- [Deserialization](deserialization) +- [Serialization](#serialization) +- [Deserialization](#deserialization) ### Serialization @@ -143,7 +145,7 @@ serializer.serialie(dict_data, schema_content, logging_enable=True) ### More sample code -Please find further examples in the [samples](./samples) directory demonstrating common Azure Schema Registry Avro Serializer scenarios. +Please find further examples in the [samples][sr_avro_samples] directory demonstrating common Azure Schema Registry Avro Serializer scenarios. ## Contributing @@ -159,8 +161,14 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + [pip]: https://pypi.org/project/pip/ +[pypi]: https://pypi.org/project/azure-schemaregistry-avroserializer [python]: https://www.python.org/downloads/ [azure_core]: https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/README.md [azure_sub]: https://azure.microsoft.com/free/ [python_logging]: https://docs.python.org/3/library/logging.html +[sr_avro_samples]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples +[api_reference]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry-avroserializer/latest/index.html +[source_code]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer +[change_log]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index 001cfd2d45cc..367d84aea99b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -24,10 +24,9 @@ # # -------------------------------------------------------------------------- from io import BytesIO -from typing import Any, Dict, Union, TYPE_CHECKING +from typing import Any, Dict, Union import avro -from azure.schemaregistry import SchemaRegistryClient, SerializationType from ._avro_serializer import AvroObjectSerializer @@ -39,16 +38,16 @@ class SchemaRegistryAvroSerializer(object): :param schema_registry: The schema registry client which is used to register schema and retrieve schema from the service. - :type schema_registry: SchemaRegistryClient + :type schema_registry: ~azure.schemaregistry.SchemaRegistryClient :param str schema_group: Schema group under which schema should be registered. :keyword str codec: The writer codec. If None, let the avro library decides. """ def __init__(self, schema_registry, schema_group, **kwargs): - # type: (SchemaRegistryClient, str, Any) -> None + # type: ("SchemaRegistryClient", str, Any) -> None self._schema_group = schema_group self._avro_serializer = AvroObjectSerializer(codec=kwargs.get("codec")) - self._schema_registry_client = schema_registry # type: SchemaRegistryClient + self._schema_registry_client = schema_registry # type: "SchemaRegistryClient" self._id_to_schema = {} self._schema_to_id = {} self._user_input_schema_cache = {} @@ -89,7 +88,7 @@ def _get_schema_id(self, schema_name, schema, **kwargs): schema_id = self._schema_registry_client.register_schema( self._schema_group, schema_name, - SerializationType.AVRO, + "Avro", schema_str, **kwargs ).schema_id diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md index 972766cadf0b..5c235b8bccba 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md @@ -14,7 +14,7 @@ These are code samples that show common scenario operations with the Schema Regi Several Schema Registry Avro Serializer Python SDK samples are available to you in the SDK's GitHub repository. These samples provide example code for additional scenarios commonly encountered while working with Schema Registry Avro Serializer: -* [avro_serializer.py](./avro_serializer.py) - Examples for common Schema Registry Avro Serializer tasks: +* [avro_serializer.py][avro_serializer_sample] - Examples for common Schema Registry Avro Serializer tasks: * Serialize data according to the given schema * Deserialize data @@ -42,5 +42,9 @@ pip install azure-schemaregistry-avroserializer azure-identity ## Next steps -Check out the [API reference documentation](https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry-avro-serializer/latest/index.html) to learn more about +Check out the [API reference documentation][api_reference] to learn more about what you can do with the Azure Schema Registry Avro Serializer library. + + +[avro_serializer_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py +[api_reference]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry-avroserializer/latest/index.html diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml deleted file mode 100644 index e8db561effb4..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml +++ /dev/null @@ -1,104 +0,0 @@ -interactions: -- request: - body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"testschemabe93dcc6\", - \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", - \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], - \"name\": \"favorite_color\"}]}"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '271' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User?api-version=2017-04 - response: - body: - string: '{"id":"2bd850ebc8e6415aa89684ce8fc333c3"}' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:39:02 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 2bd850ebc8e6415aa89684ce8fc333c3 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2bd850ebc8e6415aa89684ce8fc333c3?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User/versions?api-version=2017-04 - status: - code: 200 - message: OK -- request: - body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"testschemabe93dcc6\", - \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", - \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], - \"name\": \"favorite_color\"}]}"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '271' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User?api-version=2017-04 - response: - body: - string: '{"id":"2bd850ebc8e6415aa89684ce8fc333c3"}' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:39:03 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 2bd850ebc8e6415aa89684ce8fc333c3 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2bd850ebc8e6415aa89684ce8fc333c3?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/testschemabe93dcc6.User/versions?api-version=2017-04 - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py index db1e468b23c2..f1f78608d0d0 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py @@ -32,14 +32,8 @@ SCHEMA_REGISTRY_ENDPOINT_PARAM = "schemaregistry_endpoint" SCHEMA_REGISTRY_GROUP_PARAM = "schemaregistry_group" -SCHEMA_REGISTRY_TENANT_ID_PARAM = "schemaregistry_tenant_id" -SCHEMA_REGISTRY_CLIENT_ID_PARAM = "schemaregistry_client_id" -SCHEMA_REGISTRY_CLIENT_SECRET_PARAM = "schemaregistry_client_secret" SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME = 'SCHEMA_REGISTRY_ENDPOINT' SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME = 'SCHEMA_REGISTRY_GROUP' -AZURE_TENANT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_TENANT_ID' -AZURE_CLIENT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_ID' -AZURE_CLIENT_SECRET_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_SECRET' class SchemaRegistryNamespacePreparer(AzureMgmtPreparer): @@ -68,10 +62,11 @@ def create_resource(self, name, **kwargs): SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME] } + else: return { - SCHEMA_REGISTRY_ENDPOINT_PARAM: "fake.servicebus.windows.net", - SCHEMA_REGISTRY_GROUP_PARAM: "fake-group" + SCHEMA_REGISTRY_ENDPOINT_PARAM: "sr-playground.servicebus.windows.net", + SCHEMA_REGISTRY_GROUP_PARAM: "azsdk_python_test_group" } def remove_resource(self, name, **kwargs): diff --git a/sdk/schemaregistry/azure-schemaregistry/README.md b/sdk/schemaregistry/azure-schemaregistry/README.md index 61bfb236cfdd..8ef6b8790650 100644 --- a/sdk/schemaregistry/azure-schemaregistry/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/README.md @@ -3,6 +3,8 @@ Azure Schema Registry is a service that provides the ability to store and retrieve different types of schemas such as Avro, Json, etc. +[Source code][source_code] | [Package (PyPi)][pypi] | [API reference documentation][api_reference] | [Samples][sr_samples] | [Changelog][change_log] + ## Getting started ### Install the package @@ -41,9 +43,9 @@ schema_registry_client = SchemaRegistryClient(endpoint, credential) The following sections provide several code snippets covering some of the most common Schema Registry tasks, including: -- [Register a schema](register-a-schema) -- [Get the schema by id](get-the-schema-by-id) -- [Get the id of a schema](get-the-id-of-a-schema) +- [Register a schema](#register-a-schema) +- [Get the schema by id](#get-the-schema-by-id) +- [Get the id of a schema](#get-the-id-of-a-schema) ### Register a schema @@ -168,7 +170,7 @@ schema_registry_client.get_schema(schema_id, logging_enable=True) ### More sample code -Please take a look at the [samples](./samples) directory for detailed examples of how to use this library to register and retrieve schema to/from Schema Registry. +Please take a look at the [samples][sr_samples] directory for detailed examples of how to use this library to register and retrieve schema to/from Schema Registry. ## Contributing @@ -184,8 +186,14 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + [pip]: https://pypi.org/project/pip/ +[pypi]: https://pypi.org/project/azure-schemaregistry [python]: https://www.python.org/downloads/ [azure_core]: https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/README.md [azure_sub]: https://azure.microsoft.com/free/ -[python_logging]: https://docs.python.org/3/library/logging.html \ No newline at end of file +[python_logging]: https://docs.python.org/3/library/logging.html +[sr_samples]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/samples +[api_reference]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry/latest/index.html +[source_code]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry +[change_log]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/README.md b/sdk/schemaregistry/azure-schemaregistry/samples/README.md index 826f4ddaadb1..a8eddb04a14e 100644 --- a/sdk/schemaregistry/azure-schemaregistry/samples/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/samples/README.md @@ -16,7 +16,7 @@ and require Python 3.5 or later. Several Schema Registry Python SDK samples are available to you in the SDK's GitHub repository. These samples provide example code for additional scenarios commonly encountered while working with Schema Registry: -* [schema_registry.py](./sync_samples/schema_registry.py) ([async version](./async_samples/schema_registry_async.py)) - Examples for common Schema Registry tasks: +* [schema_registry.py][schema_registry_sample] ([async version][schema_registry_async_sample]) - Examples for common Schema Registry tasks: * Register a schema * Get schema by id * Get schema id @@ -45,5 +45,10 @@ pip install azure-schemaregistry azure-identity ## Next steps -Check out the [API reference documentation](https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry/latest/index.html) to learn more about +Check out the [API reference documentation][api_reference] to learn more about what you can do with the Azure Schema Registry client library. + + +[schema_registry_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/schema_registry.py +[schema_registry_async_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/schema_registry_async.py +[api_reference]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry/latest/index.html diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic.yaml deleted file mode 100644 index 7f21ba6b7f6e..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic.yaml +++ /dev/null @@ -1,98 +0,0 @@ -interactions: -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4?api-version=2017-04 - response: - body: - string: '{"id":"42278eb1818b4a16bc2580ed5f70e417"}' - headers: - content-type: application/json - date: Tue, 25 Aug 2020 03:07:09 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions/1?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: 42278eb1818b4a16bc2580ed5f70e417 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4?api-version=2017-04 -- request: - body: null - headers: - Accept: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 - response: - body: - string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - content-type: application/json - date: Tue, 25 Aug 2020 03:07:09 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions/1?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: 42278eb1818b4a16bc2580ed5f70e417 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4?api-version=2017-04 - response: - body: - string: '{"id":"42278eb1818b4a16bc2580ed5f70e417"}' - headers: - content-type: application/json - date: Tue, 25 Aug 2020 03:07:09 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions/1?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: 42278eb1818b4a16bc2580ed5f70e417 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/42278eb1818b4a16bc2580ed5f70e417?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-20071b22-55d4-4e59-a4b1-ff3f47ffeec4?api-version=2017-04 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml deleted file mode 100644 index 7ef9979f0e00..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml +++ /dev/null @@ -1,98 +0,0 @@ -interactions: -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579?api-version=2017-04 - response: - body: - string: '{"id":"6f12bc50b6944b8997def314d7b9a2de"}' - headers: - content-type: application/json - date: Thu, 03 Sep 2020 21:36:03 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions/1?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: 6f12bc50b6944b8997def314d7b9a2de - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579?api-version=2017-04 -- request: - body: null - headers: - Accept: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 - response: - body: - string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - content-type: application/json - date: Thu, 03 Sep 2020 21:36:03 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions/1?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: 6f12bc50b6944b8997def314d7b9a2de - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579?api-version=2017-04 - response: - body: - string: '{"id":"6f12bc50b6944b8997def314d7b9a2de"}' - headers: - content-type: application/json - date: Thu, 03 Sep 2020 21:36:05 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions/1?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: 6f12bc50b6944b8997def314d7b9a2de - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/6f12bc50b6944b8997def314d7b9a2de?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8727ce01-362e-4bcf-985e-d4ba90072579?api-version=2017-04 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml deleted file mode 100644 index d6aec1d501db..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml +++ /dev/null @@ -1,51 +0,0 @@ -interactions: -- request: - body: null - headers: - Accept: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 - response: - body: - string: 400SubCode=40000, UnknownType:The request - is invalid. [MGResponseHttpError=BadRequest]. TrackingId:89f17396-2d5e-49d4-aaaa-402ce6f5257b_G5, - SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/a, - Timestamp:2020-09-03T21:36:06 - headers: - content-type: application/xml; charset=utf-8 - date: Thu, 03 Sep 2020 21:36:06 GMT - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 400 - message: Bad Request - url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 -- request: - body: null - headers: - Accept: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 - response: - body: - string: 404Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - does not exist. TrackingId:80afec87-e5b0-4843-b1d7-1e51ed76f6f5_G5, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, - Timestamp:2020-09-03T21:36:07 - headers: - content-type: application/xml; charset=utf-8 - date: Thu, 03 Sep 2020 21:36:06 GMT - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 404 - message: Not Found - url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update.yaml deleted file mode 100644 index cfae03285dbc..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update.yaml +++ /dev/null @@ -1,98 +0,0 @@ -interactions: -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d?api-version=2017-04 - response: - body: - string: '{"id":"fd3af450331b427a90260487b6a593f1"}' - headers: - content-type: application/json - date: Tue, 25 Aug 2020 03:11:20 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions/1?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: fd3af450331b427a90260487b6a593f1 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/fd3af450331b427a90260487b6a593f1?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d?api-version=2017-04 -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Content-Length: - - '244' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d?api-version=2017-04 - response: - body: - string: '{"id":"5a04b4ccfd9e4dc6bc98adcd04785031"}' - headers: - content-type: application/json - date: Tue, 25 Aug 2020 03:11:20 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions/2?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: 5a04b4ccfd9e4dc6bc98adcd04785031 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5a04b4ccfd9e4dc6bc98adcd04785031?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '2' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d?api-version=2017-04 -- request: - body: null - headers: - Accept: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/5a04b4ccfd9e4dc6bc98adcd04785031?api-version=2017-04 - response: - body: - string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' - headers: - content-type: application/json - date: Tue, 25 Aug 2020 03:11:21 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions/2?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: 5a04b4ccfd9e4dc6bc98adcd04785031 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5a04b4ccfd9e4dc6bc98adcd04785031?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '2' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-95fcb46a-0f9d-4adb-bb4b-1ad881f78f3d/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/5a04b4ccfd9e4dc6bc98adcd04785031?api-version=2017-04 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml deleted file mode 100644 index b13e7af1bc15..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml +++ /dev/null @@ -1,98 +0,0 @@ -interactions: -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91?api-version=2017-04 - response: - body: - string: '{"id":"b03b1e31b79f4af8b285428e3ed76814"}' - headers: - content-type: application/json - date: Thu, 03 Sep 2020 21:36:13 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions/1?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: b03b1e31b79f4af8b285428e3ed76814 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b03b1e31b79f4af8b285428e3ed76814?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '1' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91?api-version=2017-04 -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Content-Length: - - '244' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91?api-version=2017-04 - response: - body: - string: '{"id":"c3f0bdbe0b3d475ba7fe0bd437d9c272"}' - headers: - content-type: application/json - date: Thu, 03 Sep 2020 21:36:13 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions/2?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: c3f0bdbe0b3d475ba7fe0bd437d9c272 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c3f0bdbe0b3d475ba7fe0bd437d9c272?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '2' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91?api-version=2017-04 -- request: - body: null - headers: - Accept: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/c3f0bdbe0b3d475ba7fe0bd437d9c272?api-version=2017-04 - response: - body: - string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' - headers: - content-type: application/json - date: Thu, 03 Sep 2020 21:36:13 GMT - location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions/2?api-version=2017-04 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - x-schema-id: c3f0bdbe0b3d475ba7fe0bd437d9c272 - x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c3f0bdbe0b3d475ba7fe0bd437d9c272?api-version=2017-04 - x-schema-type: Avro - x-schema-version: '2' - x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-6cfe0c61-b6fb-42fb-93cd-24cd8b7d4e91/versions?api-version=2017-04 - status: - code: 200 - message: OK - url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/c3f0bdbe0b3d475ba7fe0bd437d9c272?api-version=2017-04 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py index 29e8df3cc26a..4341d0c444bd 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py @@ -19,6 +19,7 @@ # -------------------------------------------------------------------------- import pytest import uuid +import os from azure.schemaregistry.aio import SchemaRegistryClient from azure.identity.aio import ClientSecretCredential @@ -26,15 +27,83 @@ from schemaregistry_preparer import SchemaRegistryPreparer from devtools_testutils import AzureTestCase +from devtools_testutils.azure_testcase import _is_autorest_v3 +from azure.core.credentials import AccessToken class SchemaRegistryAsyncTests(AzureTestCase): + class AsyncFakeCredential(object): + async def get_token(self, *scopes, **kwargs): + return AccessToken('fake_token', 2527537086) + + async def close(self): + pass + + def create_basic_client(self, client_class, **kwargs): + # This is the patch for creating client using aio identity + + tenant_id = os.environ.get("AZURE_TENANT_ID", None) + client_id = os.environ.get("AZURE_CLIENT_ID", None) + secret = os.environ.get("AZURE_CLIENT_SECRET", None) + + if tenant_id and client_id and secret and self.is_live: + if _is_autorest_v3(client_class): + # Create azure-identity class + from azure.identity.aio import ClientSecretCredential + credentials = ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=secret + ) + else: + # Create msrestazure class + from msrestazure.azure_active_directory import ServicePrincipalCredentials + credentials = ServicePrincipalCredentials( + tenant=tenant_id, + client_id=client_id, + secret=secret + ) + else: + if _is_autorest_v3(client_class): + credentials = self.AsyncFakeCredential() + #credentials = self.settings.get_azure_core_credentials() + else: + credentials = self.settings.get_credentials() + + # Real client creation + # FIXME decide what is the final argument for that + # if self.is_playback(): + # kwargs.setdefault("polling_interval", 0) + if _is_autorest_v3(client_class): + kwargs.setdefault("logging_enable", True) + client = client_class( + credential=credentials, + **kwargs + ) + else: + client = client_class( + credentials=credentials, + **kwargs + ) + + if self.is_playback(): + try: + client._config.polling_interval = 0 # FIXME in azure-mgmt-core, make this a kwargs + except AttributeError: + pass + + if hasattr(client, "config"): # Autorest v2 + if self.is_playback(): + client.config.long_running_operation_timeout = 0 + client.config.enable_http_logger = True + return client + @SchemaRegistryPreparer() async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) async with client: - schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_name = 'test-schema-basic-async' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -61,12 +130,13 @@ async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_ assert returned_schema_properties.location_by_id is not None assert returned_schema_properties.version == 1 assert returned_schema_properties.serialization_type == "Avro" + await client._generated_client._config.credential.close() @SchemaRegistryPreparer() async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) async with client: - schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_name = 'test-schema-update-async' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -95,13 +165,26 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry assert new_schema.schema_content == schema_str_new assert new_schema.schema_properties.version == 2 assert new_schema.schema_properties.serialization_type == "Avro" + await client._generated_client._config.credential.close() + + @SchemaRegistryPreparer() + async def test_schema_same_twice_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): + client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) + schema_name = 'test-schema-twice-async' + str(uuid.uuid4())[0:8] + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":["int","null"]},{"name":"city","type":["string","null"]}]}""" + serialization_type = "Avro" + async with client: + schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + schema_properties_second = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + assert schema_properties.schema_id == schema_properties_second.schema_id + await client._generated_client._config.credential.close() @SchemaRegistryPreparer() async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) async with client, credential: - schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_name = 'test-schema-negative-async' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ClientAuthenticationError): @@ -111,11 +194,12 @@ async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpo async def test_schema_negative_wrong_endpoint_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint="nonexist.servicebus.windows.net") async with client: - schema_name = 'test-schema-' + str(uuid.uuid4()) + schema_name = 'test-schema-nonexist-async' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ServiceRequestError): await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) + await client._generated_client._config.credential.close() @SchemaRegistryPreparer() async def test_schema_negative_no_schema_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): @@ -126,3 +210,4 @@ async def test_schema_negative_no_schema_async(self, schemaregistry_endpoint, sc with pytest.raises(HttpResponseError): await client.get_schema('a' * 32) + await client._generated_client._config.credential.close() diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_register_schema.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_register_schema.yaml deleted file mode 100644 index 25f94760b004..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_register_schema.yaml +++ /dev/null @@ -1,140 +0,0 @@ -interactions: -- request: - body: '"somestr"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '9' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43?api-version=2017-04 - response: - body: - string: '{"id":"2cc9a5eb78484e1cac2e5b547b0374ef"}' - headers: - content-type: - - application/json - date: - - Tue, 25 Aug 2020 00:43:02 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 2cc9a5eb78484e1cac2e5b547b0374ef - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2cc9a5eb78484e1cac2e5b547b0374ef?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions?api-version=2017-04 - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/2cc9a5eb78484e1cac2e5b547b0374ef?api-version=2017-04 - response: - body: - string: '"somestr"' - headers: - content-type: - - application/json - date: - - Tue, 25 Aug 2020 00:43:02 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 2cc9a5eb78484e1cac2e5b547b0374ef - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2cc9a5eb78484e1cac2e5b547b0374ef?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions?api-version=2017-04 - status: - code: 200 - message: OK -- request: - body: '"somestr"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '9' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43?api-version=2017-04 - response: - body: - string: '{"id":"2cc9a5eb78484e1cac2e5b547b0374ef"}' - headers: - content-type: - - application/json - date: - - Tue, 25 Aug 2020 00:43:04 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 2cc9a5eb78484e1cac2e5b547b0374ef - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/2cc9a5eb78484e1cac2e5b547b0374ef?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-8d6c5e4e-116b-4c3d-b13f-e5b55e4c4f43/versions?api-version=2017-04 - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml deleted file mode 100644 index c36bcb7474c7..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml +++ /dev/null @@ -1,140 +0,0 @@ -interactions: -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569?api-version=2017-04 - response: - body: - string: '{"id":"5b6a64e2029b43f3b212445a13ad04e1"}' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:34:48 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 5b6a64e2029b43f3b212445a13ad04e1 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5b6a64e2029b43f3b212445a13ad04e1?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions?api-version=2017-04 - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/5b6a64e2029b43f3b212445a13ad04e1?api-version=2017-04 - response: - body: - string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:34:48 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 5b6a64e2029b43f3b212445a13ad04e1 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5b6a64e2029b43f3b212445a13ad04e1?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions?api-version=2017-04 - status: - code: 200 - message: OK -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - avro - method: POST - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569?api-version=2017-04 - response: - body: - string: '{"id":"5b6a64e2029b43f3b212445a13ad04e1"}' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:34:49 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 5b6a64e2029b43f3b212445a13ad04e1 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/5b6a64e2029b43f3b212445a13ad04e1?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-a03183d0-26be-4681-b545-a09eba63b569/versions?api-version=2017-04 - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml deleted file mode 100644 index 777942e82949..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml +++ /dev/null @@ -1,67 +0,0 @@ -interactions: -- request: - body: null - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 - response: - body: - string: 400SubCode=40000, UnknownType:The request - is invalid. [MGResponseHttpError=BadRequest]. TrackingId:94d3d37a-2ce8-4284-ae34-fba5253f4e8e_G2, - SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/a, - Timestamp:2020-09-03T21:34:50 - headers: - content-type: - - application/xml; charset=utf-8 - date: - - Thu, 03 Sep 2020 21:34:49 GMT - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 400 - message: Bad Request -- request: - body: null - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 - response: - body: - string: 404Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - does not exist. TrackingId:d5c92586-7471-41e1-8e91-7bd497a1828d_G2, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, - Timestamp:2020-09-03T21:34:51 - headers: - content-type: - - application/xml; charset=utf-8 - date: - - Thu, 03 Sep 2020 21:34:51 GMT - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 404 - message: Not Found -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml deleted file mode 100644 index bfef2ced2c46..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml +++ /dev/null @@ -1,98 +0,0 @@ -interactions: -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":[\"int\",\"null\"]},{\"name\":\"city\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '223' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0?api-version=2017-04 - response: - body: - string: '{"id":"c4ff58960b344a89a3d2a1d70388a921"}' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:34:57 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - c4ff58960b344a89a3d2a1d70388a921 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c4ff58960b344a89a3d2a1d70388a921?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0/versions?api-version=2017-04 - status: - code: 200 - message: OK -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":[\"int\",\"null\"]},{\"name\":\"city\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '223' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0?api-version=2017-04 - response: - body: - string: '{"id":"c4ff58960b344a89a3d2a1d70388a921"}' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:34:57 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - c4ff58960b344a89a3d2a1d70388a921 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c4ff58960b344a89a3d2a1d70388a921?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-f203ca32-d22c-4fd3-85f4-f34378cc93e0/versions?api-version=2017-04 - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml deleted file mode 100644 index c9c63163e0c0..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml +++ /dev/null @@ -1,140 +0,0 @@ -interactions: -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '245' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b?api-version=2017-04 - response: - body: - string: '{"id":"6defbafaf03c4480a9d70d5dbe78b8fb"}' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:35:00 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions/1?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 6defbafaf03c4480a9d70d5dbe78b8fb - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/6defbafaf03c4480a9d70d5dbe78b8fb?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '1' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions?api-version=2017-04 - status: - code: 200 - message: OK -- request: - body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '244' - Content-Type: - - application/json - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - X-Schema-Type: - - Avro - method: PUT - uri: https://sr-playground.servicebus.windows.net/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b?api-version=2017-04 - response: - body: - string: '{"id":"274d0cbe53ed4002b0857d61a9139e78"}' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:35:00 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions/2?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 274d0cbe53ed4002b0857d61a9139e78 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/274d0cbe53ed4002b0857d61a9139e78?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '2' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions?api-version=2017-04 - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) - method: GET - uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/274d0cbe53ed4002b0857d61a9139e78?api-version=2017-04 - response: - body: - string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' - headers: - content-type: - - application/json - date: - - Thu, 03 Sep 2020 21:35:01 GMT - location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions/2?api-version=2017-04 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - x-schema-id: - - 274d0cbe53ed4002b0857d61a9139e78 - x-schema-id-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/274d0cbe53ed4002b0857d61a9139e78?api-version=2017-04 - x-schema-type: - - Avro - x-schema-version: - - '2' - x-schema-versions-location: - - https://sr-playground.servicebus.windows.net:443/$schemagroups/yuling-test-group/schemas/test-schema-28cfcd92-8f38-461b-be52-fdd0b8b2647b/versions?api-version=2017-04 - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py index db1e468b23c2..511e2b8c147c 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/schemaregistry_preparer.py @@ -32,14 +32,8 @@ SCHEMA_REGISTRY_ENDPOINT_PARAM = "schemaregistry_endpoint" SCHEMA_REGISTRY_GROUP_PARAM = "schemaregistry_group" -SCHEMA_REGISTRY_TENANT_ID_PARAM = "schemaregistry_tenant_id" -SCHEMA_REGISTRY_CLIENT_ID_PARAM = "schemaregistry_client_id" -SCHEMA_REGISTRY_CLIENT_SECRET_PARAM = "schemaregistry_client_secret" SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME = 'SCHEMA_REGISTRY_ENDPOINT' SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME = 'SCHEMA_REGISTRY_GROUP' -AZURE_TENANT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_TENANT_ID' -AZURE_CLIENT_ID_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_ID' -AZURE_CLIENT_SECRET_ENV_KEY_NAME = 'SCHEMA_REGISTRY_AZURE_CLIENT_SECRET' class SchemaRegistryNamespacePreparer(AzureMgmtPreparer): @@ -70,8 +64,8 @@ def create_resource(self, name, **kwargs): } else: return { - SCHEMA_REGISTRY_ENDPOINT_PARAM: "fake.servicebus.windows.net", - SCHEMA_REGISTRY_GROUP_PARAM: "fake-group" + SCHEMA_REGISTRY_ENDPOINT_PARAM: "sr-playground.servicebus.windows.net", + SCHEMA_REGISTRY_GROUP_PARAM: "azsdk_python_test_group" } def remove_resource(self, name, **kwargs): diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index 9c882f12bba5..5627efc83b60 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -33,7 +33,7 @@ class SchemaRegistryTests(AzureTestCase): @SchemaRegistryPreparer() def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - schema_name = 'test-schema-basic' + schema_name = 'test-schema-basic' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -64,7 +64,7 @@ def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, **kwa @SchemaRegistryPreparer() def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - schema_name = 'test-schema-update' + schema_name = 'test-schema-update' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -97,7 +97,7 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, **kw @SchemaRegistryPreparer() def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - schema_name = 'test-schema-twice' + schema_name = 'test-schema-twice' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":["int","null"]},{"name":"city","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -108,7 +108,7 @@ def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - schema_name = 'test-schema-negative' + schema_name = 'test-schema-negative' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ClientAuthenticationError): @@ -117,7 +117,7 @@ def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemar @SchemaRegistryPreparer() def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint="nonexist.servicebus.windows.net") - schema_name = 'test-schema-nonexist' + schema_name = 'test-schema-nonexist' + str(uuid.uuid4())[0:8] schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ServiceRequestError): From 8fbed9006533f7436a557b72ef68211c29468c98 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Fri, 4 Sep 2020 10:10:04 -0700 Subject: [PATCH 33/35] add share requirement --- shared_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/shared_requirements.txt b/shared_requirements.txt index 4cd58d2e5a4f..cfc5544216d2 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -82,6 +82,7 @@ azure-mgmt-trafficmanager~=0.50.0 azure-mgmt-web~=0.35.0 azure-nspkg azure-keyvault-nspkg +azure-schemaregistry<2.0.0,>=1.0.0b1 azure-search-nspkg azure-security-nspkg azure-synapse-nspkg From 01a39a7ade429c42e04c59291034c81264d4a7ef Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Fri, 4 Sep 2020 10:21:26 -0700 Subject: [PATCH 34/35] update avro dependency --- sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py | 2 +- shared_requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py index 76e9259ef602..c97916e14530 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py @@ -66,6 +66,6 @@ packages=find_packages(exclude=exclude_packages), install_requires=[ 'azure-schemaregistry<2.0.0,>=1.0.0b1', - 'avro<2.0.0,>=1.0.0' + 'avro<2.0.0,>=1.10.0' ] ) diff --git a/shared_requirements.txt b/shared_requirements.txt index cfc5544216d2..d5d154f6c5df 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -112,6 +112,7 @@ aiodns>=2.0 python-dateutil>=2.8.0 six>=1.6 isodate>=0.6.0 +avro<2.0.0,>=1.10.0 #override azure azure-keyvault~=1.0 #override azure-mgmt-core azure-core<2.0.0,>=1.7.0 #override azure-appconfiguration azure-core<2.0.0,>=1.0.0 From bde3c246df825f2bd782f04156156cc57a23a130 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Fri, 4 Sep 2020 15:12:22 -0700 Subject: [PATCH 35/35] pr feedback and livetest update --- .../avroserializer/_avro_serializer.py | 20 +-- .../serializer/avroserializer/_constants.py | 36 +++++ .../_schema_registry_avro_serializer.py | 10 +- ...ializer.test_basic_sr_avro_serializer.yaml | 104 +++++++++++++ .../tests/test_avro_serializer.py | 4 +- ...egistry_async.test_schema_basic_async.yaml | 98 ++++++++++++ ....test_schema_negative_no_schema_async.yaml | 51 +++++++ ...ry_async.test_schema_same_twice_async.yaml | 70 +++++++++ ...gistry_async.test_schema_update_async.yaml | 98 ++++++++++++ .../async_tests/test_schema_registry_async.py | 17 ++- ...est_schema_registry.test_schema_basic.yaml | 140 ++++++++++++++++++ ...gistry.test_schema_negative_no_schema.yaml | 67 +++++++++ ...chema_registry.test_schema_same_twice.yaml | 98 ++++++++++++ ...st_schema_registry.test_schema_update.yaml | 140 ++++++++++++++++++ .../tests/test_schema_registry.py | 16 +- 15 files changed, 936 insertions(+), 33 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_constants.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_same_twice_async.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py index a3fc8d3ad6ba..a190bffe5467 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py @@ -52,8 +52,10 @@ def serialize( https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema :param data: An object to serialize :type data: ObjectType - :param schema: A Avro RecordSchema + :param schema: An Avro RecordSchema :type schema: Union[str, bytes, avro.schema.Schema] + :returns: Encoded bytes + :rtype: bytes """ if not schema: raise ValueError("Schema is required in Avro serializer.") @@ -68,11 +70,9 @@ def serialize( self._schema_writer_cache[str(schema)] = writer stream = BytesIO() - - writer.write(data, BinaryEncoder(stream)) - encoded_data = stream.getvalue() - - stream.close() + with stream: + writer.write(data, BinaryEncoder(stream)) + encoded_data = stream.getvalue() return encoded_data def deserialize( @@ -85,7 +85,7 @@ def deserialize( Return type will be ignored, since the schema is deduced from the provided bytes. :param data: A stream of bytes or bytes directly :type data: BinaryIO or bytes - :param schema: A Avro RecordSchema + :param schema: An Avro RecordSchema :type schema: Union[str, bytes, avro.schema.Schema] :returns: An instantiated object :rtype: ObjectType @@ -102,8 +102,8 @@ def deserialize( reader = DatumReader(writers_schema=schema) self._schema_reader_cache[str(schema)] = reader - bin_decoder = BinaryDecoder(data) - decoded_data = reader.read(bin_decoder) - data.close() + with data: + bin_decoder = BinaryDecoder(data) + decoded_data = reader.read(bin_decoder) return decoded_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_constants.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_constants.py new file mode 100644 index 000000000000..a95ec7dd5088 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_constants.py @@ -0,0 +1,36 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +# encoded payload is consisted of: +# 0~3: 4 bytes denoting record format identifier +# 4~35: 32 bytes denoting schema id +# 36~END: encode data + +RECORD_FORMAT_IDENTIFIER_START_INDEX = 0 +RECORD_FORMAT_IDENTIFIER_LENGTH = 4 +SCHEMA_ID_START_INDEX = 4 +SCHEMA_ID_LENGTH = 32 +DATA_START_INDEX = 36 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index 367d84aea99b..a09e7e6bd9d0 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -27,7 +27,7 @@ from typing import Any, Dict, Union import avro - +from ._constants import SCHEMA_ID_START_INDEX, SCHEMA_ID_LENGTH, DATA_START_INDEX from ._avro_serializer import AvroObjectSerializer @@ -116,7 +116,9 @@ def _get_schema(self, schema_id, **kwargs): def serialize(self, data, schema, **kwargs): # type: (Dict[str, Any], Union[str, bytes], Any) -> bytes """ - Encode dict data with the given schema. + Encode dict data with the given schema. The returns bytes are consisted of: The first 4 bytes + denoting record format identifier. The following 32 bytes denoting schema id returned by schema registry + service. The remaining bytes are the real data payload. :param data: The dict data to be encoded. :param schema: The schema used to encode the data. @@ -155,8 +157,8 @@ def deserialize(self, data, **kwargs): :rtype: Dict[str, Any] """ # record_format_identifier = data[0:4] # The first 4 bytes are retained for future record format identifier. - schema_id = data[4:36].decode('utf-8') + schema_id = data[SCHEMA_ID_START_INDEX:(SCHEMA_ID_START_INDEX + SCHEMA_ID_LENGTH)].decode('utf-8') schema_content = self._get_schema(schema_id, **kwargs) - dict_data = self._avro_serializer.deserialize(data[36:], schema_content) + dict_data = self._avro_serializer.deserialize(data[DATA_START_INDEX:], schema_content) return dict_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml new file mode 100644 index 000000000000..c23e6224d483 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer.yaml @@ -0,0 +1,104 @@ +interactions: +- request: + body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", + \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], + \"name\": \"favorite_color\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '265' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/example.avro.User?api-version=2017-04 + response: + body: + string: '{"id":"ec9977eee8d84c178db2bcb9f9685ac8"}' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:09:09 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/example.avro.User/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - ec9977eee8d84c178db2bcb9f9685ac8 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/ec9977eee8d84c178db2bcb9f9685ac8?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/example.avro.User/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", + \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], + \"name\": \"favorite_color\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '265' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: POST + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/example.avro.User?api-version=2017-04 + response: + body: + string: '{"id":"ec9977eee8d84c178db2bcb9f9685ac8"}' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:09:09 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/example.avro.User/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - ec9977eee8d84c178db2bcb9f9685ac8 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/ec9977eee8d84c178db2bcb9f9685ac8?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/example.avro.User/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py index 31adadb1c08d..31ceb1933385 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -78,9 +78,7 @@ def test_basic_sr_avro_serializer(self, schemaregistry_endpoint, schemaregistry_ sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) sr_avro_serializer = SchemaRegistryAvroSerializer(sr_client, schemaregistry_group) - random_schema_namespace = "testschema" + str(uuid.uuid4())[0:8] - schema_str = "{\"namespace\":\"" + random_schema_namespace + "\"" +\ - ""","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml new file mode 100644 index 000000000000..c78f2730ba15 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_basic_async.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482?api-version=2017-04 + response: + body: + string: '{"id":"b8503db5451844488672458ba137c27b"}' + headers: + content-type: application/json + date: Fri, 04 Sep 2020 22:07:30 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: b8503db5451844488672458ba137c27b + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b8503db5451844488672458ba137c27b?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482?api-version=2017-04 +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/b8503db5451844488672458ba137c27b?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: application/json + date: Fri, 04 Sep 2020 22:07:30 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: b8503db5451844488672458ba137c27b + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b8503db5451844488672458ba137c27b?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/b8503db5451844488672458ba137c27b?api-version=2017-04 +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: POST + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482?api-version=2017-04 + response: + body: + string: '{"id":"b8503db5451844488672458ba137c27b"}' + headers: + content-type: application/json + date: Fri, 04 Sep 2020 22:07:31 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: b8503db5451844488672458ba137c27b + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/b8503db5451844488672458ba137c27b?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic-asynce5e1482?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml new file mode 100644 index 000000000000..66be60cfac3c --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_negative_no_schema_async.yaml @@ -0,0 +1,51 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 + response: + body: + string: 400SubCode=40000, UnknownType:The request + is invalid. [MGResponseHttpError=BadRequest]. TrackingId:67733d61-7b5b-4e51-9b06-72a6afba9bdf_G1, + SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/a, + Timestamp:2020-09-04T22:07:32 + headers: + content-type: application/xml; charset=utf-8 + date: Fri, 04 Sep 2020 22:07:32 GMT + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 400 + message: Bad Request + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 + response: + body: + string: 404Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + does not exist. TrackingId:17234c8c-2982-4d72-b05d-28df0d550d73_G1, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, + Timestamp:2020-09-04T22:07:33 + headers: + content-type: application/xml; charset=utf-8 + date: Fri, 04 Sep 2020 22:07:32 GMT + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 404 + message: Not Found + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_same_twice_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_same_twice_async.yaml new file mode 100644 index 000000000000..b0a1a139e1c5 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_same_twice_async.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":[\"int\",\"null\"]},{\"name\":\"city\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '223' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice-async7bfd16a1?api-version=2017-04 + response: + body: + string: '{"id":"7d0ff6342f0648b4a89829010b152e4f"}' + headers: + content-type: application/json + date: Fri, 04 Sep 2020 22:07:39 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice-async7bfd16a1/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 7d0ff6342f0648b4a89829010b152e4f + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/7d0ff6342f0648b4a89829010b152e4f?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice-async7bfd16a1/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice-async7bfd16a1?api-version=2017-04 +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":[\"int\",\"null\"]},{\"name\":\"city\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '223' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice-async7bfd16a1?api-version=2017-04 + response: + body: + string: '{"id":"7d0ff6342f0648b4a89829010b152e4f"}' + headers: + content-type: application/json + date: Fri, 04 Sep 2020 22:07:39 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice-async7bfd16a1/versions/1?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 7d0ff6342f0648b4a89829010b152e4f + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/7d0ff6342f0648b4a89829010b152e4f?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '1' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice-async7bfd16a1/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice-async7bfd16a1?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml new file mode 100644 index 000000000000..316f7c90d7de --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/recordings/test_schema_registry_async.test_schema_update_async.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503?api-version=2017-04 + response: + body: + string: '{"id":"a960c5bf612b4f928b409ab2474de907"}' + headers: + content-type: application/json + date: Fri, 04 Sep 2020 22:07:41 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503/versions/9?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: a960c5bf612b4f928b409ab2474de907 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/a960c5bf612b4f928b409ab2474de907?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '9' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503?api-version=2017-04 +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Content-Length: + - '244' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503?api-version=2017-04 + response: + body: + string: '{"id":"767b2ebdbe4e4ac2bfa48f58e0fa7365"}' + headers: + content-type: application/json + date: Fri, 04 Sep 2020 22:07:41 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503/versions/10?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 767b2ebdbe4e4ac2bfa48f58e0fa7365 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/767b2ebdbe4e4ac2bfa48f58e0fa7365?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '10' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503?api-version=2017-04 +- request: + body: null + headers: + Accept: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/767b2ebdbe4e4ac2bfa48f58e0fa7365?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: application/json + date: Fri, 04 Sep 2020 22:07:42 GMT + location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503/versions/10?api-version=2017-04 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + x-schema-id: 767b2ebdbe4e4ac2bfa48f58e0fa7365 + x-schema-id-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/767b2ebdbe4e4ac2bfa48f58e0fa7365?api-version=2017-04 + x-schema-type: Avro + x-schema-version: '10' + x-schema-versions-location: https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update-async24591503/versions?api-version=2017-04 + status: + code: 200 + message: OK + url: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/767b2ebdbe4e4ac2bfa48f58e0fa7365?api-version=2017-04 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py index 4341d0c444bd..267549748aae 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py @@ -31,6 +31,7 @@ from azure.core.credentials import AccessToken + class SchemaRegistryAsyncTests(AzureTestCase): class AsyncFakeCredential(object): @@ -103,7 +104,7 @@ def create_basic_client(self, client_class, **kwargs): async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) async with client: - schema_name = 'test-schema-basic-async' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-basic-async') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -136,7 +137,7 @@ async def test_schema_basic_async(self, schemaregistry_endpoint, schemaregistry_ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) async with client: - schema_name = 'test-schema-update-async' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-update-async') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = await client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -144,7 +145,7 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry assert schema_properties.schema_id is not None assert schema_properties.location is not None assert schema_properties.location_by_id is not None - assert schema_properties.version == 1 + assert schema_properties.version != 0 assert schema_properties.serialization_type == "Avro" schema_str_new = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_food","type":["string","null"]}]}""" @@ -153,7 +154,7 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry assert new_schema_properties.schema_id is not None assert new_schema_properties.location is not None assert new_schema_properties.location_by_id is not None - assert new_schema_properties.version == 2 + assert new_schema_properties.version == schema_properties.version + 1 assert new_schema_properties.serialization_type == "Avro" new_schema = await client.get_schema(schema_id=new_schema_properties.schema_id) @@ -163,14 +164,14 @@ async def test_schema_update_async(self, schemaregistry_endpoint, schemaregistry assert new_schema.schema_properties.location is not None assert new_schema.schema_properties.location_by_id is not None assert new_schema.schema_content == schema_str_new - assert new_schema.schema_properties.version == 2 + assert new_schema.schema_properties.version == schema_properties.version + 1 assert new_schema.schema_properties.serialization_type == "Avro" await client._generated_client._config.credential.close() @SchemaRegistryPreparer() async def test_schema_same_twice_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - schema_name = 'test-schema-twice-async' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-twice-async') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":["int","null"]},{"name":"city","type":["string","null"]}]}""" serialization_type = "Avro" async with client: @@ -184,7 +185,7 @@ async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpo credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) async with client, credential: - schema_name = 'test-schema-negative-async' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-negative-async') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ClientAuthenticationError): @@ -194,7 +195,7 @@ async def test_schema_negative_wrong_credential_async(self, schemaregistry_endpo async def test_schema_negative_wrong_endpoint_async(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint="nonexist.servicebus.windows.net") async with client: - schema_name = 'test-schema-nonexist-async' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-nonexist-async') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ServiceRequestError): diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml new file mode 100644 index 000000000000..f5a3ae16820b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic31c70f88?api-version=2017-04 + response: + body: + string: '{"id":"4a24a2d868ff4d42bce7889243ba055c"}' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:02:04 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic31c70f88/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 4a24a2d868ff4d42bce7889243ba055c + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/4a24a2d868ff4d42bce7889243ba055c?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic31c70f88/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/4a24a2d868ff4d42bce7889243ba055c?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:02:06 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic31c70f88/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 4a24a2d868ff4d42bce7889243ba055c + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/4a24a2d868ff4d42bce7889243ba055c?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic31c70f88/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: POST + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic31c70f88?api-version=2017-04 + response: + body: + string: '{"id":"4a24a2d868ff4d42bce7889243ba055c"}' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:02:06 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic31c70f88/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 4a24a2d868ff4d42bce7889243ba055c + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/4a24a2d868ff4d42bce7889243ba055c?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-basic31c70f88/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml new file mode 100644 index 000000000000..34ab6e84f860 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/a?api-version=2017-04 + response: + body: + string: 400SubCode=40000, UnknownType:The request + is invalid. [MGResponseHttpError=BadRequest]. TrackingId:8fffa255-f448-4289-9c6b-4fabe270883b_G1, + SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/a, + Timestamp:2020-09-04T22:02:07 + headers: + content-type: + - application/xml; charset=utf-8 + date: + - Fri, 04 Sep 2020 22:02:07 GMT + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 400 + message: Bad Request +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?api-version=2017-04 + response: + body: + string: 404Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + does not exist. TrackingId:0f6141ae-5a35-4219-a634-fe3533912dcc_G1, SystemTracker:sr-playground.servicebus.windows.net:$schemagroups/getSchemaById/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, + Timestamp:2020-09-04T22:02:08 + headers: + content-type: + - application/xml; charset=utf-8 + date: + - Fri, 04 Sep 2020 22:02:07 GMT + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml new file mode 100644 index 000000000000..50bba1757a86 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_same_twice.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":[\"int\",\"null\"]},{\"name\":\"city\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '223' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice863b11a7?api-version=2017-04 + response: + body: + string: '{"id":"07a69522b495454e80c9dfee0bf43cc3"}' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:02:14 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice863b11a7/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 07a69522b495454e80c9dfee0bf43cc3 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/07a69522b495454e80c9dfee0bf43cc3?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice863b11a7/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":[\"int\",\"null\"]},{\"name\":\"city\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '223' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice863b11a7?api-version=2017-04 + response: + body: + string: '{"id":"07a69522b495454e80c9dfee0bf43cc3"}' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:02:14 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice863b11a7/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 07a69522b495454e80c9dfee0bf43cc3 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/07a69522b495454e80c9dfee0bf43cc3?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-twice863b11a7/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml new file mode 100644 index 000000000000..64cf664e72d2 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_update.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '245' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-update423f1009?api-version=2017-04 + response: + body: + string: '{"id":"c105c5cd1be740e38c05f35fb630f5ff"}' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:02:16 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update423f1009/versions/5?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - c105c5cd1be740e38c05f35fb630f5ff + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/c105c5cd1be740e38c05f35fb630f5ff?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '5' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update423f1009/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '244' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://sr-playground.servicebus.windows.net/$schemagroups/azsdk_python_test_group/schemas/test-schema-update423f1009?api-version=2017-04 + response: + body: + string: '{"id":"9f19901a6d2d4fd0bc79549d3a908764"}' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:02:16 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update423f1009/versions/6?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 9f19901a6d2d4fd0bc79549d3a908764 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/9f19901a6d2d4fd0bc79549d3a908764?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '6' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update423f1009/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.7.7 (Windows-10-10.0.19041-SP0) + method: GET + uri: https://sr-playground.servicebus.windows.net/$schemagroups/getSchemaById/9f19901a6d2d4fd0bc79549d3a908764?api-version=2017-04 + response: + body: + string: '"{\"namespace\":\"example.avro\",\"type\":\"record\",\"name\":\"User\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_food\",\"type\":[\"string\",\"null\"]}]}"' + headers: + content-type: + - application/json + date: + - Fri, 04 Sep 2020 22:02:17 GMT + location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update423f1009/versions/6?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 9f19901a6d2d4fd0bc79549d3a908764 + x-schema-id-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/getschemabyid/9f19901a6d2d4fd0bc79549d3a908764?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '6' + x-schema-versions-location: + - https://sr-playground.servicebus.windows.net:443/$schemagroups/azsdk_python_test_group/schemas/test-schema-update423f1009/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index 5627efc83b60..29b490a7577b 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -33,7 +33,7 @@ class SchemaRegistryTests(AzureTestCase): @SchemaRegistryPreparer() def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - schema_name = 'test-schema-basic' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-basic') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -64,7 +64,7 @@ def test_schema_basic(self, schemaregistry_endpoint, schemaregistry_group, **kwa @SchemaRegistryPreparer() def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - schema_name = 'test-schema-update' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-update') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -72,7 +72,7 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, **kw assert schema_properties.schema_id is not None assert schema_properties.location is not None assert schema_properties.location_by_id is not None - assert schema_properties.version == 1 + assert schema_properties.version != 0 assert schema_properties.serialization_type == "Avro" schema_str_new = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_food","type":["string","null"]}]}""" @@ -81,7 +81,7 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, **kw assert new_schema_properties.schema_id is not None assert new_schema_properties.location is not None assert new_schema_properties.location_by_id is not None - assert new_schema_properties.version == 2 + assert new_schema_properties.version == schema_properties.version + 1 assert new_schema_properties.serialization_type == "Avro" new_schema = client.get_schema(schema_id=new_schema_properties.schema_id) @@ -91,13 +91,13 @@ def test_schema_update(self, schemaregistry_endpoint, schemaregistry_group, **kw assert new_schema.schema_properties.location is not None assert new_schema.schema_properties.location_by_id is not None assert new_schema.schema_content == schema_str_new - assert new_schema.schema_properties.version == 2 + assert new_schema.schema_properties.version == schema_properties.version + 1 assert new_schema.schema_properties.serialization_type == "Avro" @SchemaRegistryPreparer() def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - schema_name = 'test-schema-twice' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-twice') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":["int","null"]},{"name":"city","type":["string","null"]}]}""" serialization_type = "Avro" schema_properties = client.register_schema(schemaregistry_group, schema_name, serialization_type, schema_str) @@ -108,7 +108,7 @@ def test_schema_same_twice(self, schemaregistry_endpoint, schemaregistry_group, def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): credential = ClientSecretCredential(tenant_id="fake", client_id="fake", client_secret="fake") client = SchemaRegistryClient(endpoint=schemaregistry_endpoint, credential=credential) - schema_name = 'test-schema-negative' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-negative') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ClientAuthenticationError): @@ -117,7 +117,7 @@ def test_schema_negative_wrong_credential(self, schemaregistry_endpoint, schemar @SchemaRegistryPreparer() def test_schema_negative_wrong_endpoint(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): client = self.create_basic_client(SchemaRegistryClient, endpoint="nonexist.servicebus.windows.net") - schema_name = 'test-schema-nonexist' + str(uuid.uuid4())[0:8] + schema_name = self.get_resource_name('test-schema-nonexist') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" serialization_type = "Avro" with pytest.raises(ServiceRequestError):