From c039186d4aff5872cc711f9ea49bc92453d58183 Mon Sep 17 00:00:00 2001 From: irina-herciu <irina.tanasa@endava.com> Date: Tue, 1 Apr 2025 19:39:42 +0300 Subject: [PATCH 1/3] Create e5fd3631-e192-4dcb-931b-69d45722ae02.json add change file Allow to set DynamoDBEntryConversion per table --- .../e5fd3631-e192-4dcb-931b-69d45722ae02.json | 11 +++++ .../Conversion/DynamoDBEntryConversion.cs | 8 +++- .../DynamoDBv2/Custom/DataModel/Attributes.cs | 21 ++++++++- .../DynamoDBv2/Custom/DataModel/Configs.cs | 33 +++++++++++--- .../Custom/DataModel/InternalModel.cs | 17 +++++++ .../IntegrationTests/DataModelTests.cs | 44 +++++++++++++++++++ 6 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 generator/.DevConfigs/e5fd3631-e192-4dcb-931b-69d45722ae02.json diff --git a/generator/.DevConfigs/e5fd3631-e192-4dcb-931b-69d45722ae02.json b/generator/.DevConfigs/e5fd3631-e192-4dcb-931b-69d45722ae02.json new file mode 100644 index 000000000000..f2707bb13f51 --- /dev/null +++ b/generator/.DevConfigs/e5fd3631-e192-4dcb-931b-69d45722ae02.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "DynamoDBv2", + "type": "patch", + "changeLogMessages": [ + "Allow to set DynamoDBEntryConversion per table." + ] + } + ] +} \ No newline at end of file diff --git a/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs b/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs index 43236d04dbc7..d88cb51a501b 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs @@ -30,11 +30,17 @@ namespace Amazon.DynamoDBv2 { + /// <summary> /// Available conversion schemas. /// </summary> - internal enum ConversionSchema + public enum ConversionSchema { + /// <summary> + /// No schema set. + /// </summary> + Unset = -1, + /// <summary> /// Default schema before 2014 L, M, BOOL, NULL support /// diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Attributes.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Attributes.cs index 525ffa78ba2c..d5d6c5c1b9e7 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Attributes.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Attributes.cs @@ -52,12 +52,19 @@ public sealed class DynamoDBTableAttribute : DynamoDBAttribute /// </summary> public bool LowerCamelCaseProperties { get; set; } + /// <summary> + /// Gets and sets the Conversion Schema property. + /// </summary> + public ConversionSchema Conversion { get; } + /// <summary> /// Construct an instance of DynamoDBTableAttribute /// </summary> /// <param name="tableName"></param> public DynamoDBTableAttribute(string tableName) - : this(tableName, false) { } + : this(tableName, false, ConversionSchema.Unset) + { + } /// <summary> /// Construct an instance of DynamoDBTableAttribute @@ -65,9 +72,21 @@ public DynamoDBTableAttribute(string tableName) /// <param name="tableName"></param> /// <param name="lowerCamelCaseProperties"></param> public DynamoDBTableAttribute(string tableName, bool lowerCamelCaseProperties) + : this(tableName, lowerCamelCaseProperties, ConversionSchema.Unset) + { + } + + /// <summary> + /// Construct an instance of DynamoDBTableAttribute + /// </summary> + /// <param name="tableName"></param> + /// <param name="lowerCamelCaseProperties"></param> + /// <param name="conversion"></param> + public DynamoDBTableAttribute(string tableName, bool lowerCamelCaseProperties, ConversionSchema conversion) { TableName = tableName; LowerCamelCaseProperties = lowerCamelCaseProperties; + Conversion = conversion; } } diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs index 4e7686c9069d..ea28f04a6050 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs @@ -432,7 +432,7 @@ public DynamoDBFlatConfig(DynamoDBOperationConfig operationConfig, DynamoDBConte bool ignoreNullValues = operationConfig.IgnoreNullValues ?? contextConfig.IgnoreNullValues ?? false; bool retrieveDateTimeInUtc = operationConfig.RetrieveDateTimeInUtc ?? contextConfig.RetrieveDateTimeInUtc ?? true; bool isEmptyStringValueEnabled = operationConfig.IsEmptyStringValueEnabled ?? contextConfig.IsEmptyStringValueEnabled ?? false; - DynamoDBEntryConversion conversion = operationConfig.Conversion ?? contextConfig.Conversion ?? DynamoDBEntryConversion.CurrentConversion; + DynamoDBEntryConversion conversion = contextConfig.Conversion ?? DynamoDBEntryConversion.CurrentConversion; string tableNamePrefix = operationConfig.TableNamePrefix ?? contextConfig.TableNamePrefix ?? string.Empty; // These properties can only be set at the operation level @@ -463,7 +463,8 @@ public DynamoDBFlatConfig(DynamoDBOperationConfig operationConfig, DynamoDBConte IndexName = indexName; QueryFilter = queryFilter; ConditionalOperator = conditionalOperator; - Conversion = conversion; + ContextConversion = conversion; + OperationConversion = operationConfig.Conversion; MetadataCachingMode = metadataCachingMode; DisableFetchingTableMetadata = disableFetchingTableMetadata; RetrieveDateTimeInUtc = retrieveDateTimeInUtc; @@ -471,7 +472,7 @@ public DynamoDBFlatConfig(DynamoDBOperationConfig operationConfig, DynamoDBConte State = new OperationState(); } - + /// <summary> /// Property that directs DynamoDBContext to use consistent reads. /// If property is not set, behavior defaults to non-consistent reads. @@ -552,10 +553,29 @@ public DynamoDBFlatConfig(DynamoDBOperationConfig operationConfig, DynamoDBConte public List<ScanCondition> QueryFilter { get; set; } /// <summary> - /// Conversion specification which controls how conversion between + /// Entity Conversion specification which controls how conversion between /// .NET and DynamoDB types happens. /// </summary> - public DynamoDBEntryConversion Conversion { get; set; } + public DynamoDBEntryConversion ItemConversion { get; set; } + + + /// <summary> + /// Operation Conversion specification which controls how conversion between + /// .NET and DynamoDB types happens. + /// </summary> + private DynamoDBEntryConversion OperationConversion { get; } + + /// <summary> + /// Context Conversion specification which controls how conversion between + /// .NET and DynamoDB types happens. + /// </summary> + public DynamoDBEntryConversion Conversion => OperationConversion ?? ItemConversion ?? ContextConversion; + + /// <summary> + /// Context Conversion specification which controls how conversion between + /// .NET and DynamoDB types happens. + /// </summary> + private DynamoDBEntryConversion ContextConversion { get; } /// <inheritdoc cref="DynamoDBContextConfig.DisableFetchingTableMetadata"/> public bool DisableFetchingTableMetadata { get; set; } @@ -564,7 +584,7 @@ public DynamoDBFlatConfig(DynamoDBOperationConfig operationConfig, DynamoDBConte public bool RetrieveDateTimeInUtc { get; set; } // Checks if the IndexName is set on the config - internal bool IsIndexOperation { get { return !string.IsNullOrEmpty(IndexName); } } + internal bool IsIndexOperation => !string.IsNullOrEmpty(IndexName); // State of the operation using this config internal OperationState State { get; private set; } @@ -584,6 +604,7 @@ public DynamoDBFlatConfig(DynamoDBOperationConfig operationConfig, DynamoDBConte /// </remarks> public string DerivedTypeAttributeName { get; set; } + public class OperationState { private CircularReferenceTracking referenceTracking; diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/InternalModel.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/InternalModel.cs index 0b7d47943567..0cfc7965f16c 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/InternalModel.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/InternalModel.cs @@ -428,6 +428,9 @@ internal class ItemStorageConfig // indexName to GSIConfig mapping public Dictionary<string, GSIConfig> IndexNameToGSIMapping { get; set; } + // entity conversion + public DynamoDBEntryConversion Conversion { get; set; } + public bool StorePolymorphicTypes => this.PolymorphicTypesStorageConfig.Any(); @@ -735,6 +738,10 @@ public ItemStorageConfig GetConfig([DynamicallyAccessedMembers(InternalConstants if (tableCache.Cache.TryGetValue(actualTableName, out config)) { + if (flatConfig == null) + throw new ArgumentNullException("flatConfig"); + + flatConfig.ItemConversion = config.Conversion; return config; } } @@ -759,6 +766,8 @@ public ItemStorageConfig GetConfig([DynamicallyAccessedMembers(InternalConstants if (tableCache == null) { var baseStorageConfig = CreateStorageConfig(type, actualTableName: null, flatConfig); + flatConfig.ItemConversion = baseStorageConfig.Conversion; + tableCache = new ConfigTableCache(baseStorageConfig); Cache[type] = tableCache; } @@ -780,6 +789,7 @@ public ItemStorageConfig GetConfig([DynamicallyAccessedMembers(InternalConstants } config = CreateStorageConfig(type, actualTableName, flatConfig); + flatConfig.ItemConversion = config.Conversion; tableCache.Cache[actualTableName] = config; return config; @@ -858,6 +868,13 @@ private static void PopulateConfigFromType(ItemStorageConfig config, [Dynamicall if (string.IsNullOrEmpty(tableAttribute.TableName)) throw new InvalidOperationException("DynamoDBTableAttribute.Table is empty or null"); config.TableName = tableAttribute.TableName; config.LowerCamelCaseProperties = tableAttribute.LowerCamelCaseProperties; + + config.Conversion = tableAttribute.Conversion switch + { + ConversionSchema.V1 => DynamoDBEntryConversion.V1, + ConversionSchema.V2 => DynamoDBEntryConversion.V2, + _ => config.Conversion + }; } string tableAlias; diff --git a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs index 9677280def08..4d3e85ffa066 100644 --- a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs +++ b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs @@ -1114,6 +1114,45 @@ private void TestContextConversions() #pragma warning restore CS0618 // Re-enable the warning } + { + +#pragma warning disable CS0618 // Disable the warning for the deprecated DynamoDBContext constructors + ProductV2 productV2 = new ProductV2 + { + Id = 1, + Name = "CloudSpotter", + CompanyName = "CloudsAreGrate", + Price = 1200, + TagSet = new HashSet<string> { "Prod", "1.0" }, + CurrentStatus = Status.Active, + FormerStatus = Status.Upcoming, + Supports = Support.Unix | Support.Windows, + PreviousSupport = null, + InternalId = "T1000", + IsPublic = true, + AlwaysN = true, + Rating = 4, + Components = new List<string> { "Code", "Coffee" }, + KeySizes = new List<byte> { 16, 64, 128 }, + CompanyInfo = new CompanyInfo + { + Name = "MyCloud", + Founded = new DateTime(1994, 7, 6), + Revenue = 9001 + } + }; + + using (var contextV1 = new DynamoDBContext(Client, new DynamoDBContextConfig { Conversion = conversionV1 })) + { + var docV1 = contextV1.ToDocument(productV2, new ToDocumentConfig { Conversion = conversionV1 }); + var docV2 = contextV1.ToDocument(productV2, new ToDocumentConfig { }); + VerifyConversions(docV1, docV2); + } + +#pragma warning restore CS0618 // Re-enable the warning + + } + // Introduce a circular reference and try to serialize { product.CompanyInfo = new CompanyInfo @@ -2632,6 +2671,11 @@ public object FromEntry(DynamoDBEntry entry) } } + [DynamoDBTable("HashTable", false, ConversionSchema.V2)] + public class ProductV2 : Product + { + } + /// <summary> /// Class representing items in the table [TableNamePrefix]HashTable /// </summary> From c7bad739c4016fcf3a01ea8a7fc3e2307abacdf0 Mon Sep 17 00:00:00 2001 From: irina-herciu <irina.tanasa@endava.com> Date: Mon, 7 Apr 2025 13:50:32 +0300 Subject: [PATCH 2/3] address PR feedback --- .../Conversion/DynamoDBEntryConversion.cs | 57 ++++++++++--------- .../DynamoDBv2/Custom/DataModel/Attributes.cs | 10 +++- .../DynamoDBv2/Custom/DataModel/Configs.cs | 6 +- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs b/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs index d88cb51a501b..a6d894555bcf 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs @@ -32,46 +32,49 @@ namespace Amazon.DynamoDBv2 { /// <summary> - /// Available conversion schemas. + /// Specifies the conversion schema used to map the types to DynamoDB types. + /// This schema influences how items are serialized and deserialized when interacting with DynamoDB. /// </summary> public enum ConversionSchema { /// <summary> - /// No schema set. + /// Indicates that no schema has been explicitly set. /// </summary> Unset = -1, /// <summary> - /// Default schema before 2014 L, M, BOOL, NULL support - /// - /// The following .NET types are converted into the following DynamoDB types: - /// Number types (byte, int, float, decimal, etc.) are converted to N - /// String and char are converted to S - /// Bool is converted to N (0=false, 1=true) - /// DateTime and Guid are converto to S - /// MemoryStream and byte[] are converted to B - /// List, HashSet, and array of numerics types are converted to NS - /// List, HashSet, and array of string-based types are converted to SS - /// List, HashSet, and array of binary-based types are converted to BS - /// Dictionary{string,object} are converted to M + /// Legacy conversion schema (and current default for context-level configurations). + /// + /// This schema pre-dates support for native DynamoDB types such as L (list), M (map), BOOL, and NULL. + /// Common .NET type mappings: + /// - Number types (byte, int, float, decimal, etc.) → DynamoDB N (number) + /// - string, char → S (string) + /// - bool → N ("0" for false, "1" for true) + /// - DateTime, Guid → S (string) + /// - MemoryStream, byte[] → B (binary) + /// - List, HashSet, array of numeric types → NS (number set) + /// - List, HashSet, array of string types → SS (string set) + /// - List, HashSet, array of binary types → BS (binary set) + /// - Dictionary{string, object} → M (map) /// </summary> V1 = 0, /// <summary> - /// Schema fully supporting 2014 L, M, BOOL, NULL additions + /// Enhanced conversion schema that supports native DynamoDB types including L (list), M (map), BOOL, and NULL. /// - /// The following .NET types are converted into the following DynamoDB types: - /// Number types (byte, int, float, decimal, etc.) are converted to N - /// String and char are converted to S - /// Bool is converted to BOOL - /// DateTime and Guid are converto to S - /// MemoryStream and byte[] are converted to B - /// HashSet of numerics types are converted to NS - /// HashSet of string-based types are converted to SS - /// HashSet of binary-based types are converted to BS - /// List and array of numerics, string-based types, and binary-based types - /// are converted to L type. - /// Dictionary{string,object} are converted to M + /// Common .NET type mappings: + /// - Number types (byte, int, float, decimal, etc.) → DynamoDB N (number) + /// - string, char → S (string) + /// - bool → BOOL + /// - DateTime, Guid → S (string) + /// - MemoryStream, byte[] → B (binary) + /// - HashSet of numeric types → NS (number set) + /// - HashSet of string types → SS (string set) + /// - HashSet of binary types → BS (binary set) + /// - List, array (numeric, string, binary types) → L (list) + /// - Dictionary{string, object} → M (map) + /// + /// Recommended for applications that need full fidelity with native DynamoDB types. /// </summary> V2 = 1, } diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Attributes.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Attributes.cs index d5d6c5c1b9e7..5d3affa53f91 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Attributes.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Attributes.cs @@ -53,9 +53,15 @@ public sealed class DynamoDBTableAttribute : DynamoDBAttribute public bool LowerCamelCaseProperties { get; set; } /// <summary> - /// Gets and sets the Conversion Schema property. + /// Gets and sets the <see cref="ConversionSchema"/> used for mapping between .NET and DynamoDB types. + /// + /// The conversion schema determines how types are serialized and deserialized during data persistence. + /// When resolving the effective schema, the following precedence is applied: + /// 1. If set on the operation configuration, it takes the highest precedence. + /// 2. If not set on the operation, but specified at the table level, the table configuration is used. + /// 3. If neither is set, the context-level configuration is used as the default fallback. /// </summary> - public ConversionSchema Conversion { get; } + public ConversionSchema Conversion { get; set; } /// <summary> /// Construct an instance of DynamoDBTableAttribute diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs index ea28f04a6050..0f6bc30780d5 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs @@ -553,8 +553,10 @@ public DynamoDBFlatConfig(DynamoDBOperationConfig operationConfig, DynamoDBConte public List<ScanCondition> QueryFilter { get; set; } /// <summary> - /// Entity Conversion specification which controls how conversion between - /// .NET and DynamoDB types happens. + /// Specifies the conversion behavior for .NET objects (entities) mapped to DynamoDB items. + /// + /// This setting controls how conversion between .NET and DynamoDB types happens + /// on classes annotated with <see cref="DynamoDBTableAttribute"/> /// </summary> public DynamoDBEntryConversion ItemConversion { get; set; } From 50bc4ca91c43d045e97eaeb9ab21a8202ce99c32 Mon Sep 17 00:00:00 2001 From: irina-herciu <irina.tanasa@endava.com> Date: Tue, 8 Apr 2025 10:51:09 +0300 Subject: [PATCH 3/3] update ConversionSchema documentation --- .../Conversion/DynamoDBEntryConversion.cs | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs b/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs index a6d894555bcf..5a121a3324b8 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/Conversion/DynamoDBEntryConversion.cs @@ -47,15 +47,17 @@ public enum ConversionSchema /// /// This schema pre-dates support for native DynamoDB types such as L (list), M (map), BOOL, and NULL. /// Common .NET type mappings: - /// - Number types (byte, int, float, decimal, etc.) → DynamoDB N (number) - /// - string, char → S (string) - /// - bool → N ("0" for false, "1" for true) - /// - DateTime, Guid → S (string) - /// - MemoryStream, byte[] → B (binary) - /// - List, HashSet, array of numeric types → NS (number set) - /// - List, HashSet, array of string types → SS (string set) - /// - List, HashSet, array of binary types → BS (binary set) - /// - Dictionary{string, object} → M (map) + /// <ul> + /// <li><para>Number types (byte, int, float, decimal, etc.) → DynamoDB N (number)</para></li> + /// <li><para>string, char → S (string)</para></li> + /// <li><para>bool → N ("0" for false, "1" for true)</para></li> + /// <li><para>DateTime, Guid → S (string)</para></li> + /// <li><para>MemoryStream, byte[] → B (binary)</para></li> + /// <li><para>List, HashSet, array of numeric types → NS (number set)</para></li> + /// <li><para>List, HashSet, array of string types → SS (string set)</para></li> + /// <li><para>List, HashSet, array of binary types → BS (binary set)</para></li> + /// <li><para>Dictionary{string, object} → M (map)</para></li> + /// </ul> /// </summary> V1 = 0, @@ -63,17 +65,18 @@ public enum ConversionSchema /// Enhanced conversion schema that supports native DynamoDB types including L (list), M (map), BOOL, and NULL. /// /// Common .NET type mappings: - /// - Number types (byte, int, float, decimal, etc.) → DynamoDB N (number) - /// - string, char → S (string) - /// - bool → BOOL - /// - DateTime, Guid → S (string) - /// - MemoryStream, byte[] → B (binary) - /// - HashSet of numeric types → NS (number set) - /// - HashSet of string types → SS (string set) - /// - HashSet of binary types → BS (binary set) - /// - List, array (numeric, string, binary types) → L (list) - /// - Dictionary{string, object} → M (map) - /// + /// <ul> + /// <li><para>Number types (byte, int, float, decimal, etc.) → DynamoDB N (number)</para></li> + /// <li><para>string, char → S (string)</para></li> + /// <li><para>bool → BOOL</para></li> + /// <li><para>DateTime, Guid → S (string)</para></li> + /// <li><para>MemoryStream, byte[] → B (binary)</para></li> + /// <li><para>HashSet of numeric types → NS (number set)</para></li> + /// <li><para>HashSet of string types → SS (string set)</para></li> + /// <li><para>HashSet of binary types → BS (binary set)</para></li> + /// <li><para>List, array (numeric, string, binary types) → L (list)</para></li> + /// <li><para>Dictionary{string, object} → M (map)</para></li> + /// </ul> /// Recommended for applications that need full fidelity with native DynamoDB types. /// </summary> V2 = 1,