From ea2c33cbbbcc4ec12de043ee428ad26bf9e6da92 Mon Sep 17 00:00:00 2001 From: DrFaust92 Date: Tue, 25 Aug 2020 11:09:13 +0300 Subject: [PATCH 1/4] add support for `scan_all` and `scan_rate` ddb target attributes --- aws/resource_aws_glue_crawler.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/aws/resource_aws_glue_crawler.go b/aws/resource_aws_glue_crawler.go index fc416850d1e7..84da8f1e007c 100644 --- a/aws/resource_aws_glue_crawler.go +++ b/aws/resource_aws_glue_crawler.go @@ -127,6 +127,17 @@ func resourceAwsGlueCrawler() *schema.Resource { Type: schema.TypeString, Required: true, }, + "scan_all": { + Type: schema.TypeBool, + Optional: true, + Default: true, + }, + "scan_rate": { + Type: schema.TypeFloat, + Optional: true, + Computed: true, + ValidateFunc: validation.FloatBetween(0.1, 1.5), + }, }, }, }, @@ -373,7 +384,12 @@ func expandGlueDynamoDBTargets(targets []interface{}) []*glue.DynamoDBTarget { func expandGlueDynamoDBTarget(cfg map[string]interface{}) *glue.DynamoDBTarget { target := &glue.DynamoDBTarget{ - Path: aws.String(cfg["path"].(string)), + Path: aws.String(cfg["path"].(string)), + ScanAll: aws.Bool(cfg["scan_all"].(bool)), + } + + if v, ok := cfg["scan_rate"].(float64); ok { + target.ScanRate = aws.Float64(v) } return target @@ -615,6 +631,8 @@ func flattenGlueDynamoDBTargets(dynamodbTargets []*glue.DynamoDBTarget) []map[st for _, dynamodbTarget := range dynamodbTargets { attrs := make(map[string]interface{}) attrs["path"] = aws.StringValue(dynamodbTarget.Path) + attrs["scan_all"] = aws.BoolValue(dynamodbTarget.ScanAll) + attrs["scan_rate"] = aws.Float64Value(dynamodbTarget.ScanRate) result = append(result, attrs) } From 65658f92101d01d5e22ff08828d3c6e64841f3d4 Mon Sep 17 00:00:00 2001 From: DrFaust92 Date: Tue, 25 Aug 2020 12:19:12 +0300 Subject: [PATCH 2/4] add test for scan all --- aws/resource_aws_glue_crawler.go | 3 +- aws/resource_aws_glue_crawler_test.go | 74 +++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/aws/resource_aws_glue_crawler.go b/aws/resource_aws_glue_crawler.go index 84da8f1e007c..eea2d81366a5 100644 --- a/aws/resource_aws_glue_crawler.go +++ b/aws/resource_aws_glue_crawler.go @@ -135,7 +135,6 @@ func resourceAwsGlueCrawler() *schema.Resource { "scan_rate": { Type: schema.TypeFloat, Optional: true, - Computed: true, ValidateFunc: validation.FloatBetween(0.1, 1.5), }, }, @@ -388,7 +387,7 @@ func expandGlueDynamoDBTarget(cfg map[string]interface{}) *glue.DynamoDBTarget { ScanAll: aws.Bool(cfg["scan_all"].(bool)), } - if v, ok := cfg["scan_rate"].(float64); ok { + if v, ok := cfg["scan_rate"].(float64); ok && v != 0 { target.ScanRate = aws.Float64(v) } diff --git a/aws/resource_aws_glue_crawler_test.go b/aws/resource_aws_glue_crawler_test.go index 5370a3fc5822..0e56de05d250 100644 --- a/aws/resource_aws_glue_crawler_test.go +++ b/aws/resource_aws_glue_crawler_test.go @@ -80,6 +80,7 @@ func TestAccAWSGlueCrawler_DynamodbTarget(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "description", ""), resource.TestCheckResourceAttr(resourceName, "dynamodb_target.#", "1"), resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "true"), resource.TestCheckResourceAttr(resourceName, "jdbc_target.#", "0"), resource.TestCheckResourceAttr(resourceName, "name", rName), resource.TestCheckResourceAttr(resourceName, "role", rName), @@ -103,6 +104,7 @@ func TestAccAWSGlueCrawler_DynamodbTarget(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "description", ""), resource.TestCheckResourceAttr(resourceName, "dynamodb_target.#", "1"), resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table2"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "true"), resource.TestCheckResourceAttr(resourceName, "jdbc_target.#", "0"), resource.TestCheckResourceAttr(resourceName, "name", rName), resource.TestCheckResourceAttr(resourceName, "role", rName), @@ -124,6 +126,49 @@ func TestAccAWSGlueCrawler_DynamodbTarget(t *testing.T) { }) } +func TestAccAWSGlueCrawler_DynamodbTarget_scanAll(t *testing.T) { + var crawler glue.Crawler + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_glue_crawler.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueCrawlerDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueCrawlerConfig_DynamodbTargetScanAll(rName, "table1", false), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueCrawlerExists(resourceName, &crawler), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "false"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccGlueCrawlerConfig_DynamodbTargetScanAll(rName, "table1", true), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueCrawlerExists(resourceName, &crawler), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "true"), + ), + }, + { + Config: testAccGlueCrawlerConfig_DynamodbTargetScanAll(rName, "table1", false), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueCrawlerExists(resourceName, &crawler), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "false"), + ), + }, + }, + }) +} + func TestAccAWSGlueCrawler_JdbcTarget(t *testing.T) { var crawler glue.Crawler rName := acctest.RandomWithPrefix("tf-acc-test") @@ -1211,21 +1256,42 @@ resource "aws_glue_crawler" "test" { func testAccGlueCrawlerConfig_DynamodbTarget(rName, path string) string { return testAccGlueCrawlerConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { - name = %q + name = %[1]q } resource "aws_glue_crawler" "test" { depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] database_name = aws_glue_catalog_database.test.name - name = %q + name = %[1]q role = aws_iam_role.test.name dynamodb_target { - path = %q + path = %[2]q } } -`, rName, rName, path) +`, rName, path) +} + +func testAccGlueCrawlerConfig_DynamodbTargetScanAll(rName, path string, scanAll bool) string { + return testAccGlueCrawlerConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_catalog_database" "test" { + name = %[1]q +} + +resource "aws_glue_crawler" "test" { + depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] + + database_name = aws_glue_catalog_database.test.name + name = %[1]q + role = aws_iam_role.test.name + + dynamodb_target { + path = %[2]q + scan_all = %[3]t + } +} +`, rName, path) } func testAccGlueCrawlerConfig_JdbcTarget(rName, path string) string { From dbc7fba3656d010f8ba232332eaad96e2d885075 Mon Sep 17 00:00:00 2001 From: DrFaust92 Date: Tue, 25 Aug 2020 12:25:12 +0300 Subject: [PATCH 3/4] add test for scan rate --- aws/resource_aws_glue_crawler_test.go | 66 ++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/aws/resource_aws_glue_crawler_test.go b/aws/resource_aws_glue_crawler_test.go index 0e56de05d250..513d9b254c12 100644 --- a/aws/resource_aws_glue_crawler_test.go +++ b/aws/resource_aws_glue_crawler_test.go @@ -169,6 +169,49 @@ func TestAccAWSGlueCrawler_DynamodbTarget_scanAll(t *testing.T) { }) } +func TestAccAWSGlueCrawler_DynamodbTarget_scanRate(t *testing.T) { + var crawler glue.Crawler + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_glue_crawler.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueCrawlerDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueCrawlerConfig_DynamodbTargetScanRate(rName, "table1", 0.5), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueCrawlerExists(resourceName, &crawler), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_rate", "0.5"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccGlueCrawlerConfig_DynamodbTargetScanRate(rName, "table1", 1.5), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueCrawlerExists(resourceName, &crawler), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_rate", "0.5"), + ), + }, + { + Config: testAccGlueCrawlerConfig_DynamodbTargetScanRate(rName, "table1", 0.5), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueCrawlerExists(resourceName, &crawler), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_rate", "0.5"), + ), + }, + }, + }) +} + func TestAccAWSGlueCrawler_JdbcTarget(t *testing.T) { var crawler glue.Crawler rName := acctest.RandomWithPrefix("tf-acc-test") @@ -1291,7 +1334,28 @@ resource "aws_glue_crawler" "test" { scan_all = %[3]t } } -`, rName, path) +`, rName, path, scanAll) +} + +func testAccGlueCrawlerConfig_DynamodbTargetScanRate(rName, path string, scanRate float64) string { + return testAccGlueCrawlerConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_catalog_database" "test" { + name = %[1]q +} + +resource "aws_glue_crawler" "test" { + depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] + + database_name = aws_glue_catalog_database.test.name + name = %[1]q + role = aws_iam_role.test.name + + dynamodb_target { + path = %[2]q + scan_all = %[3]f + } +} +`, rName, path, scanRate) } func testAccGlueCrawlerConfig_JdbcTarget(rName, path string) string { From c7ecdf4488c865ffad14919bfcf3a86a42c82bfd Mon Sep 17 00:00:00 2001 From: DrFaust92 Date: Tue, 25 Aug 2020 12:57:21 +0300 Subject: [PATCH 4/4] add docs for ddb target arguments --- aws/resource_aws_glue_crawler_test.go | 6 +++--- website/docs/r/glue_crawler.html.markdown | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/aws/resource_aws_glue_crawler_test.go b/aws/resource_aws_glue_crawler_test.go index 513d9b254c12..80d3d90ac94e 100644 --- a/aws/resource_aws_glue_crawler_test.go +++ b/aws/resource_aws_glue_crawler_test.go @@ -197,7 +197,7 @@ func TestAccAWSGlueCrawler_DynamodbTarget_scanRate(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueCrawlerExists(resourceName, &crawler), resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"), - resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_rate", "0.5"), + resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_rate", "1.5"), ), }, { @@ -1351,8 +1351,8 @@ resource "aws_glue_crawler" "test" { role = aws_iam_role.test.name dynamodb_target { - path = %[2]q - scan_all = %[3]f + path = %[2]q + scan_rate = %[3]g } } `, rName, path, scanRate) diff --git a/website/docs/r/glue_crawler.html.markdown b/website/docs/r/glue_crawler.html.markdown index 512cffa5edaa..6b9083b16a69 100644 --- a/website/docs/r/glue_crawler.html.markdown +++ b/website/docs/r/glue_crawler.html.markdown @@ -108,6 +108,8 @@ The following arguments are supported: ### dynamodb_target Argument Reference * `path` - (Required) The name of the DynamoDB table to crawl. +* `scan_all` - (Optional) Indicates whether to scan all the records, or to sample rows from the table. Scanning all the records can take a long time when the table is not a high throughput table. defaults to `true`. +* `scan_rate` - (Optional) The percentage of the configured read capacity units to use by the AWS Glue crawler. The valid values are null or a value between 0.1 to 1.5. ### jdbc_target Argument Reference