From 61081d12ae234f7b5981eb51115308b2a4e7b9f3 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Wed, 15 Jan 2025 20:52:43 +0000 Subject: [PATCH 1/9] backport of commit 855f2e940a53ec428696c571e7f04a191e0063dd --- internal/command/junit/junit.go | 47 +++++++++++++++++-- .../1pass-1fail/expected-output.xml | 9 ++-- .../junit-output/1pass-1fail/main.tftest.hcl | 10 +++- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index 3ae7bfa8158a..e0accd707354 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -16,6 +16,11 @@ import ( "github.com/hashicorp/terraform/internal/configs/configload" "github.com/hashicorp/terraform/internal/moduletest" "github.com/hashicorp/terraform/internal/tfdiags" + "github.com/zclconf/go-cty/cty" +) + +var ( + failedTestSummary = "Test assertion failed" ) // TestJUnitXMLFile produces a JUnit XML file at the conclusion of a test @@ -211,11 +216,20 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source Body: body, } case moduletest.Fail: + var diagsStr strings.Builder + var failedAssertion tfdiags.Diagnostic + for _, diag := range run.Diagnostics { + // Find the diag resulting from a failed assertion + if diag.Description().Summary == failedTestSummary { + diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80)) + failedAssertion = diag + break + } + } + body := getFailBody(run, failedAssertion) testCase.Failure = &withMessage{ Message: "Test run failed", - // FIXME: What's a useful thing to report in the body - // here? A summary of the statuses from all of the - // checkable objects in the configuration? + Body: body, } case moduletest.Error: var diagsStr strings.Builder @@ -284,6 +298,33 @@ func getSkipDetails(runIndex int, file *moduletest.File, suiteStopped bool) (str return "", "" } +func getFailBody(run *moduletest.Run, diag tfdiags.Diagnostic) string { + testCtx := diag.FromExpr() + + // Identify which assertion failed out of multiple assertions in a run block + var failedIndex int + var found bool + for i, assertion := range run.Config.CheckRules { + condition, diag := assertion.Condition.Value(testCtx.EvalContext) + if diag.HasErrors() { + return "" + } + + if condition.RawEquals(cty.BoolVal(false)) { + failedIndex = i + 1 // index 1 + found = true + break + } + } + + if found { + return fmt.Sprintf("Test failed on assertion %d of %d", failedIndex, len(run.Config.CheckRules)) + } + + // Unhandled case + return "" +} + func suiteFilesAsSortedList(files map[string]*moduletest.File) []*moduletest.File { fileNames := make([]string, len(files)) i := 0 diff --git a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml index e05aa399adb5..f491695ee8e1 100644 --- a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml +++ b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml @@ -1,17 +1,16 @@ - + diff --git a/internal/command/testdata/test/junit-output/1pass-1fail/main.tftest.hcl b/internal/command/testdata/test/junit-output/1pass-1fail/main.tftest.hcl index 04b7bc709d99..e62d28e32a2b 100644 --- a/internal/command/testdata/test/junit-output/1pass-1fail/main.tftest.hcl +++ b/internal/command/testdata/test/junit-output/1pass-1fail/main.tftest.hcl @@ -1,7 +1,15 @@ run "failing_assertion" { + assert { + condition = local.number == 10 + error_message = "assertion 1 should pass" + } assert { condition = local.number < 0 - error_message = "local variable 'number' has a value greater than zero, so this assertion will fail" + error_message = "local variable 'number' has a value greater than zero, so assertion 2 will fail" + } + assert { + condition = local.number == 10 + error_message = "assertion 3 should pass" } } From 696220b965db76eb86f4c9d1385b4704da3ce8b2 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Wed, 15 Jan 2025 20:56:10 +0000 Subject: [PATCH 2/9] backport of commit 132b8bda6d6c3ec058e98c3b34adf823b4a2b22d --- internal/command/junit/junit.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index e0accd707354..4c7022beed58 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -216,12 +216,10 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source Body: body, } case moduletest.Fail: - var diagsStr strings.Builder var failedAssertion tfdiags.Diagnostic for _, diag := range run.Diagnostics { // Find the diag resulting from a failed assertion if diag.Description().Summary == failedTestSummary { - diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80)) failedAssertion = diag break } From f6fc9978753143af9c9cd6e70d3be2a4b8da9acd Mon Sep 17 00:00:00 2001 From: Sarah French Date: Wed, 15 Jan 2025 21:30:20 +0000 Subject: [PATCH 3/9] backport of commit d8da87ee8237ab7e00d7fa52472c11b9380b4c56 --- internal/command/junit/junit.go | 22 +++++++++++++------ .../1pass-1fail/expected-output.xml | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index 4c7022beed58..c093a7b16a56 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -224,9 +224,9 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source break } } - body := getFailBody(run, failedAssertion) + message, body := getFailDetails(run, failedAssertion) testCase.Failure = &withMessage{ - Message: "Test run failed", + Message: message, Body: body, } case moduletest.Error: @@ -296,7 +296,9 @@ func getSkipDetails(runIndex int, file *moduletest.File, suiteStopped bool) (str return "", "" } -func getFailBody(run *moduletest.Run, diag tfdiags.Diagnostic) string { +// getFailDetails uses data from the diagnostic raised by the failing test to create +// a message and body text for the element in the XML +func getFailDetails(run *moduletest.Run, diag tfdiags.Diagnostic) (string, string) { testCtx := diag.FromExpr() // Identify which assertion failed out of multiple assertions in a run block @@ -305,22 +307,28 @@ func getFailBody(run *moduletest.Run, diag tfdiags.Diagnostic) string { for i, assertion := range run.Config.CheckRules { condition, diag := assertion.Condition.Value(testCtx.EvalContext) if diag.HasErrors() { - return "" + return "", "" } if condition.RawEquals(cty.BoolVal(false)) { - failedIndex = i + 1 // index 1 + failedIndex = i found = true break } } if found { - return fmt.Sprintf("Test failed on assertion %d of %d", failedIndex, len(run.Config.CheckRules)) + message, diag := run.Config.CheckRules[failedIndex].ErrorMessage.Value(testCtx.EvalContext) + if diag.HasErrors() { + return "", "" + } + + body := fmt.Sprintf("Test failed on assertion %d of %d", failedIndex+1, len(run.Config.CheckRules)) + return message.AsString(), body } // Unhandled case - return "" + return "", "" } func suiteFilesAsSortedList(files map[string]*moduletest.File) []*moduletest.File { diff --git a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml index f491695ee8e1..d3fd0039ba9e 100644 --- a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml +++ b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml @@ -1,7 +1,7 @@ - + Date: Thu, 16 Jan 2025 11:19:03 +0000 Subject: [PATCH 4/9] backport of commit b0ea07849db8277d8cf85d6ae6a3cc23b0cff091 --- internal/command/junit/junit.go | 48 +++++---------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index c093a7b16a56..8d89e657877a 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -16,7 +16,6 @@ import ( "github.com/hashicorp/terraform/internal/configs/configload" "github.com/hashicorp/terraform/internal/moduletest" "github.com/hashicorp/terraform/internal/tfdiags" - "github.com/zclconf/go-cty/cty" ) var ( @@ -217,17 +216,17 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source } case moduletest.Fail: var failedAssertion tfdiags.Diagnostic - for _, diag := range run.Diagnostics { + for _, d := range run.Diagnostics { // Find the diag resulting from a failed assertion - if diag.Description().Summary == failedTestSummary { - failedAssertion = diag + if d.Description().Summary == failedTestSummary { + failedAssertion = d break } } - message, body := getFailDetails(run, failedAssertion) + testCase.Failure = &withMessage{ - Message: message, - Body: body, + Message: failedAssertion.Description().Detail, + Body: format.DiagnosticPlain(failedAssertion, sources, 80), } case moduletest.Error: var diagsStr strings.Builder @@ -296,41 +295,6 @@ func getSkipDetails(runIndex int, file *moduletest.File, suiteStopped bool) (str return "", "" } -// getFailDetails uses data from the diagnostic raised by the failing test to create -// a message and body text for the element in the XML -func getFailDetails(run *moduletest.Run, diag tfdiags.Diagnostic) (string, string) { - testCtx := diag.FromExpr() - - // Identify which assertion failed out of multiple assertions in a run block - var failedIndex int - var found bool - for i, assertion := range run.Config.CheckRules { - condition, diag := assertion.Condition.Value(testCtx.EvalContext) - if diag.HasErrors() { - return "", "" - } - - if condition.RawEquals(cty.BoolVal(false)) { - failedIndex = i - found = true - break - } - } - - if found { - message, diag := run.Config.CheckRules[failedIndex].ErrorMessage.Value(testCtx.EvalContext) - if diag.HasErrors() { - return "", "" - } - - body := fmt.Sprintf("Test failed on assertion %d of %d", failedIndex+1, len(run.Config.CheckRules)) - return message.AsString(), body - } - - // Unhandled case - return "", "" -} - func suiteFilesAsSortedList(files map[string]*moduletest.File) []*moduletest.File { fileNames := make([]string, len(files)) i := 0 From 4332f2ad581b2ad05dda95ea9704fdbe87afe73e Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 16 Jan 2025 11:28:54 +0000 Subject: [PATCH 5/9] backport of commit 5aea43eeb0dafbb4471b86af6f64fd223bad1aef --- internal/command/junit/junit.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index 8d89e657877a..b07c4bf82371 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -193,6 +193,14 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source for i, run := range file.Runs { // Each run is a "test case". + // By creating a map of diags we can delete them as they're used below + // This helps to identify diags that are only appropriate to include in + // the "system-err" element + diagsMap := make(map[int]tfdiags.Diagnostic, len(run.Diagnostics)) + for i, diag := range run.Diagnostics { + diagsMap[i] = diag + } + testCase := testCase{ Name: run.Name, @@ -216,10 +224,11 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source } case moduletest.Fail: var failedAssertion tfdiags.Diagnostic - for _, d := range run.Diagnostics { + for key, d := range diagsMap { // Find the diag resulting from a failed assertion if d.Description().Summary == failedTestSummary { failedAssertion = d + delete(diagsMap, key) break } } @@ -230,23 +239,25 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source } case moduletest.Error: var diagsStr strings.Builder - for _, diag := range run.Diagnostics { - diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80)) + for key, d := range diagsMap { + diagsStr.WriteString(format.DiagnosticPlain(d, sources, 80)) + delete(diagsMap, key) } testCase.Error = &withMessage{ Message: "Encountered an error", Body: diagsStr.String(), } } - if len(run.Diagnostics) != 0 && testCase.Error == nil { + if len(diagsMap) != 0 && testCase.Error == nil { // If we have diagnostics but the outcome wasn't an error // then we're presumably holding diagnostics that didn't // cause the test to error, such as warnings. We'll place // those into the "system-err" element instead, so that // they'll be reported _somewhere_ at least. var diagsStr strings.Builder - for _, diag := range run.Diagnostics { + for key, diag := range diagsMap { diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80)) + delete(diagsMap, key) } testCase.Stderr = &withMessage{ Body: diagsStr.String(), From 9962e57299ee4b3c8e4ce3071907e5164e05c927 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 16 Jan 2025 11:29:08 +0000 Subject: [PATCH 6/9] backport of commit 080505c2031d2f34e4b087138324f578b3ee1ff8 --- .../test/junit-output/1pass-1fail/expected-output.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml index d3fd0039ba9e..6ef5906622be 100644 --- a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml +++ b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml @@ -1,8 +1,7 @@ - - +]]> From 9b440a4485dd469402c36e57524a1006707153ff Mon Sep 17 00:00:00 2001 From: Sarah French Date: Wed, 22 Jan 2025 12:38:33 +0000 Subject: [PATCH 7/9] backport of commit b6798ae702022eefdf524c914bedc1dfaa78a58c --- internal/command/junit/junit.go | 113 ++++++++++++------ internal/command/junit/junit_internal_test.go | 81 +++++++++++++ .../1pass-1fail/expected-output.xml | 2 +- 3 files changed, 160 insertions(+), 36 deletions(-) diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index b07c4bf82371..c029642f5834 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -193,14 +193,6 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source for i, run := range file.Runs { // Each run is a "test case". - // By creating a map of diags we can delete them as they're used below - // This helps to identify diags that are only appropriate to include in - // the "system-err" element - diagsMap := make(map[int]tfdiags.Diagnostic, len(run.Diagnostics)) - for i, diag := range run.Diagnostics { - diagsMap[i] = diag - } - testCase := testCase{ Name: run.Name, @@ -223,46 +215,64 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source Body: body, } case moduletest.Fail: - var failedAssertion tfdiags.Diagnostic - for key, d := range diagsMap { - // Find the diag resulting from a failed assertion - if d.Description().Summary == failedTestSummary { - failedAssertion = d - delete(diagsMap, key) - break + // When the test fails we only use error diags that originate from failing assertions + var failedAssertions tfdiags.Diagnostics + for _, d := range run.Diagnostics { + if d.Severity() == tfdiags.Error && d.Description().Summary == failedTestSummary { + failedAssertions = failedAssertions.Append(d) } } testCase.Failure = &withMessage{ - Message: failedAssertion.Description().Detail, - Body: format.DiagnosticPlain(failedAssertion, sources, 80), + Message: failureMessage(failedAssertions, len(run.Config.CheckRules)), + Body: getDiagString(failedAssertions, sources), } + case moduletest.Error: - var diagsStr strings.Builder - for key, d := range diagsMap { - diagsStr.WriteString(format.DiagnosticPlain(d, sources, 80)) - delete(diagsMap, key) + // When the test errors we use all diags with Error severity + var errDiags tfdiags.Diagnostics + for _, d := range run.Diagnostics { + if d.Severity() == tfdiags.Error { + errDiags = errDiags.Append(d) + } } + testCase.Error = &withMessage{ Message: "Encountered an error", - Body: diagsStr.String(), + Body: getDiagString(errDiags, sources), } } - if len(diagsMap) != 0 && testCase.Error == nil { - // If we have diagnostics but the outcome wasn't an error - // then we're presumably holding diagnostics that didn't - // cause the test to error, such as warnings. We'll place - // those into the "system-err" element instead, so that - // they'll be reported _somewhere_ at least. - var diagsStr strings.Builder - for key, diag := range diagsMap { - diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80)) - delete(diagsMap, key) + + // Determine if there are diagnostics left unused by the switch block above + // that should be included in the "system-err" element + if len(run.Diagnostics) > 0 { + var systemErrDiags tfdiags.Diagnostics + + if run.Status == moduletest.Error && run.Diagnostics.HasWarnings() { + // If the test case errored, then all Error diags are in the "error" element + // Therefore we'd only need to include warnings in "system-err" + systemErrDiags = getWarnings(run.Diagnostics) } - testCase.Stderr = &withMessage{ - Body: diagsStr.String(), + + if run.Status != moduletest.Error { + // If a test hasn't errored then we need to find all diagnostics that aren't due + // to a failing assertion in a test (these are already displayed in the "failure" element) + + // Collect diags not due to failed assertions, both errors and warnings + for _, d := range run.Diagnostics { + if d.Description().Summary != failedTestSummary { + systemErrDiags = systemErrDiags.Append(d) + } + } + } + + if len(systemErrDiags) > 0 { + testCase.Stderr = &withMessage{ + Body: getDiagString(systemErrDiags, sources), + } } } + enc.EncodeElement(&testCase, xml.StartElement{ Name: caseName, }) @@ -275,7 +285,19 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source return buf.Bytes(), nil } -// getSkipDetails checks data about the test suite, file and runs to determine why a given run was skipped +func failureMessage(failedAssertions tfdiags.Diagnostics, checkCount int) string { + if len(failedAssertions) == 0 { + return "" + } + + if len(failedAssertions) == 1 { + // Slightly different format if only single assertion failure + return fmt.Sprintf("%d of %d assertions failed: %s", len(failedAssertions), checkCount, failedAssertions[0].Description().Detail) + } + + // Handle multiple assertion failures + return fmt.Sprintf("%d of %d assertions failed, including: %s", len(failedAssertions), checkCount, failedAssertions[0].Description().Detail) +} // Test can be skipped due to: // 1. terraform test recieving an interrupt from users; all unstarted tests will be skipped // 2. A previous run in a file has failed, causing subsequent run blocks to be skipped @@ -321,3 +343,24 @@ func suiteFilesAsSortedList(files map[string]*moduletest.File) []*moduletest.Fil } return sortedFiles } + +func getDiagString(diags tfdiags.Diagnostics, sources map[string][]byte) string { + var diagsStr strings.Builder + for _, d := range diags { + diagsStr.WriteString(format.DiagnosticPlain(d, sources, 80)) + } + return diagsStr.String() +} + +func getWarnings(diags tfdiags.Diagnostics) tfdiags.Diagnostics { + // Collect warnings + warnDiags := tfdiags.Diagnostics{} + if diags.HasWarnings() { + for _, d := range diags { + if d.Severity() == tfdiags.Warning { + warnDiags = warnDiags.Append(d) + } + } + } + return warnDiags +} diff --git a/internal/command/junit/junit_internal_test.go b/internal/command/junit/junit_internal_test.go index 41ab23f82fd3..3e67afc7a472 100644 --- a/internal/command/junit/junit_internal_test.go +++ b/internal/command/junit/junit_internal_test.go @@ -8,7 +8,10 @@ import ( "os" "testing" + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/terraform/internal/moduletest" + "github.com/hashicorp/terraform/internal/tfdiags" ) func Test_TestJUnitXMLFile_save(t *testing.T) { @@ -67,6 +70,66 @@ func Test_TestJUnitXMLFile_save(t *testing.T) { } } + +func Test_getWarnings(t *testing.T) { + + errorDiag := &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "error", + Detail: "this is an error", + } + + warnDiag := &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "warning", + Detail: "this is a warning", + } + + cases := map[string]struct { + diags tfdiags.Diagnostics + expected tfdiags.Diagnostics + }{ + "empty diags": { + diags: tfdiags.Diagnostics{}, + expected: tfdiags.Diagnostics{}, + }, + "nil diags": { + diags: nil, + expected: tfdiags.Diagnostics{}, + }, + "all error diags": { + diags: func() tfdiags.Diagnostics { + var d tfdiags.Diagnostics + d = d.Append(errorDiag, errorDiag, errorDiag) + return d + }(), + expected: tfdiags.Diagnostics{}, + }, + "mixture of error and warning diags": { + diags: func() tfdiags.Diagnostics { + var d tfdiags.Diagnostics + d = d.Append(errorDiag, errorDiag, warnDiag) // 1 warning + return d + }(), + expected: func() tfdiags.Diagnostics { + var d tfdiags.Diagnostics + d = d.Append(warnDiag) // 1 warning + return d + }(), + }, + } + + for tn, tc := range cases { + t.Run(tn, func(t *testing.T) { + warnings := getWarnings(tc.diags) + + if diff := cmp.Diff(tc.expected, warnings, cmp.Comparer(diagnosticComparer)); diff != "" { + t.Errorf("wrong diagnostics\n%s", diff) + } + }) + } +} + func Test_suiteFilesAsSortedList(t *testing.T) { cases := map[string]struct { Suite *moduletest.Suite @@ -151,3 +214,21 @@ func Test_suiteFilesAsSortedList(t *testing.T) { }) } } + +// From internal/backend/remote-state/s3/testing_test.go +// diagnosticComparer is a Comparer function for use with cmp.Diff to compare two tfdiags.Diagnostic values +func diagnosticComparer(l, r tfdiags.Diagnostic) bool { + if l.Severity() != r.Severity() { + return false + } + if l.Description() != r.Description() { + return false + } + + lp := tfdiags.GetAttribute(l) + rp := tfdiags.GetAttribute(r) + if len(lp) != len(rp) { + return false + } + return lp.Equals(rp) +} diff --git a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml index 6ef5906622be..5a3d04227844 100644 --- a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml +++ b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml @@ -1,7 +1,7 @@ - Date: Wed, 22 Jan 2025 12:40:40 +0000 Subject: [PATCH 8/9] backport of commit add24c64289e096a98281c2f84fb006299626e6c --- internal/command/junit/junit.go | 31 ++++++++++--------- internal/command/junit/junit_internal_test.go | 1 - 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index c029642f5834..22eeb72f4461 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -207,13 +207,12 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source testCase.RunTime = execMeta.Duration.Seconds() testCase.Timestamp = execMeta.StartTimestamp() } + + // Depending on run status, add either of: "skipped", "failure", or "error" elements switch run.Status { case moduletest.Skip: - message, body := getSkipDetails(i, file, suiteRunnerStopped) - testCase.Skipped = &withMessage{ - Message: message, - Body: body, - } + testCase.Skipped = skipDetails(i, file, suiteRunnerStopped) + case moduletest.Fail: // When the test fails we only use error diags that originate from failing assertions var failedAssertions tfdiags.Diagnostics @@ -298,17 +297,20 @@ func failureMessage(failedAssertions tfdiags.Diagnostics, checkCount int) string // Handle multiple assertion failures return fmt.Sprintf("%d of %d assertions failed, including: %s", len(failedAssertions), checkCount, failedAssertions[0].Description().Detail) } + +// skipDetails checks data about the test suite, file and runs to determine why a given run was skipped // Test can be skipped due to: // 1. terraform test recieving an interrupt from users; all unstarted tests will be skipped // 2. A previous run in a file has failed, causing subsequent run blocks to be skipped -func getSkipDetails(runIndex int, file *moduletest.File, suiteStopped bool) (string, string) { +// The returned value is used to set content in the "skipped" element +func skipDetails(runIndex int, file *moduletest.File, suiteStopped bool) *withMessage { if suiteStopped { // Test suite experienced an interrupt // This block only handles graceful Stop interrupts, as Cancel interrupts will prevent a JUnit file being produced at all - message := "Testcase skipped due to an interrupt" - body := "Terraform received an interrupt and stopped gracefully. This caused all remaining testcases to be skipped" - - return message, body + return &withMessage{ + Message: "Testcase skipped due to an interrupt", + Body: "Terraform received an interrupt and stopped gracefully. This caused all remaining testcases to be skipped", + } } if file.Status == moduletest.Error { @@ -317,15 +319,16 @@ func getSkipDetails(runIndex int, file *moduletest.File, suiteStopped bool) (str for i := runIndex; i >= 0; i-- { if file.Runs[i].Status == moduletest.Error { // Skipped due to error in previous run within the file - message := "Testcase skipped due to a previous testcase error" - body := fmt.Sprintf("Previous testcase %q ended in error, which caused the remaining tests in the file to be skipped", file.Runs[i].Name) - return message, body + return &withMessage{ + Message: "Testcase skipped due to a previous testcase error", + Body: fmt.Sprintf("Previous testcase %q ended in error, which caused the remaining tests in the file to be skipped", file.Runs[i].Name), + } } } } // Unhandled case: This results in with no attributes or body - return "", "" + return nil } func suiteFilesAsSortedList(files map[string]*moduletest.File) []*moduletest.File { diff --git a/internal/command/junit/junit_internal_test.go b/internal/command/junit/junit_internal_test.go index 3e67afc7a472..c6ca196cd42a 100644 --- a/internal/command/junit/junit_internal_test.go +++ b/internal/command/junit/junit_internal_test.go @@ -70,7 +70,6 @@ func Test_TestJUnitXMLFile_save(t *testing.T) { } } - func Test_getWarnings(t *testing.T) { errorDiag := &hcl.Diagnostic{ From d4bb505dca23d0b95742dd459988da045bb6c00b Mon Sep 17 00:00:00 2001 From: Sarah French Date: Wed, 22 Jan 2025 13:46:10 +0000 Subject: [PATCH 9/9] backport of commit 0bbcfdda1dffcceb740bac9139ecd85f9f26a3c7 --- internal/command/junit/junit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index 22eeb72f4461..2e6040ee1ba8 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -328,7 +328,7 @@ func skipDetails(runIndex int, file *moduletest.File, suiteStopped bool) *withMe } // Unhandled case: This results in with no attributes or body - return nil + return &withMessage{} } func suiteFilesAsSortedList(files map[string]*moduletest.File) []*moduletest.File {