diff --git a/CHANGES.txt b/CHANGES.txt index 443d69673b..ffea1f6ec5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ Current +Fixed: GITHUB-2674: Run onTestSkipped for each value from data provider (Krishnan Mahadevan) Fixed: GITHUB-2672: Log real stacktrace when test times out. (cdalexndr) Fixed: GITHUB-2669: A failed retry with ITestContext will lose the ITestContext. (Nan Liang) Fixed: GITHUB-2643: assertEquals(Set,Set) now ignores ordering as it did before. (Elis Edlund) diff --git a/testng-core/src/main/java/org/testng/CommandLineArgs.java b/testng-core/src/main/java/org/testng/CommandLineArgs.java index d3a752a226..32be0f3901 100644 --- a/testng-core/src/main/java/org/testng/CommandLineArgs.java +++ b/testng-core/src/main/java/org/testng/CommandLineArgs.java @@ -242,4 +242,13 @@ public class CommandLineArgs { description = "Comma separated fully qualified class names of listeners that should be skipped from being wired in via Service Loaders.") public Boolean overrideIncludedMethods = false; + + public static final String INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING = + "-includeAllDataDrivenTestsWhenSkipping"; + + @Parameter( + names = INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING, + description = + "Should TestNG report all iterations of a data driven test as individual skips, in-case of upstream failures.") + public Boolean includeAllDataDrivenTestsWhenSkipping = false; } diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java index 585c86360e..158c224fdd 100644 --- a/testng-core/src/main/java/org/testng/TestNG.java +++ b/testng-core/src/main/java/org/testng/TestNG.java @@ -582,6 +582,14 @@ public void addMethodSelector(XmlMethodSelector selector) { m_selectors.add(selector); } + public void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTestsAsSkipped) { + this.m_configuration.setReportAllDataDrivenTestsAsSkipped(reportAllDataDrivenTestsAsSkipped); + } + + public boolean getReportAllDataDrivenTestsAsSkipped() { + return this.m_configuration.getReportAllDataDrivenTestsAsSkipped(); + } + /** * Set the suites file names to be run by this TestNG object. This method tries to load and parse * the specified TestNG suite xml files. If a file is missing, it is ignored. @@ -1404,6 +1412,7 @@ public static TestNG privateMain(String[] argv, ITestListener listener) { * @param cla The command line parameters */ protected void configure(CommandLineArgs cla) { + setReportAllDataDrivenTestsAsSkipped(cla.includeAllDataDrivenTestsWhenSkipping); if (cla.verbose != null) { setVerbose(cla.verbose); } @@ -1613,6 +1622,10 @@ public void configure(Map cmdLineArgs) { result.xmlPathInJar = (String) cmdLineArgs.get(CommandLineArgs.XML_PATH_IN_JAR); result.junit = (Boolean) cmdLineArgs.get(CommandLineArgs.JUNIT); result.mixed = (Boolean) cmdLineArgs.get(CommandLineArgs.MIXED); + Object tmpValue = cmdLineArgs.get(CommandLineArgs.INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING); + if (tmpValue != null) { + result.includeAllDataDrivenTestsWhenSkipping = Boolean.parseBoolean(tmpValue.toString()); + } result.skipFailedInvocationCounts = (Boolean) cmdLineArgs.get(CommandLineArgs.SKIP_FAILED_INVOCATION_COUNTS); result.failIfAllTestsSkipped = diff --git a/testng-core/src/main/java/org/testng/internal/Configuration.java b/testng-core/src/main/java/org/testng/internal/Configuration.java index e1a36d14d2..c4a4c604f1 100644 --- a/testng-core/src/main/java/org/testng/internal/Configuration.java +++ b/testng-core/src/main/java/org/testng/internal/Configuration.java @@ -33,6 +33,8 @@ public class Configuration implements IConfiguration { private IInjectorFactory injectorFactory = new GuiceBackedInjectorFactory(); private boolean overrideIncludedMethods = false; + private boolean includeAllDataDrivenTestsWhenSkipping; + public Configuration() { init(new JDK15AnnotationFinder(new DefaultAnnotationTransformer())); } @@ -144,4 +146,14 @@ public boolean getOverrideIncludedMethods() { public void setOverrideIncludedMethods(boolean overrideIncludedMethods) { this.overrideIncludedMethods = overrideIncludedMethods; } + + @Override + public void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTestsAsSkipped) { + this.includeAllDataDrivenTestsWhenSkipping = reportAllDataDrivenTestsAsSkipped; + } + + @Override + public boolean getReportAllDataDrivenTestsAsSkipped() { + return this.includeAllDataDrivenTestsWhenSkipping; + } } diff --git a/testng-core/src/main/java/org/testng/internal/IConfiguration.java b/testng-core/src/main/java/org/testng/internal/IConfiguration.java index 6e81649668..9ab92d87d5 100644 --- a/testng-core/src/main/java/org/testng/internal/IConfiguration.java +++ b/testng-core/src/main/java/org/testng/internal/IConfiguration.java @@ -49,4 +49,10 @@ default boolean addExecutionListenerIfAbsent(IExecutionListener l) { boolean getOverrideIncludedMethods(); void setOverrideIncludedMethods(boolean overrideIncludedMethods); + + default void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTestsAsSkipped) {} + + default boolean getReportAllDataDrivenTestsAsSkipped() { + return false; + } } diff --git a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java index c7115c9d1d..4d180aa0c4 100644 --- a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java +++ b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java @@ -7,6 +7,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -16,6 +17,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; import org.testng.DataProviderHolder; import org.testng.DataProviderInvocationException; @@ -37,6 +39,7 @@ import org.testng.SuiteRunner; import org.testng.TestException; import org.testng.TestNGException; +import org.testng.collections.CollectionUtils; import org.testng.collections.Lists; import org.testng.collections.Maps; import org.testng.collections.Sets; @@ -104,12 +107,48 @@ public List invokeTestMethods( // // Not okToProceed. Test is being skipped // - ITestResult result = - registerSkippedTestResult( - testMethod, System.currentTimeMillis(), new Throwable(okToProceed)); - m_notifier.addSkippedTest(testMethod, result); - InvokedMethod invokedMethod = new InvokedMethod(System.currentTimeMillis(), result); - invokeListenersForSkippedTestResult(result, invokedMethod); + List results = new ArrayList<>(); + Consumer resultProcessor = + result -> { + m_notifier.addSkippedTest(testMethod, result); + InvokedMethod invokedMethod = new InvokedMethod(System.currentTimeMillis(), result); + invokeListenersForSkippedTestResult(result, invokedMethod); + }; + boolean reportAllDataDrivenTestsAsSkipped = + m_configuration.getReportAllDataDrivenTestsAsSkipped(); + if (reportAllDataDrivenTestsAsSkipped && testMethod.isDataDriven()) { + ParameterHandler handler = + new ParameterHandler( + m_configuration.getObjectFactory(), + annotationFinder(), + buildDataProviderHolder(), + 1); + + ParameterBag bag = + handler.createParameters( + testMethod, Maps.newHashMap(), Maps.newHashMap(), context, instance); + Iterator allParamValues = Objects.requireNonNull(bag.parameterHolder).parameters; + Iterable allParameterValues = CollectionUtils.asIterable(allParamValues); + for (Object[] next : allParameterValues) { + if (next == null) { + continue; + } + Method m = testMethod.getConstructorOrMethod().getMethod(); + Object[] parameterValues = Parameters.injectParameters(next, m, context); + ITestResult result = + registerSkippedTestResult( + testMethod, System.currentTimeMillis(), new Throwable(okToProceed)); + result.setParameters(parameterValues); + resultProcessor.accept(result); + results.add(result); + } + } else { + ITestResult result = + registerSkippedTestResult( + testMethod, System.currentTimeMillis(), new Throwable(okToProceed)); + resultProcessor.accept(result); + results.add(result); + } testMethod.incrementCurrentInvocationCount(); GroupConfigMethodArguments args = new Builder() @@ -119,7 +158,7 @@ public List invokeTestMethods( .withParameters(parameters) .build(); this.invoker.invokeAfterGroupsConfigurations(args); - return Collections.singletonList(result); + return Collections.unmodifiableList(results); } // For invocationCount > 1 and threadPoolSize > 1 run this method in its own pool thread. diff --git a/testng-core/src/test/java/test/skip/ReasonForSkipTest.java b/testng-core/src/test/java/test/skip/ReasonForSkipTest.java index 0ea4b10975..c479f21b73 100644 --- a/testng-core/src/test/java/test/skip/ReasonForSkipTest.java +++ b/testng-core/src/test/java/test/skip/ReasonForSkipTest.java @@ -5,13 +5,16 @@ import java.util.Arrays; import java.util.Map; +import org.testng.CommandLineArgs; import org.testng.ITestResult; import org.testng.TestNG; import org.testng.annotations.Test; import org.testng.collections.Maps; import org.testng.xml.XmlSuite; +import test.InvokedMethodNameListener; import test.SimpleBaseTest; import test.skip.github1967.TestClassSample; +import test.skip.issue2674.ConfigAwareTestNG; public class ReasonForSkipTest extends SimpleBaseTest { @@ -103,6 +106,31 @@ public void testEnsureTestStatusIsSetProperlyForSkippedTests() { assertThat(actual).containsAllEntriesOf(expected); } + @Test(description = "GITHUB-2674") + public void ensureUpstreamFailuresTriggerSkipsForAllDataProviderValues() { + TestNG testng = create(test.skip.issue2674.TestClassSample.class); + testng.setReportAllDataDrivenTestsAsSkipped(true); + InvokedMethodNameListener listener = new InvokedMethodNameListener(); + testng.addListener(listener); + testng.run(); + assertThat(listener.getSkippedMethodNames()) + .containsExactly("test2(iPhone,13)", "test2(iPhone-Pro,12)"); + } + + @Test(description = "GITHUB-2674") + public void ensureUpstreamFailuresTriggerSkipsForAllDataProviderValuesViaCmdLineArgs() { + CommandLineArgs cli = new CommandLineArgs(); + cli.includeAllDataDrivenTestsWhenSkipping = true; + ConfigAwareTestNG testng = new ConfigAwareTestNG(); + testng.setTestClasses(new Class[] {test.skip.issue2674.TestClassSample.class}); + testng.configure(cli); + InvokedMethodNameListener listener = new InvokedMethodNameListener(); + testng.addListener(listener); + testng.run(); + assertThat(listener.getSkippedMethodNames()) + .containsExactly("test2(iPhone,13)", "test2(iPhone-Pro,12)"); + } + private static void runTest(Map expected, Class... clazz) { TestNG testng = create(clazz); ReasonReporter reporter = new ReasonReporter(); diff --git a/testng-core/src/test/java/test/skip/issue2674/ConfigAwareTestNG.java b/testng-core/src/test/java/test/skip/issue2674/ConfigAwareTestNG.java new file mode 100644 index 0000000000..70daf4e401 --- /dev/null +++ b/testng-core/src/test/java/test/skip/issue2674/ConfigAwareTestNG.java @@ -0,0 +1,12 @@ +package test.skip.issue2674; + +import org.testng.CommandLineArgs; +import org.testng.TestNG; + +public class ConfigAwareTestNG extends TestNG { + + @Override + public void configure(CommandLineArgs cla) { + super.configure(cla); + } +} diff --git a/testng-core/src/test/java/test/skip/issue2674/TestClassSample.java b/testng-core/src/test/java/test/skip/issue2674/TestClassSample.java new file mode 100644 index 0000000000..831266ce72 --- /dev/null +++ b/testng-core/src/test/java/test/skip/issue2674/TestClassSample.java @@ -0,0 +1,22 @@ +package test.skip.issue2674; + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class TestClassSample { + @Test + void test1() { + Assert.fail(); + } + + @Test( + dataProvider = "items", + dependsOnMethods = {"test1"}) + void test2(String model, int variant) {} + + @DataProvider(name = "items") + Object[][] items() { + return new Object[][] {{"iPhone", 13}, {"iPhone-Pro", 12}}; + } +}