diff --git a/docs/command-line.md b/docs/command-line.md
index f8ac6374b8..0faeee16f9 100644
--- a/docs/command-line.md
+++ b/docs/command-line.md
@@ -19,6 +19,7 @@
[Specify a seed for the Random Number Generator](#specify-a-seed-for-the-random-number-generator)
[Identify framework and version according to the libIdentify standard](#identify-framework-and-version-according-to-the-libidentify-standard)
[Wait for key before continuing](#wait-for-key-before-continuing)
+[Skip benchmarks](#skip-benchmarks)
[Specify the number of benchmark samples to collect](#specify-the-number-of-benchmark-samples-to-collect)
[Specify the number of resamples for bootstrapping](#specify-the-number-of-resamples-for-bootstrapping)
[Specify the confidence-interval for bootstrapping](#specify-the-confidence-interval-for-bootstrapping)
@@ -62,6 +63,7 @@ Click one of the following links to take you straight to that option - or scroll
` --rng-seed`
` --libidentify`
` --wait-for-keypress`
+ ` --skip-benchmarks`
` --benchmark-samples`
` --benchmark-resamples`
` --benchmark-confidence-interval`
@@ -370,6 +372,16 @@ See [The LibIdentify repo for more information and examples](https://github.com/
Will cause the executable to print a message and wait until the return/ enter key is pressed before continuing -
either before running any tests, after running all tests - or both, depending on the argument.
+
+## Skip all benchmarks
+
--skip-benchmarks
+
+> [Introduced](https://github.com/catchorg/Catch2/issues/2408) in Catch X.Y.Z.
+
+This flag tells Catch2 to skip running all benchmarks. Benchmarks in this
+case mean code blocks in `BENCHMARK` and `BENCHMARK_ADVANCED` macros, not
+test cases with the `[!benchmark]` tag.
+
## Specify the number of benchmark samples to collect
--benchmark-samples <# of samples>
diff --git a/src/catch2/benchmark/catch_benchmark.hpp b/src/catch2/benchmark/catch_benchmark.hpp
index 28664b613e..66e3670f60 100644
--- a/src/catch2/benchmark/catch_benchmark.hpp
+++ b/src/catch2/benchmark/catch_benchmark.hpp
@@ -95,8 +95,11 @@ namespace Catch {
// sets lambda to be used in fun *and* executes benchmark!
template ::value, int> = 0>
Benchmark & operator=(Fun func) {
- fun = Detail::BenchmarkFunction(func);
- run();
+ auto const* cfg = getCurrentContext().getConfig();
+ if (!cfg->skipBenchmarks()) {
+ fun = Detail::BenchmarkFunction(func);
+ run();
+ }
return *this;
}
diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp
index 9e56e9fe70..e3ce6b65c0 100644
--- a/src/catch2/catch_config.cpp
+++ b/src/catch2/catch_config.cpp
@@ -152,6 +152,7 @@ namespace Catch {
bool Config::showInvisibles() const { return m_data.showInvisibles; }
Verbosity Config::verbosity() const { return m_data.verbosity; }
+ bool Config::skipBenchmarks() const { return m_data.skipBenchmarks; }
bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; }
unsigned int Config::benchmarkSamples() const { return m_data.benchmarkSamples; }
double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; }
diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp
index 8823accdf2..a3dc010b70 100644
--- a/src/catch2/catch_config.hpp
+++ b/src/catch2/catch_config.hpp
@@ -64,6 +64,7 @@ namespace Catch {
unsigned int shardCount = 1;
unsigned int shardIndex = 0;
+ bool skipBenchmarks = false;
bool benchmarkNoAnalysis = false;
unsigned int benchmarkSamples = 100;
double benchmarkConfidenceInterval = 0.95;
@@ -129,6 +130,7 @@ namespace Catch {
int abortAfter() const override;
bool showInvisibles() const override;
Verbosity verbosity() const override;
+ bool skipBenchmarks() const override;
bool benchmarkNoAnalysis() const override;
unsigned int benchmarkSamples() const override;
double benchmarkConfidenceInterval() const override;
diff --git a/src/catch2/interfaces/catch_interfaces_config.hpp b/src/catch2/interfaces/catch_interfaces_config.hpp
index 7e3c9d0b81..5e358650a4 100644
--- a/src/catch2/interfaces/catch_interfaces_config.hpp
+++ b/src/catch2/interfaces/catch_interfaces_config.hpp
@@ -88,6 +88,7 @@ namespace Catch {
virtual std::vector const& getSectionsToRun() const = 0;
virtual Verbosity verbosity() const = 0;
+ virtual bool skipBenchmarks() const = 0;
virtual bool benchmarkNoAnalysis() const = 0;
virtual unsigned int benchmarkSamples() const = 0;
virtual double benchmarkConfidenceInterval() const = 0;
diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp
index 59b06de9f3..f2f638bc17 100644
--- a/src/catch2/internal/catch_commandline.cpp
+++ b/src/catch2/internal/catch_commandline.cpp
@@ -296,6 +296,9 @@ namespace Catch {
| Opt( setWaitForKeypress, "never|start|exit|both" )
["--wait-for-keypress"]
( "waits for a keypress before exiting" )
+ | Opt( config.skipBenchmarks)
+ ["--skip-benchmarks"]
+ ( "disable running benchmarks")
| Opt( config.benchmarkSamples, "samples" )
["--benchmark-samples"]
( "number of samples to collect (default: 100)" )
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 911f385710..627928f154 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -359,6 +359,18 @@ add_test(NAME CheckConvenienceHeaders
${PYTHON_EXECUTABLE} ${CATCH_DIR}/tools/scripts/checkConvenienceHeaders.py
)
+add_test(NAME "Benchmarking::SkipBenchmarkMacros"
+ COMMAND
+ $ "Skip benchmark macros"
+ --reporter console
+ --skip-benchmarks
+)
+set_tests_properties("Benchmarking::SkipBenchmarkMacros"
+ PROPERTIES
+ PASS_REGULAR_EXPRESSION "All tests passed \\(2 assertions in 1 test case\\)"
+ FAIL_REGULAR_EXPRESSION "benchmark name"
+)
+
add_test(NAME "Benchmarking::FailureReporting::OptimizedOut"
COMMAND
diff --git a/tests/SelfTest/UsageTests/Benchmark.tests.cpp b/tests/SelfTest/UsageTests/Benchmark.tests.cpp
index 14b001079a..ffedc9a2f6 100644
--- a/tests/SelfTest/UsageTests/Benchmark.tests.cpp
+++ b/tests/SelfTest/UsageTests/Benchmark.tests.cpp
@@ -152,3 +152,20 @@ TEST_CASE("Benchmark containers", "[!benchmark]") {
};
}
}
+
+TEST_CASE("Skip benchmark macros", "[!benchmark]") {
+ std::vector v;
+ BENCHMARK("fill vector") {
+ v.emplace_back(1);
+ v.emplace_back(2);
+ v.emplace_back(3);
+ };
+ REQUIRE(v.size() == 0);
+
+ std::size_t counter{0};
+ BENCHMARK_ADVANCED("construct vector")(Catch::Benchmark::Chronometer meter) {
+ std::vector> storage(meter.runs());
+ meter.measure([&](int i) { storage[i].construct("thing"); counter++; });
+ };
+ REQUIRE(counter == 0);
+}