No public description

PiperOrigin-RevId: 732274940

centipede/centipede_callbacks.cc

@@ -114,6 +114,7 @@ std::string CentipedeCallbacks::ConstructRunnerFlags(
       "CENTIPEDE_RUNNER_FLAGS=",
       absl::StrCat("timeout_per_input=", env_.timeout_per_input),
       absl::StrCat("timeout_per_batch=", env_.timeout_per_batch),
+      absl::StrCat("watchdog_abort_timeout=", env_.watchdog_abort_timeout),
       absl::StrCat("address_space_limit_mb=", env_.address_space_limit_mb),
       absl::StrCat("rss_limit_mb=", env_.rss_limit_mb),
       absl::StrCat("stack_limit_kb=", env_.stack_limit_kb),

centipede/environment.h

@@ -67,6 +67,7 @@ struct Environment {
   size_t stack_limit_kb = 0;
   size_t timeout_per_input = 60;
   size_t timeout_per_batch = 0;
+  size_t watchdog_abort_timeout = 0;
   absl::Time stop_at = absl::InfiniteFuture();
   bool fork_server = true;
   bool full_sync = false;

centipede/environment_flags.cc

@@ -157,6 +157,12 @@ ABSL_FLAG(size_t, timeout_per_batch, Environment::Default().timeout_per_batch,
           "finish within --timeout_per_batch seconds. The default is computed "
           "as a function of --timeout_per_input * --batch_size. Support may "
           "vary depending on the runner.");
+ABSL_FLAG(
+    size_t, watchdog_abort_timeout,
+    Environment::Default().watchdog_abort_timeout,
+    "If not zero, the runner's watchdog thread will forcefully abort if "
+    "the process is still running for too long after a previous abort "
+    "attempt. This is useful to prevent hangs (e.g. during stacktrace dumps).");
 ABSL_FLAG(absl::Time, stop_at, Environment::Default().stop_at,
           "Stop fuzzing in all shards (--total_shards) at approximately this "
           "time in ISO-8601/RFC-3339 format, e.g. 2023-04-06T23:35:02Z. "
@@ -462,6 +468,7 @@ Environment CreateEnvironmentFromFlags(const std::vector<std::string> &argv) {
       /*stack_limit_kb=*/absl::GetFlag(FLAGS_stack_limit_kb),
       /*timeout_per_input=*/absl::GetFlag(FLAGS_timeout_per_input),
       /*timeout_per_batch=*/absl::GetFlag(FLAGS_timeout_per_batch),
+      /*watchdog_abort_timeout=*/absl::GetFlag(FLAGS_watchdog_abort_timeout),
       /*stop_at=*/
       GetStopAtTime(absl::GetFlag(FLAGS_stop_at),
                     absl::GetFlag(FLAGS_stop_after)),

centipede/runner.cc

@@ -211,6 +211,26 @@ static uint64_t TimeInUsec() {
   return tv.tv_sec * kUsecInSec + tv.tv_usec;
 }
 
+static void CheckWatchdogAbortTimeout(uint64_t curr_time) {
+  // No timeout is set, ignore.
+  if (state.run_time_flags.watchdog_abort_timeout == 0) return;
+  // Watchdog did not invoke abort yet, ignore.
+  if (state.watchdog_abort_start_time == 0) return;
+  const uint64_t ongoing_abort_duration =
+      curr_time - state.watchdog_abort_start_time;
+  // The runner is still running unexpectedly long after we started
+  // aborting.
+  if (ongoing_abort_duration > state.run_time_flags.watchdog_abort_timeout) {
+    fprintf(stderr,
+            "========= Watchdog Abort timer exceeded: %" PRIu64 " > %" PRIu64
+            " (sec); std::abort() invoked.\n",
+            ongoing_abort_duration,
+            state.run_time_flags.watchdog_abort_timeout);
+    fprintf(stderr, "Aborting the runner in the watchdog thread.\n");
+    std::abort();
+  }
+}
+
 static void CheckWatchdogLimits() {
   const uint64_t curr_time = time(nullptr);
   struct Resource {
@@ -253,6 +273,7 @@ static void CheckWatchdogLimits() {
       // `RunOneInput()` after all the work is done.
       static std::atomic<bool> already_handling_failure = false;
       if (!already_handling_failure.exchange(true)) {
+        state.watchdog_abort_start_time = time(nullptr);
         fprintf(stderr,
                 "========= %s exceeded: %" PRIu64 " > %" PRIu64
                 " (%s); exiting\n",
@@ -272,6 +293,7 @@ static void CheckWatchdogLimits() {
       }
     }
   }
+  CheckWatchdogAbortTimeout(curr_time);
 }
 
 // Watchdog thread. Periodically checks if it's time to abort due to a

centipede/runner.h

@@ -67,6 +67,7 @@ struct RunTimeFlags {
   uint64_t use_auto_dictionary : 1;
   std::atomic<uint64_t> timeout_per_input;
   uint64_t timeout_per_batch;
+  uint64_t watchdog_abort_timeout;
   std::atomic<uint64_t> stack_limit_kb;
   std::atomic<uint64_t> rss_limit_mb;
   uint64_t crossover_level;
@@ -172,6 +173,7 @@ struct GlobalRunnerState {
       /*use_auto_dictionary=*/HasFlag(":use_auto_dictionary:"),
       /*timeout_per_input=*/HasIntFlag(":timeout_per_input=", 0),
       /*timeout_per_batch=*/HasIntFlag(":timeout_per_batch=", 0),
+      /*watchdog_abort_timeout=*/HasIntFlag(":watchdog_abort_timeout=", 0),
       /*stack_limit_kb=*/HasIntFlag(":stack_limit_kb=", 0),
       /*rss_limit_mb=*/HasIntFlag(":rss_limit_mb=", 0),
       /*crossover_level=*/HasIntFlag(":crossover_level=", 50),
@@ -345,6 +347,9 @@ struct GlobalRunnerState {
   // Per-batch timer. Initially, zero. ResetInputTimer() sets it to the current
   // time before the first input and never resets it.
   std::atomic<time_t> batch_start_time;
+  // Initially, zero. Watchdog thread sets it to the current time when it
+  // attempts to abort the runner main thread.
+  std::atomic<time_t> watchdog_abort_start_time;
 
   // The Watchdog thread sets this to true.
   std::atomic<bool> watchdog_thread_started;