feat(reactor): Add hooks for end of event dispatching

Add support for hooks that are called after all i/o and timer events
have been dispatched. These hooks are called, and only if, work done
in the cycle is greater than zero. And they are always called before
EndOfCycleNoWait hooks.

SDB-6783

toolbox/io/Reactor.cpp

@@ -60,7 +60,7 @@ int Reactor::poll(CyclTime now, Duration timeout)
 
     // If timeout is zero then the wait_until time should also be zero to signify no wait.
     MonoTime wait_until{};
-    if (!is_zero(timeout) && hooks_.empty()) {
+    if (!is_zero(timeout) && end_of_cycle_no_wait_hooks.empty()) {
         const MonoTime next
             = next_expiry(timeout == NoTimeout ? MonoClock::max() : now.mono_time() + timeout);
         if (next > now.mono_time()) {
@@ -98,7 +98,10 @@ int Reactor::poll(CyclTime now, Duration timeout)
         work += tqs_[Low].dispatch(now);
     }
     // End of cycle hooks.
-    io::dispatch(now, hooks_);
+    if (work > 0) {
+        io::dispatch(now, end_of_event_dispatch_hooks_);
+    }
+    io::dispatch(now, end_of_cycle_no_wait_hooks);
     return work;
 }
 

toolbox/io/Reactor.hpp

@@ -34,6 +34,17 @@ using IoSlot = BasicSlot<CyclTime, int, unsigned>;
 class TOOLBOX_API Reactor : public Waker {
   public:
     using Event = EpollEvent;
+    // HookType describes the kind of hook.
+    enum class HookType : int {
+        // EndOfCycleNoWait hooks are called at the end of the Reactor cycle.
+        // The Reactor cycle will not wait for i/o and/or timer events
+        // while any of these hooks are installed.
+        EndOfCycleNoWait = 1,
+        // EndOfEventDispatch hooks are called after all i/o and timer events have been dispatched.
+        // These hooks are called, and only if, work done in the cycle is greater than zero.
+        // And they are always called before EndOfCycleNoWait hooks.
+        EndOfEventDispatch = 2,
+    };
     class Handle {
       public:
         Handle(Reactor& reactor, int fd, int sid)
@@ -139,7 +150,17 @@ class TOOLBOX_API Reactor : public Waker {
     }
     // clang-format on
 
-    void add_hook(Hook& hook) noexcept { hooks_.push_back(hook); }
+    void add_hook(Hook& hook, HookType ht = HookType::EndOfCycleNoWait) noexcept
+    {
+        switch (ht) {
+        case HookType::EndOfCycleNoWait:
+            end_of_cycle_no_wait_hooks.push_back(hook);
+            break;
+        case HookType::EndOfEventDispatch:
+            end_of_event_dispatch_hooks_.push_back(hook);
+            break;
+        }
+    }
     /// Poll for I/O and timer events.
     /// The thread-local cycle time is unconditionally updated after the call to epoll() returns.
     /// Returns the number of events signalled.
@@ -171,7 +192,7 @@ class TOOLBOX_API Reactor : public Waker {
     static_assert(static_cast<int>(Priority::Low) == 1);
     TimerPool tp_;
     std::array<TimerQueue, 2> tqs_{tp_, tp_};
-    HookList hooks_;
+    HookList end_of_cycle_no_wait_hooks, end_of_event_dispatch_hooks_;
 };
 
 } // namespace io