LCOV - code coverage report
Current view: top level - corosio/native/detail/reactor - reactor_scheduler.hpp (source / functions) Coverage Total Hit Missed
Test: coverage_remapped.info Lines: 98.3 % 404 397 7
Test Date: 2026-11-16 20:48:52 Functions: 98.1 % 54 53 1

           TLA  Line data    Source code
       1                 : //
       2                 : // Copyright (c) 2026 Steve Gerbino
       3                 : //
       4                 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
       5                 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
       6                 : //
       7                 : // Official repository: https://github.com/cppalliance/corosio
       8                 : //
       9                 : 
      10                 : #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
      11                 : #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
      12                 : 
      13                 : #include <boost/corosio/detail/config.hpp>
      14                 : #include <boost/corosio/detail/object_ref.hpp>
      15                 : #include <boost/capy/ex/execution_context.hpp>
      16                 : #include <boost/capy/ex/abandon.hpp>
      17                 : 
      18                 : #include <boost/corosio/detail/ready_queue.hpp>
      19                 : #include <boost/corosio/detail/scheduler.hpp>
      20                 : #include <boost/corosio/detail/scheduler_op.hpp>
      21                 : #include <boost/corosio/detail/thread_local_ptr.hpp>
      22                 : #include <boost/corosio/detail/timer_service.hpp>
      23                 : 
      24                 : #include <atomic>
      25                 : #include <chrono>
      26                 : #include <coroutine>
      27                 : #include <cstddef>
      28                 : #include <cstdint>
      29                 : #include <limits>
      30                 : #include <memory>
      31                 : 
      32                 : #include <boost/corosio/detail/conditionally_enabled_mutex.hpp>
      33                 : #include <boost/corosio/detail/conditionally_enabled_event.hpp>
      34                 : 
      35                 : namespace boost::corosio::detail {
      36                 : 
      37                 : // Forward declarations
      38                 : class reactor_scheduler;
      39                 : 
      40                 : /** A deregistered descriptor whose owner must outlive the reactor pass
      41                 :     that may still hold it.
      42                 : 
      43                 :     An event the reactor has already read from the kernel can name a
      44                 :     descriptor that another thread closes before the reactor queues it.
      45                 :     The closing thread hands over a reference instead of dropping it, so
      46                 :     the owner is neither freed nor recycled until that pass ends.
      47                 : */
      48                 : struct reactor_retirable
      49                 : {
      50                 :     // All four fields are guarded by the scheduler's mutex.
      51                 : 
      52                 :     /// Next in the scheduler's retired list.
      53                 :     reactor_retirable* retired_next_ = nullptr;
      54                 : 
      55                 :     /// Owner reference held until the pass ends.
      56                 :     object_ref retired_ref_;
      57                 : 
      58                 :     /// Next in a settling thread's list of references to drop.
      59                 :     reactor_retirable* dropped_next_ = nullptr;
      60                 : 
      61                 :     /// Owner reference waiting to be dropped outside the mutex.
      62                 :     object_ref dropped_ref_;
      63                 : 
      64                 :     /// Move @p keep into a queued completion, if one is pending.
      65                 :     virtual void hand_off(object_ref& keep) noexcept = 0;
      66                 : 
      67                 : protected:
      68 HIT       12564 :     ~reactor_retirable() = default;
      69                 : };
      70                 : 
      71                 : /** Per-thread state for a reactor scheduler.
      72                 : 
      73                 :     Each thread running a scheduler's event loop has one of these
      74                 :     on a thread-local stack. It holds a private work queue and
      75                 :     inline completion budget for speculative I/O fast paths.
      76                 : */
      77                 : struct BOOST_COROSIO_SYMBOL_VISIBLE reactor_scheduler_context
      78                 : {
      79                 :     /// Scheduler this context belongs to.
      80                 :     reactor_scheduler const* key;
      81                 : 
      82                 :     /// Next context frame on this thread's stack.
      83                 :     reactor_scheduler_context* next;
      84                 : 
      85                 :     /// Private work queue for reduced contention.
      86                 :     ready_queue private_queue;
      87                 : 
      88                 :     /// Unflushed work count for the private queue.
      89                 :     std::int64_t private_outstanding_work;
      90                 : 
      91                 :     /// Remaining inline completions allowed this cycle.
      92                 :     int inline_budget;
      93                 : 
      94                 :     /// Maximum inline budget (adaptive, 2-16).
      95                 :     int inline_budget_max;
      96                 : 
      97                 :     /// True if no other thread absorbed queued work last cycle.
      98                 :     bool unassisted;
      99                 : 
     100                 :     /// Construct a context frame linked to @a n.
     101                 :     reactor_scheduler_context(
     102                 :         reactor_scheduler const* k, reactor_scheduler_context* n);
     103                 : };
     104                 : 
     105                 : /// Thread-local context stack for reactor schedulers.
     106                 : inline thread_local_ptr<reactor_scheduler_context> reactor_context_stack;
     107                 : 
     108                 : /// Find the context frame for a scheduler on this thread.
     109                 : inline reactor_scheduler_context*
     110          913711 : reactor_find_context(reactor_scheduler const* self) noexcept
     111                 : {
     112          914258 :     for (auto* c = reactor_context_stack.get(); c != nullptr; c = c->next)
     113                 :     {
     114          850469 :         if (c->key == self)
     115          849922 :             return c;
     116                 :     }
     117           63789 :     return nullptr;
     118                 : }
     119                 : 
     120                 : /** Non-template base for reactor-backed scheduler implementations.
     121                 : 
     122                 :     Provides the complete threading model shared by epoll, kqueue,
     123                 :     and select schedulers: signal state machine, inline completion
     124                 :     budget, work counting, run/poll methods, and the do_one event
     125                 :     loop.
     126                 : 
     127                 :     Derived classes provide platform-specific hooks by overriding:
     128                 :     - `run_task(lock, ctx)` to run the reactor poll
     129                 :     - `interrupt_reactor()` to wake a blocked reactor
     130                 : 
     131                 :     De-templated from the original CRTP design to eliminate
     132                 :     duplicate instantiations when multiple backends are compiled
     133                 :     into the same binary. Virtual dispatch for run_task (called
     134                 :     once per reactor cycle, before a blocking syscall) has
     135                 :     negligible overhead.
     136                 : 
     137                 :     @par Thread Safety
     138                 :     All public member functions are thread-safe.
     139                 : */
     140                 : class reactor_scheduler : public scheduler
     141                 : {
     142                 : public:
     143                 :     using context_type = reactor_scheduler_context;
     144                 :     using mutex_type   = conditionally_enabled_mutex;
     145                 :     using lock_type    = mutex_type::scoped_lock;
     146                 :     using event_type   = conditionally_enabled_event;
     147                 : 
     148                 :     /// Post a coroutine for deferred execution.
     149                 :     void post(std::coroutine_handle<> h) const override;
     150                 : 
     151                 :     /// Post a scheduler operation for deferred execution.
     152                 :     void post(scheduler_op* h) const override;
     153                 : 
     154                 :     /** Release an owner reference once no reactor pass can hold `d`.
     155                 : 
     156                 :         Called after a descriptor is deregistered. If a reactor pass is
     157                 :         running, `keep` is parked on `d` until that pass ends. Then, or
     158                 :         at once if no pass is running, it moves to `d`'s queued
     159                 :         invocation if `d` is queued, and is dropped otherwise.
     160                 : 
     161                 :         @param d The deregistered descriptor.
     162                 :         @param keep A reference to the descriptor's owner.
     163                 :     */
     164                 :     void retire_descriptor(reactor_retirable& d, object_ref keep) const;
     165                 : 
     166                 :     /// Post a continuation for deferred execution.
     167                 :     void post(capy::continuation&) const override;
     168                 : 
     169                 :     /// Return true if called from a thread running this scheduler.
     170                 :     bool running_in_this_thread() const noexcept override;
     171                 : 
     172                 :     /// Request the scheduler to stop dispatching handlers.
     173                 :     void stop() override;
     174                 : 
     175                 :     /// Return true if the scheduler has been stopped.
     176                 :     bool stopped() const noexcept override;
     177                 : 
     178                 :     /// Reset the stopped state so `run()` can resume.
     179                 :     void restart() override;
     180                 : 
     181                 :     /// Run the event loop until no work remains.
     182                 :     std::size_t run() override;
     183                 : 
     184                 :     /// Run until one handler completes or no work remains.
     185                 :     std::size_t run_one() override;
     186                 : 
     187                 :     /// Run until one handler completes or @a usec elapses.
     188                 :     std::size_t wait_one(long usec) override;
     189                 : 
     190                 :     /// Run ready handlers without blocking.
     191                 :     std::size_t poll() override;
     192                 : 
     193                 :     /// Run at most one ready handler without blocking.
     194                 :     std::size_t poll_one() override;
     195                 : 
     196                 :     /// Increment the outstanding work count.
     197                 :     void work_started() noexcept override;
     198                 : 
     199                 :     /// Decrement the outstanding work count, stopping on zero.
     200                 :     void work_finished() noexcept override;
     201                 : 
     202                 :     /** Reset the thread's inline completion budget.
     203                 : 
     204                 :         Called at the start of each posted completion handler to
     205                 :         grant a fresh budget for speculative inline completions.
     206                 :     */
     207                 :     void reset_inline_budget() const noexcept;
     208                 : 
     209                 :     /** Consume one unit of inline budget if available.
     210                 : 
     211                 :         @return True if budget was available and consumed.
     212                 :     */
     213                 :     bool try_consume_inline_budget() const noexcept;
     214                 : 
     215                 :     /** Offset a forthcoming work_finished from work_cleanup.
     216                 : 
     217                 :         Called by descriptor_state when all I/O returned EAGAIN and
     218                 :         no handler will be executed. Must be called from a scheduler
     219                 :         thread.
     220                 :     */
     221                 :     /** Note that an op was parked outside the usual registration path.
     222                 : 
     223                 :         A reactor that builds its poll set from parked ops must rebuild
     224                 :         it; one the kernel tracks needs nothing.
     225                 :     */
     226             495 :     virtual void notify_reparked() const {}
     227                 : 
     228                 :     void compensating_work_started() const noexcept;
     229                 : 
     230                 :     /** Post an operation whose work is already counted.
     231                 : 
     232                 :         The posting thread touches nothing of the scheduler after this
     233                 :         returns, unlike a @ref post followed by @ref work_finished: once
     234                 :         posted, the operation can complete and let its context be
     235                 :         destroyed.
     236                 : 
     237                 :         @pre work_started() was called for @p op.
     238                 : 
     239                 :         @param op The operation to post.
     240                 :     */
     241                 :     void post_counted(scheduler_op* op) const;
     242                 : 
     243                 :     /** Post completed operations for deferred invocation.
     244                 : 
     245                 :         If called from a thread running this scheduler, operations
     246                 :         go to the thread's private queue (fast path). Otherwise,
     247                 :         operations are added to the global queue under mutex and a
     248                 :         waiter is signaled.
     249                 : 
     250                 :         @pre work_started() must have been called for each operation.
     251                 : 
     252                 :         @param ops Queue of operations to post.
     253                 :     */
     254                 :     void post_deferred_completions(ready_queue& ops) const;
     255                 : 
     256                 :     /** Apply runtime configuration to the scheduler.
     257                 : 
     258                 :         Called by `io_context` after construction. Values that do
     259                 :         not apply to this backend are silently ignored.
     260                 : 
     261                 :         @pre `max_events` is in [ 1, `INT_MAX` ] and `budget_max` is at
     262                 :             most `INT_MAX`; `io_context` validates both.
     263                 : 
     264                 :         @param max_events  Event buffer size for epoll/kqueue.
     265                 :         @param budget_init Starting inline completion budget.
     266                 :         @param budget_max  Hard ceiling on adaptive budget ramp-up.
     267                 :         @param unassisted  Budget when single-threaded.
     268                 :     */
     269                 :     virtual void configure_reactor(
     270                 :         unsigned max_events,
     271                 :         unsigned budget_init,
     272                 :         unsigned budget_max,
     273                 :         unsigned unassisted);
     274                 : 
     275                 :     /// Return the configured initial inline budget.
     276           11283 :     unsigned inline_budget_initial() const noexcept
     277                 :     {
     278           11283 :         return inline_budget_initial_;
     279                 :     }
     280                 : 
     281                 :     /// Return true when scheduler locking is disabled (fully-lockless tier).
     282            1460 :     bool scheduler_locking_disabled() const noexcept override
     283                 :     {
     284            1460 :         return scheduler_locking_disabled_;
     285                 :     }
     286                 : 
     287            4672 :     void configure_threading(threading_config cfg) noexcept override
     288                 :     {
     289            4672 :         scheduler_locking_disabled_ = !cfg.scheduler_locking;
     290                 :         // reactor_io_locking takes effect at descriptor registration (see the
     291                 :         // register_descriptor overrides), not here.
     292            4672 :         reactor_io_locking_ = cfg.reactor_io_locking;
     293            4672 :         one_thread_         = cfg.one_thread;
     294            4672 :         mutex_.set_enabled(cfg.scheduler_locking);
     295            4672 :         cond_.set_enabled(cfg.scheduler_locking);
     296            4672 :     }
     297                 : 
     298                 : protected:
     299                 :     bool scheduler_locking_disabled_ = false;
     300                 :     bool reactor_io_locking_         = true;
     301                 :     bool one_thread_                 = false;
     302                 : 
     303            4688 :     reactor_scheduler() = default;
     304                 : 
     305                 :     /// Destroy, destroying whatever was posted after the drain.
     306                 :     ~reactor_scheduler() override;
     307                 : 
     308                 :     /** Drain completed_ops during shutdown.
     309                 : 
     310                 :         Pops all operations from the global queue and destroys them,
     311                 :         skipping the task sentinel. Signals all waiting threads.
     312                 :         Derived classes call this from their shutdown() override
     313                 :         before performing platform-specific cleanup.
     314                 :     */
     315                 :     void shutdown_drain();
     316                 : 
     317                 :     /// Destroy every queued operation without running it.
     318                 :     void destroy_queued(lock_type& lock) noexcept;
     319                 : 
     320                 :     /// RAII guard that re-inserts the task sentinel after `run_task`.
     321                 :     struct task_cleanup
     322                 :     {
     323                 :         reactor_scheduler const* sched;
     324                 :         lock_type* lock;
     325                 :         context_type& ctx;
     326                 :         ~task_cleanup();
     327                 :     };
     328                 : 
     329                 :     mutable mutex_type mutex_{true};
     330                 :     mutable event_type cond_{true};
     331                 :     mutable ready_queue completed_ops_;
     332                 :     mutable std::atomic<std::int64_t> outstanding_work_{0};
     333                 :     std::atomic<bool> stopped_{false};
     334                 :     mutable std::atomic<bool> task_running_{false};
     335                 :     mutable bool task_interrupted_ = false;
     336                 : 
     337                 :     // Descriptors retired during the running reactor pass (guarded by
     338                 :     // mutex_).
     339                 :     mutable reactor_retirable* retired_ = nullptr;
     340                 : 
     341                 :     /// Settle every descriptor retired before the current pass ended.
     342                 :     /// @pre `lock` holds `mutex_`; it is held again on return.
     343                 :     void settle_retired(lock_type& lock) const;
     344                 : 
     345                 :     /// Move what an enclosing handler on this thread queued privately
     346                 :     /// to the shared queue. @pre `mutex_` is held.
     347                 :     void adopt_enclosing_work(context_type const& ctx);
     348                 : 
     349                 :     // Runtime-configurable reactor tuning parameters.
     350                 :     // Defaults match the library's built-in values.
     351                 :     unsigned max_events_per_poll_   = 128;
     352                 :     unsigned inline_budget_initial_ = 2;
     353                 :     unsigned inline_budget_max_     = 16;
     354                 :     unsigned unassisted_budget_     = 4;
     355                 : 
     356                 :     /// Bit 0 of `state_`: set when the condvar should be signaled.
     357                 :     static constexpr std::size_t signaled_bit = 1;
     358                 : 
     359                 :     /// Increment per waiting thread in `state_`.
     360                 :     static constexpr std::size_t waiter_increment = 2;
     361                 :     mutable std::size_t state_                    = 0;
     362                 : 
     363                 :     /// Sentinel op that triggers a reactor poll when dequeued.
     364                 :     struct task_op final : scheduler_op
     365                 :     {
     366                 :         // LCOV_EXCL_START: the sentinel is intercepted by pointer
     367                 :         // identity; its virtuals exist for vtable completeness.
     368                 :         void operator()() override {}
     369                 :         void destroy() override {}
     370                 :         // LCOV_EXCL_STOP
     371                 :     };
     372                 :     task_op task_op_;
     373                 : 
     374                 :     /** Run the platform-specific reactor poll.
     375                 : 
     376                 :         @par Postconditions
     377                 :         `lock` is owned on return, however the poll ended. An
     378                 :         implementation that unlocks around the blocking call owes the
     379                 :         caller a matching re-acquire on every path out, including the
     380                 :         errors it retries rather than reports.
     381                 :     */
     382                 :     virtual void
     383                 :     run_task(lock_type& lock, context_type& ctx, long timeout_us) = 0;
     384                 : 
     385                 :     /// Wake a blocked reactor (e.g. write to eventfd or pipe).
     386                 :     virtual void interrupt_reactor() const = 0;
     387                 : 
     388                 :     /// Wake the reactor whenever the earliest timer changes.
     389                 :     void on_timer_service_attached(timer_service& svc) override;
     390                 : 
     391                 : private:
     392                 :     struct work_cleanup
     393                 :     {
     394                 :         reactor_scheduler* sched;
     395                 :         lock_type* lock;
     396                 :         context_type& ctx;
     397                 :         ~work_cleanup();
     398                 :     };
     399                 : 
     400                 :     std::size_t do_one(lock_type& lock, long timeout_us, context_type& ctx);
     401                 : 
     402                 :     void signal_all(lock_type& lock) const;
     403                 :     bool maybe_unlock_and_signal_one(lock_type& lock) const;
     404                 :     bool unlock_and_signal_one(lock_type& lock) const;
     405                 :     void clear_signal() const;
     406                 :     void wait_for_signal(lock_type& lock) const;
     407                 :     void wait_for_signal_for(lock_type& lock, long timeout_us) const;
     408                 :     void wake_one_thread_and_unlock(lock_type& lock) const;
     409                 : };
     410                 : 
     411                 : /** RAII guard that pushes/pops a scheduler context frame.
     412                 : 
     413                 :     On construction, pushes a new context frame onto the
     414                 :     thread-local stack. On destruction, drains any remaining
     415                 :     private queue items to the global queue and pops the frame.
     416                 : */
     417                 : struct reactor_thread_context_guard
     418                 : {
     419                 :     /// The context frame managed by this guard.
     420                 :     reactor_scheduler_context frame_;
     421                 : 
     422                 :     /// Marks the scheduler innermost for the executor's fast check.
     423                 :     running_scheduler_guard running_;
     424                 : 
     425                 :     /// Construct the guard, pushing a frame for @a sched.
     426           11283 :     explicit reactor_thread_context_guard(
     427                 :         reactor_scheduler const* sched) noexcept
     428           11283 :         : frame_(sched, reactor_context_stack.get())
     429           11283 :         , running_(sched)
     430                 :     {
     431           11283 :         reactor_context_stack.set(&frame_);
     432           11283 :     }
     433                 : 
     434                 :     /** Destroy the guard, popping the frame.
     435                 : 
     436                 :         The private queue is empty here by invariant: work_cleanup and
     437                 :         task_cleanup splice it to the global queue after every handler
     438                 :         and every reactor pass.
     439                 :     */
     440           11283 :     ~reactor_thread_context_guard() noexcept
     441                 :     {
     442           11283 :         reactor_context_stack.set(frame_.next);
     443           11283 :     }
     444                 : };
     445                 : 
     446                 : // ---- Inline implementations ------------------------------------------------
     447                 : 
     448           11283 : inline reactor_scheduler_context::reactor_scheduler_context(
     449           11283 :     reactor_scheduler const* k, reactor_scheduler_context* n)
     450           11283 :     : key(k)
     451           11283 :     , next(n)
     452           11283 :     , private_outstanding_work(0)
     453           11283 :     , inline_budget(0)
     454           11283 :     , inline_budget_max(static_cast<int>(k->inline_budget_initial()))
     455           11283 :     , unassisted(false)
     456                 : {
     457           11283 : }
     458                 : 
     459                 : inline void
     460              50 : reactor_scheduler::configure_reactor(
     461                 :     unsigned max_events,
     462                 :     unsigned budget_init,
     463                 :     unsigned budget_max,
     464                 :     unsigned unassisted)
     465                 : {
     466              50 :     BOOST_COROSIO_ASSERT(
     467                 :         max_events >= 1 &&
     468                 :         max_events <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
     469              50 :     BOOST_COROSIO_ASSERT(
     470                 :         budget_max <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
     471                 : 
     472                 :     // Clamp initial and unassisted to budget_max.
     473              50 :     if (budget_init > budget_max)
     474              20 :         budget_init = budget_max;
     475              50 :     if (unassisted > budget_max)
     476              20 :         unassisted = budget_max;
     477                 : 
     478              50 :     max_events_per_poll_   = max_events;
     479              50 :     inline_budget_initial_ = budget_init;
     480              50 :     inline_budget_max_     = budget_max;
     481              50 :     unassisted_budget_     = unassisted;
     482              50 : }
     483                 : 
     484                 : inline void
     485           98780 : reactor_scheduler::reset_inline_budget() const noexcept
     486                 : {
     487                 :     // When budget is disabled (max==0), all paths below would no-op
     488                 :     // (inline_budget stays 0). Skip the TLS lookup entirely.
     489           98780 :     if (inline_budget_max_ == 0)
     490              58 :         return;
     491           98722 :     if (auto* ctx = reactor_find_context(this))
     492                 :     {
     493                 :         // Cap when no other thread absorbed queued work
     494           98722 :         if (ctx->unassisted)
     495                 :         {
     496           97927 :             ctx->inline_budget_max = static_cast<int>(unassisted_budget_);
     497           97927 :             ctx->inline_budget     = static_cast<int>(unassisted_budget_);
     498           97927 :             return;
     499                 :         }
     500                 :         // Ramp up when previous cycle fully consumed budget.
     501                 :         // max(1, ...) ensures the doubling escapes zero.
     502                 :         // Doubled in a wider type: the cap may be as large as INT_MAX.
     503             795 :         if (ctx->inline_budget == 0)
     504                 :         {
     505                 :             long long const doubled =
     506             263 :                 2LL * (std::max)(1, ctx->inline_budget_max);
     507             263 :             long long const cap    = inline_budget_max_;
     508             263 :             ctx->inline_budget_max = static_cast<int>((std::min)(doubled, cap));
     509                 :         }
     510             532 :         else if (ctx->inline_budget < ctx->inline_budget_max)
     511                 :         {
     512             174 :             ctx->inline_budget_max = static_cast<int>(inline_budget_initial_);
     513                 :         }
     514             795 :         ctx->inline_budget = ctx->inline_budget_max;
     515                 :     }
     516                 : }
     517                 : 
     518                 : inline bool
     519          409704 : reactor_scheduler::try_consume_inline_budget() const noexcept
     520                 : {
     521          409704 :     if (inline_budget_max_ == 0)
     522              44 :         return false;
     523          409660 :     if (auto* ctx = reactor_find_context(this))
     524                 :     {
     525          409656 :         if (ctx->inline_budget > 0)
     526                 :         {
     527          327464 :             --ctx->inline_budget;
     528          327464 :             return true;
     529                 :         }
     530                 :     }
     531           82196 :     return false;
     532                 : }
     533                 : 
     534                 : inline void
     535            3862 : reactor_scheduler::post(std::coroutine_handle<> h) const
     536                 : {
     537                 :     struct post_handler final : scheduler_op
     538                 :     {
     539                 :         std::coroutine_handle<> h_;
     540                 : 
     541            3862 :         explicit post_handler(std::coroutine_handle<> h) : h_(h) {}
     542            7724 :         ~post_handler() override = default;
     543                 : 
     544            3790 :         void operator()() override
     545                 :         {
     546            3790 :             auto saved = h_;
     547            3790 :             delete this;
     548            3790 :             saved.resume();
     549            3774 :         }
     550                 : 
     551              72 :         void destroy() override
     552                 :         {
     553                 :             // A bare handle carries no owner to unwind from: it is
     554                 :             // destroyed directly.
     555              72 :             auto saved = h_;
     556              72 :             delete this;
     557              72 :             saved.destroy();
     558              72 :         }
     559                 :     };
     560                 : 
     561            3862 :     auto ph = std::make_unique<post_handler>(h);
     562                 : 
     563            3862 :     if (auto* ctx = reactor_find_context(this))
     564                 :     {
     565              34 :         ++ctx->private_outstanding_work;
     566              34 :         ctx->private_queue.push(ph.release());
     567              34 :         return;
     568                 :     }
     569                 : 
     570            3828 :     outstanding_work_.fetch_add(1, std::memory_order_relaxed);
     571                 : 
     572            3828 :     lock_type lock(mutex_);
     573            3828 :     completed_ops_.push(ph.release());
     574            3828 :     wake_one_thread_and_unlock(lock);
     575            3862 : }
     576                 : 
     577                 : inline void
     578          105540 : reactor_scheduler::post(scheduler_op* h) const
     579                 : {
     580          105540 :     if (auto* ctx = reactor_find_context(this))
     581                 :     {
     582          104005 :         ++ctx->private_outstanding_work;
     583          104005 :         ctx->private_queue.push(h);
     584          104005 :         return;
     585                 :     }
     586                 : 
     587            1535 :     outstanding_work_.fetch_add(1, std::memory_order_relaxed);
     588                 : 
     589            1535 :     lock_type lock(mutex_);
     590            1535 :     completed_ops_.push(h);
     591            1535 :     wake_one_thread_and_unlock(lock);
     592            1535 : }
     593                 : 
     594                 : inline void
     595          236933 : reactor_scheduler::post(capy::continuation& c) const
     596                 : {
     597          236933 :     if (auto* ctx = reactor_find_context(this))
     598                 :     {
     599          204199 :         ++ctx->private_outstanding_work;
     600          204199 :         ctx->private_queue.push(c);
     601          204199 :         return;
     602                 :     }
     603                 : 
     604           32734 :     outstanding_work_.fetch_add(1, std::memory_order_relaxed);
     605                 : 
     606           32734 :     lock_type lock(mutex_);
     607           32734 :     completed_ops_.push(c);
     608           32734 :     wake_one_thread_and_unlock(lock);
     609           32734 : }
     610                 : 
     611                 : inline bool
     612           24297 : reactor_scheduler::running_in_this_thread() const noexcept
     613                 : {
     614           24297 :     return reactor_find_context(this) != nullptr;
     615                 : }
     616                 : 
     617                 : inline void
     618            8972 : reactor_scheduler::stop()
     619                 : {
     620            8972 :     lock_type lock(mutex_);
     621            8972 :     if (!stopped_.load(std::memory_order_acquire))
     622                 :     {
     623            7906 :         stopped_.store(true, std::memory_order_release);
     624            7906 :         signal_all(lock);
     625            7906 :         interrupt_reactor();
     626                 :     }
     627            8972 : }
     628                 : 
     629                 : inline bool
     630            1296 : reactor_scheduler::stopped() const noexcept
     631                 : {
     632            1296 :     return stopped_.load(std::memory_order_acquire);
     633                 : }
     634                 : 
     635                 : inline void
     636            6127 : reactor_scheduler::restart()
     637                 : {
     638            6127 :     stopped_.store(false, std::memory_order_release);
     639            6127 : }
     640                 : 
     641                 : inline std::size_t
     642            6956 : reactor_scheduler::run()
     643                 : {
     644           13912 :     if (outstanding_work_.load(std::memory_order_acquire) == 0)
     645                 :     {
     646             267 :         stop();
     647             267 :         return 0;
     648                 :     }
     649                 : 
     650            6689 :     reactor_thread_context_guard ctx(this);
     651            6689 :     lock_type lock(mutex_);
     652            6689 :     adopt_enclosing_work(ctx.frame_);
     653                 : 
     654            6689 :     std::size_t n = 0;
     655                 :     for (;;)
     656                 :     {
     657          382775 :         if (!do_one(lock, -1, ctx.frame_))
     658            6684 :             break;
     659          376086 :         if (n != (std::numeric_limits<std::size_t>::max)())
     660          376086 :             ++n;
     661          376086 :         if (!lock.owns_lock())
     662           86075 :             lock.lock();
     663                 :     }
     664            6684 :     return n;
     665            6694 : }
     666                 : 
     667                 : inline std::size_t
     668             146 : reactor_scheduler::run_one()
     669                 : {
     670             292 :     if (outstanding_work_.load(std::memory_order_acquire) == 0)
     671                 :     {
     672               5 :         stop();
     673               5 :         return 0;
     674                 :     }
     675                 : 
     676             141 :     reactor_thread_context_guard ctx(this);
     677             141 :     lock_type lock(mutex_);
     678             141 :     adopt_enclosing_work(ctx.frame_);
     679             141 :     return do_one(lock, -1, ctx.frame_);
     680             143 : }
     681                 : 
     682                 : inline std::size_t
     683            3756 : reactor_scheduler::wait_one(long usec)
     684                 : {
     685            7512 :     if (outstanding_work_.load(std::memory_order_acquire) == 0)
     686                 :     {
     687             830 :         stop();
     688             830 :         return 0;
     689                 :     }
     690                 : 
     691            2926 :     reactor_thread_context_guard ctx(this);
     692            2926 :     lock_type lock(mutex_);
     693            2926 :     adopt_enclosing_work(ctx.frame_);
     694            2926 :     return do_one(lock, usec, ctx.frame_);
     695            2934 : }
     696                 : 
     697                 : inline std::size_t
     698            1479 : reactor_scheduler::poll()
     699                 : {
     700            2958 :     if (outstanding_work_.load(std::memory_order_acquire) == 0)
     701                 :     {
     702              31 :         stop();
     703              31 :         return 0;
     704                 :     }
     705                 : 
     706            1448 :     reactor_thread_context_guard ctx(this);
     707            1448 :     lock_type lock(mutex_);
     708            1448 :     adopt_enclosing_work(ctx.frame_);
     709                 : 
     710            1448 :     std::size_t n = 0;
     711                 :     for (;;)
     712                 :     {
     713            9628 :         if (!do_one(lock, 0, ctx.frame_))
     714            1446 :             break;
     715            8180 :         if (n != (std::numeric_limits<std::size_t>::max)())
     716            8180 :             ++n;
     717            8180 :         if (!lock.owns_lock())
     718            8176 :             lock.lock();
     719                 :     }
     720            1446 :     return n;
     721            1450 : }
     722                 : 
     723                 : inline std::size_t
     724              86 : reactor_scheduler::poll_one()
     725                 : {
     726             172 :     if (outstanding_work_.load(std::memory_order_acquire) == 0)
     727                 :     {
     728               7 :         stop();
     729               7 :         return 0;
     730                 :     }
     731                 : 
     732              79 :     reactor_thread_context_guard ctx(this);
     733              79 :     lock_type lock(mutex_);
     734              79 :     adopt_enclosing_work(ctx.frame_);
     735              79 :     return do_one(lock, 0, ctx.frame_);
     736              81 : }
     737                 : 
     738                 : inline void
     739           67300 : reactor_scheduler::work_started() noexcept
     740                 : {
     741           67300 :     outstanding_work_.fetch_add(1, std::memory_order_relaxed);
     742           67300 : }
     743                 : 
     744                 : inline void
     745          112125 : reactor_scheduler::work_finished() noexcept
     746                 : {
     747          224250 :     if (outstanding_work_.fetch_sub(1, std::memory_order_acq_rel) == 1)
     748            7735 :         stop();
     749          112125 : }
     750                 : 
     751                 : inline void
     752           32699 : reactor_scheduler::compensating_work_started() const noexcept
     753                 : {
     754           32699 :     auto* ctx = reactor_find_context(this);
     755           32699 :     if (ctx)
     756           32697 :         ++ctx->private_outstanding_work;
     757           32699 : }
     758                 : 
     759                 : inline void
     760            1996 : reactor_scheduler::post_counted(scheduler_op* op) const
     761                 : {
     762            1996 :     if (auto* ctx = reactor_find_context(this))
     763                 :     {
     764             607 :         ctx->private_queue.push(op);
     765             607 :         return;
     766                 :     }
     767                 : 
     768            1389 :     lock_type lock(mutex_);
     769            1389 :     completed_ops_.push(op);
     770            1389 :     wake_one_thread_and_unlock(lock);
     771            1389 : }
     772                 : 
     773                 : inline void
     774           16293 : reactor_scheduler::post_deferred_completions(ready_queue& ops) const
     775                 : {
     776           16293 :     if (ops.empty())
     777           16293 :         return;
     778                 : 
     779               2 :     if (auto* ctx = reactor_find_context(this))
     780                 :     {
     781               2 :         ctx->private_queue.splice(ops);
     782               2 :         return;
     783                 :     }
     784                 : 
     785 MIS           0 :     lock_type lock(mutex_);
     786               0 :     completed_ops_.splice(ops);
     787               0 :     wake_one_thread_and_unlock(lock);
     788               0 : }
     789                 : 
     790                 : inline void
     791 HIT       23178 : reactor_scheduler::retire_descriptor(
     792                 :     reactor_retirable& d, object_ref keep) const
     793                 : {
     794           23178 :     lock_type lock(mutex_);
     795           23178 :     if (!task_running_.load(std::memory_order_relaxed))
     796                 :     {
     797                 :         // No pass can hold an event, but an earlier one may have
     798                 :         // queued d.
     799           17820 :         d.hand_off(keep);
     800           17820 :         return;
     801                 :     }
     802                 :     // Already parked for this pass: the held reference suffices.
     803            5358 :     if (d.retired_ref_)
     804             126 :         return;
     805            5232 :     d.retired_ref_  = std::move(keep);
     806            5232 :     d.retired_next_ = retired_;
     807            5232 :     retired_        = &d;
     808           23178 : }
     809                 : 
     810                 : inline void
     811           11283 : reactor_scheduler::adopt_enclosing_work(context_type const& ctx)
     812                 : {
     813                 :     // A run nested in a handler would otherwise never see what that
     814                 :     // handler posted: it stays in the handler's frame until it returns.
     815           11289 :     for (auto* c = ctx.next; c != nullptr; c = c->next)
     816                 :     {
     817              12 :         if (c->key != this)
     818               6 :             continue;
     819               6 :         if (c->private_outstanding_work > 0)
     820                 :         {
     821               6 :             outstanding_work_.fetch_add(
     822                 :                 c->private_outstanding_work, std::memory_order_relaxed);
     823               6 :             c->private_outstanding_work = 0;
     824                 :         }
     825               6 :         completed_ops_.splice(c->private_queue);
     826               6 :         return;
     827                 :     }
     828                 : }
     829                 : 
     830                 : inline void
     831          292837 : reactor_scheduler::settle_retired(lock_type& lock) const
     832                 : {
     833                 :     // Handed off under the lock, since a later pass may retire the same
     834                 :     // descriptor again as soon as it is released. A reference that is
     835                 :     // not handed off may be the owner's last, and its release recycles
     836                 :     // through the pool mutex, so it is dropped with the lock released.
     837          292837 :     reactor_retirable* dropped = nullptr;
     838          298069 :     while (auto* d = retired_)
     839                 :     {
     840            5232 :         retired_         = d->retired_next_;
     841            5232 :         d->retired_next_ = nullptr;
     842            5232 :         auto keep        = std::move(d->retired_ref_);
     843            5232 :         d->hand_off(keep);
     844                 :         // A drop already pending keeps the owner alive, so this one
     845                 :         // cannot be the last and is released here.
     846            5232 :         if (!keep || d->dropped_ref_)
     847             376 :             continue;
     848            4856 :         d->dropped_ref_  = std::move(keep);
     849            4856 :         d->dropped_next_ = dropped;
     850            4856 :         dropped          = d;
     851           10464 :     }
     852          297693 :     while (auto* d = dropped)
     853                 :     {
     854            4856 :         dropped          = d->dropped_next_;
     855            4856 :         d->dropped_next_ = nullptr;
     856            4856 :         auto keep        = std::move(d->dropped_ref_);
     857            4856 :         lock.unlock();
     858            4856 :         keep.reset();
     859            4856 :         lock.lock();
     860            4856 :     }
     861          292837 : }
     862                 : 
     863                 : inline void
     864             407 : reactor_scheduler::on_timer_service_attached(timer_service& svc)
     865                 : {
     866             407 :     svc.set_on_earliest_changed(timer_service::callback(this, [](void* p) {
     867            3324 :         static_cast<reactor_scheduler*>(p)->interrupt_reactor();
     868            3324 :     }));
     869             407 : }
     870                 : 
     871                 : inline void
     872            4672 : reactor_scheduler::shutdown_drain()
     873                 : {
     874            4672 :     wait_cancels_posting();
     875                 : 
     876            4672 :     lock_type lock(mutex_);
     877            4672 :     settle_retired(lock);
     878            4672 :     destroy_queued(lock);
     879            4672 :     signal_all(lock);
     880            4672 : }
     881                 : 
     882            4688 : inline reactor_scheduler::~reactor_scheduler()
     883                 : {
     884                 :     // Services give up their work in shutdown(), but anything posted
     885                 :     // after the drain, such as from a destructor, is still destroyed.
     886            4688 :     lock_type lock(mutex_);
     887            4688 :     destroy_queued(lock);
     888            4688 : }
     889                 : 
     890                 : inline void
     891            9360 : reactor_scheduler::destroy_queued(lock_type& lock) noexcept
     892                 : {
     893           23485 :     while (auto e = completed_ops_.pop())
     894                 :     {
     895           14125 :         if (ready_is_continuation(e))
     896                 :         {
     897            7844 :             lock.unlock();
     898            7844 :             if (auto* c = ready_as_cont(e); c->h)
     899            7844 :                 capy::abandon(*c);
     900            7844 :             lock.lock();
     901                 :         }
     902                 :         else
     903                 :         {
     904            6281 :             auto* op = ready_as_op(e);
     905            6281 :             if (op == &task_op_)
     906            4669 :                 continue;
     907            1612 :             lock.unlock();
     908            1612 :             op->destroy();
     909            1612 :             lock.lock();
     910                 :         }
     911           14125 :     }
     912            9360 : }
     913                 : 
     914                 : inline void
     915           12578 : reactor_scheduler::signal_all(lock_type&) const
     916                 : {
     917           12578 :     state_ |= signaled_bit;
     918           12578 :     cond_.notify_all();
     919           12578 : }
     920                 : 
     921                 : inline bool
     922           40911 : reactor_scheduler::maybe_unlock_and_signal_one(lock_type& lock) const
     923                 : {
     924           40911 :     state_ |= signaled_bit;
     925           40911 :     if (state_ > signaled_bit)
     926                 :     {
     927                 :         // Signalled before unlocking: a post from another context's
     928                 :         // thread may be the last touch of this scheduler before the
     929                 :         // posted work runs, finishes and lets it be destroyed.
     930             204 :         cond_.notify_one();
     931             204 :         lock.unlock();
     932             204 :         return true;
     933                 :     }
     934           40707 :     return false;
     935                 : }
     936                 : 
     937                 : inline bool
     938          636179 : reactor_scheduler::unlock_and_signal_one(lock_type& lock) const
     939                 : {
     940          636179 :     state_ |= signaled_bit;
     941          636179 :     bool have_waiters = state_ > signaled_bit;
     942          636179 :     lock.unlock();
     943          636179 :     if (have_waiters)
     944            2868 :         cond_.notify_one();
     945          636179 :     return have_waiters;
     946                 : }
     947                 : 
     948                 : inline void
     949             939 : reactor_scheduler::clear_signal() const
     950                 : {
     951             939 :     state_ &= ~signaled_bit;
     952             939 : }
     953                 : 
     954                 : inline void
     955             907 : reactor_scheduler::wait_for_signal(lock_type& lock) const
     956                 : {
     957            1888 :     while ((state_ & signaled_bit) == 0)
     958                 :     {
     959             981 :         state_ += waiter_increment;
     960             981 :         cond_.wait(lock);
     961             981 :         state_ -= waiter_increment;
     962                 :     }
     963             907 : }
     964                 : 
     965                 : inline void
     966              32 : reactor_scheduler::wait_for_signal_for(lock_type& lock, long timeout_us) const
     967                 : {
     968              32 :     if ((state_ & signaled_bit) == 0)
     969                 :     {
     970              32 :         state_ += waiter_increment;
     971              32 :         cond_.wait_for(lock, std::chrono::microseconds(timeout_us));
     972              32 :         state_ -= waiter_increment;
     973                 :     }
     974              32 : }
     975                 : 
     976                 : inline void
     977           39486 : reactor_scheduler::wake_one_thread_and_unlock(lock_type& lock) const
     978                 : {
     979           39486 :     if (maybe_unlock_and_signal_one(lock))
     980             173 :         return;
     981                 : 
     982           39313 :     if (task_running_.load(std::memory_order_relaxed) && !task_interrupted_)
     983                 :     {
     984            1720 :         task_interrupted_ = true;
     985                 :         // Under the lock, for the same reason as the signal above.
     986            1720 :         interrupt_reactor();
     987            1720 :         lock.unlock();
     988                 :     }
     989                 :     else
     990                 :     {
     991           37593 :         lock.unlock();
     992                 :     }
     993                 : }
     994                 : 
     995          387184 : inline reactor_scheduler::work_cleanup::~work_cleanup()
     996                 : {
     997          387184 :     std::int64_t produced = ctx.private_outstanding_work;
     998          387184 :     if (produced > 1)
     999             929 :         sched->outstanding_work_.fetch_add(
    1000                 :             produced - 1, std::memory_order_relaxed);
    1001          386255 :     else if (produced < 1)
    1002           65418 :         sched->work_finished();
    1003          387184 :     ctx.private_outstanding_work = 0;
    1004                 : 
    1005          387184 :     if (!ctx.private_queue.empty())
    1006                 :     {
    1007          290383 :         lock->lock();
    1008          290383 :         sched->completed_ops_.splice(ctx.private_queue);
    1009                 :     }
    1010          387184 : }
    1011                 : 
    1012          288168 : inline reactor_scheduler::task_cleanup::~task_cleanup()
    1013                 : {
    1014          288168 :     if (ctx.private_outstanding_work > 0)
    1015                 :     {
    1016           10403 :         sched->outstanding_work_.fetch_add(
    1017           10403 :             ctx.private_outstanding_work, std::memory_order_relaxed);
    1018           10403 :         ctx.private_outstanding_work = 0;
    1019                 :     }
    1020                 : 
    1021          288168 :     if (!ctx.private_queue.empty())
    1022                 :     {
    1023           10403 :         if (!lock->owns_lock())
    1024 MIS           0 :             lock->lock();
    1025 HIT       10403 :         sched->completed_ops_.splice(ctx.private_queue);
    1026                 :     }
    1027          288168 : }
    1028                 : 
    1029                 : inline std::size_t
    1030          395549 : reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx)
    1031                 : {
    1032                 :     for (;;)
    1033                 :     {
    1034          683228 :         if (stopped_.load(std::memory_order_acquire))
    1035            6928 :             return 0;
    1036                 : 
    1037          676300 :         auto const e     = completed_ops_.pop();
    1038          676300 :         scheduler_op* op = ready_is_continuation(e) ? nullptr : ready_as_op(e);
    1039                 : 
    1040                 :         // Handle reactor sentinel — time to poll for I/O
    1041          676300 :         if (op == &task_op_)
    1042                 :         {
    1043          288168 :             bool more_handlers = !completed_ops_.empty();
    1044                 : 
    1045          325149 :             if (!more_handlers &&
    1046           73962 :                 outstanding_work_.load(std::memory_order_acquire) == 0)
    1047                 :             {
    1048 MIS           0 :                 completed_ops_.push(&task_op_);
    1049               0 :                 return 0;
    1050                 :             }
    1051                 : 
    1052 HIT      288168 :             long task_timeout_us = more_handlers ? 0 : timeout_us;
    1053          288168 :             task_interrupted_    = task_timeout_us == 0;
    1054          288168 :             task_running_.store(true, std::memory_order_release);
    1055                 : 
    1056                 :             // Wake a peer to take the pending handlers while this thread
    1057                 :             // polls the reactor; skipped when one_thread_ (no peer exists).
    1058          288168 :             if (more_handlers && !one_thread_)
    1059          251176 :                 unlock_and_signal_one(lock);
    1060                 : 
    1061                 :             try
    1062                 :             {
    1063          288168 :                 run_task(lock, ctx, task_timeout_us);
    1064                 :             }
    1065               3 :             catch (...)
    1066                 :             {
    1067               3 :                 task_running_.store(false, std::memory_order_relaxed);
    1068               3 :                 throw;
    1069               3 :             }
    1070                 : 
    1071          288165 :             task_running_.store(false, std::memory_order_relaxed);
    1072          288165 :             bool const idle = completed_ops_.empty();
    1073          288165 :             completed_ops_.push(&task_op_);
    1074          288165 :             settle_retired(lock);
    1075                 :             // A bounded call blocks at most once, then finishes as a poll
    1076                 :             // does, running what its pass queued.
    1077          288165 :             if (timeout_us > 0)
    1078            2712 :                 timeout_us = 0;
    1079                 :             // A poll ends once a pass leaves nothing queued; going round
    1080                 :             // again would only repeat the pass. A peer that found the
    1081                 :             // queue empty during the pass may be asleep, so it is woken
    1082                 :             // to take the sentinel.
    1083          288165 :             if (timeout_us == 0 && idle)
    1084                 :             {
    1085            1425 :                 maybe_unlock_and_signal_one(lock);
    1086            1425 :                 return 0;
    1087                 :             }
    1088          286740 :             continue;
    1089          286740 :         }
    1090                 : 
    1091                 :         // Handle ready entry (op or continuation)
    1092          388132 :         if (e != nullptr)
    1093                 :         {
    1094          387184 :             bool more = !completed_ops_.empty();
    1095                 : 
    1096          387184 :             if (more && !one_thread_)
    1097                 :             {
    1098                 :                 // Wake a peer for the remaining work; unassisted if none
    1099                 :                 // was parked to take it.
    1100          385003 :                 ctx.unassisted = !unlock_and_signal_one(lock);
    1101                 :             }
    1102                 :             else
    1103                 :             {
    1104                 :                 // No peer to wake (one_thread_, or nothing more queued).
    1105            2181 :                 ctx.unassisted = more;
    1106            2181 :                 lock.unlock();
    1107                 :             }
    1108                 : 
    1109          387184 :             [[maybe_unused]] work_cleanup on_exit{this, &lock, ctx};
    1110                 : 
    1111          387184 :             if (ready_is_continuation(e))
    1112          229089 :                 ready_as_cont(e)->h.resume();
    1113                 :             else
    1114          158095 :                 (*op)();
    1115          387168 :             return 1;
    1116          387184 :         }
    1117                 : 
    1118            1896 :         if (outstanding_work_.load(std::memory_order_acquire) == 0 ||
    1119                 :             timeout_us == 0)
    1120               9 :             return 0;
    1121                 : 
    1122             939 :         clear_signal();
    1123             939 :         if (timeout_us < 0)
    1124                 :         {
    1125             907 :             wait_for_signal(lock);
    1126                 :         }
    1127                 :         else
    1128                 :         {
    1129              32 :             wait_for_signal_for(lock, timeout_us);
    1130                 :             // A bounded call blocks once, so its caller can check its
    1131                 :             // deadline; another thread may hold the reactor for good.
    1132              32 :             timeout_us = 0;
    1133                 :         }
    1134          287679 :     }
    1135                 : }
    1136                 : 
    1137                 : } // namespace boost::corosio::detail
    1138                 : 
    1139                 : #endif // BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
        

Generated by: LCOV version 2.3