include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp

98.3% Lines (397 / 404, 2 excl) 100.0% Functions (52 / 52, 2 excl)
reactor_scheduler.hpp
f(x) Functions (54)
Function Calls Lines Blocks
boost::corosio::detail::reactor_retirable::~reactor_retirable() :68 12564x 100.0% 100.0% boost::corosio::detail::reactor_find_context(boost::corosio::detail::reactor_scheduler const*) :110 913711x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::notify_reparked() const :226 495x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::inline_budget_initial() const :276 11283x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::scheduler_locking_disabled() const :282 1460x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::configure_threading(boost::corosio::detail::scheduler::threading_config) :287 4672x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::reactor_scheduler() :303 4688x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::task_op::operator()() :368 – – – boost::corosio::detail::reactor_scheduler::task_op::destroy() :369 – – – boost::corosio::detail::reactor_thread_context_guard::reactor_thread_context_guard(boost::corosio::detail::reactor_scheduler const*) :426 11283x 100.0% 100.0% boost::corosio::detail::reactor_thread_context_guard::~reactor_thread_context_guard() :440 11283x 100.0% 100.0% boost::corosio::detail::reactor_scheduler_context::reactor_scheduler_context(boost::corosio::detail::reactor_scheduler const*, boost::corosio::detail::reactor_scheduler_context*) :448 11283x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::configure_reactor(unsigned int, unsigned int, unsigned int, unsigned int) :460 50x 100.0% 85.0% boost::corosio::detail::reactor_scheduler::reset_inline_budget() const :485 98780x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::try_consume_inline_budget() const :519 409704x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const :535 3862x 100.0% 84.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::post_handler(std::__n4861::coroutine_handle<void>) :541 3862x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::~post_handler() :542 7724x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::operator()() :544 3790x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::destroy() :551 72x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(boost::corosio::detail::scheduler_op*) const :578 105540x 100.0% 87.0% boost::corosio::detail::reactor_scheduler::post(boost::capy::continuation&) const :595 236933x 100.0% 87.0% boost::corosio::detail::reactor_scheduler::running_in_this_thread() const :612 24297x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::stop() :618 8972x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::stopped() const :630 1296x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::restart() :636 6127x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::run() :642 6956x 100.0% 86.0% boost::corosio::detail::reactor_scheduler::run_one() :668 146x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::wait_one(long) :683 3756x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::poll() :698 1479x 100.0% 86.0% boost::corosio::detail::reactor_scheduler::poll_one() :724 86x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::work_started() :739 67300x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::work_finished() :745 112125x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::compensating_work_started() const :752 32699x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post_counted(boost::corosio::detail::scheduler_op*) const :760 1996x 100.0% 85.0% boost::corosio::detail::reactor_scheduler::post_deferred_completions(boost::corosio::detail::ready_queue&) const :774 16293x 60.0% 59.0% boost::corosio::detail::reactor_scheduler::retire_descriptor(boost::corosio::detail::reactor_retirable&, boost::corosio::detail::object_ref) const :791 23178x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::adopt_enclosing_work(boost::corosio::detail::reactor_scheduler_context const&) :811 11283x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::settle_retired(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :831 292837x 100.0% 94.0% boost::corosio::detail::reactor_scheduler::on_timer_service_attached(boost::corosio::detail::timer_service&) :864 407x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::on_timer_service_attached(boost::corosio::detail::timer_service&)::{lambda(void*)#1}::operator()(void*) const :866 3324x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::shutdown_drain() :872 4672x 100.0% 80.0% boost::corosio::detail::reactor_scheduler::~reactor_scheduler() :882 4688x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::destroy_queued(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) :891 9360x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::signal_all(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :915 12578x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::maybe_unlock_and_signal_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :922 40911x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::unlock_and_signal_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :938 636179x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::clear_signal() const :949 939x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wait_for_signal(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :955 907x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wait_for_signal_for(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&, long) const :966 32x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wake_one_thread_and_unlock(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :977 39486x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::work_cleanup::~work_cleanup() :995 387184x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::task_cleanup::~task_cleanup() :1012 288168x 90.0% 91.0% boost::corosio::detail::reactor_scheduler::do_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&, long, boost::corosio::detail::reactor_scheduler_context&) :1030 395549x 96.2% 86.0%
Line TLA Hits Source Code
1 //
2 // Copyright (c) 2026 Steve Gerbino
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 //
7 // Official repository: https://github.com/cppalliance/corosio
8 //
9
10 #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
11 #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
12
13 #include <boost/corosio/detail/config.hpp>
14 #include <boost/corosio/detail/object_ref.hpp>
15 #include <boost/capy/ex/execution_context.hpp>
16 #include <boost/capy/ex/abandon.hpp>
17
18 #include <boost/corosio/detail/ready_queue.hpp>
19 #include <boost/corosio/detail/scheduler.hpp>
20 #include <boost/corosio/detail/scheduler_op.hpp>
21 #include <boost/corosio/detail/thread_local_ptr.hpp>
22 #include <boost/corosio/detail/timer_service.hpp>
23
24 #include <atomic>
25 #include <chrono>
26 #include <coroutine>
27 #include <cstddef>
28 #include <cstdint>
29 #include <limits>
30 #include <memory>
31
32 #include <boost/corosio/detail/conditionally_enabled_mutex.hpp>
33 #include <boost/corosio/detail/conditionally_enabled_event.hpp>
34
35 namespace boost::corosio::detail {
36
37 // Forward declarations
38 class reactor_scheduler;
39
40 /** A deregistered descriptor whose owner must outlive the reactor pass
41 that may still hold it.
42
43 An event the reactor has already read from the kernel can name a
44 descriptor that another thread closes before the reactor queues it.
45 The closing thread hands over a reference instead of dropping it, so
46 the owner is neither freed nor recycled until that pass ends.
47 */
48 struct reactor_retirable
49 {
50 // All four fields are guarded by the scheduler's mutex.
51
52 /// Next in the scheduler's retired list.
53 reactor_retirable* retired_next_ = nullptr;
54
55 /// Owner reference held until the pass ends.
56 object_ref retired_ref_;
57
58 /// Next in a settling thread's list of references to drop.
59 reactor_retirable* dropped_next_ = nullptr;
60
61 /// Owner reference waiting to be dropped outside the mutex.
62 object_ref dropped_ref_;
63
64 /// Move @p keep into a queued completion, if one is pending.
65 virtual void hand_off(object_ref& keep) noexcept = 0;
66
67 protected:
68 12564x ~reactor_retirable() = default;
69 };
70
71 /** Per-thread state for a reactor scheduler.
72
73 Each thread running a scheduler's event loop has one of these
74 on a thread-local stack. It holds a private work queue and
75 inline completion budget for speculative I/O fast paths.
76 */
77 struct BOOST_COROSIO_SYMBOL_VISIBLE reactor_scheduler_context
78 {
79 /// Scheduler this context belongs to.
80 reactor_scheduler const* key;
81
82 /// Next context frame on this thread's stack.
83 reactor_scheduler_context* next;
84
85 /// Private work queue for reduced contention.
86 ready_queue private_queue;
87
88 /// Unflushed work count for the private queue.
89 std::int64_t private_outstanding_work;
90
91 /// Remaining inline completions allowed this cycle.
92 int inline_budget;
93
94 /// Maximum inline budget (adaptive, 2-16).
95 int inline_budget_max;
96
97 /// True if no other thread absorbed queued work last cycle.
98 bool unassisted;
99
100 /// Construct a context frame linked to @a n.
101 reactor_scheduler_context(
102 reactor_scheduler const* k, reactor_scheduler_context* n);
103 };
104
105 /// Thread-local context stack for reactor schedulers.
106 inline thread_local_ptr<reactor_scheduler_context> reactor_context_stack;
107
108 /// Find the context frame for a scheduler on this thread.
109 inline reactor_scheduler_context*
110 913711x reactor_find_context(reactor_scheduler const* self) noexcept
111 {
112 914258x for (auto* c = reactor_context_stack.get(); c != nullptr; c = c->next)
113 {
114 850469x if (c->key == self)
115 849922x return c;
116 }
117 63789x return nullptr;
118 }
119
120 /** Non-template base for reactor-backed scheduler implementations.
121
122 Provides the complete threading model shared by epoll, kqueue,
123 and select schedulers: signal state machine, inline completion
124 budget, work counting, run/poll methods, and the do_one event
125 loop.
126
127 Derived classes provide platform-specific hooks by overriding:
128 - `run_task(lock, ctx)` to run the reactor poll
129 - `interrupt_reactor()` to wake a blocked reactor
130
131 De-templated from the original CRTP design to eliminate
132 duplicate instantiations when multiple backends are compiled
133 into the same binary. Virtual dispatch for run_task (called
134 once per reactor cycle, before a blocking syscall) has
135 negligible overhead.
136
137 @par Thread Safety
138 All public member functions are thread-safe.
139 */
140 class reactor_scheduler : public scheduler
141 {
142 public:
143 using context_type = reactor_scheduler_context;
144 using mutex_type = conditionally_enabled_mutex;
145 using lock_type = mutex_type::scoped_lock;
146 using event_type = conditionally_enabled_event;
147
148 /// Post a coroutine for deferred execution.
149 void post(std::coroutine_handle<> h) const override;
150
151 /// Post a scheduler operation for deferred execution.
152 void post(scheduler_op* h) const override;
153
154 /** Release an owner reference once no reactor pass can hold `d`.
155
156 Called after a descriptor is deregistered. If a reactor pass is
157 running, `keep` is parked on `d` until that pass ends. Then, or
158 at once if no pass is running, it moves to `d`'s queued
159 invocation if `d` is queued, and is dropped otherwise.
160
161 @param d The deregistered descriptor.
162 @param keep A reference to the descriptor's owner.
163 */
164 void retire_descriptor(reactor_retirable& d, object_ref keep) const;
165
166 /// Post a continuation for deferred execution.
167 void post(capy::continuation&) const override;
168
169 /// Return true if called from a thread running this scheduler.
170 bool running_in_this_thread() const noexcept override;
171
172 /// Request the scheduler to stop dispatching handlers.
173 void stop() override;
174
175 /// Return true if the scheduler has been stopped.
176 bool stopped() const noexcept override;
177
178 /// Reset the stopped state so `run()` can resume.
179 void restart() override;
180
181 /// Run the event loop until no work remains.
182 std::size_t run() override;
183
184 /// Run until one handler completes or no work remains.
185 std::size_t run_one() override;
186
187 /// Run until one handler completes or @a usec elapses.
188 std::size_t wait_one(long usec) override;
189
190 /// Run ready handlers without blocking.
191 std::size_t poll() override;
192
193 /// Run at most one ready handler without blocking.
194 std::size_t poll_one() override;
195
196 /// Increment the outstanding work count.
197 void work_started() noexcept override;
198
199 /// Decrement the outstanding work count, stopping on zero.
200 void work_finished() noexcept override;
201
202 /** Reset the thread's inline completion budget.
203
204 Called at the start of each posted completion handler to
205 grant a fresh budget for speculative inline completions.
206 */
207 void reset_inline_budget() const noexcept;
208
209 /** Consume one unit of inline budget if available.
210
211 @return True if budget was available and consumed.
212 */
213 bool try_consume_inline_budget() const noexcept;
214
215 /** Offset a forthcoming work_finished from work_cleanup.
216
217 Called by descriptor_state when all I/O returned EAGAIN and
218 no handler will be executed. Must be called from a scheduler
219 thread.
220 */
221 /** Note that an op was parked outside the usual registration path.
222
223 A reactor that builds its poll set from parked ops must rebuild
224 it; one the kernel tracks needs nothing.
225 */
226 495x virtual void notify_reparked() const {}
227
228 void compensating_work_started() const noexcept;
229
230 /** Post an operation whose work is already counted.
231
232 The posting thread touches nothing of the scheduler after this
233 returns, unlike a @ref post followed by @ref work_finished: once
234 posted, the operation can complete and let its context be
235 destroyed.
236
237 @pre work_started() was called for @p op.
238
239 @param op The operation to post.
240 */
241 void post_counted(scheduler_op* op) const;
242
243 /** Post completed operations for deferred invocation.
244
245 If called from a thread running this scheduler, operations
246 go to the thread's private queue (fast path). Otherwise,
247 operations are added to the global queue under mutex and a
248 waiter is signaled.
249
250 @pre work_started() must have been called for each operation.
251
252 @param ops Queue of operations to post.
253 */
254 void post_deferred_completions(ready_queue& ops) const;
255
256 /** Apply runtime configuration to the scheduler.
257
258 Called by `io_context` after construction. Values that do
259 not apply to this backend are silently ignored.
260
261 @pre `max_events` is in [ 1, `INT_MAX` ] and `budget_max` is at
262 most `INT_MAX`; `io_context` validates both.
263
264 @param max_events Event buffer size for epoll/kqueue.
265 @param budget_init Starting inline completion budget.
266 @param budget_max Hard ceiling on adaptive budget ramp-up.
267 @param unassisted Budget when single-threaded.
268 */
269 virtual void configure_reactor(
270 unsigned max_events,
271 unsigned budget_init,
272 unsigned budget_max,
273 unsigned unassisted);
274
275 /// Return the configured initial inline budget.
276 11283x unsigned inline_budget_initial() const noexcept
277 {
278 11283x return inline_budget_initial_;
279 }
280
281 /// Return true when scheduler locking is disabled (fully-lockless tier).
282 1460x bool scheduler_locking_disabled() const noexcept override
283 {
284 1460x return scheduler_locking_disabled_;
285 }
286
287 4672x void configure_threading(threading_config cfg) noexcept override
288 {
289 4672x scheduler_locking_disabled_ = !cfg.scheduler_locking;
290 // reactor_io_locking takes effect at descriptor registration (see the
291 // register_descriptor overrides), not here.
292 4672x reactor_io_locking_ = cfg.reactor_io_locking;
293 4672x one_thread_ = cfg.one_thread;
294 4672x mutex_.set_enabled(cfg.scheduler_locking);
295 4672x cond_.set_enabled(cfg.scheduler_locking);
296 4672x }
297
298 protected:
299 bool scheduler_locking_disabled_ = false;
300 bool reactor_io_locking_ = true;
301 bool one_thread_ = false;
302
303 4688x reactor_scheduler() = default;
304
305 /// Destroy, destroying whatever was posted after the drain.
306 ~reactor_scheduler() override;
307
308 /** Drain completed_ops during shutdown.
309
310 Pops all operations from the global queue and destroys them,
311 skipping the task sentinel. Signals all waiting threads.
312 Derived classes call this from their shutdown() override
313 before performing platform-specific cleanup.
314 */
315 void shutdown_drain();
316
317 /// Destroy every queued operation without running it.
318 void destroy_queued(lock_type& lock) noexcept;
319
320 /// RAII guard that re-inserts the task sentinel after `run_task`.
321 struct task_cleanup
322 {
323 reactor_scheduler const* sched;
324 lock_type* lock;
325 context_type& ctx;
326 ~task_cleanup();
327 };
328
329 mutable mutex_type mutex_{true};
330 mutable event_type cond_{true};
331 mutable ready_queue completed_ops_;
332 mutable std::atomic<std::int64_t> outstanding_work_{0};
333 std::atomic<bool> stopped_{false};
334 mutable std::atomic<bool> task_running_{false};
335 mutable bool task_interrupted_ = false;
336
337 // Descriptors retired during the running reactor pass (guarded by
338 // mutex_).
339 mutable reactor_retirable* retired_ = nullptr;
340
341 /// Settle every descriptor retired before the current pass ended.
342 /// @pre `lock` holds `mutex_`; it is held again on return.
343 void settle_retired(lock_type& lock) const;
344
345 /// Move what an enclosing handler on this thread queued privately
346 /// to the shared queue. @pre `mutex_` is held.
347 void adopt_enclosing_work(context_type const& ctx);
348
349 // Runtime-configurable reactor tuning parameters.
350 // Defaults match the library's built-in values.
351 unsigned max_events_per_poll_ = 128;
352 unsigned inline_budget_initial_ = 2;
353 unsigned inline_budget_max_ = 16;
354 unsigned unassisted_budget_ = 4;
355
356 /// Bit 0 of `state_`: set when the condvar should be signaled.
357 static constexpr std::size_t signaled_bit = 1;
358
359 /// Increment per waiting thread in `state_`.
360 static constexpr std::size_t waiter_increment = 2;
361 mutable std::size_t state_ = 0;
362
363 /// Sentinel op that triggers a reactor poll when dequeued.
364 struct task_op final : scheduler_op
365 {
366 // LCOV_EXCL_START: the sentinel is intercepted by pointer
367 // identity; its virtuals exist for vtable completeness.
368 − void operator()() override {}
369 − void destroy() override {}
370 // LCOV_EXCL_STOP
371 };
372 task_op task_op_;
373
374 /** Run the platform-specific reactor poll.
375
376 @par Postconditions
377 `lock` is owned on return, however the poll ended. An
378 implementation that unlocks around the blocking call owes the
379 caller a matching re-acquire on every path out, including the
380 errors it retries rather than reports.
381 */
382 virtual void
383 run_task(lock_type& lock, context_type& ctx, long timeout_us) = 0;
384
385 /// Wake a blocked reactor (e.g. write to eventfd or pipe).
386 virtual void interrupt_reactor() const = 0;
387
388 /// Wake the reactor whenever the earliest timer changes.
389 void on_timer_service_attached(timer_service& svc) override;
390
391 private:
392 struct work_cleanup
393 {
394 reactor_scheduler* sched;
395 lock_type* lock;
396 context_type& ctx;
397 ~work_cleanup();
398 };
399
400 std::size_t do_one(lock_type& lock, long timeout_us, context_type& ctx);
401
402 void signal_all(lock_type& lock) const;
403 bool maybe_unlock_and_signal_one(lock_type& lock) const;
404 bool unlock_and_signal_one(lock_type& lock) const;
405 void clear_signal() const;
406 void wait_for_signal(lock_type& lock) const;
407 void wait_for_signal_for(lock_type& lock, long timeout_us) const;
408 void wake_one_thread_and_unlock(lock_type& lock) const;
409 };
410
411 /** RAII guard that pushes/pops a scheduler context frame.
412
413 On construction, pushes a new context frame onto the
414 thread-local stack. On destruction, drains any remaining
415 private queue items to the global queue and pops the frame.
416 */
417 struct reactor_thread_context_guard
418 {
419 /// The context frame managed by this guard.
420 reactor_scheduler_context frame_;
421
422 /// Marks the scheduler innermost for the executor's fast check.
423 running_scheduler_guard running_;
424
425 /// Construct the guard, pushing a frame for @a sched.
426 11283x explicit reactor_thread_context_guard(
427 reactor_scheduler const* sched) noexcept
428 11283x : frame_(sched, reactor_context_stack.get())
429 11283x , running_(sched)
430 {
431 11283x reactor_context_stack.set(&frame_);
432 11283x }
433
434 /** Destroy the guard, popping the frame.
435
436 The private queue is empty here by invariant: work_cleanup and
437 task_cleanup splice it to the global queue after every handler
438 and every reactor pass.
439 */
440 11283x ~reactor_thread_context_guard() noexcept
441 {
442 11283x reactor_context_stack.set(frame_.next);
443 11283x }
444 };
445
446 // ---- Inline implementations ------------------------------------------------
447
448 11283x inline reactor_scheduler_context::reactor_scheduler_context(
449 11283x reactor_scheduler const* k, reactor_scheduler_context* n)
450 11283x : key(k)
451 11283x , next(n)
452 11283x , private_outstanding_work(0)
453 11283x , inline_budget(0)
454 11283x , inline_budget_max(static_cast<int>(k->inline_budget_initial()))
455 11283x , unassisted(false)
456 {
457 11283x }
458
459 inline void
460 50x reactor_scheduler::configure_reactor(
461 unsigned max_events,
462 unsigned budget_init,
463 unsigned budget_max,
464 unsigned unassisted)
465 {
466 50x BOOST_COROSIO_ASSERT(
467 max_events >= 1 &&
468 max_events <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
469 50x BOOST_COROSIO_ASSERT(
470 budget_max <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
471
472 // Clamp initial and unassisted to budget_max.
473 50x if (budget_init > budget_max)
474 20x budget_init = budget_max;
475 50x if (unassisted > budget_max)
476 20x unassisted = budget_max;
477
478 50x max_events_per_poll_ = max_events;
479 50x inline_budget_initial_ = budget_init;
480 50x inline_budget_max_ = budget_max;
481 50x unassisted_budget_ = unassisted;
482 50x }
483
484 inline void
485 98780x reactor_scheduler::reset_inline_budget() const noexcept
486 {
487 // When budget is disabled (max==0), all paths below would no-op
488 // (inline_budget stays 0). Skip the TLS lookup entirely.
489 98780x if (inline_budget_max_ == 0)
490 58x return;
491 98722x if (auto* ctx = reactor_find_context(this))
492 {
493 // Cap when no other thread absorbed queued work
494 98722x if (ctx->unassisted)
495 {
496 97927x ctx->inline_budget_max = static_cast<int>(unassisted_budget_);
497 97927x ctx->inline_budget = static_cast<int>(unassisted_budget_);
498 97927x return;
499 }
500 // Ramp up when previous cycle fully consumed budget.
501 // max(1, ...) ensures the doubling escapes zero.
502 // Doubled in a wider type: the cap may be as large as INT_MAX.
503 795x if (ctx->inline_budget == 0)
504 {
505 long long const doubled =
506 263x 2LL * (std::max)(1, ctx->inline_budget_max);
507 263x long long const cap = inline_budget_max_;
508 263x ctx->inline_budget_max = static_cast<int>((std::min)(doubled, cap));
509 }
510 532x else if (ctx->inline_budget < ctx->inline_budget_max)
511 {
512 174x ctx->inline_budget_max = static_cast<int>(inline_budget_initial_);
513 }
514 795x ctx->inline_budget = ctx->inline_budget_max;
515 }
516 }
517
518 inline bool
519 409704x reactor_scheduler::try_consume_inline_budget() const noexcept
520 {
521 409704x if (inline_budget_max_ == 0)
522 44x return false;
523 409660x if (auto* ctx = reactor_find_context(this))
524 {
525 409656x if (ctx->inline_budget > 0)
526 {
527 327464x --ctx->inline_budget;
528 327464x return true;
529 }
530 }
531 82196x return false;
532 }
533
534 inline void
535 3862x reactor_scheduler::post(std::coroutine_handle<> h) const
536 {
537 struct post_handler final : scheduler_op
538 {
539 std::coroutine_handle<> h_;
540
541 3862x explicit post_handler(std::coroutine_handle<> h) : h_(h) {}
542 7724x ~post_handler() override = default;
543
544 3790x void operator()() override
545 {
546 3790x auto saved = h_;
547 3790x delete this;
548 3790x saved.resume();
549 3774x }
550
551 72x void destroy() override
552 {
553 // A bare handle carries no owner to unwind from: it is
554 // destroyed directly.
555 72x auto saved = h_;
556 72x delete this;
557 72x saved.destroy();
558 72x }
559 };
560
561 3862x auto ph = std::make_unique<post_handler>(h);
562
563 3862x if (auto* ctx = reactor_find_context(this))
564 {
565 34x ++ctx->private_outstanding_work;
566 34x ctx->private_queue.push(ph.release());
567 34x return;
568 }
569
570 3828x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
571
572 3828x lock_type lock(mutex_);
573 3828x completed_ops_.push(ph.release());
574 3828x wake_one_thread_and_unlock(lock);
575 3862x }
576
577 inline void
578 105540x reactor_scheduler::post(scheduler_op* h) const
579 {
580 105540x if (auto* ctx = reactor_find_context(this))
581 {
582 104005x ++ctx->private_outstanding_work;
583 104005x ctx->private_queue.push(h);
584 104005x return;
585 }
586
587 1535x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
588
589 1535x lock_type lock(mutex_);
590 1535x completed_ops_.push(h);
591 1535x wake_one_thread_and_unlock(lock);
592 1535x }
593
594 inline void
595 236933x reactor_scheduler::post(capy::continuation& c) const
596 {
597 236933x if (auto* ctx = reactor_find_context(this))
598 {
599 204199x ++ctx->private_outstanding_work;
600 204199x ctx->private_queue.push(c);
601 204199x return;
602 }
603
604 32734x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
605
606 32734x lock_type lock(mutex_);
607 32734x completed_ops_.push(c);
608 32734x wake_one_thread_and_unlock(lock);
609 32734x }
610
611 inline bool
612 24297x reactor_scheduler::running_in_this_thread() const noexcept
613 {
614 24297x return reactor_find_context(this) != nullptr;
615 }
616
617 inline void
618 8972x reactor_scheduler::stop()
619 {
620 8972x lock_type lock(mutex_);
621 8972x if (!stopped_.load(std::memory_order_acquire))
622 {
623 7906x stopped_.store(true, std::memory_order_release);
624 7906x signal_all(lock);
625 7906x interrupt_reactor();
626 }
627 8972x }
628
629 inline bool
630 1296x reactor_scheduler::stopped() const noexcept
631 {
632 1296x return stopped_.load(std::memory_order_acquire);
633 }
634
635 inline void
636 6127x reactor_scheduler::restart()
637 {
638 6127x stopped_.store(false, std::memory_order_release);
639 6127x }
640
641 inline std::size_t
642 6956x reactor_scheduler::run()
643 {
644 13912x if (outstanding_work_.load(std::memory_order_acquire) == 0)
645 {
646 267x stop();
647 267x return 0;
648 }
649
650 6689x reactor_thread_context_guard ctx(this);
651 6689x lock_type lock(mutex_);
652 6689x adopt_enclosing_work(ctx.frame_);
653
654 6689x std::size_t n = 0;
655 for (;;)
656 {
657 382775x if (!do_one(lock, -1, ctx.frame_))
658 6684x break;
659 376086x if (n != (std::numeric_limits<std::size_t>::max)())
660 376086x ++n;
661 376086x if (!lock.owns_lock())
662 86075x lock.lock();
663 }
664 6684x return n;
665 6694x }
666
667 inline std::size_t
668 146x reactor_scheduler::run_one()
669 {
670 292x if (outstanding_work_.load(std::memory_order_acquire) == 0)
671 {
672 5x stop();
673 5x return 0;
674 }
675
676 141x reactor_thread_context_guard ctx(this);
677 141x lock_type lock(mutex_);
678 141x adopt_enclosing_work(ctx.frame_);
679 141x return do_one(lock, -1, ctx.frame_);
680 143x }
681
682 inline std::size_t
683 3756x reactor_scheduler::wait_one(long usec)
684 {
685 7512x if (outstanding_work_.load(std::memory_order_acquire) == 0)
686 {
687 830x stop();
688 830x return 0;
689 }
690
691 2926x reactor_thread_context_guard ctx(this);
692 2926x lock_type lock(mutex_);
693 2926x adopt_enclosing_work(ctx.frame_);
694 2926x return do_one(lock, usec, ctx.frame_);
695 2934x }
696
697 inline std::size_t
698 1479x reactor_scheduler::poll()
699 {
700 2958x if (outstanding_work_.load(std::memory_order_acquire) == 0)
701 {
702 31x stop();
703 31x return 0;
704 }
705
706 1448x reactor_thread_context_guard ctx(this);
707 1448x lock_type lock(mutex_);
708 1448x adopt_enclosing_work(ctx.frame_);
709
710 1448x std::size_t n = 0;
711 for (;;)
712 {
713 9628x if (!do_one(lock, 0, ctx.frame_))
714 1446x break;
715 8180x if (n != (std::numeric_limits<std::size_t>::max)())
716 8180x ++n;
717 8180x if (!lock.owns_lock())
718 8176x lock.lock();
719 }
720 1446x return n;
721 1450x }
722
723 inline std::size_t
724 86x reactor_scheduler::poll_one()
725 {
726 172x if (outstanding_work_.load(std::memory_order_acquire) == 0)
727 {
728 7x stop();
729 7x return 0;
730 }
731
732 79x reactor_thread_context_guard ctx(this);
733 79x lock_type lock(mutex_);
734 79x adopt_enclosing_work(ctx.frame_);
735 79x return do_one(lock, 0, ctx.frame_);
736 81x }
737
738 inline void
739 67300x reactor_scheduler::work_started() noexcept
740 {
741 67300x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
742 67300x }
743
744 inline void
745 112125x reactor_scheduler::work_finished() noexcept
746 {
747 224250x if (outstanding_work_.fetch_sub(1, std::memory_order_acq_rel) == 1)
748 7735x stop();
749 112125x }
750
751 inline void
752 32699x reactor_scheduler::compensating_work_started() const noexcept
753 {
754 32699x auto* ctx = reactor_find_context(this);
755 32699x if (ctx)
756 32697x ++ctx->private_outstanding_work;
757 32699x }
758
759 inline void
760 1996x reactor_scheduler::post_counted(scheduler_op* op) const
761 {
762 1996x if (auto* ctx = reactor_find_context(this))
763 {
764 607x ctx->private_queue.push(op);
765 607x return;
766 }
767
768 1389x lock_type lock(mutex_);
769 1389x completed_ops_.push(op);
770 1389x wake_one_thread_and_unlock(lock);
771 1389x }
772
773 inline void
774 16293x reactor_scheduler::post_deferred_completions(ready_queue& ops) const
775 {
776 16293x if (ops.empty())
777 16293x return;
778
779 2x if (auto* ctx = reactor_find_context(this))
780 {
781 2x ctx->private_queue.splice(ops);
782 2x return;
783 }
784
785 ✗ lock_type lock(mutex_);
786 ✗ completed_ops_.splice(ops);
787 ✗ wake_one_thread_and_unlock(lock);
788 ✗ }
789
790 inline void
791 23178x reactor_scheduler::retire_descriptor(
792 reactor_retirable& d, object_ref keep) const
793 {
794 23178x lock_type lock(mutex_);
795 23178x if (!task_running_.load(std::memory_order_relaxed))
796 {
797 // No pass can hold an event, but an earlier one may have
798 // queued d.
799 17820x d.hand_off(keep);
800 17820x return;
801 }
802 // Already parked for this pass: the held reference suffices.
803 5358x if (d.retired_ref_)
804 126x return;
805 5232x d.retired_ref_ = std::move(keep);
806 5232x d.retired_next_ = retired_;
807 5232x retired_ = &d;
808 23178x }
809
810 inline void
811 11283x reactor_scheduler::adopt_enclosing_work(context_type const& ctx)
812 {
813 // A run nested in a handler would otherwise never see what that
814 // handler posted: it stays in the handler's frame until it returns.
815 11289x for (auto* c = ctx.next; c != nullptr; c = c->next)
816 {
817 12x if (c->key != this)
818 6x continue;
819 6x if (c->private_outstanding_work > 0)
820 {
821 6x outstanding_work_.fetch_add(
822 c->private_outstanding_work, std::memory_order_relaxed);
823 6x c->private_outstanding_work = 0;
824 }
825 6x completed_ops_.splice(c->private_queue);
826 6x return;
827 }
828 }
829
830 inline void
831 292837x reactor_scheduler::settle_retired(lock_type& lock) const
832 {
833 // Handed off under the lock, since a later pass may retire the same
834 // descriptor again as soon as it is released. A reference that is
835 // not handed off may be the owner's last, and its release recycles
836 // through the pool mutex, so it is dropped with the lock released.
837 292837x reactor_retirable* dropped = nullptr;
838 298069x while (auto* d = retired_)
839 {
840 5232x retired_ = d->retired_next_;
841 5232x d->retired_next_ = nullptr;
842 5232x auto keep = std::move(d->retired_ref_);
843 5232x d->hand_off(keep);
844 // A drop already pending keeps the owner alive, so this one
845 // cannot be the last and is released here.
846 5232x if (!keep || d->dropped_ref_)
847 376x continue;
848 4856x d->dropped_ref_ = std::move(keep);
849 4856x d->dropped_next_ = dropped;
850 4856x dropped = d;
851 10464x }
852 297693x while (auto* d = dropped)
853 {
854 4856x dropped = d->dropped_next_;
855 4856x d->dropped_next_ = nullptr;
856 4856x auto keep = std::move(d->dropped_ref_);
857 4856x lock.unlock();
858 4856x keep.reset();
859 4856x lock.lock();
860 4856x }
861 292837x }
862
863 inline void
864 407x reactor_scheduler::on_timer_service_attached(timer_service& svc)
865 {
866 407x svc.set_on_earliest_changed(timer_service::callback(this, [](void* p) {
867 3324x static_cast<reactor_scheduler*>(p)->interrupt_reactor();
868 3324x }));
869 407x }
870
871 inline void
872 4672x reactor_scheduler::shutdown_drain()
873 {
874 4672x wait_cancels_posting();
875
876 4672x lock_type lock(mutex_);
877 4672x settle_retired(lock);
878 4672x destroy_queued(lock);
879 4672x signal_all(lock);
880 4672x }
881
882 4688x inline reactor_scheduler::~reactor_scheduler()
883 {
884 // Services give up their work in shutdown(), but anything posted
885 // after the drain, such as from a destructor, is still destroyed.
886 4688x lock_type lock(mutex_);
887 4688x destroy_queued(lock);
888 4688x }
889
890 inline void
891 9360x reactor_scheduler::destroy_queued(lock_type& lock) noexcept
892 {
893 23485x while (auto e = completed_ops_.pop())
894 {
895 14125x if (ready_is_continuation(e))
896 {
897 7844x lock.unlock();
898 7844x if (auto* c = ready_as_cont(e); c->h)
899 7844x capy::abandon(*c);
900 7844x lock.lock();
901 }
902 else
903 {
904 6281x auto* op = ready_as_op(e);
905 6281x if (op == &task_op_)
906 4669x continue;
907 1612x lock.unlock();
908 1612x op->destroy();
909 1612x lock.lock();
910 }
911 14125x }
912 9360x }
913
914 inline void
915 12578x reactor_scheduler::signal_all(lock_type&) const
916 {
917 12578x state_ |= signaled_bit;
918 12578x cond_.notify_all();
919 12578x }
920
921 inline bool
922 40911x reactor_scheduler::maybe_unlock_and_signal_one(lock_type& lock) const
923 {
924 40911x state_ |= signaled_bit;
925 40911x if (state_ > signaled_bit)
926 {
927 // Signalled before unlocking: a post from another context's
928 // thread may be the last touch of this scheduler before the
929 // posted work runs, finishes and lets it be destroyed.
930 204x cond_.notify_one();
931 204x lock.unlock();
932 204x return true;
933 }
934 40707x return false;
935 }
936
937 inline bool
938 636179x reactor_scheduler::unlock_and_signal_one(lock_type& lock) const
939 {
940 636179x state_ |= signaled_bit;
941 636179x bool have_waiters = state_ > signaled_bit;
942 636179x lock.unlock();
943 636179x if (have_waiters)
944 2868x cond_.notify_one();
945 636179x return have_waiters;
946 }
947
948 inline void
949 939x reactor_scheduler::clear_signal() const
950 {
951 939x state_ &= ~signaled_bit;
952 939x }
953
954 inline void
955 907x reactor_scheduler::wait_for_signal(lock_type& lock) const
956 {
957 1888x while ((state_ & signaled_bit) == 0)
958 {
959 981x state_ += waiter_increment;
960 981x cond_.wait(lock);
961 981x state_ -= waiter_increment;
962 }
963 907x }
964
965 inline void
966 32x reactor_scheduler::wait_for_signal_for(lock_type& lock, long timeout_us) const
967 {
968 32x if ((state_ & signaled_bit) == 0)
969 {
970 32x state_ += waiter_increment;
971 32x cond_.wait_for(lock, std::chrono::microseconds(timeout_us));
972 32x state_ -= waiter_increment;
973 }
974 32x }
975
976 inline void
977 39486x reactor_scheduler::wake_one_thread_and_unlock(lock_type& lock) const
978 {
979 39486x if (maybe_unlock_and_signal_one(lock))
980 173x return;
981
982 39313x if (task_running_.load(std::memory_order_relaxed) && !task_interrupted_)
983 {
984 1720x task_interrupted_ = true;
985 // Under the lock, for the same reason as the signal above.
986 1720x interrupt_reactor();
987 1720x lock.unlock();
988 }
989 else
990 {
991 37593x lock.unlock();
992 }
993 }
994
995 387184x inline reactor_scheduler::work_cleanup::~work_cleanup()
996 {
997 387184x std::int64_t produced = ctx.private_outstanding_work;
998 387184x if (produced > 1)
999 929x sched->outstanding_work_.fetch_add(
1000 produced - 1, std::memory_order_relaxed);
1001 386255x else if (produced < 1)
1002 65418x sched->work_finished();
1003 387184x ctx.private_outstanding_work = 0;
1004
1005 387184x if (!ctx.private_queue.empty())
1006 {
1007 290383x lock->lock();
1008 290383x sched->completed_ops_.splice(ctx.private_queue);
1009 }
1010 387184x }
1011
1012 288168x inline reactor_scheduler::task_cleanup::~task_cleanup()
1013 {
1014 288168x if (ctx.private_outstanding_work > 0)
1015 {
1016 10403x sched->outstanding_work_.fetch_add(
1017 10403x ctx.private_outstanding_work, std::memory_order_relaxed);
1018 10403x ctx.private_outstanding_work = 0;
1019 }
1020
1021 288168x if (!ctx.private_queue.empty())
1022 {
1023 10403x if (!lock->owns_lock())
1024 ✗ lock->lock();
1025 10403x sched->completed_ops_.splice(ctx.private_queue);
1026 }
1027 288168x }
1028
1029 inline std::size_t
1030 395549x reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx)
1031 {
1032 for (;;)
1033 {
1034 683228x if (stopped_.load(std::memory_order_acquire))
1035 6928x return 0;
1036
1037 676300x auto const e = completed_ops_.pop();
1038 676300x scheduler_op* op = ready_is_continuation(e) ? nullptr : ready_as_op(e);
1039
1040 // Handle reactor sentinel — time to poll for I/O
1041 676300x if (op == &task_op_)
1042 {
1043 288168x bool more_handlers = !completed_ops_.empty();
1044
1045 325149x if (!more_handlers &&
1046 73962x outstanding_work_.load(std::memory_order_acquire) == 0)
1047 {
1048 ✗ completed_ops_.push(&task_op_);
1049 ✗ return 0;
1050 }
1051
1052 288168x long task_timeout_us = more_handlers ? 0 : timeout_us;
1053 288168x task_interrupted_ = task_timeout_us == 0;
1054 288168x task_running_.store(true, std::memory_order_release);
1055
1056 // Wake a peer to take the pending handlers while this thread
1057 // polls the reactor; skipped when one_thread_ (no peer exists).
1058 288168x if (more_handlers && !one_thread_)
1059 251176x unlock_and_signal_one(lock);
1060
1061 try
1062 {
1063 288168x run_task(lock, ctx, task_timeout_us);
1064 }
1065 3x catch (...)
1066 {
1067 3x task_running_.store(false, std::memory_order_relaxed);
1068 3x throw;
1069 3x }
1070
1071 288165x task_running_.store(false, std::memory_order_relaxed);
1072 288165x bool const idle = completed_ops_.empty();
1073 288165x completed_ops_.push(&task_op_);
1074 288165x settle_retired(lock);
1075 // A bounded call blocks at most once, then finishes as a poll
1076 // does, running what its pass queued.
1077 288165x if (timeout_us > 0)
1078 2712x timeout_us = 0;
1079 // A poll ends once a pass leaves nothing queued; going round
1080 // again would only repeat the pass. A peer that found the
1081 // queue empty during the pass may be asleep, so it is woken
1082 // to take the sentinel.
1083 288165x if (timeout_us == 0 && idle)
1084 {
1085 1425x maybe_unlock_and_signal_one(lock);
1086 1425x return 0;
1087 }
1088 286740x continue;
1089 286740x }
1090
1091 // Handle ready entry (op or continuation)
1092 388132x if (e != nullptr)
1093 {
1094 387184x bool more = !completed_ops_.empty();
1095
1096 387184x if (more && !one_thread_)
1097 {
1098 // Wake a peer for the remaining work; unassisted if none
1099 // was parked to take it.
1100 385003x ctx.unassisted = !unlock_and_signal_one(lock);
1101 }
1102 else
1103 {
1104 // No peer to wake (one_thread_, or nothing more queued).
1105 2181x ctx.unassisted = more;
1106 2181x lock.unlock();
1107 }
1108
1109 387184x [[maybe_unused]] work_cleanup on_exit{this, &lock, ctx};
1110
1111 387184x if (ready_is_continuation(e))
1112 229089x ready_as_cont(e)->h.resume();
1113 else
1114 158095x (*op)();
1115 387168x return 1;
1116 387184x }
1117
1118 1896x if (outstanding_work_.load(std::memory_order_acquire) == 0 ||
1119 timeout_us == 0)
1120 9x return 0;
1121
1122 939x clear_signal();
1123 939x if (timeout_us < 0)
1124 {
1125 907x wait_for_signal(lock);
1126 }
1127 else
1128 {
1129 32x wait_for_signal_for(lock, timeout_us);
1130 // A bounded call blocks once, so its caller can check its
1131 // deadline; another thread may hold the reactor for good.
1132 32x timeout_us = 0;
1133 }
1134 287679x }
1135 }
1136
1137 } // namespace boost::corosio::detail
1138
1139 #endif // BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
1140