TLA Line data Source code
1 : //
2 : // Copyright (c) 2026 Steve Gerbino
3 : //
4 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 : //
7 : // Official repository: https://github.com/cppalliance/corosio
8 : //
9 :
10 : #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
11 : #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
12 :
13 : #include <boost/corosio/detail/config.hpp>
14 : #include <boost/corosio/detail/object_ref.hpp>
15 : #include <boost/capy/ex/execution_context.hpp>
16 : #include <boost/capy/ex/abandon.hpp>
17 :
18 : #include <boost/corosio/detail/ready_queue.hpp>
19 : #include <boost/corosio/detail/scheduler.hpp>
20 : #include <boost/corosio/detail/scheduler_op.hpp>
21 : #include <boost/corosio/detail/thread_local_ptr.hpp>
22 : #include <boost/corosio/detail/timer_service.hpp>
23 :
24 : #include <atomic>
25 : #include <chrono>
26 : #include <coroutine>
27 : #include <cstddef>
28 : #include <cstdint>
29 : #include <limits>
30 : #include <memory>
31 :
32 : #include <boost/corosio/detail/conditionally_enabled_mutex.hpp>
33 : #include <boost/corosio/detail/conditionally_enabled_event.hpp>
34 :
35 : namespace boost::corosio::detail {
36 :
37 : // Forward declarations
38 : class reactor_scheduler;
39 :
40 : /** A deregistered descriptor whose owner must outlive the reactor pass
41 : that may still hold it.
42 :
43 : An event the reactor has already read from the kernel can name a
44 : descriptor that another thread closes before the reactor queues it.
45 : The closing thread hands over a reference instead of dropping it, so
46 : the owner is neither freed nor recycled until that pass ends.
47 : */
48 : struct reactor_retirable
49 : {
50 : // All four fields are guarded by the scheduler's mutex.
51 :
52 : /// Next in the scheduler's retired list.
53 : reactor_retirable* retired_next_ = nullptr;
54 :
55 : /// Owner reference held until the pass ends.
56 : object_ref retired_ref_;
57 :
58 : /// Next in a settling thread's list of references to drop.
59 : reactor_retirable* dropped_next_ = nullptr;
60 :
61 : /// Owner reference waiting to be dropped outside the mutex.
62 : object_ref dropped_ref_;
63 :
64 : /// Move @p keep into a queued completion, if one is pending.
65 : virtual void hand_off(object_ref& keep) noexcept = 0;
66 :
67 : protected:
68 HIT 12564 : ~reactor_retirable() = default;
69 : };
70 :
71 : /** Per-thread state for a reactor scheduler.
72 :
73 : Each thread running a scheduler's event loop has one of these
74 : on a thread-local stack. It holds a private work queue and
75 : inline completion budget for speculative I/O fast paths.
76 : */
77 : struct BOOST_COROSIO_SYMBOL_VISIBLE reactor_scheduler_context
78 : {
79 : /// Scheduler this context belongs to.
80 : reactor_scheduler const* key;
81 :
82 : /// Next context frame on this thread's stack.
83 : reactor_scheduler_context* next;
84 :
85 : /// Private work queue for reduced contention.
86 : ready_queue private_queue;
87 :
88 : /// Unflushed work count for the private queue.
89 : std::int64_t private_outstanding_work;
90 :
91 : /// Remaining inline completions allowed this cycle.
92 : int inline_budget;
93 :
94 : /// Maximum inline budget (adaptive, 2-16).
95 : int inline_budget_max;
96 :
97 : /// True if no other thread absorbed queued work last cycle.
98 : bool unassisted;
99 :
100 : /// Construct a context frame linked to @a n.
101 : reactor_scheduler_context(
102 : reactor_scheduler const* k, reactor_scheduler_context* n);
103 : };
104 :
105 : /// Thread-local context stack for reactor schedulers.
106 : inline thread_local_ptr<reactor_scheduler_context> reactor_context_stack;
107 :
108 : /// Find the context frame for a scheduler on this thread.
109 : inline reactor_scheduler_context*
110 913711 : reactor_find_context(reactor_scheduler const* self) noexcept
111 : {
112 914258 : for (auto* c = reactor_context_stack.get(); c != nullptr; c = c->next)
113 : {
114 850469 : if (c->key == self)
115 849922 : return c;
116 : }
117 63789 : return nullptr;
118 : }
119 :
120 : /** Non-template base for reactor-backed scheduler implementations.
121 :
122 : Provides the complete threading model shared by epoll, kqueue,
123 : and select schedulers: signal state machine, inline completion
124 : budget, work counting, run/poll methods, and the do_one event
125 : loop.
126 :
127 : Derived classes provide platform-specific hooks by overriding:
128 : - `run_task(lock, ctx)` to run the reactor poll
129 : - `interrupt_reactor()` to wake a blocked reactor
130 :
131 : De-templated from the original CRTP design to eliminate
132 : duplicate instantiations when multiple backends are compiled
133 : into the same binary. Virtual dispatch for run_task (called
134 : once per reactor cycle, before a blocking syscall) has
135 : negligible overhead.
136 :
137 : @par Thread Safety
138 : All public member functions are thread-safe.
139 : */
140 : class reactor_scheduler : public scheduler
141 : {
142 : public:
143 : using context_type = reactor_scheduler_context;
144 : using mutex_type = conditionally_enabled_mutex;
145 : using lock_type = mutex_type::scoped_lock;
146 : using event_type = conditionally_enabled_event;
147 :
148 : /// Post a coroutine for deferred execution.
149 : void post(std::coroutine_handle<> h) const override;
150 :
151 : /// Post a scheduler operation for deferred execution.
152 : void post(scheduler_op* h) const override;
153 :
154 : /** Release an owner reference once no reactor pass can hold `d`.
155 :
156 : Called after a descriptor is deregistered. If a reactor pass is
157 : running, `keep` is parked on `d` until that pass ends. Then, or
158 : at once if no pass is running, it moves to `d`'s queued
159 : invocation if `d` is queued, and is dropped otherwise.
160 :
161 : @param d The deregistered descriptor.
162 : @param keep A reference to the descriptor's owner.
163 : */
164 : void retire_descriptor(reactor_retirable& d, object_ref keep) const;
165 :
166 : /// Post a continuation for deferred execution.
167 : void post(capy::continuation&) const override;
168 :
169 : /// Return true if called from a thread running this scheduler.
170 : bool running_in_this_thread() const noexcept override;
171 :
172 : /// Request the scheduler to stop dispatching handlers.
173 : void stop() override;
174 :
175 : /// Return true if the scheduler has been stopped.
176 : bool stopped() const noexcept override;
177 :
178 : /// Reset the stopped state so `run()` can resume.
179 : void restart() override;
180 :
181 : /// Run the event loop until no work remains.
182 : std::size_t run() override;
183 :
184 : /// Run until one handler completes or no work remains.
185 : std::size_t run_one() override;
186 :
187 : /// Run until one handler completes or @a usec elapses.
188 : std::size_t wait_one(long usec) override;
189 :
190 : /// Run ready handlers without blocking.
191 : std::size_t poll() override;
192 :
193 : /// Run at most one ready handler without blocking.
194 : std::size_t poll_one() override;
195 :
196 : /// Increment the outstanding work count.
197 : void work_started() noexcept override;
198 :
199 : /// Decrement the outstanding work count, stopping on zero.
200 : void work_finished() noexcept override;
201 :
202 : /** Reset the thread's inline completion budget.
203 :
204 : Called at the start of each posted completion handler to
205 : grant a fresh budget for speculative inline completions.
206 : */
207 : void reset_inline_budget() const noexcept;
208 :
209 : /** Consume one unit of inline budget if available.
210 :
211 : @return True if budget was available and consumed.
212 : */
213 : bool try_consume_inline_budget() const noexcept;
214 :
215 : /** Offset a forthcoming work_finished from work_cleanup.
216 :
217 : Called by descriptor_state when all I/O returned EAGAIN and
218 : no handler will be executed. Must be called from a scheduler
219 : thread.
220 : */
221 : /** Note that an op was parked outside the usual registration path.
222 :
223 : A reactor that builds its poll set from parked ops must rebuild
224 : it; one the kernel tracks needs nothing.
225 : */
226 495 : virtual void notify_reparked() const {}
227 :
228 : void compensating_work_started() const noexcept;
229 :
230 : /** Post an operation whose work is already counted.
231 :
232 : The posting thread touches nothing of the scheduler after this
233 : returns, unlike a @ref post followed by @ref work_finished: once
234 : posted, the operation can complete and let its context be
235 : destroyed.
236 :
237 : @pre work_started() was called for @p op.
238 :
239 : @param op The operation to post.
240 : */
241 : void post_counted(scheduler_op* op) const;
242 :
243 : /** Post completed operations for deferred invocation.
244 :
245 : If called from a thread running this scheduler, operations
246 : go to the thread's private queue (fast path). Otherwise,
247 : operations are added to the global queue under mutex and a
248 : waiter is signaled.
249 :
250 : @pre work_started() must have been called for each operation.
251 :
252 : @param ops Queue of operations to post.
253 : */
254 : void post_deferred_completions(ready_queue& ops) const;
255 :
256 : /** Apply runtime configuration to the scheduler.
257 :
258 : Called by `io_context` after construction. Values that do
259 : not apply to this backend are silently ignored.
260 :
261 : @pre `max_events` is in [ 1, `INT_MAX` ] and `budget_max` is at
262 : most `INT_MAX`; `io_context` validates both.
263 :
264 : @param max_events Event buffer size for epoll/kqueue.
265 : @param budget_init Starting inline completion budget.
266 : @param budget_max Hard ceiling on adaptive budget ramp-up.
267 : @param unassisted Budget when single-threaded.
268 : */
269 : virtual void configure_reactor(
270 : unsigned max_events,
271 : unsigned budget_init,
272 : unsigned budget_max,
273 : unsigned unassisted);
274 :
275 : /// Return the configured initial inline budget.
276 11283 : unsigned inline_budget_initial() const noexcept
277 : {
278 11283 : return inline_budget_initial_;
279 : }
280 :
281 : /// Return true when scheduler locking is disabled (fully-lockless tier).
282 1460 : bool scheduler_locking_disabled() const noexcept override
283 : {
284 1460 : return scheduler_locking_disabled_;
285 : }
286 :
287 4672 : void configure_threading(threading_config cfg) noexcept override
288 : {
289 4672 : scheduler_locking_disabled_ = !cfg.scheduler_locking;
290 : // reactor_io_locking takes effect at descriptor registration (see the
291 : // register_descriptor overrides), not here.
292 4672 : reactor_io_locking_ = cfg.reactor_io_locking;
293 4672 : one_thread_ = cfg.one_thread;
294 4672 : mutex_.set_enabled(cfg.scheduler_locking);
295 4672 : cond_.set_enabled(cfg.scheduler_locking);
296 4672 : }
297 :
298 : protected:
299 : bool scheduler_locking_disabled_ = false;
300 : bool reactor_io_locking_ = true;
301 : bool one_thread_ = false;
302 :
303 4688 : reactor_scheduler() = default;
304 :
305 : /// Destroy, destroying whatever was posted after the drain.
306 : ~reactor_scheduler() override;
307 :
308 : /** Drain completed_ops during shutdown.
309 :
310 : Pops all operations from the global queue and destroys them,
311 : skipping the task sentinel. Signals all waiting threads.
312 : Derived classes call this from their shutdown() override
313 : before performing platform-specific cleanup.
314 : */
315 : void shutdown_drain();
316 :
317 : /// Destroy every queued operation without running it.
318 : void destroy_queued(lock_type& lock) noexcept;
319 :
320 : /// RAII guard that re-inserts the task sentinel after `run_task`.
321 : struct task_cleanup
322 : {
323 : reactor_scheduler const* sched;
324 : lock_type* lock;
325 : context_type& ctx;
326 : ~task_cleanup();
327 : };
328 :
329 : mutable mutex_type mutex_{true};
330 : mutable event_type cond_{true};
331 : mutable ready_queue completed_ops_;
332 : mutable std::atomic<std::int64_t> outstanding_work_{0};
333 : std::atomic<bool> stopped_{false};
334 : mutable std::atomic<bool> task_running_{false};
335 : mutable bool task_interrupted_ = false;
336 :
337 : // Descriptors retired during the running reactor pass (guarded by
338 : // mutex_).
339 : mutable reactor_retirable* retired_ = nullptr;
340 :
341 : /// Settle every descriptor retired before the current pass ended.
342 : /// @pre `lock` holds `mutex_`; it is held again on return.
343 : void settle_retired(lock_type& lock) const;
344 :
345 : /// Move what an enclosing handler on this thread queued privately
346 : /// to the shared queue. @pre `mutex_` is held.
347 : void adopt_enclosing_work(context_type const& ctx);
348 :
349 : // Runtime-configurable reactor tuning parameters.
350 : // Defaults match the library's built-in values.
351 : unsigned max_events_per_poll_ = 128;
352 : unsigned inline_budget_initial_ = 2;
353 : unsigned inline_budget_max_ = 16;
354 : unsigned unassisted_budget_ = 4;
355 :
356 : /// Bit 0 of `state_`: set when the condvar should be signaled.
357 : static constexpr std::size_t signaled_bit = 1;
358 :
359 : /// Increment per waiting thread in `state_`.
360 : static constexpr std::size_t waiter_increment = 2;
361 : mutable std::size_t state_ = 0;
362 :
363 : /// Sentinel op that triggers a reactor poll when dequeued.
364 : struct task_op final : scheduler_op
365 : {
366 : // LCOV_EXCL_START: the sentinel is intercepted by pointer
367 : // identity; its virtuals exist for vtable completeness.
368 : void operator()() override {}
369 : void destroy() override {}
370 : // LCOV_EXCL_STOP
371 : };
372 : task_op task_op_;
373 :
374 : /** Run the platform-specific reactor poll.
375 :
376 : @par Postconditions
377 : `lock` is owned on return, however the poll ended. An
378 : implementation that unlocks around the blocking call owes the
379 : caller a matching re-acquire on every path out, including the
380 : errors it retries rather than reports.
381 : */
382 : virtual void
383 : run_task(lock_type& lock, context_type& ctx, long timeout_us) = 0;
384 :
385 : /// Wake a blocked reactor (e.g. write to eventfd or pipe).
386 : virtual void interrupt_reactor() const = 0;
387 :
388 : /// Wake the reactor whenever the earliest timer changes.
389 : void on_timer_service_attached(timer_service& svc) override;
390 :
391 : private:
392 : struct work_cleanup
393 : {
394 : reactor_scheduler* sched;
395 : lock_type* lock;
396 : context_type& ctx;
397 : ~work_cleanup();
398 : };
399 :
400 : std::size_t do_one(lock_type& lock, long timeout_us, context_type& ctx);
401 :
402 : void signal_all(lock_type& lock) const;
403 : bool maybe_unlock_and_signal_one(lock_type& lock) const;
404 : bool unlock_and_signal_one(lock_type& lock) const;
405 : void clear_signal() const;
406 : void wait_for_signal(lock_type& lock) const;
407 : void wait_for_signal_for(lock_type& lock, long timeout_us) const;
408 : void wake_one_thread_and_unlock(lock_type& lock) const;
409 : };
410 :
411 : /** RAII guard that pushes/pops a scheduler context frame.
412 :
413 : On construction, pushes a new context frame onto the
414 : thread-local stack. On destruction, drains any remaining
415 : private queue items to the global queue and pops the frame.
416 : */
417 : struct reactor_thread_context_guard
418 : {
419 : /// The context frame managed by this guard.
420 : reactor_scheduler_context frame_;
421 :
422 : /// Marks the scheduler innermost for the executor's fast check.
423 : running_scheduler_guard running_;
424 :
425 : /// Construct the guard, pushing a frame for @a sched.
426 11283 : explicit reactor_thread_context_guard(
427 : reactor_scheduler const* sched) noexcept
428 11283 : : frame_(sched, reactor_context_stack.get())
429 11283 : , running_(sched)
430 : {
431 11283 : reactor_context_stack.set(&frame_);
432 11283 : }
433 :
434 : /** Destroy the guard, popping the frame.
435 :
436 : The private queue is empty here by invariant: work_cleanup and
437 : task_cleanup splice it to the global queue after every handler
438 : and every reactor pass.
439 : */
440 11283 : ~reactor_thread_context_guard() noexcept
441 : {
442 11283 : reactor_context_stack.set(frame_.next);
443 11283 : }
444 : };
445 :
446 : // ---- Inline implementations ------------------------------------------------
447 :
448 11283 : inline reactor_scheduler_context::reactor_scheduler_context(
449 11283 : reactor_scheduler const* k, reactor_scheduler_context* n)
450 11283 : : key(k)
451 11283 : , next(n)
452 11283 : , private_outstanding_work(0)
453 11283 : , inline_budget(0)
454 11283 : , inline_budget_max(static_cast<int>(k->inline_budget_initial()))
455 11283 : , unassisted(false)
456 : {
457 11283 : }
458 :
459 : inline void
460 50 : reactor_scheduler::configure_reactor(
461 : unsigned max_events,
462 : unsigned budget_init,
463 : unsigned budget_max,
464 : unsigned unassisted)
465 : {
466 50 : BOOST_COROSIO_ASSERT(
467 : max_events >= 1 &&
468 : max_events <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
469 50 : BOOST_COROSIO_ASSERT(
470 : budget_max <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
471 :
472 : // Clamp initial and unassisted to budget_max.
473 50 : if (budget_init > budget_max)
474 20 : budget_init = budget_max;
475 50 : if (unassisted > budget_max)
476 20 : unassisted = budget_max;
477 :
478 50 : max_events_per_poll_ = max_events;
479 50 : inline_budget_initial_ = budget_init;
480 50 : inline_budget_max_ = budget_max;
481 50 : unassisted_budget_ = unassisted;
482 50 : }
483 :
484 : inline void
485 98780 : reactor_scheduler::reset_inline_budget() const noexcept
486 : {
487 : // When budget is disabled (max==0), all paths below would no-op
488 : // (inline_budget stays 0). Skip the TLS lookup entirely.
489 98780 : if (inline_budget_max_ == 0)
490 58 : return;
491 98722 : if (auto* ctx = reactor_find_context(this))
492 : {
493 : // Cap when no other thread absorbed queued work
494 98722 : if (ctx->unassisted)
495 : {
496 97927 : ctx->inline_budget_max = static_cast<int>(unassisted_budget_);
497 97927 : ctx->inline_budget = static_cast<int>(unassisted_budget_);
498 97927 : return;
499 : }
500 : // Ramp up when previous cycle fully consumed budget.
501 : // max(1, ...) ensures the doubling escapes zero.
502 : // Doubled in a wider type: the cap may be as large as INT_MAX.
503 795 : if (ctx->inline_budget == 0)
504 : {
505 : long long const doubled =
506 263 : 2LL * (std::max)(1, ctx->inline_budget_max);
507 263 : long long const cap = inline_budget_max_;
508 263 : ctx->inline_budget_max = static_cast<int>((std::min)(doubled, cap));
509 : }
510 532 : else if (ctx->inline_budget < ctx->inline_budget_max)
511 : {
512 174 : ctx->inline_budget_max = static_cast<int>(inline_budget_initial_);
513 : }
514 795 : ctx->inline_budget = ctx->inline_budget_max;
515 : }
516 : }
517 :
518 : inline bool
519 409704 : reactor_scheduler::try_consume_inline_budget() const noexcept
520 : {
521 409704 : if (inline_budget_max_ == 0)
522 44 : return false;
523 409660 : if (auto* ctx = reactor_find_context(this))
524 : {
525 409656 : if (ctx->inline_budget > 0)
526 : {
527 327464 : --ctx->inline_budget;
528 327464 : return true;
529 : }
530 : }
531 82196 : return false;
532 : }
533 :
534 : inline void
535 3862 : reactor_scheduler::post(std::coroutine_handle<> h) const
536 : {
537 : struct post_handler final : scheduler_op
538 : {
539 : std::coroutine_handle<> h_;
540 :
541 3862 : explicit post_handler(std::coroutine_handle<> h) : h_(h) {}
542 7724 : ~post_handler() override = default;
543 :
544 3790 : void operator()() override
545 : {
546 3790 : auto saved = h_;
547 3790 : delete this;
548 3790 : saved.resume();
549 3774 : }
550 :
551 72 : void destroy() override
552 : {
553 : // A bare handle carries no owner to unwind from: it is
554 : // destroyed directly.
555 72 : auto saved = h_;
556 72 : delete this;
557 72 : saved.destroy();
558 72 : }
559 : };
560 :
561 3862 : auto ph = std::make_unique<post_handler>(h);
562 :
563 3862 : if (auto* ctx = reactor_find_context(this))
564 : {
565 34 : ++ctx->private_outstanding_work;
566 34 : ctx->private_queue.push(ph.release());
567 34 : return;
568 : }
569 :
570 3828 : outstanding_work_.fetch_add(1, std::memory_order_relaxed);
571 :
572 3828 : lock_type lock(mutex_);
573 3828 : completed_ops_.push(ph.release());
574 3828 : wake_one_thread_and_unlock(lock);
575 3862 : }
576 :
577 : inline void
578 105540 : reactor_scheduler::post(scheduler_op* h) const
579 : {
580 105540 : if (auto* ctx = reactor_find_context(this))
581 : {
582 104005 : ++ctx->private_outstanding_work;
583 104005 : ctx->private_queue.push(h);
584 104005 : return;
585 : }
586 :
587 1535 : outstanding_work_.fetch_add(1, std::memory_order_relaxed);
588 :
589 1535 : lock_type lock(mutex_);
590 1535 : completed_ops_.push(h);
591 1535 : wake_one_thread_and_unlock(lock);
592 1535 : }
593 :
594 : inline void
595 236933 : reactor_scheduler::post(capy::continuation& c) const
596 : {
597 236933 : if (auto* ctx = reactor_find_context(this))
598 : {
599 204199 : ++ctx->private_outstanding_work;
600 204199 : ctx->private_queue.push(c);
601 204199 : return;
602 : }
603 :
604 32734 : outstanding_work_.fetch_add(1, std::memory_order_relaxed);
605 :
606 32734 : lock_type lock(mutex_);
607 32734 : completed_ops_.push(c);
608 32734 : wake_one_thread_and_unlock(lock);
609 32734 : }
610 :
611 : inline bool
612 24297 : reactor_scheduler::running_in_this_thread() const noexcept
613 : {
614 24297 : return reactor_find_context(this) != nullptr;
615 : }
616 :
617 : inline void
618 8972 : reactor_scheduler::stop()
619 : {
620 8972 : lock_type lock(mutex_);
621 8972 : if (!stopped_.load(std::memory_order_acquire))
622 : {
623 7906 : stopped_.store(true, std::memory_order_release);
624 7906 : signal_all(lock);
625 7906 : interrupt_reactor();
626 : }
627 8972 : }
628 :
629 : inline bool
630 1296 : reactor_scheduler::stopped() const noexcept
631 : {
632 1296 : return stopped_.load(std::memory_order_acquire);
633 : }
634 :
635 : inline void
636 6127 : reactor_scheduler::restart()
637 : {
638 6127 : stopped_.store(false, std::memory_order_release);
639 6127 : }
640 :
641 : inline std::size_t
642 6956 : reactor_scheduler::run()
643 : {
644 13912 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
645 : {
646 267 : stop();
647 267 : return 0;
648 : }
649 :
650 6689 : reactor_thread_context_guard ctx(this);
651 6689 : lock_type lock(mutex_);
652 6689 : adopt_enclosing_work(ctx.frame_);
653 :
654 6689 : std::size_t n = 0;
655 : for (;;)
656 : {
657 382775 : if (!do_one(lock, -1, ctx.frame_))
658 6684 : break;
659 376086 : if (n != (std::numeric_limits<std::size_t>::max)())
660 376086 : ++n;
661 376086 : if (!lock.owns_lock())
662 86075 : lock.lock();
663 : }
664 6684 : return n;
665 6694 : }
666 :
667 : inline std::size_t
668 146 : reactor_scheduler::run_one()
669 : {
670 292 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
671 : {
672 5 : stop();
673 5 : return 0;
674 : }
675 :
676 141 : reactor_thread_context_guard ctx(this);
677 141 : lock_type lock(mutex_);
678 141 : adopt_enclosing_work(ctx.frame_);
679 141 : return do_one(lock, -1, ctx.frame_);
680 143 : }
681 :
682 : inline std::size_t
683 3756 : reactor_scheduler::wait_one(long usec)
684 : {
685 7512 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
686 : {
687 830 : stop();
688 830 : return 0;
689 : }
690 :
691 2926 : reactor_thread_context_guard ctx(this);
692 2926 : lock_type lock(mutex_);
693 2926 : adopt_enclosing_work(ctx.frame_);
694 2926 : return do_one(lock, usec, ctx.frame_);
695 2934 : }
696 :
697 : inline std::size_t
698 1479 : reactor_scheduler::poll()
699 : {
700 2958 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
701 : {
702 31 : stop();
703 31 : return 0;
704 : }
705 :
706 1448 : reactor_thread_context_guard ctx(this);
707 1448 : lock_type lock(mutex_);
708 1448 : adopt_enclosing_work(ctx.frame_);
709 :
710 1448 : std::size_t n = 0;
711 : for (;;)
712 : {
713 9628 : if (!do_one(lock, 0, ctx.frame_))
714 1446 : break;
715 8180 : if (n != (std::numeric_limits<std::size_t>::max)())
716 8180 : ++n;
717 8180 : if (!lock.owns_lock())
718 8176 : lock.lock();
719 : }
720 1446 : return n;
721 1450 : }
722 :
723 : inline std::size_t
724 86 : reactor_scheduler::poll_one()
725 : {
726 172 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
727 : {
728 7 : stop();
729 7 : return 0;
730 : }
731 :
732 79 : reactor_thread_context_guard ctx(this);
733 79 : lock_type lock(mutex_);
734 79 : adopt_enclosing_work(ctx.frame_);
735 79 : return do_one(lock, 0, ctx.frame_);
736 81 : }
737 :
738 : inline void
739 67300 : reactor_scheduler::work_started() noexcept
740 : {
741 67300 : outstanding_work_.fetch_add(1, std::memory_order_relaxed);
742 67300 : }
743 :
744 : inline void
745 112125 : reactor_scheduler::work_finished() noexcept
746 : {
747 224250 : if (outstanding_work_.fetch_sub(1, std::memory_order_acq_rel) == 1)
748 7735 : stop();
749 112125 : }
750 :
751 : inline void
752 32699 : reactor_scheduler::compensating_work_started() const noexcept
753 : {
754 32699 : auto* ctx = reactor_find_context(this);
755 32699 : if (ctx)
756 32697 : ++ctx->private_outstanding_work;
757 32699 : }
758 :
759 : inline void
760 1996 : reactor_scheduler::post_counted(scheduler_op* op) const
761 : {
762 1996 : if (auto* ctx = reactor_find_context(this))
763 : {
764 607 : ctx->private_queue.push(op);
765 607 : return;
766 : }
767 :
768 1389 : lock_type lock(mutex_);
769 1389 : completed_ops_.push(op);
770 1389 : wake_one_thread_and_unlock(lock);
771 1389 : }
772 :
773 : inline void
774 16293 : reactor_scheduler::post_deferred_completions(ready_queue& ops) const
775 : {
776 16293 : if (ops.empty())
777 16293 : return;
778 :
779 2 : if (auto* ctx = reactor_find_context(this))
780 : {
781 2 : ctx->private_queue.splice(ops);
782 2 : return;
783 : }
784 :
785 MIS 0 : lock_type lock(mutex_);
786 0 : completed_ops_.splice(ops);
787 0 : wake_one_thread_and_unlock(lock);
788 0 : }
789 :
790 : inline void
791 HIT 23178 : reactor_scheduler::retire_descriptor(
792 : reactor_retirable& d, object_ref keep) const
793 : {
794 23178 : lock_type lock(mutex_);
795 23178 : if (!task_running_.load(std::memory_order_relaxed))
796 : {
797 : // No pass can hold an event, but an earlier one may have
798 : // queued d.
799 17820 : d.hand_off(keep);
800 17820 : return;
801 : }
802 : // Already parked for this pass: the held reference suffices.
803 5358 : if (d.retired_ref_)
804 126 : return;
805 5232 : d.retired_ref_ = std::move(keep);
806 5232 : d.retired_next_ = retired_;
807 5232 : retired_ = &d;
808 23178 : }
809 :
810 : inline void
811 11283 : reactor_scheduler::adopt_enclosing_work(context_type const& ctx)
812 : {
813 : // A run nested in a handler would otherwise never see what that
814 : // handler posted: it stays in the handler's frame until it returns.
815 11289 : for (auto* c = ctx.next; c != nullptr; c = c->next)
816 : {
817 12 : if (c->key != this)
818 6 : continue;
819 6 : if (c->private_outstanding_work > 0)
820 : {
821 6 : outstanding_work_.fetch_add(
822 : c->private_outstanding_work, std::memory_order_relaxed);
823 6 : c->private_outstanding_work = 0;
824 : }
825 6 : completed_ops_.splice(c->private_queue);
826 6 : return;
827 : }
828 : }
829 :
830 : inline void
831 292837 : reactor_scheduler::settle_retired(lock_type& lock) const
832 : {
833 : // Handed off under the lock, since a later pass may retire the same
834 : // descriptor again as soon as it is released. A reference that is
835 : // not handed off may be the owner's last, and its release recycles
836 : // through the pool mutex, so it is dropped with the lock released.
837 292837 : reactor_retirable* dropped = nullptr;
838 298069 : while (auto* d = retired_)
839 : {
840 5232 : retired_ = d->retired_next_;
841 5232 : d->retired_next_ = nullptr;
842 5232 : auto keep = std::move(d->retired_ref_);
843 5232 : d->hand_off(keep);
844 : // A drop already pending keeps the owner alive, so this one
845 : // cannot be the last and is released here.
846 5232 : if (!keep || d->dropped_ref_)
847 376 : continue;
848 4856 : d->dropped_ref_ = std::move(keep);
849 4856 : d->dropped_next_ = dropped;
850 4856 : dropped = d;
851 10464 : }
852 297693 : while (auto* d = dropped)
853 : {
854 4856 : dropped = d->dropped_next_;
855 4856 : d->dropped_next_ = nullptr;
856 4856 : auto keep = std::move(d->dropped_ref_);
857 4856 : lock.unlock();
858 4856 : keep.reset();
859 4856 : lock.lock();
860 4856 : }
861 292837 : }
862 :
863 : inline void
864 407 : reactor_scheduler::on_timer_service_attached(timer_service& svc)
865 : {
866 407 : svc.set_on_earliest_changed(timer_service::callback(this, [](void* p) {
867 3324 : static_cast<reactor_scheduler*>(p)->interrupt_reactor();
868 3324 : }));
869 407 : }
870 :
871 : inline void
872 4672 : reactor_scheduler::shutdown_drain()
873 : {
874 4672 : wait_cancels_posting();
875 :
876 4672 : lock_type lock(mutex_);
877 4672 : settle_retired(lock);
878 4672 : destroy_queued(lock);
879 4672 : signal_all(lock);
880 4672 : }
881 :
882 4688 : inline reactor_scheduler::~reactor_scheduler()
883 : {
884 : // Services give up their work in shutdown(), but anything posted
885 : // after the drain, such as from a destructor, is still destroyed.
886 4688 : lock_type lock(mutex_);
887 4688 : destroy_queued(lock);
888 4688 : }
889 :
890 : inline void
891 9360 : reactor_scheduler::destroy_queued(lock_type& lock) noexcept
892 : {
893 23485 : while (auto e = completed_ops_.pop())
894 : {
895 14125 : if (ready_is_continuation(e))
896 : {
897 7844 : lock.unlock();
898 7844 : if (auto* c = ready_as_cont(e); c->h)
899 7844 : capy::abandon(*c);
900 7844 : lock.lock();
901 : }
902 : else
903 : {
904 6281 : auto* op = ready_as_op(e);
905 6281 : if (op == &task_op_)
906 4669 : continue;
907 1612 : lock.unlock();
908 1612 : op->destroy();
909 1612 : lock.lock();
910 : }
911 14125 : }
912 9360 : }
913 :
914 : inline void
915 12578 : reactor_scheduler::signal_all(lock_type&) const
916 : {
917 12578 : state_ |= signaled_bit;
918 12578 : cond_.notify_all();
919 12578 : }
920 :
921 : inline bool
922 40911 : reactor_scheduler::maybe_unlock_and_signal_one(lock_type& lock) const
923 : {
924 40911 : state_ |= signaled_bit;
925 40911 : if (state_ > signaled_bit)
926 : {
927 : // Signalled before unlocking: a post from another context's
928 : // thread may be the last touch of this scheduler before the
929 : // posted work runs, finishes and lets it be destroyed.
930 204 : cond_.notify_one();
931 204 : lock.unlock();
932 204 : return true;
933 : }
934 40707 : return false;
935 : }
936 :
937 : inline bool
938 636179 : reactor_scheduler::unlock_and_signal_one(lock_type& lock) const
939 : {
940 636179 : state_ |= signaled_bit;
941 636179 : bool have_waiters = state_ > signaled_bit;
942 636179 : lock.unlock();
943 636179 : if (have_waiters)
944 2868 : cond_.notify_one();
945 636179 : return have_waiters;
946 : }
947 :
948 : inline void
949 939 : reactor_scheduler::clear_signal() const
950 : {
951 939 : state_ &= ~signaled_bit;
952 939 : }
953 :
954 : inline void
955 907 : reactor_scheduler::wait_for_signal(lock_type& lock) const
956 : {
957 1888 : while ((state_ & signaled_bit) == 0)
958 : {
959 981 : state_ += waiter_increment;
960 981 : cond_.wait(lock);
961 981 : state_ -= waiter_increment;
962 : }
963 907 : }
964 :
965 : inline void
966 32 : reactor_scheduler::wait_for_signal_for(lock_type& lock, long timeout_us) const
967 : {
968 32 : if ((state_ & signaled_bit) == 0)
969 : {
970 32 : state_ += waiter_increment;
971 32 : cond_.wait_for(lock, std::chrono::microseconds(timeout_us));
972 32 : state_ -= waiter_increment;
973 : }
974 32 : }
975 :
976 : inline void
977 39486 : reactor_scheduler::wake_one_thread_and_unlock(lock_type& lock) const
978 : {
979 39486 : if (maybe_unlock_and_signal_one(lock))
980 173 : return;
981 :
982 39313 : if (task_running_.load(std::memory_order_relaxed) && !task_interrupted_)
983 : {
984 1720 : task_interrupted_ = true;
985 : // Under the lock, for the same reason as the signal above.
986 1720 : interrupt_reactor();
987 1720 : lock.unlock();
988 : }
989 : else
990 : {
991 37593 : lock.unlock();
992 : }
993 : }
994 :
995 387184 : inline reactor_scheduler::work_cleanup::~work_cleanup()
996 : {
997 387184 : std::int64_t produced = ctx.private_outstanding_work;
998 387184 : if (produced > 1)
999 929 : sched->outstanding_work_.fetch_add(
1000 : produced - 1, std::memory_order_relaxed);
1001 386255 : else if (produced < 1)
1002 65418 : sched->work_finished();
1003 387184 : ctx.private_outstanding_work = 0;
1004 :
1005 387184 : if (!ctx.private_queue.empty())
1006 : {
1007 290383 : lock->lock();
1008 290383 : sched->completed_ops_.splice(ctx.private_queue);
1009 : }
1010 387184 : }
1011 :
1012 288168 : inline reactor_scheduler::task_cleanup::~task_cleanup()
1013 : {
1014 288168 : if (ctx.private_outstanding_work > 0)
1015 : {
1016 10403 : sched->outstanding_work_.fetch_add(
1017 10403 : ctx.private_outstanding_work, std::memory_order_relaxed);
1018 10403 : ctx.private_outstanding_work = 0;
1019 : }
1020 :
1021 288168 : if (!ctx.private_queue.empty())
1022 : {
1023 10403 : if (!lock->owns_lock())
1024 MIS 0 : lock->lock();
1025 HIT 10403 : sched->completed_ops_.splice(ctx.private_queue);
1026 : }
1027 288168 : }
1028 :
1029 : inline std::size_t
1030 395549 : reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx)
1031 : {
1032 : for (;;)
1033 : {
1034 683228 : if (stopped_.load(std::memory_order_acquire))
1035 6928 : return 0;
1036 :
1037 676300 : auto const e = completed_ops_.pop();
1038 676300 : scheduler_op* op = ready_is_continuation(e) ? nullptr : ready_as_op(e);
1039 :
1040 : // Handle reactor sentinel — time to poll for I/O
1041 676300 : if (op == &task_op_)
1042 : {
1043 288168 : bool more_handlers = !completed_ops_.empty();
1044 :
1045 325149 : if (!more_handlers &&
1046 73962 : outstanding_work_.load(std::memory_order_acquire) == 0)
1047 : {
1048 MIS 0 : completed_ops_.push(&task_op_);
1049 0 : return 0;
1050 : }
1051 :
1052 HIT 288168 : long task_timeout_us = more_handlers ? 0 : timeout_us;
1053 288168 : task_interrupted_ = task_timeout_us == 0;
1054 288168 : task_running_.store(true, std::memory_order_release);
1055 :
1056 : // Wake a peer to take the pending handlers while this thread
1057 : // polls the reactor; skipped when one_thread_ (no peer exists).
1058 288168 : if (more_handlers && !one_thread_)
1059 251176 : unlock_and_signal_one(lock);
1060 :
1061 : try
1062 : {
1063 288168 : run_task(lock, ctx, task_timeout_us);
1064 : }
1065 3 : catch (...)
1066 : {
1067 3 : task_running_.store(false, std::memory_order_relaxed);
1068 3 : throw;
1069 3 : }
1070 :
1071 288165 : task_running_.store(false, std::memory_order_relaxed);
1072 288165 : bool const idle = completed_ops_.empty();
1073 288165 : completed_ops_.push(&task_op_);
1074 288165 : settle_retired(lock);
1075 : // A bounded call blocks at most once, then finishes as a poll
1076 : // does, running what its pass queued.
1077 288165 : if (timeout_us > 0)
1078 2712 : timeout_us = 0;
1079 : // A poll ends once a pass leaves nothing queued; going round
1080 : // again would only repeat the pass. A peer that found the
1081 : // queue empty during the pass may be asleep, so it is woken
1082 : // to take the sentinel.
1083 288165 : if (timeout_us == 0 && idle)
1084 : {
1085 1425 : maybe_unlock_and_signal_one(lock);
1086 1425 : return 0;
1087 : }
1088 286740 : continue;
1089 286740 : }
1090 :
1091 : // Handle ready entry (op or continuation)
1092 388132 : if (e != nullptr)
1093 : {
1094 387184 : bool more = !completed_ops_.empty();
1095 :
1096 387184 : if (more && !one_thread_)
1097 : {
1098 : // Wake a peer for the remaining work; unassisted if none
1099 : // was parked to take it.
1100 385003 : ctx.unassisted = !unlock_and_signal_one(lock);
1101 : }
1102 : else
1103 : {
1104 : // No peer to wake (one_thread_, or nothing more queued).
1105 2181 : ctx.unassisted = more;
1106 2181 : lock.unlock();
1107 : }
1108 :
1109 387184 : [[maybe_unused]] work_cleanup on_exit{this, &lock, ctx};
1110 :
1111 387184 : if (ready_is_continuation(e))
1112 229089 : ready_as_cont(e)->h.resume();
1113 : else
1114 158095 : (*op)();
1115 387168 : return 1;
1116 387184 : }
1117 :
1118 1896 : if (outstanding_work_.load(std::memory_order_acquire) == 0 ||
1119 : timeout_us == 0)
1120 9 : return 0;
1121 :
1122 939 : clear_signal();
1123 939 : if (timeout_us < 0)
1124 : {
1125 907 : wait_for_signal(lock);
1126 : }
1127 : else
1128 : {
1129 32 : wait_for_signal_for(lock, timeout_us);
1130 : // A bounded call blocks once, so its caller can check its
1131 : // deadline; another thread may hold the reactor for good.
1132 32 : timeout_us = 0;
1133 : }
1134 287679 : }
1135 : }
1136 :
1137 : } // namespace boost::corosio::detail
1138 :
1139 : #endif // BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
|