TLA Line data Source code
1 : //
2 : // Copyright (c) 2026 Steve Gerbino
3 : // Copyright (c) 2026 Michael Vandeberg
4 : //
5 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
6 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7 : //
8 : // Official repository: https://github.com/cppalliance/corosio
9 : //
10 :
11 : #ifndef BOOST_COROSIO_NATIVE_DETAIL_EPOLL_EPOLL_SCHEDULER_HPP
12 : #define BOOST_COROSIO_NATIVE_DETAIL_EPOLL_EPOLL_SCHEDULER_HPP
13 :
14 : #include <boost/corosio/detail/platform.hpp>
15 :
16 : #if BOOST_COROSIO_HAS_EPOLL
17 :
18 : #include <boost/corosio/detail/config.hpp>
19 : #include <boost/capy/ex/execution_context.hpp>
20 :
21 : #include <boost/corosio/native/detail/reactor/reactor_scheduler.hpp>
22 : #include <boost/corosio/native/detail/reactor/reactor_signal_pipe.hpp>
23 :
24 : #include <boost/corosio/native/detail/epoll/epoll_traits.hpp>
25 : #include <boost/corosio/detail/timer_service.hpp>
26 : #include <boost/corosio/native/detail/make_err.hpp>
27 :
28 : #include <boost/corosio/detail/except.hpp>
29 :
30 : #include <atomic>
31 : #include <chrono>
32 : #include <cstdint>
33 : #include <limits>
34 : #include <mutex>
35 : #include <vector>
36 :
37 : #include <errno.h>
38 : #include <sys/epoll.h>
39 : #include <sys/eventfd.h>
40 : #include <sys/timerfd.h>
41 : #include <unistd.h>
42 :
43 : namespace boost::corosio::detail {
44 :
45 : /** Linux scheduler using epoll for I/O multiplexing.
46 :
47 : This scheduler implements the scheduler interface using Linux epoll
48 : for efficient I/O event notification. It uses a single reactor model
49 : where one thread runs epoll_wait while other threads
50 : wait on a condition variable for handler work. This design provides:
51 :
52 : - Handler parallelism: N posted handlers can execute on N threads
53 : - No thundering herd: condition_variable wakes exactly one thread
54 : - IOCP parity: Behavior matches Windows I/O completion port semantics
55 :
56 : When threads call run(), they first try to execute queued handlers.
57 : If the queue is empty and no reactor is running, one thread becomes
58 : the reactor and runs epoll_wait. Other threads wait on a condition
59 : variable until handlers are available.
60 :
61 : @par Thread Safety
62 : All public member functions are thread-safe.
63 : */
64 : class BOOST_COROSIO_DECL epoll_scheduler final : public reactor_scheduler
65 : {
66 : public:
67 : /** Construct the scheduler.
68 :
69 : Creates an epoll instance and an eventfd for reactor
70 : interruption. The timerfd for timer expiry is created with the
71 : context's first timer.
72 :
73 : @param ctx Reference to the owning execution_context.
74 : @param concurrency_hint Hint for expected thread count (unused).
75 : */
76 : epoll_scheduler(capy::execution_context& ctx, int concurrency_hint = -1);
77 :
78 : /// Destroy the scheduler.
79 : ~epoll_scheduler() override;
80 :
81 : epoll_scheduler(epoll_scheduler const&) = delete;
82 : epoll_scheduler& operator=(epoll_scheduler const&) = delete;
83 :
84 : /// Shut down the scheduler, draining pending operations.
85 : void shutdown() override;
86 :
87 : /// Apply runtime configuration, resizing the event buffer.
88 : void configure_reactor(
89 : unsigned max_events,
90 : unsigned budget_init,
91 : unsigned budget_max,
92 : unsigned unassisted) override;
93 :
94 : /** Return the epoll file descriptor.
95 :
96 : Used by socket services to register file descriptors
97 : for I/O event notification.
98 :
99 : @return The epoll file descriptor.
100 : */
101 : int epoll_fd() const noexcept
102 : {
103 : return epoll_fd_;
104 : }
105 :
106 : /** Register a descriptor for persistent monitoring.
107 :
108 : The fd is registered once and stays registered until explicitly
109 : deregistered. Events are dispatched via reactor_descriptor_state which
110 : tracks pending read/write/connect operations.
111 :
112 : @param fd The file descriptor to register.
113 : @param desc Pointer to descriptor data (stored in epoll_event.data.ptr).
114 :
115 : @return The error if registration fails, otherwise a default
116 : constructed error code.
117 : */
118 : std::error_code
119 : register_descriptor(int fd, reactor_descriptor_state* desc) const;
120 :
121 : /// No-op: write readiness is watched from registration on.
122 : std::error_code
123 HIT 5811 : ensure_write_registered(int, reactor_descriptor_state*) const noexcept
124 : {
125 5811 : return {};
126 : }
127 :
128 : /** Deregister a persistently registered descriptor.
129 :
130 : @param fd The file descriptor to deregister.
131 : */
132 : void deregister_descriptor(int fd) const;
133 :
134 : /// Watch the read end of the POSIX signal self-pipe (see scheduler.hpp).
135 287 : [[nodiscard]] std::error_code register_signal_reader(int read_fd) override
136 : {
137 287 : return register_descriptor(read_fd, signal_pipe_reader_.arm());
138 : }
139 :
140 : private:
141 : void run_task(lock_type& lock, context_type& ctx, long timeout_us) override;
142 : void interrupt_reactor() const override;
143 : void update_timerfd() const;
144 : static int timer_timeout_ms(timer_service& svc, int timeout_ms) noexcept;
145 : void on_timer_service_attached(timer_service& svc) override;
146 :
147 : int epoll_fd_;
148 : int event_fd_;
149 : int timer_fd_;
150 :
151 : // Watches the global signal self-pipe's read end (armed lazily by
152 : // register_signal_reader on the first signal registration).
153 : reactor_signal_pipe_reader signal_pipe_reader_;
154 :
155 : // Edge-triggered eventfd state
156 : mutable std::atomic<bool> eventfd_armed_{false};
157 :
158 : // Set when the earliest timer changes; flushed before epoll_wait
159 : mutable std::atomic<bool> timerfd_stale_{false};
160 :
161 : // Event buffer sized from max_events_per_poll_ (set at construction,
162 : // resized by configure_reactor via io_context_options).
163 : std::vector<epoll_event> event_buffer_;
164 : };
165 :
166 2611 : inline epoll_scheduler::epoll_scheduler(capy::execution_context&, int)
167 2611 : : epoll_fd_(-1)
168 2611 : , event_fd_(-1)
169 2611 : , timer_fd_(-1)
170 5222 : , event_buffer_(max_events_per_poll_)
171 : {
172 2611 : epoll_fd_ = ::epoll_create1(EPOLL_CLOEXEC);
173 2611 : if (epoll_fd_ < 0)
174 3 : detail::throw_system_error(make_err(errno), "epoll_create1");
175 :
176 2608 : event_fd_ = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
177 2608 : if (event_fd_ < 0)
178 : {
179 3 : int errn = errno;
180 3 : ::close(epoll_fd_);
181 3 : detail::throw_system_error(make_err(errn), "eventfd");
182 : }
183 :
184 2605 : epoll_event ev{};
185 2605 : ev.events = EPOLLIN | EPOLLET;
186 2605 : ev.data.ptr = nullptr;
187 2605 : if (::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, event_fd_, &ev) < 0)
188 : {
189 3 : int errn = errno;
190 3 : ::close(event_fd_);
191 3 : ::close(epoll_fd_);
192 3 : detail::throw_system_error(make_err(errn), "epoll_ctl");
193 : }
194 :
195 2602 : completed_ops_.push(&task_op_);
196 2629 : }
197 :
198 : inline void
199 546 : epoll_scheduler::on_timer_service_attached(timer_service& svc)
200 : {
201 : // Without a timerfd the epoll_wait timeout drives the timers: the
202 : // first timer is often made where a throw can only terminate.
203 546 : int fd = ::timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC);
204 546 : if (fd >= 0)
205 : {
206 : // Set before registration, which orders it for every thread
207 : // the timerfd's events can wake.
208 545 : timer_fd_ = fd;
209 545 : epoll_event timer_ev{};
210 545 : timer_ev.events = EPOLLIN | EPOLLERR;
211 545 : timer_ev.data.ptr = &timer_fd_;
212 545 : if (::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &timer_ev) < 0)
213 : {
214 1 : ::close(fd);
215 1 : timer_fd_ = -1;
216 : }
217 : }
218 546 : svc.set_on_earliest_changed(timer_service::callback(this, [](void* p) {
219 4725 : auto* self = static_cast<epoll_scheduler*>(p);
220 4725 : self->timerfd_stale_.store(true, std::memory_order_release);
221 4725 : self->interrupt_reactor();
222 4725 : }));
223 546 : }
224 :
225 5204 : inline epoll_scheduler::~epoll_scheduler()
226 : {
227 2602 : if (timer_fd_ >= 0)
228 544 : ::close(timer_fd_);
229 2602 : if (event_fd_ >= 0)
230 2602 : ::close(event_fd_);
231 2602 : if (epoll_fd_ >= 0)
232 2602 : ::close(epoll_fd_);
233 5204 : }
234 :
235 : inline void
236 2602 : epoll_scheduler::shutdown()
237 : {
238 2602 : shutdown_drain();
239 :
240 2602 : if (event_fd_ >= 0)
241 2602 : interrupt_reactor();
242 2602 : }
243 :
244 : inline void
245 30 : epoll_scheduler::configure_reactor(
246 : unsigned max_events,
247 : unsigned budget_init,
248 : unsigned budget_max,
249 : unsigned unassisted)
250 : {
251 30 : reactor_scheduler::configure_reactor(
252 : max_events, budget_init, budget_max, unassisted);
253 30 : event_buffer_.resize(max_events_per_poll_);
254 30 : }
255 :
256 : inline std::error_code
257 13721 : epoll_scheduler::register_descriptor(
258 : int fd, reactor_descriptor_state* desc) const
259 : {
260 13721 : epoll_event ev{};
261 13721 : ev.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLERR | EPOLLHUP;
262 13721 : ev.data.ptr = desc;
263 :
264 : // Initialized before the add: another run thread's reactor can
265 : // dispatch an event for the descriptor as soon as it is added.
266 : // A completion queued by an earlier registration of the same state
267 : // may still be running, so the fields it reads change under its
268 : // mutex, and the mutex mode is only written when it differs.
269 13721 : if (desc->mutex.enabled() != reactor_io_locking_)
270 3 : desc->mutex.set_enabled(reactor_io_locking_);
271 : {
272 13721 : conditionally_enabled_mutex::scoped_lock lock(desc->mutex);
273 13721 : desc->registered_events = ev.events;
274 13721 : desc->unpollable = false;
275 13721 : desc->fd = fd;
276 13721 : desc->scheduler_ = this;
277 13721 : desc->ready_events_.store(0, std::memory_order_release);
278 : // object_ref_ is not touched: a queued invocation from an
279 : // earlier registration still owns the reference in it.
280 13721 : desc->read_ready = false;
281 13721 : desc->write_ready = false;
282 13721 : }
283 :
284 13721 : if (::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &ev) < 0)
285 : {
286 12 : int const err = errno;
287 12 : conditionally_enabled_mutex::scoped_lock lock(desc->mutex);
288 12 : desc->registered_events = 0;
289 : // EPERM: a file type epoll cannot watch. Its I/O does not
290 : // block, so adopt it unwatched, as asio does.
291 12 : if (err != EPERM)
292 7 : return make_err(err);
293 5 : desc->unpollable = true;
294 12 : }
295 13714 : return {};
296 : }
297 :
298 : inline void
299 13423 : epoll_scheduler::deregister_descriptor(int fd) const
300 : {
301 13423 : ::epoll_ctl(epoll_fd_, EPOLL_CTL_DEL, fd, nullptr);
302 13423 : }
303 :
304 : inline void
305 13518 : epoll_scheduler::interrupt_reactor() const
306 : {
307 13518 : bool expected = false;
308 13518 : if (eventfd_armed_.compare_exchange_strong(
309 : expected, true, std::memory_order_release,
310 : std::memory_order_relaxed))
311 : {
312 11291 : std::uint64_t val = 1;
313 11291 : if (::write(event_fd_, &val, sizeof(val)) < 0)
314 : {
315 : // The flag is what coalesces later interrupts into a byte
316 : // already in the eventfd; a write that failed put no byte
317 : // there, so leaving it armed would swallow every interrupt
318 : // that follows. Disarming keeps the cost to the interrupts
319 : // already in flight -- the next one arms and writes again,
320 : // instead of every one after this coalescing into a byte
321 : // that does not exist.
322 2 : eventfd_armed_.store(false, std::memory_order_release);
323 : }
324 : }
325 13518 : }
326 :
327 : inline void
328 12019 : epoll_scheduler::update_timerfd() const
329 : {
330 12019 : auto* tsvc = timers();
331 12019 : if (!tsvc || timer_fd_ < 0)
332 6 : return;
333 12013 : auto nearest = tsvc->nearest_expiry();
334 :
335 12013 : itimerspec ts{};
336 12013 : int flags = 0;
337 :
338 12013 : if (nearest == timer_service::time_point::max())
339 : {
340 : // No timers — disarm by setting to 0 (relative)
341 : }
342 : else
343 : {
344 10726 : auto now = std::chrono::steady_clock::now();
345 10726 : if (nearest <= now)
346 : {
347 : // Use 1ns instead of 0 — zero disarms the timerfd
348 1257 : ts.it_value.tv_nsec = 1;
349 : }
350 : else
351 : {
352 9469 : auto nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(
353 9469 : nearest - now)
354 9469 : .count();
355 9469 : ts.it_value.tv_sec = nsec / 1000000000;
356 9469 : ts.it_value.tv_nsec = nsec % 1000000000;
357 9469 : if (ts.it_value.tv_sec == 0 && ts.it_value.tv_nsec == 0)
358 MIS 0 : ts.it_value.tv_nsec = 1;
359 : }
360 : }
361 :
362 HIT 12013 : if (::timerfd_settime(timer_fd_, flags, &ts, nullptr) < 0)
363 1 : detail::throw_system_error(make_err(errno), "timerfd_settime");
364 : }
365 :
366 : // Rounded up, so the wait never wakes before the timer is due.
367 : inline int
368 4 : epoll_scheduler::timer_timeout_ms(timer_service& svc, int timeout_ms) noexcept
369 : {
370 4 : auto nearest = svc.nearest_expiry();
371 4 : if (nearest == timer_service::time_point::max())
372 MIS 0 : return timeout_ms;
373 HIT 4 : auto now = std::chrono::steady_clock::now();
374 4 : if (nearest <= now)
375 MIS 0 : return 0;
376 : auto ms =
377 HIT 4 : std::chrono::ceil<std::chrono::milliseconds>(nearest - now).count();
378 8 : ms = (std::min)(ms,
379 4 : static_cast<decltype(ms)>(
380 4 : (std::numeric_limits<int>::max)()));
381 4 : if (timeout_ms >= 0 && timeout_ms < ms)
382 MIS 0 : return timeout_ms;
383 HIT 4 : return static_cast<int>(ms);
384 : }
385 :
386 : inline void
387 147072 : epoll_scheduler::run_task(lock_type& lock, context_type& ctx, long timeout_us)
388 : {
389 : int timeout_ms;
390 147072 : if (task_interrupted_)
391 126618 : timeout_ms = 0;
392 20454 : else if (timeout_us < 0)
393 19986 : timeout_ms = -1;
394 : else
395 468 : timeout_ms = static_cast<int>((timeout_us + 999) / 1000);
396 :
397 147072 : if (lock.owns_lock())
398 21089 : lock.unlock();
399 :
400 147072 : task_cleanup on_exit{this, &lock, ctx};
401 :
402 : // Flush deferred timerfd programming before blocking
403 147072 : if (timerfd_stale_.exchange(false, std::memory_order_acquire))
404 4061 : update_timerfd();
405 :
406 : // timer_fd_ is read only once the service is seen, after which it
407 : // no longer changes.
408 147071 : auto* tsvc = timers();
409 147071 : bool const poll_timers = tsvc && timer_fd_ < 0;
410 147071 : if (poll_timers && timeout_ms != 0)
411 4 : timeout_ms = timer_timeout_ms(*tsvc, timeout_ms);
412 :
413 147071 : int nfds = ::epoll_wait(
414 147071 : epoll_fd_, event_buffer_.data(), static_cast<int>(event_buffer_.size()),
415 : timeout_ms);
416 :
417 147071 : if (nfds < 0 && errno != EINTR)
418 1 : detail::throw_system_error(make_err(errno), "epoll_wait");
419 :
420 : // The timerfd is armed relative to when it was set, so it can fire a
421 : // little after a deadline. A pass that doesn't block checks the
422 : // clock instead, so poll() never misses a timer that is already due.
423 147070 : bool check_timers = poll_timers;
424 147070 : if (!check_timers && tsvc && timeout_ms == 0)
425 : {
426 18638 : auto const nearest = tsvc->nearest_expiry();
427 36908 : check_timers = nearest != (timer_service::time_point::max)() &&
428 36908 : nearest <= std::chrono::steady_clock::now();
429 : }
430 147070 : ready_queue local_ops;
431 :
432 205686 : for (int i = 0; i < nfds; ++i)
433 : {
434 58616 : if (event_buffer_[i].data.ptr == nullptr)
435 : {
436 : std::uint64_t val;
437 : // NOLINTNEXTLINE(clang-analyzer-unix.BlockInCriticalSection)
438 8687 : [[maybe_unused]] auto r = ::read(event_fd_, &val, sizeof(val));
439 8687 : eventfd_armed_.store(false, std::memory_order_relaxed);
440 8687 : continue;
441 8687 : }
442 :
443 49929 : if (event_buffer_[i].data.ptr == &timer_fd_)
444 : {
445 : std::uint64_t expirations;
446 : // NOLINTNEXTLINE(clang-analyzer-unix.BlockInCriticalSection)
447 : [[maybe_unused]] auto r =
448 7954 : ::read(timer_fd_, &expirations, sizeof(expirations));
449 7954 : check_timers = true;
450 7954 : continue;
451 7954 : }
452 :
453 : auto* desc =
454 41975 : static_cast<reactor_descriptor_state*>(event_buffer_[i].data.ptr);
455 :
456 : // EPOLLHUP maps to no reactor_event_* bit, so a HUP the widening
457 : // below did not translate would take no branch in
458 : // invoke_deferred_io() and, the registration being
459 : // edge-triggered, never get another chance.
460 : //
461 : // HUP without OUT means a non-socket: a pipe or tty whose peer
462 : // closed, with or without data still unread (HUP alone, or
463 : // IN|HUP). Treat it as readable, writable and faulted, the way
464 : // poll() and io_uring report it, so a parked error wait
465 : // completes too. Sockets carry at least OUT with HUP (a fresh
466 : // unconnected stream socket reports HUP|OUT), so they take the
467 : // plain widening, where forcing IN costs at most one spurious
468 : // EAGAIN.
469 41975 : std::uint32_t ev = event_buffer_[i].events;
470 41975 : if ((ev & (EPOLLHUP | EPOLLOUT)) == EPOLLHUP)
471 8 : ev |= EPOLLIN | EPOLLOUT | EPOLLERR;
472 41967 : else if (ev & EPOLLHUP)
473 3280 : ev |= EPOLLIN | EPOLLOUT;
474 41975 : desc->add_ready_events(ev);
475 :
476 41975 : bool expected = false;
477 41975 : if (desc->is_enqueued_.compare_exchange_strong(
478 : expected, true, std::memory_order_release,
479 : std::memory_order_relaxed))
480 : {
481 41966 : local_ops.push(desc);
482 : }
483 : }
484 :
485 147070 : if (check_timers)
486 : {
487 7958 : if (auto* svc = timers())
488 7958 : svc->process_expired();
489 7958 : update_timerfd();
490 : }
491 :
492 147070 : lock.lock();
493 :
494 147070 : completed_ops_.splice(local_ops);
495 147072 : }
496 :
497 : } // namespace boost::corosio::detail
498 :
499 : #endif // BOOST_COROSIO_HAS_EPOLL
500 :
501 : #endif // BOOST_COROSIO_NATIVE_DETAIL_EPOLL_EPOLL_SCHEDULER_HPP
|