Ruby: thread_pthread.c Source File

Ruby 3.0.5p211 (2022-11-24 revision ba5cf0f7c52d4d35cc6a173c89eda98ceffa2dcf)

thread_pthread.c
Go to the documentation of this file.
1/* -*-c-*- */
2/**********************************************************************
3
4 thread_pthread.c -
5
6 $Author$
7
8 Copyright (C) 2004-2007 Koichi Sasada
9
10**********************************************************************/
11
12#ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION
13
14#include "gc.h"
15#include "mjit.h"
16
17#ifdef HAVE_SYS_RESOURCE_H
18#include <sys/resource.h>
19#endif
20#ifdef HAVE_THR_STKSEGMENT
21#include <thread.h>
22#endif
23#if HAVE_FCNTL_H
24#include <fcntl.h>
25#elif HAVE_SYS_FCNTL_H
26#include <sys/fcntl.h>
27#endif
28#ifdef HAVE_SYS_PRCTL_H
29#include <sys/prctl.h>
30#endif
31#if defined(HAVE_SYS_TIME_H)
32#include <sys/time.h>
33#endif
34#if defined(__HAIKU__)
35#include <kernel/OS.h>
36#endif
37#include <time.h>
38#include <signal.h>
39
40#if defined(HAVE_SYS_EVENTFD_H) && defined(HAVE_EVENTFD)
41# define USE_EVENTFD (1)
42# include <sys/eventfd.h>
43#else
44# define USE_EVENTFD (0)
45#endif
46
47#if defined(SIGVTALRM) && !defined(__CYGWIN__)
48# define USE_UBF_LIST 1
49#endif
50
51/*
52 * UBF_TIMER and ubf_list both use SIGVTALRM.
53 *
54 * UBF_TIMER has NOTHING to do with thread timeslices (TIMER_INTERRUPT_MASK)
55 *
56 * UBF_TIMER is to close TOCTTOU signal race on programs where we
57 * cannot rely on GVL contention (vm->gvl.timer) to perform wakeups
58 * while a thread is doing blocking I/O on sockets or pipes. With
59 * rb_thread_call_without_gvl and similar functions:
60 *
61 * (1) Check interrupts.
62 * (2) release GVL.
63 * (2a) signal received
64 * (3) call func with data1 (blocks for a long time without ubf_timer)
65 * (4) acquire GVL.
66 * Other Ruby threads can not run in parallel any more.
67 * (5) Check interrupts.
68 *
69 * We need UBF_TIMER to break out of (3) if (2a) happens.
70 *
71 * ubf_list wakeups may be triggered on gvl_yield.
72 *
73 * If we have vm->gvl.timer (on GVL contention), we don't need UBF_TIMER
74 * as it can perform the same tasks while doing timeslices.
75 */
76#define UBF_TIMER_NONE 0
77#define UBF_TIMER_POSIX 1
78#define UBF_TIMER_PTHREAD 2
79
80#ifndef UBF_TIMER
81# if defined(HAVE_TIMER_SETTIME) && defined(HAVE_TIMER_CREATE) && \
82 defined(CLOCK_MONOTONIC) && defined(USE_UBF_LIST)
83 /* preferred */
84# define UBF_TIMER UBF_TIMER_POSIX
85# elif defined(USE_UBF_LIST)
86 /* safe, but inefficient */
87# define UBF_TIMER UBF_TIMER_PTHREAD
88# else
89 /* we'll be racy without SIGVTALRM for ubf_list */
90# define UBF_TIMER UBF_TIMER_NONE
91# endif
92#endif
93
94enum rtimer_state {
95 /* alive, after timer_create: */
96 RTIMER_DISARM,
97 RTIMER_ARMING,
98 RTIMER_ARMED,
99
100 RTIMER_DEAD
101};
102
103#if UBF_TIMER == UBF_TIMER_POSIX
104static const struct itimerspec zero;
105static struct {
106 rb_atomic_t state; /* rtimer_state */
107 rb_pid_t owner;
108 timer_t timerid;
109} timer_posix = {
110 /* .state = */ RTIMER_DEAD,
111};
112
113#elif UBF_TIMER == UBF_TIMER_PTHREAD
114static void *timer_pthread_fn(void *);
115static struct {
116 int low[2];
117 rb_atomic_t armed; /* boolean */
118 rb_pid_t owner;
119 pthread_t thid;
120} timer_pthread = {
121 { -1, -1 },
122};
123#endif
124
125static const rb_hrtime_t *sigwait_timeout(rb_thread_t *, int sigwait_fd,
126 const rb_hrtime_t *,
127 int *drained_p);
128static void ubf_timer_disarm(void);
129static void threadptr_trap_interrupt(rb_thread_t *);
130static void clear_thread_cache_altstack(void);
131static void ubf_wakeup_all_threads(void);
132static int ubf_threads_empty(void);
133
134#define TIMER_THREAD_CREATED_P() (signal_self_pipe.owner_process == getpid())
135
136/* for testing, and in case we come across a platform w/o pipes: */
137#define BUSY_WAIT_SIGNALS (0)
138
139/*
140 * sigwait_th is the thread which owns sigwait_fd and sleeps on it
141 * (using ppoll). MJIT worker can be sigwait_th==0, so we initialize
142 * it to THREAD_INVALID at startup and fork time. It is the ONLY thread
143 * allowed to read from sigwait_fd, otherwise starvation can occur.
144 */
145#define THREAD_INVALID ((const rb_thread_t *)-1)
146static const rb_thread_t *sigwait_th;
147
148#ifdef HAVE_SCHED_YIELD
149#define native_thread_yield() (void)sched_yield()
150#else
151#define native_thread_yield() ((void)0)
152#endif
153
154#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
155 defined(CLOCK_REALTIME) && defined(CLOCK_MONOTONIC) && \
156 defined(HAVE_CLOCK_GETTIME)
157static pthread_condattr_t condattr_mono;
158static pthread_condattr_t *condattr_monotonic = &condattr_mono;
159#else
160static const void *const condattr_monotonic = NULL;
161#endif
162
163/* 100ms. 10ms is too small for user level thread scheduling
164 * on recent Linux (tested on 2.6.35)
165 */
166#define TIME_QUANTUM_MSEC (100)
167#define TIME_QUANTUM_USEC (TIME_QUANTUM_MSEC * 1000)
168#define TIME_QUANTUM_NSEC (TIME_QUANTUM_USEC * 1000)
169
170static rb_hrtime_t native_cond_timeout(rb_nativethread_cond_t *, rb_hrtime_t);
171static int native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, const rb_hrtime_t *abs);
172
173/*
174 * Designate the next gvl.timer thread, favor the last thread in
175 * the waitq since it will be in waitq longest
176 */
177static int
178designate_timer_thread(rb_global_vm_lock_t *gvl)
179{
180 native_thread_data_t *last;
181
182 last = list_tail(&gvl->waitq, native_thread_data_t, node.ubf);
183 if (last) {
184 rb_native_cond_signal(&last->cond.gvlq);
185 return TRUE;
186 }
187 return FALSE;
188}
189
190/*
191 * We become designated timer thread to kick vm->gvl.owner
192 * periodically. Continue on old timeout if it expired.
193 */
194static void
195do_gvl_timer(rb_global_vm_lock_t *gvl, rb_thread_t *th)
196{
197 rb_vm_t *vm = GET_VM();
198 static rb_hrtime_t abs;
199 native_thread_data_t *nd = &th->native_thread_data;
200
201 gvl->timer = th;
202
203 /* take over wakeups from UBF_TIMER */
204 ubf_timer_disarm();
205
206 if (gvl->timer_err == ETIMEDOUT) {
207 abs = native_cond_timeout(&nd->cond.gvlq, TIME_QUANTUM_NSEC);
208 }
209 gvl->timer_err = native_cond_timedwait(&nd->cond.gvlq, &gvl->lock, &abs);
210
211 ubf_wakeup_all_threads();
212 ruby_sigchld_handler(vm);
213
214 if (UNLIKELY(rb_signal_buff_size())) {
215 if (th == vm->ractor.main_thread) {
216 RUBY_VM_SET_TRAP_INTERRUPT(th->ec);
217 }
218 else {
219 threadptr_trap_interrupt(vm->ractor.main_thread);
220 }
221 }
222
223 /*
224 * Timeslice. Warning: the process may fork while this
225 * thread is contending for GVL:
226 */
227 if (gvl->owner) {
228 // strictly speaking, accessing "gvl->owner" is not thread-safe
229 RUBY_VM_SET_TIMER_INTERRUPT(gvl->owner->ec);
230 }
231 gvl->timer = 0;
232}
233
234static void
235gvl_acquire_common(rb_global_vm_lock_t *gvl, rb_thread_t *th)
236{
237 if (gvl->owner) {
238 native_thread_data_t *nd = &th->native_thread_data;
239
240 VM_ASSERT(th->unblock.func == 0 &&
241 "we must not be in ubf_list and GVL waitq at the same time");
242
243 list_add_tail(&gvl->waitq, &nd->node.gvl);
244
245 do {
246 if (!gvl->timer) {
247 do_gvl_timer(gvl, th);
248 }
249 else {
250 rb_native_cond_wait(&nd->cond.gvlq, &gvl->lock);
251 }
252 } while (gvl->owner);
253
254 list_del_init(&nd->node.gvl);
255
256 if (gvl->need_yield) {
257 gvl->need_yield = 0;
258 rb_native_cond_signal(&gvl->switch_cond);
259 }
260 }
261 else { /* reset timer if uncontended */
262 gvl->timer_err = ETIMEDOUT;
263 }
264 gvl->owner = th;
265 if (!gvl->timer) {
266 if (!designate_timer_thread(gvl) && !ubf_threads_empty()) {
267 rb_thread_wakeup_timer_thread(-1);
268 }
269 }
270}
271
272static void
273gvl_acquire(rb_global_vm_lock_t *gvl, rb_thread_t *th)
274{
275 rb_native_mutex_lock(&gvl->lock);
276 gvl_acquire_common(gvl, th);
277 rb_native_mutex_unlock(&gvl->lock);
278}
279
280static const native_thread_data_t *
281gvl_release_common(rb_global_vm_lock_t *gvl)
282{
283 native_thread_data_t *next;
284 gvl->owner = 0;
285 next = list_top(&gvl->waitq, native_thread_data_t, node.ubf);
286 if (next) rb_native_cond_signal(&next->cond.gvlq);
287
288 return next;
289}
290
291static void
292gvl_release(rb_global_vm_lock_t *gvl)
293{
294 rb_native_mutex_lock(&gvl->lock);
295 gvl_release_common(gvl);
296 rb_native_mutex_unlock(&gvl->lock);
297}
298
299static void
300gvl_yield(rb_global_vm_lock_t *gvl, rb_thread_t *th)
301{
302 const native_thread_data_t *next;
303
304 /*
305 * Perhaps other threads are stuck in blocking region w/o GVL, too,
306 * (perhaps looping in io_close_fptr) so we kick them:
307 */
308 ubf_wakeup_all_threads();
309 rb_native_mutex_lock(&gvl->lock);
310 next = gvl_release_common(gvl);
311
312 /* An another thread is processing GVL yield. */
313 if (UNLIKELY(gvl->wait_yield)) {
314 while (gvl->wait_yield)
315 rb_native_cond_wait(&gvl->switch_wait_cond, &gvl->lock);
316 }
317 else if (next) {
318 /* Wait until another thread task takes GVL. */
319 gvl->need_yield = 1;
320 gvl->wait_yield = 1;
321 while (gvl->need_yield)
322 rb_native_cond_wait(&gvl->switch_cond, &gvl->lock);
323 gvl->wait_yield = 0;
324 rb_native_cond_broadcast(&gvl->switch_wait_cond);
325 }
326 else {
327 rb_native_mutex_unlock(&gvl->lock);
328 native_thread_yield();
329 rb_native_mutex_lock(&gvl->lock);
330 rb_native_cond_broadcast(&gvl->switch_wait_cond);
331 }
332 gvl_acquire_common(gvl, th);
333 rb_native_mutex_unlock(&gvl->lock);
334}
335
336void
337 rb_gvl_init(rb_global_vm_lock_t *gvl)
338{
339 rb_native_mutex_initialize(&gvl->lock);
340 rb_native_cond_initialize(&gvl->switch_cond);
341 rb_native_cond_initialize(&gvl->switch_wait_cond);
342 list_head_init(&gvl->waitq);
343 gvl->owner = 0;
344 gvl->timer = 0;
345 gvl->timer_err = ETIMEDOUT;
346 gvl->need_yield = 0;
347 gvl->wait_yield = 0;
348}
349
350static void
351gvl_destroy(rb_global_vm_lock_t *gvl)
352{
353 /*
354 * only called once at VM shutdown (not atfork), another thread
355 * may still grab vm->gvl.lock when calling gvl_release at
356 * the end of thread_start_func_2
357 */
358 if (0) {
359 rb_native_cond_destroy(&gvl->switch_wait_cond);
360 rb_native_cond_destroy(&gvl->switch_cond);
361 rb_native_mutex_destroy(&gvl->lock);
362 }
363 clear_thread_cache_altstack();
364}
365
366#if defined(HAVE_WORKING_FORK)
367static void thread_cache_reset(void);
368static void
369gvl_atfork(rb_global_vm_lock_t *gvl)
370{
371 thread_cache_reset();
372 rb_gvl_init(gvl);
373 gvl_acquire(gvl, GET_THREAD());
374}
375#endif
376
377#define NATIVE_MUTEX_LOCK_DEBUG 0
378
379static void
380mutex_debug(const char *msg, void *lock)
381{
382 if (NATIVE_MUTEX_LOCK_DEBUG) {
383 int r;
384 static pthread_mutex_t dbglock = PTHREAD_MUTEX_INITIALIZER;
385
386 if ((r = pthread_mutex_lock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
387 fprintf(stdout, "%s: %p\n", msg, lock);
388 if ((r = pthread_mutex_unlock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
389 }
390}
391
392void
393 rb_native_mutex_lock(pthread_mutex_t *lock)
394{
395 int r;
396 mutex_debug("lock", lock);
397 if ((r = pthread_mutex_lock(lock)) != 0) {
398 rb_bug_errno("pthread_mutex_lock", r);
399 }
400}
401
402void
403 rb_native_mutex_unlock(pthread_mutex_t *lock)
404{
405 int r;
406 mutex_debug("unlock", lock);
407 if ((r = pthread_mutex_unlock(lock)) != 0) {
408 rb_bug_errno("pthread_mutex_unlock", r);
409 }
410}
411
412int
413 rb_native_mutex_trylock(pthread_mutex_t *lock)
414{
415 int r;
416 mutex_debug("trylock", lock);
417 if ((r = pthread_mutex_trylock(lock)) != 0) {
418 if (r == EBUSY) {
419 return EBUSY;
420 }
421 else {
422 rb_bug_errno("pthread_mutex_trylock", r);
423 }
424 }
425 return 0;
426}
427
428void
429 rb_native_mutex_initialize(pthread_mutex_t *lock)
430{
431 int r = pthread_mutex_init(lock, 0);
432 mutex_debug("init", lock);
433 if (r != 0) {
434 rb_bug_errno("pthread_mutex_init", r);
435 }
436}
437
438void
439 rb_native_mutex_destroy(pthread_mutex_t *lock)
440{
441 int r = pthread_mutex_destroy(lock);
442 mutex_debug("destroy", lock);
443 if (r != 0) {
444 rb_bug_errno("pthread_mutex_destroy", r);
445 }
446}
447
448void
449 rb_native_cond_initialize(rb_nativethread_cond_t *cond)
450{
451 int r = pthread_cond_init(cond, condattr_monotonic);
452 if (r != 0) {
453 rb_bug_errno("pthread_cond_init", r);
454 }
455}
456
457void
458 rb_native_cond_destroy(rb_nativethread_cond_t *cond)
459{
460 int r = pthread_cond_destroy(cond);
461 if (r != 0) {
462 rb_bug_errno("pthread_cond_destroy", r);
463 }
464}
465
466/*
467 * In OS X 10.7 (Lion), pthread_cond_signal and pthread_cond_broadcast return
468 * EAGAIN after retrying 8192 times. You can see them in the following page:
469 *
470 * http://www.opensource.apple.com/source/Libc/Libc-763.11/pthreads/pthread_cond.c
471 *
472 * The following rb_native_cond_signal and rb_native_cond_broadcast functions
473 * need to retrying until pthread functions don't return EAGAIN.
474 */
475
476void
477 rb_native_cond_signal(rb_nativethread_cond_t *cond)
478{
479 int r;
480 do {
481 r = pthread_cond_signal(cond);
482 } while (r == EAGAIN);
483 if (r != 0) {
484 rb_bug_errno("pthread_cond_signal", r);
485 }
486}
487
488void
489 rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
490{
491 int r;
492 do {
493 r = pthread_cond_broadcast(cond);
494 } while (r == EAGAIN);
495 if (r != 0) {
496 rb_bug_errno("rb_native_cond_broadcast", r);
497 }
498}
499
500void
501 rb_native_cond_wait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex)
502{
503 int r = pthread_cond_wait(cond, mutex);
504 if (r != 0) {
505 rb_bug_errno("pthread_cond_wait", r);
506 }
507}
508
509static int
510native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, const rb_hrtime_t *abs)
511{
512 int r;
513 struct timespec ts;
514
515 /*
516 * An old Linux may return EINTR. Even though POSIX says
517 * "These functions shall not return an error code of [EINTR]".
518 * http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_cond_timedwait.html
519 * Let's hide it from arch generic code.
520 */
521 do {
522 rb_hrtime2timespec(&ts, abs);
523 r = pthread_cond_timedwait(cond, mutex, &ts);
524 } while (r == EINTR);
525
526 if (r != 0 && r != ETIMEDOUT) {
527 rb_bug_errno("pthread_cond_timedwait", r);
528 }
529
530 return r;
531}
532
533void
534 rb_native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, unsigned long msec)
535{
536 rb_hrtime_t hrmsec = native_cond_timeout(cond, RB_HRTIME_PER_MSEC * msec);
537 native_cond_timedwait(cond, mutex, &hrmsec);
538}
539
540static rb_hrtime_t
541native_cond_timeout(rb_nativethread_cond_t *cond, const rb_hrtime_t rel)
542{
543 if (condattr_monotonic) {
544 return rb_hrtime_add(rb_hrtime_now(), rel);
545 }
546 else {
547 struct timespec ts;
548
549 rb_timespec_now(&ts);
550 return rb_hrtime_add(rb_timespec2hrtime(&ts), rel);
551 }
552}
553
554#define native_cleanup_push pthread_cleanup_push
555#define native_cleanup_pop pthread_cleanup_pop
556
557#ifdef RB_THREAD_LOCAL_SPECIFIER
558static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread;
559#else
560static pthread_key_t ruby_native_thread_key;
561#endif
562
563static void
564null_func(int i)
565{
566 /* null */
567}
568
569static rb_thread_t *
570ruby_thread_from_native(void)
571{
572#ifdef RB_THREAD_LOCAL_SPECIFIER
573 return ruby_native_thread;
574#else
575 return pthread_getspecific(ruby_native_thread_key);
576#endif
577}
578
579static int
580ruby_thread_set_native(rb_thread_t *th)
581{
582 if (th && th->ec) {
583 rb_ractor_set_current_ec(th->ractor, th->ec);
584 }
585#ifdef RB_THREAD_LOCAL_SPECIFIER
586 ruby_native_thread = th;
587 return 1;
588#else
589 return pthread_setspecific(ruby_native_thread_key, th) == 0;
590#endif
591}
592
593static void native_thread_init(rb_thread_t *th);
594
595void
596 Init_native_thread(rb_thread_t *th)
597{
598#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
599 if (condattr_monotonic) {
600 int r = pthread_condattr_init(condattr_monotonic);
601 if (r == 0) {
602 r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC);
603 }
604 if (r) condattr_monotonic = NULL;
605 }
606#endif
607
608#ifndef RB_THREAD_LOCAL_SPECIFIER
609 if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) {
610 rb_bug("pthread_key_create failed (ruby_native_thread_key)");
611 }
612 if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) {
613 rb_bug("pthread_key_create failed (ruby_current_ec_key)");
614 }
615#endif
616 th->thread_id = pthread_self();
617 ruby_thread_set_native(th);
618 fill_thread_id_str(th);
619 native_thread_init(th);
620 posix_signal(SIGVTALRM, null_func);
621}
622
623static void
624native_thread_init(rb_thread_t *th)
625{
626 native_thread_data_t *nd = &th->native_thread_data;
627
628#ifdef USE_UBF_LIST
629 list_node_init(&nd->node.ubf);
630#endif
631 rb_native_cond_initialize(&nd->cond.gvlq);
632 if (&nd->cond.gvlq != &nd->cond.intr)
633 rb_native_cond_initialize(&nd->cond.intr);
634}
635
636#ifndef USE_THREAD_CACHE
637#define USE_THREAD_CACHE 1
638#endif
639
640static void
641native_thread_destroy(rb_thread_t *th)
642{
643 native_thread_data_t *nd = &th->native_thread_data;
644
645 rb_native_cond_destroy(&nd->cond.gvlq);
646 if (&nd->cond.gvlq != &nd->cond.intr)
647 rb_native_cond_destroy(&nd->cond.intr);
648
649 /*
650 * prevent false positive from ruby_thread_has_gvl_p if that
651 * gets called from an interposing function wrapper
652 */
653 if (USE_THREAD_CACHE)
654 ruby_thread_set_native(0);
655}
656
657#if USE_THREAD_CACHE
658static rb_thread_t *register_cached_thread_and_wait(void *);
659#endif
660
661#if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP
662#define STACKADDR_AVAILABLE 1
663#elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP
664#define STACKADDR_AVAILABLE 1
665#undef MAINSTACKADDR_AVAILABLE
666#define MAINSTACKADDR_AVAILABLE 1
667void *pthread_get_stackaddr_np(pthread_t);
668size_t pthread_get_stacksize_np(pthread_t);
669#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
670#define STACKADDR_AVAILABLE 1
671#elif defined HAVE_PTHREAD_GETTHRDS_NP
672#define STACKADDR_AVAILABLE 1
673#elif defined __HAIKU__
674#define STACKADDR_AVAILABLE 1
675#endif
676
677#ifndef MAINSTACKADDR_AVAILABLE
678# ifdef STACKADDR_AVAILABLE
679# define MAINSTACKADDR_AVAILABLE 1
680# else
681# define MAINSTACKADDR_AVAILABLE 0
682# endif
683#endif
684#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack)
685# define get_main_stack(addr, size) get_stack(addr, size)
686#endif
687
688#ifdef STACKADDR_AVAILABLE
689/*
690 * Get the initial address and size of current thread's stack
691 */
692static int
693get_stack(void **addr, size_t *size)
694{
695#define CHECK_ERR(expr) \
696 {int err = (expr); if (err) return err;}
697#ifdef HAVE_PTHREAD_GETATTR_NP /* Linux */
698 pthread_attr_t attr;
699 size_t guard = 0;
700 STACK_GROW_DIR_DETECTION;
701 CHECK_ERR(pthread_getattr_np(pthread_self(), &attr));
702# ifdef HAVE_PTHREAD_ATTR_GETSTACK
703 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
704 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
705# else
706 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
707 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
708# endif
709# ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE
710 CHECK_ERR(pthread_attr_getguardsize(&attr, &guard));
711 *size -= guard;
712# else
713 *size -= getpagesize();
714# endif
715 pthread_attr_destroy(&attr);
716#elif defined HAVE_PTHREAD_ATTR_GET_NP /* FreeBSD, DragonFly BSD, NetBSD */
717 pthread_attr_t attr;
718 CHECK_ERR(pthread_attr_init(&attr));
719 CHECK_ERR(pthread_attr_get_np(pthread_self(), &attr));
720# ifdef HAVE_PTHREAD_ATTR_GETSTACK
721 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
722# else
723 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
724 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
725# endif
726 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
727 pthread_attr_destroy(&attr);
728#elif (defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP) /* MacOS X */
729 pthread_t th = pthread_self();
730 *addr = pthread_get_stackaddr_np(th);
731 *size = pthread_get_stacksize_np(th);
732#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
733 stack_t stk;
734# if defined HAVE_THR_STKSEGMENT /* Solaris */
735 CHECK_ERR(thr_stksegment(&stk));
736# else /* OpenBSD */
737 CHECK_ERR(pthread_stackseg_np(pthread_self(), &stk));
738# endif
739 *addr = stk.ss_sp;
740 *size = stk.ss_size;
741#elif defined HAVE_PTHREAD_GETTHRDS_NP /* AIX */
742 pthread_t th = pthread_self();
743 struct __pthrdsinfo thinfo;
744 char reg[256];
745 int regsiz=sizeof(reg);
746 CHECK_ERR(pthread_getthrds_np(&th, PTHRDSINFO_QUERY_ALL,
747 &thinfo, sizeof(thinfo),
748 &reg, &regsiz));
749 *addr = thinfo.__pi_stackaddr;
750 /* Must not use thinfo.__pi_stacksize for size.
751 It is around 3KB smaller than the correct size
752 calculated by thinfo.__pi_stackend - thinfo.__pi_stackaddr. */
753 *size = thinfo.__pi_stackend - thinfo.__pi_stackaddr;
754 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
755#elif defined __HAIKU__
756 thread_info info;
757 STACK_GROW_DIR_DETECTION;
758 CHECK_ERR(get_thread_info(find_thread(NULL), &info));
759 *addr = info.stack_base;
760 *size = (uintptr_t)info.stack_end - (uintptr_t)info.stack_base;
761 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
762#else
763#error STACKADDR_AVAILABLE is defined but not implemented.
764#endif
765 return 0;
766#undef CHECK_ERR
767}
768#endif
769
770static struct {
771 rb_nativethread_id_t id;
772 size_t stack_maxsize;
773 VALUE *stack_start;
774} native_main_thread;
775
776#ifdef STACK_END_ADDRESS
777extern void *STACK_END_ADDRESS;
778#endif
779
780enum {
781 RUBY_STACK_SPACE_LIMIT = 1024 * 1024, /* 1024KB */
782 RUBY_STACK_SPACE_RATIO = 5
783};
784
785static size_t
786space_size(size_t stack_size)
787{
788 size_t space_size = stack_size / RUBY_STACK_SPACE_RATIO;
789 if (space_size > RUBY_STACK_SPACE_LIMIT) {
790 return RUBY_STACK_SPACE_LIMIT;
791 }
792 else {
793 return space_size;
794 }
795}
796
797#ifdef __linux__
798static __attribute__((noinline)) void
799reserve_stack(volatile char *limit, size_t size)
800{
801# ifdef C_ALLOCA
802# error needs alloca()
803# endif
804 struct rlimit rl;
805 volatile char buf[0x100];
806 enum {stack_check_margin = 0x1000}; /* for -fstack-check */
807
808 STACK_GROW_DIR_DETECTION;
809
810 if (!getrlimit(RLIMIT_STACK, &rl) && rl.rlim_cur == RLIM_INFINITY)
811 return;
812
813 if (size < stack_check_margin) return;
814 size -= stack_check_margin;
815
816 size -= sizeof(buf); /* margin */
817 if (IS_STACK_DIR_UPPER()) {
818 const volatile char *end = buf + sizeof(buf);
819 limit += size;
820 if (limit > end) {
821 /* |<-bottom (=limit(a)) top->|
822 * | .. |<-buf 256B |<-end | stack check |
823 * | 256B | =size= | margin (4KB)|
824 * | =size= limit(b)->| 256B | |
825 * | | alloca(sz) | | |
826 * | .. |<-buf |<-limit(c) [sz-1]->0> | |
827 */
828 size_t sz = limit - end;
829 limit = alloca(sz);
830 limit[sz-1] = 0;
831 }
832 }
833 else {
834 limit -= size;
835 if (buf > limit) {
836 /* |<-top (=limit(a)) bottom->|
837 * | .. | 256B buf->| | stack check |
838 * | 256B | =size= | margin (4KB)|
839 * | =size= limit(b)->| 256B | |
840 * | | alloca(sz) | | |
841 * | .. | buf->| limit(c)-><0> | |
842 */
843 size_t sz = buf - limit;
844 limit = alloca(sz);
845 limit[0] = 0;
846 }
847 }
848}
849#else
850# define reserve_stack(limit, size) ((void)(limit), (void)(size))
851#endif
852
853#undef ruby_init_stack
854/* Set stack bottom of Ruby implementation.
855 *
856 * You must call this function before any heap allocation by Ruby implementation.
857 * Or GC will break living objects */
858void
859 ruby_init_stack(volatile VALUE *addr)
860{
861 native_main_thread.id = pthread_self();
862
863#if MAINSTACKADDR_AVAILABLE
864 if (native_main_thread.stack_maxsize) return;
865 {
866 void* stackaddr;
867 size_t size;
868 if (get_main_stack(&stackaddr, &size) == 0) {
869 native_main_thread.stack_maxsize = size;
870 native_main_thread.stack_start = stackaddr;
871 reserve_stack(stackaddr, size);
872 goto bound_check;
873 }
874 }
875#endif
876#ifdef STACK_END_ADDRESS
877 native_main_thread.stack_start = STACK_END_ADDRESS;
878#else
879 if (!native_main_thread.stack_start ||
880 STACK_UPPER((VALUE *)(void *)&addr,
881 native_main_thread.stack_start > addr,
882 native_main_thread.stack_start < addr)) {
883 native_main_thread.stack_start = (VALUE *)addr;
884 }
885#endif
886 {
887#if defined(HAVE_GETRLIMIT)
888#if defined(PTHREAD_STACK_DEFAULT)
889# if PTHREAD_STACK_DEFAULT < RUBY_STACK_SPACE*5
890# error "PTHREAD_STACK_DEFAULT is too small"
891# endif
892 size_t size = PTHREAD_STACK_DEFAULT;
893#else
894 size_t size = RUBY_VM_THREAD_VM_STACK_SIZE;
895#endif
896 size_t space;
897 int pagesize = getpagesize();
898 struct rlimit rlim;
899 STACK_GROW_DIR_DETECTION;
900 if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
901 size = (size_t)rlim.rlim_cur;
902 }
903 addr = native_main_thread.stack_start;
904 if (IS_STACK_DIR_UPPER()) {
905 space = ((size_t)((char *)addr + size) / pagesize) * pagesize - (size_t)addr;
906 }
907 else {
908 space = (size_t)addr - ((size_t)((char *)addr - size) / pagesize + 1) * pagesize;
909 }
910 native_main_thread.stack_maxsize = space;
911#endif
912 }
913
914#if MAINSTACKADDR_AVAILABLE
915 bound_check:
916#endif
917 /* If addr is out of range of main-thread stack range estimation, */
918 /* it should be on co-routine (alternative stack). [Feature #2294] */
919 {
920 void *start, *end;
921 STACK_GROW_DIR_DETECTION;
922
923 if (IS_STACK_DIR_UPPER()) {
924 start = native_main_thread.stack_start;
925 end = (char *)native_main_thread.stack_start + native_main_thread.stack_maxsize;
926 }
927 else {
928 start = (char *)native_main_thread.stack_start - native_main_thread.stack_maxsize;
929 end = native_main_thread.stack_start;
930 }
931
932 if ((void *)addr < start || (void *)addr > end) {
933 /* out of range */
934 native_main_thread.stack_start = (VALUE *)addr;
935 native_main_thread.stack_maxsize = 0; /* unknown */
936 }
937 }
938}
939
940#define CHECK_ERR(expr) \
941 {int err = (expr); if (err) {rb_bug_errno(#expr, err);}}
942
943static int
944native_thread_init_stack(rb_thread_t *th)
945{
946 rb_nativethread_id_t curr = pthread_self();
947
948 if (pthread_equal(curr, native_main_thread.id)) {
949 th->ec->machine.stack_start = native_main_thread.stack_start;
950 th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize;
951 }
952 else {
953#ifdef STACKADDR_AVAILABLE
954 void *start;
955 size_t size;
956
957 if (get_stack(&start, &size) == 0) {
958 uintptr_t diff = (uintptr_t)start - (uintptr_t)&curr;
959 th->ec->machine.stack_start = (VALUE *)&curr;
960 th->ec->machine.stack_maxsize = size - diff;
961 }
962#else
963 rb_raise(rb_eNotImpError, "ruby engine can initialize only in the main thread");
964#endif
965 }
966
967 return 0;
968}
969
970#ifndef __CYGWIN__
971#define USE_NATIVE_THREAD_INIT 1
972#endif
973
974static void *
975thread_start_func_1(void *th_ptr)
976{
977 rb_thread_t *th = th_ptr;
978 RB_ALTSTACK_INIT(void *altstack, th->altstack);
979#if USE_THREAD_CACHE
980 thread_start:
981#endif
982 {
983#if !defined USE_NATIVE_THREAD_INIT
984 VALUE stack_start;
985#endif
986
987 fill_thread_id_str(th);
988#if defined USE_NATIVE_THREAD_INIT
989 native_thread_init_stack(th);
990#endif
991 native_thread_init(th);
992 /* run */
993#if defined USE_NATIVE_THREAD_INIT
994 thread_start_func_2(th, th->ec->machine.stack_start);
995#else
996 thread_start_func_2(th, &stack_start);
997#endif
998 }
999#if USE_THREAD_CACHE
1000 /* cache thread */
1001 if ((th = register_cached_thread_and_wait(RB_ALTSTACK(altstack))) != 0) {
1002 goto thread_start;
1003 }
1004#else
1005 RB_ALTSTACK_FREE(altstack);
1006#endif
1007 return 0;
1008}
1009
1010struct cached_thread_entry {
1011 rb_nativethread_cond_t cond;
1012 rb_nativethread_id_t thread_id;
1013 rb_thread_t *th;
1014 void *altstack;
1015 struct list_node node;
1016};
1017
1018#if USE_THREAD_CACHE
1019static rb_nativethread_lock_t thread_cache_lock = RB_NATIVETHREAD_LOCK_INIT;
1020static LIST_HEAD(cached_thread_head);
1021
1022# if defined(HAVE_WORKING_FORK)
1023static void
1024thread_cache_reset(void)
1025{
1026 rb_native_mutex_initialize(&thread_cache_lock);
1027 list_head_init(&cached_thread_head);
1028}
1029# endif
1030
1031/*
1032 * number of seconds to cache for, I think 1-5s is sufficient to obviate
1033 * the need for thread pool in many network programs (taking into account
1034 * worst case network latency across the globe) without wasting memory
1035 */
1036#ifndef THREAD_CACHE_TIME
1037# define THREAD_CACHE_TIME ((rb_hrtime_t)3 * RB_HRTIME_PER_SEC)
1038#endif
1039
1040static rb_thread_t *
1041register_cached_thread_and_wait(void *altstack)
1042{
1043 rb_hrtime_t end = THREAD_CACHE_TIME;
1044 struct cached_thread_entry entry;
1045
1046 rb_native_cond_initialize(&entry.cond);
1047 entry.altstack = altstack;
1048 entry.th = NULL;
1049 entry.thread_id = pthread_self();
1050 end = native_cond_timeout(&entry.cond, end);
1051
1052 rb_native_mutex_lock(&thread_cache_lock);
1053 {
1054 list_add(&cached_thread_head, &entry.node);
1055
1056 native_cond_timedwait(&entry.cond, &thread_cache_lock, &end);
1057
1058 if (entry.th == NULL) { /* unused */
1059 list_del(&entry.node);
1060 }
1061 }
1062 rb_native_mutex_unlock(&thread_cache_lock);
1063
1064 rb_native_cond_destroy(&entry.cond);
1065 if (!entry.th) {
1066 RB_ALTSTACK_FREE(entry.altstack);
1067 }
1068
1069 return entry.th;
1070}
1071#else
1072# if defined(HAVE_WORKING_FORK)
1073static void thread_cache_reset(void) { }
1074# endif
1075#endif
1076
1077static int
1078use_cached_thread(rb_thread_t *th)
1079{
1080#if USE_THREAD_CACHE
1081 struct cached_thread_entry *entry;
1082
1083 rb_native_mutex_lock(&thread_cache_lock);
1084 entry = list_pop(&cached_thread_head, struct cached_thread_entry, node);
1085 if (entry) {
1086 entry->th = th;
1087 /* th->thread_id must be set before signal for Thread#name= */
1088 th->thread_id = entry->thread_id;
1089 fill_thread_id_str(th);
1090 rb_native_cond_signal(&entry->cond);
1091 }
1092 rb_native_mutex_unlock(&thread_cache_lock);
1093 return !!entry;
1094#endif
1095 return 0;
1096}
1097
1098static void
1099clear_thread_cache_altstack(void)
1100{
1101#if USE_THREAD_CACHE
1102 struct cached_thread_entry *entry;
1103
1104 rb_native_mutex_lock(&thread_cache_lock);
1105 list_for_each(&cached_thread_head, entry, node) {
1106 void MAYBE_UNUSED(*altstack) = entry->altstack;
1107 entry->altstack = 0;
1108 RB_ALTSTACK_FREE(altstack);
1109 }
1110 rb_native_mutex_unlock(&thread_cache_lock);
1111#endif
1112}
1113
1114static int
1115native_thread_create(rb_thread_t *th)
1116{
1117 int err = 0;
1118
1119 if (use_cached_thread(th)) {
1120 thread_debug("create (use cached thread): %p\n", (void *)th);
1121 }
1122 else {
1123 pthread_attr_t attr;
1124 const size_t stack_size = th->vm->default_params.thread_machine_stack_size + th->vm->default_params.thread_vm_stack_size;
1125 const size_t space = space_size(stack_size);
1126
1127#ifdef USE_SIGALTSTACK
1128 th->altstack = rb_allocate_sigaltstack();
1129#endif
1130 th->ec->machine.stack_maxsize = stack_size - space;
1131
1132 CHECK_ERR(pthread_attr_init(&attr));
1133
1134# ifdef PTHREAD_STACK_MIN
1135 thread_debug("create - stack size: %lu\n", (unsigned long)stack_size);
1136 CHECK_ERR(pthread_attr_setstacksize(&attr, stack_size));
1137# endif
1138
1139# ifdef HAVE_PTHREAD_ATTR_SETINHERITSCHED
1140 CHECK_ERR(pthread_attr_setinheritsched(&attr, PTHREAD_INHERIT_SCHED));
1141# endif
1142 CHECK_ERR(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
1143
1144 err = pthread_create(&th->thread_id, &attr, thread_start_func_1, th);
1145 thread_debug("create: %p (%d)\n", (void *)th, err);
1146 /* should be done in the created thread */
1147 fill_thread_id_str(th);
1148 CHECK_ERR(pthread_attr_destroy(&attr));
1149 }
1150 return err;
1151}
1152
1153#if USE_NATIVE_THREAD_PRIORITY
1154
1155static void
1156native_thread_apply_priority(rb_thread_t *th)
1157{
1158#if defined(_POSIX_PRIORITY_SCHEDULING) && (_POSIX_PRIORITY_SCHEDULING > 0)
1159 struct sched_param sp;
1160 int policy;
1161 int priority = 0 - th->priority;
1162 int max, min;
1163 pthread_getschedparam(th->thread_id, &policy, &sp);
1164 max = sched_get_priority_max(policy);
1165 min = sched_get_priority_min(policy);
1166
1167 if (min > priority) {
1168 priority = min;
1169 }
1170 else if (max < priority) {
1171 priority = max;
1172 }
1173
1174 sp.sched_priority = priority;
1175 pthread_setschedparam(th->thread_id, policy, &sp);
1176#else
1177 /* not touched */
1178#endif
1179}
1180
1181#endif /* USE_NATIVE_THREAD_PRIORITY */
1182
1183static int
1184native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *exceptfds, struct timeval *timeout, rb_thread_t *th)
1185{
1186 return rb_fd_select(n, readfds, writefds, exceptfds, timeout);
1187}
1188
1189static void
1190ubf_pthread_cond_signal(void *ptr)
1191{
1192 rb_thread_t *th = (rb_thread_t *)ptr;
1193 thread_debug("ubf_pthread_cond_signal (%p)\n", (void *)th);
1194 rb_native_cond_signal(&th->native_thread_data.cond.intr);
1195}
1196
1197static void
1198native_cond_sleep(rb_thread_t *th, rb_hrtime_t *rel)
1199{
1200 rb_nativethread_lock_t *lock = &th->interrupt_lock;
1201 rb_nativethread_cond_t *cond = &th->native_thread_data.cond.intr;
1202
1203 /* Solaris cond_timedwait() return EINVAL if an argument is greater than
1204 * current_time + 100,000,000. So cut up to 100,000,000. This is
1205 * considered as a kind of spurious wakeup. The caller to native_sleep
1206 * should care about spurious wakeup.
1207 *
1208 * See also [Bug #1341] [ruby-core:29702]
1209 * http://download.oracle.com/docs/cd/E19683-01/816-0216/6m6ngupgv/index.html
1210 */
1211 const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC;
1212
1213 GVL_UNLOCK_BEGIN(th);
1214 {
1215 rb_native_mutex_lock(lock);
1216 th->unblock.func = ubf_pthread_cond_signal;
1217 th->unblock.arg = th;
1218
1219 if (RUBY_VM_INTERRUPTED(th->ec)) {
1220 /* interrupted. return immediate */
1221 thread_debug("native_sleep: interrupted before sleep\n");
1222 }
1223 else {
1224 if (!rel) {
1225 rb_native_cond_wait(cond, lock);
1226 }
1227 else {
1228 rb_hrtime_t end;
1229
1230 if (*rel > max) {
1231 *rel = max;
1232 }
1233
1234 end = native_cond_timeout(cond, *rel);
1235 native_cond_timedwait(cond, lock, &end);
1236 }
1237 }
1238 th->unblock.func = 0;
1239
1240 rb_native_mutex_unlock(lock);
1241 }
1242 GVL_UNLOCK_END(th);
1243
1244 thread_debug("native_sleep done\n");
1245}
1246
1247#ifdef USE_UBF_LIST
1248static LIST_HEAD(ubf_list_head);
1249static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT;
1250
1251static void
1252ubf_list_atfork(void)
1253{
1254 list_head_init(&ubf_list_head);
1255 rb_native_mutex_initialize(&ubf_list_lock);
1256}
1257
1258/* The thread 'th' is registered to be trying unblock. */
1259static void
1260register_ubf_list(rb_thread_t *th)
1261{
1262 struct list_node *node = &th->native_thread_data.node.ubf;
1263
1264 if (list_empty((struct list_head*)node)) {
1265 rb_native_mutex_lock(&ubf_list_lock);
1266 list_add(&ubf_list_head, node);
1267 rb_native_mutex_unlock(&ubf_list_lock);
1268 }
1269}
1270
1271/* The thread 'th' is unblocked. It no longer need to be registered. */
1272static void
1273unregister_ubf_list(rb_thread_t *th)
1274{
1275 struct list_node *node = &th->native_thread_data.node.ubf;
1276
1277 /* we can't allow re-entry into ubf_list_head */
1278 VM_ASSERT(th->unblock.func == 0);
1279
1280 if (!list_empty((struct list_head*)node)) {
1281 rb_native_mutex_lock(&ubf_list_lock);
1282 list_del_init(node);
1283 if (list_empty(&ubf_list_head) && !rb_signal_buff_size()) {
1284 ubf_timer_disarm();
1285 }
1286 rb_native_mutex_unlock(&ubf_list_lock);
1287 }
1288}
1289
1290/*
1291 * send a signal to intent that a target thread return from blocking syscall.
1292 * Maybe any signal is ok, but we chose SIGVTALRM.
1293 */
1294static void
1295ubf_wakeup_thread(rb_thread_t *th)
1296{
1297 thread_debug("thread_wait_queue_wakeup (%"PRI_THREAD_ID")\n", thread_id_str(th));
1298 pthread_kill(th->thread_id, SIGVTALRM);
1299}
1300
1301static void
1302ubf_select(void *ptr)
1303{
1304 rb_thread_t *th = (rb_thread_t *)ptr;
1305 rb_global_vm_lock_t *gvl = rb_ractor_gvl(th->ractor);
1306 const rb_thread_t *cur = ruby_thread_from_native(); /* may be 0 */
1307
1308 register_ubf_list(th);
1309
1310 /*
1311 * ubf_wakeup_thread() doesn't guarantee to wake up a target thread.
1312 * Therefore, we repeatedly call ubf_wakeup_thread() until a target thread
1313 * exit from ubf function. We must have a timer to perform this operation.
1314 * We use double-checked locking here because this function may be called
1315 * while vm->gvl.lock is held in do_gvl_timer.
1316 * There is also no need to start a timer if we're the designated
1317 * sigwait_th thread, otherwise we can deadlock with a thread
1318 * in unblock_function_clear.
1319 */
1320 if (cur != gvl->timer && cur != sigwait_th) {
1321 /*
1322 * Double-checked locking above was to prevent nested locking
1323 * by the SAME thread. We use trylock here to prevent deadlocks
1324 * between DIFFERENT threads
1325 */
1326 if (rb_native_mutex_trylock(&gvl->lock) == 0) {
1327 if (!gvl->timer) {
1328 rb_thread_wakeup_timer_thread(-1);
1329 }
1330 rb_native_mutex_unlock(&gvl->lock);
1331 }
1332 }
1333
1334 ubf_wakeup_thread(th);
1335}
1336
1337static int
1338ubf_threads_empty(void)
1339{
1340 return list_empty(&ubf_list_head);
1341}
1342
1343static void
1344ubf_wakeup_all_threads(void)
1345{
1346 rb_thread_t *th;
1347 native_thread_data_t *dat;
1348
1349 if (!ubf_threads_empty()) {
1350 rb_native_mutex_lock(&ubf_list_lock);
1351 list_for_each(&ubf_list_head, dat, node.ubf) {
1352 th = container_of(dat, rb_thread_t, native_thread_data);
1353 ubf_wakeup_thread(th);
1354 }
1355 rb_native_mutex_unlock(&ubf_list_lock);
1356 }
1357}
1358
1359#else /* USE_UBF_LIST */
1360#define register_ubf_list(th) (void)(th)
1361#define unregister_ubf_list(th) (void)(th)
1362#define ubf_select 0
1363static void ubf_wakeup_all_threads(void) { return; }
1364static int ubf_threads_empty(void) { return 1; }
1365#define ubf_list_atfork() do {} while (0)
1366#endif /* USE_UBF_LIST */
1367
1368#define TT_DEBUG 0
1369#define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0)
1370
1371static struct {
1372 /* pipes are closed in forked children when owner_process does not match */
1373 int normal[2]; /* [0] == sigwait_fd */
1374 int ub_main[2]; /* unblock main thread from native_ppoll_sleep */
1375
1376 /* volatile for signal handler use: */
1377 volatile rb_pid_t owner_process;
1378} signal_self_pipe = {
1379 {-1, -1},
1380 {-1, -1},
1381};
1382
1383/* only use signal-safe system calls here */
1384static void
1385rb_thread_wakeup_timer_thread_fd(int fd)
1386{
1387#if USE_EVENTFD
1388 const uint64_t buff = 1;
1389#else
1390 const char buff = '!';
1391#endif
1392 ssize_t result;
1393
1394 /* already opened */
1395 if (fd >= 0) {
1396 retry:
1397 if ((result = write(fd, &buff, sizeof(buff))) <= 0) {
1398 int e = errno;
1399 switch (e) {
1400 case EINTR: goto retry;
1401 case EAGAIN:
1402#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
1403 case EWOULDBLOCK:
1404#endif
1405 break;
1406 default:
1407 async_bug_fd("rb_thread_wakeup_timer_thread: write", e, fd);
1408 }
1409 }
1410 if (TT_DEBUG) WRITE_CONST(2, "rb_thread_wakeup_timer_thread: write\n");
1411 }
1412 else {
1413 /* ignore wakeup */
1414 }
1415}
1416
1417/*
1418 * This ensures we get a SIGVTALRM in TIME_QUANTUM_MSEC if our
1419 * process could not react to the original signal in time.
1420 */
1421static void
1422ubf_timer_arm(rb_pid_t current) /* async signal safe */
1423{
1424#if UBF_TIMER == UBF_TIMER_POSIX
1425 if ((!current || timer_posix.owner == current) &&
1426 !ATOMIC_CAS(timer_posix.state, RTIMER_DISARM, RTIMER_ARMING)) {
1427 struct itimerspec it;
1428
1429 it.it_interval.tv_sec = it.it_value.tv_sec = 0;
1430 it.it_interval.tv_nsec = it.it_value.tv_nsec = TIME_QUANTUM_NSEC;
1431
1432 if (timer_settime(timer_posix.timerid, 0, &it, 0))
1433 rb_async_bug_errno("timer_settime (arm)", errno);
1434
1435 switch (ATOMIC_CAS(timer_posix.state, RTIMER_ARMING, RTIMER_ARMED)) {
1436 case RTIMER_DISARM:
1437 /* somebody requested a disarm while we were arming */
1438 /* may race harmlessly with ubf_timer_destroy */
1439 (void)timer_settime(timer_posix.timerid, 0, &zero, 0);
1440
1441 case RTIMER_ARMING: return; /* success */
1442 case RTIMER_ARMED:
1443 /*
1444 * it is possible to have another thread disarm, and
1445 * a third thread arm finish re-arming before we get
1446 * here, so we wasted a syscall with timer_settime but
1447 * probably unavoidable in a signal handler.
1448 */
1449 return;
1450 case RTIMER_DEAD:
1451 /* may race harmlessly with ubf_timer_destroy */
1452 (void)timer_settime(timer_posix.timerid, 0, &zero, 0);
1453 return;
1454 default:
1455 rb_async_bug_errno("UBF_TIMER_POSIX unknown state", ERANGE);
1456 }
1457 }
1458#elif UBF_TIMER == UBF_TIMER_PTHREAD
1459 if (!current || current == timer_pthread.owner) {
1460 if (ATOMIC_EXCHANGE(timer_pthread.armed, 1) == 0)
1461 rb_thread_wakeup_timer_thread_fd(timer_pthread.low[1]);
1462 }
1463#endif
1464}
1465
1466void
1467 rb_thread_wakeup_timer_thread(int sig)
1468{
1469 rb_pid_t current;
1470
1471 /* non-sighandler path */
1472 if (sig <= 0) {
1473 rb_thread_wakeup_timer_thread_fd(signal_self_pipe.normal[1]);
1474 if (sig < 0) {
1475 ubf_timer_arm(0);
1476 }
1477 return;
1478 }
1479
1480 /* must be safe inside sighandler, so no mutex */
1481 current = getpid();
1482 if (signal_self_pipe.owner_process == current) {
1483 rb_thread_wakeup_timer_thread_fd(signal_self_pipe.normal[1]);
1484
1485 /*
1486 * system_working check is required because vm and main_thread are
1487 * freed during shutdown
1488 */
1489 if (system_working > 0) {
1490 volatile rb_execution_context_t *ec;
1491 rb_vm_t *vm = GET_VM();
1492 rb_thread_t *mth;
1493
1494 /*
1495 * FIXME: root VM and main_thread should be static and not
1496 * on heap for maximum safety (and startup/shutdown speed)
1497 */
1498 if (!vm) return;
1499 mth = vm->ractor.main_thread;
1500 if (!mth || system_working <= 0) return;
1501
1502 /* this relies on GC for grace period before cont_free */
1503 ec = ACCESS_ONCE(rb_execution_context_t *, mth->ec);
1504
1505 if (ec) {
1506 RUBY_VM_SET_TRAP_INTERRUPT(ec);
1507 ubf_timer_arm(current);
1508
1509 /* some ubfs can interrupt single-threaded process directly */
1510 if (vm->ubf_async_safe && mth->unblock.func) {
1511 (mth->unblock.func)(mth->unblock.arg);
1512 }
1513 }
1514 }
1515 }
1516}
1517
1518#define CLOSE_INVALIDATE_PAIR(expr) \
1519 close_invalidate_pair(expr,"close_invalidate: "#expr)
1520static void
1521close_invalidate(int *fdp, const char *msg)
1522{
1523 int fd = *fdp;
1524
1525 *fdp = -1;
1526 if (close(fd) < 0) {
1527 async_bug_fd(msg, errno, fd);
1528 }
1529}
1530
1531static void
1532close_invalidate_pair(int fds[2], const char *msg)
1533{
1534 if (USE_EVENTFD && fds[0] == fds[1]) {
1535 close_invalidate(&fds[0], msg);
1536 fds[1] = -1;
1537 }
1538 else {
1539 close_invalidate(&fds[0], msg);
1540 close_invalidate(&fds[1], msg);
1541 }
1542}
1543
1544static void
1545set_nonblock(int fd)
1546{
1547 int oflags;
1548 int err;
1549
1550 oflags = fcntl(fd, F_GETFL);
1551 if (oflags == -1)
1552 rb_sys_fail(0);
1553 oflags |= O_NONBLOCK;
1554 err = fcntl(fd, F_SETFL, oflags);
1555 if (err == -1)
1556 rb_sys_fail(0);
1557}
1558
1559/* communication pipe with timer thread and signal handler */
1560static int
1561setup_communication_pipe_internal(int pipes[2])
1562{
1563 int err;
1564
1565 if (pipes[0] >= 0 || pipes[1] >= 0) {
1566 VM_ASSERT(pipes[0] >= 0);
1567 VM_ASSERT(pipes[1] >= 0);
1568 return 0;
1569 }
1570
1571 /*
1572 * Don't bother with eventfd on ancient Linux 2.6.22..2.6.26 which were
1573 * missing EFD_* flags, they can fall back to pipe
1574 */
1575#if USE_EVENTFD && defined(EFD_NONBLOCK) && defined(EFD_CLOEXEC)
1576 pipes[0] = pipes[1] = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
1577 if (pipes[0] >= 0) {
1578 rb_update_max_fd(pipes[0]);
1579 return 0;
1580 }
1581#endif
1582
1583 err = rb_cloexec_pipe(pipes);
1584 if (err != 0) {
1585 rb_warn("pipe creation failed for timer: %s, scheduling broken",
1586 strerror(errno));
1587 return -1;
1588 }
1589 rb_update_max_fd(pipes[0]);
1590 rb_update_max_fd(pipes[1]);
1591 set_nonblock(pipes[0]);
1592 set_nonblock(pipes[1]);
1593 return 0;
1594}
1595
1596#if !defined(SET_CURRENT_THREAD_NAME) && defined(__linux__) && defined(PR_SET_NAME)
1597# define SET_CURRENT_THREAD_NAME(name) prctl(PR_SET_NAME, name)
1598#endif
1599
1600enum {
1601 THREAD_NAME_MAX =
1602#if defined(__linux__)
1603 16
1604#elif defined(__APPLE__)
1605/* Undocumented, and main thread seems unlimited */
1606 64
1607#else
1608 16
1609#endif
1610};
1611
1612static VALUE threadptr_invoke_proc_location(rb_thread_t *th);
1613
1614static void
1615native_set_thread_name(rb_thread_t *th)
1616{
1617#ifdef SET_CURRENT_THREAD_NAME
1618 VALUE loc;
1619 if (!NIL_P(loc = th->name)) {
1620 SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc));
1621 }
1622 else if ((loc = threadptr_invoke_proc_location(th)) != Qnil) {
1623 char *name, *p;
1624 char buf[THREAD_NAME_MAX];
1625 size_t len;
1626 int n;
1627
1628 name = RSTRING_PTR(RARRAY_AREF(loc, 0));
1629 p = strrchr(name, '/'); /* show only the basename of the path. */
1630 if (p && p[1])
1631 name = p + 1;
1632
1633 n = snprintf(buf, sizeof(buf), "%s:%d", name, NUM2INT(RARRAY_AREF(loc, 1)));
1634 rb_gc_force_recycle(loc); /* acts as a GC guard, too */
1635
1636 len = (size_t)n;
1637 if (len >= sizeof(buf)) {
1638 buf[sizeof(buf)-2] = '*';
1639 buf[sizeof(buf)-1] = '\0';
1640 }
1641 SET_CURRENT_THREAD_NAME(buf);
1642 }
1643#endif
1644}
1645
1646static void
1647native_set_another_thread_name(rb_nativethread_id_t thread_id, VALUE name)
1648{
1649#if defined SET_ANOTHER_THREAD_NAME || defined SET_CURRENT_THREAD_NAME
1650 char buf[THREAD_NAME_MAX];
1651 const char *s = "";
1652# if !defined SET_ANOTHER_THREAD_NAME
1653 if (!pthread_equal(pthread_self(), thread_id)) return;
1654# endif
1655 if (!NIL_P(name)) {
1656 long n;
1657 RSTRING_GETMEM(name, s, n);
1658 if (n >= (int)sizeof(buf)) {
1659 memcpy(buf, s, sizeof(buf)-1);
1660 buf[sizeof(buf)-1] = '\0';
1661 s = buf;
1662 }
1663 }
1664# if defined SET_ANOTHER_THREAD_NAME
1665 SET_ANOTHER_THREAD_NAME(thread_id, s);
1666# elif defined SET_CURRENT_THREAD_NAME
1667 SET_CURRENT_THREAD_NAME(s);
1668# endif
1669#endif
1670}
1671
1672static void
1673ubf_timer_invalidate(void)
1674{
1675#if UBF_TIMER == UBF_TIMER_PTHREAD
1676 CLOSE_INVALIDATE_PAIR(timer_pthread.low);
1677#endif
1678}
1679
1680static void
1681ubf_timer_pthread_create(rb_pid_t current)
1682{
1683#if UBF_TIMER == UBF_TIMER_PTHREAD
1684 int err;
1685 if (timer_pthread.owner == current)
1686 return;
1687
1688 if (setup_communication_pipe_internal(timer_pthread.low) < 0)
1689 return;
1690
1691 err = pthread_create(&timer_pthread.thid, 0, timer_pthread_fn, GET_VM());
1692 if (!err)
1693 timer_pthread.owner = current;
1694 else
1695 rb_warn("pthread_create failed for timer: %s, signals racy",
1696 strerror(err));
1697#endif
1698}
1699
1700static void
1701ubf_timer_create(rb_pid_t current)
1702{
1703#if UBF_TIMER == UBF_TIMER_POSIX
1704# if defined(__sun)
1705# define UBF_TIMER_CLOCK CLOCK_REALTIME
1706# else /* Tested Linux and FreeBSD: */
1707# define UBF_TIMER_CLOCK CLOCK_MONOTONIC
1708# endif
1709
1710 struct sigevent sev;
1711
1712 sev.sigev_notify = SIGEV_SIGNAL;
1713 sev.sigev_signo = SIGVTALRM;
1714 sev.sigev_value.sival_ptr = &timer_posix;
1715
1716 if (!timer_create(UBF_TIMER_CLOCK, &sev, &timer_posix.timerid)) {
1717 rb_atomic_t prev = ATOMIC_EXCHANGE(timer_posix.state, RTIMER_DISARM);
1718
1719 if (prev != RTIMER_DEAD) {
1720 rb_bug("timer_posix was not dead: %u\n", (unsigned)prev);
1721 }
1722 timer_posix.owner = current;
1723 }
1724 else {
1725 rb_warn("timer_create failed: %s, signals racy", strerror(errno));
1726 }
1727#endif
1728 if (UBF_TIMER == UBF_TIMER_PTHREAD)
1729 ubf_timer_pthread_create(current);
1730}
1731
1732static void
1733rb_thread_create_timer_thread(void)
1734{
1735 /* we only create the pipe, and lazy-spawn */
1736 rb_pid_t current = getpid();
1737 rb_pid_t owner = signal_self_pipe.owner_process;
1738
1739 if (owner && owner != current) {
1740 CLOSE_INVALIDATE_PAIR(signal_self_pipe.normal);
1741 CLOSE_INVALIDATE_PAIR(signal_self_pipe.ub_main);
1742 ubf_timer_invalidate();
1743 }
1744
1745 if (setup_communication_pipe_internal(signal_self_pipe.normal) < 0) return;
1746 if (setup_communication_pipe_internal(signal_self_pipe.ub_main) < 0) return;
1747
1748 ubf_timer_create(current);
1749 if (owner != current) {
1750 /* validate pipe on this process */
1751 sigwait_th = THREAD_INVALID;
1752 signal_self_pipe.owner_process = current;
1753 }
1754}
1755
1756static void
1757ubf_timer_disarm(void)
1758{
1759#if UBF_TIMER == UBF_TIMER_POSIX
1760 rb_atomic_t prev;
1761
1762 if (timer_posix.owner && timer_posix.owner != getpid()) return;
1763 prev = ATOMIC_CAS(timer_posix.state, RTIMER_ARMED, RTIMER_DISARM);
1764 switch (prev) {
1765 case RTIMER_DISARM: return; /* likely */
1766 case RTIMER_ARMING: return; /* ubf_timer_arm will disarm itself */
1767 case RTIMER_ARMED:
1768 if (timer_settime(timer_posix.timerid, 0, &zero, 0)) {
1769 int err = errno;
1770
1771 if (err == EINVAL) {
1772 prev = ATOMIC_CAS(timer_posix.state, RTIMER_DISARM, RTIMER_DISARM);
1773
1774 /* main thread may have killed the timer */
1775 if (prev == RTIMER_DEAD) return;
1776
1777 rb_bug_errno("timer_settime (disarm)", err);
1778 }
1779 }
1780 return;
1781 case RTIMER_DEAD: return; /* stay dead */
1782 default:
1783 rb_bug("UBF_TIMER_POSIX bad state: %u\n", (unsigned)prev);
1784 }
1785
1786#elif UBF_TIMER == UBF_TIMER_PTHREAD
1787 ATOMIC_SET(timer_pthread.armed, 0);
1788#endif
1789}
1790
1791static void
1792ubf_timer_destroy(void)
1793{
1794#if UBF_TIMER == UBF_TIMER_POSIX
1795 if (timer_posix.owner == getpid()) {
1796 rb_atomic_t expect = RTIMER_DISARM;
1797 size_t i, max = 10000000;
1798
1799 /* prevent signal handler from arming: */
1800 for (i = 0; i < max; i++) {
1801 switch (ATOMIC_CAS(timer_posix.state, expect, RTIMER_DEAD)) {
1802 case RTIMER_DISARM:
1803 if (expect == RTIMER_DISARM) goto done;
1804 expect = RTIMER_DISARM;
1805 break;
1806 case RTIMER_ARMING:
1807 native_thread_yield(); /* let another thread finish arming */
1808 expect = RTIMER_ARMED;
1809 break;
1810 case RTIMER_ARMED:
1811 if (expect == RTIMER_ARMED) {
1812 if (timer_settime(timer_posix.timerid, 0, &zero, 0))
1813 rb_bug_errno("timer_settime (destroy)", errno);
1814 goto done;
1815 }
1816 expect = RTIMER_ARMED;
1817 break;
1818 case RTIMER_DEAD:
1819 rb_bug("RTIMER_DEAD unexpected");
1820 }
1821 }
1822 rb_bug("timed out waiting for timer to arm");
1823 done:
1824 if (timer_delete(timer_posix.timerid) < 0)
1825 rb_sys_fail("timer_delete");
1826
1827 VM_ASSERT(ATOMIC_EXCHANGE(timer_posix.state, RTIMER_DEAD) == RTIMER_DEAD);
1828 }
1829#elif UBF_TIMER == UBF_TIMER_PTHREAD
1830 int err;
1831
1832 timer_pthread.owner = 0;
1833 ubf_timer_disarm();
1834 rb_thread_wakeup_timer_thread_fd(timer_pthread.low[1]);
1835 err = pthread_join(timer_pthread.thid, 0);
1836 if (err) {
1837 rb_raise(rb_eThreadError, "native_thread_join() failed (%d)", err);
1838 }
1839#endif
1840}
1841
1842static int
1843native_stop_timer_thread(void)
1844{
1845 int stopped;
1846 stopped = --system_working <= 0;
1847 if (stopped)
1848 ubf_timer_destroy();
1849
1850 if (TT_DEBUG) fprintf(stderr, "stop timer thread\n");
1851 return stopped;
1852}
1853
1854static void
1855native_reset_timer_thread(void)
1856{
1857 if (TT_DEBUG) fprintf(stderr, "reset timer thread\n");
1858}
1859
1860#ifdef HAVE_SIGALTSTACK
1861int
1862ruby_stack_overflowed_p(const rb_thread_t *th, const void *addr)
1863{
1864 void *base;
1865 size_t size;
1866 const size_t water_mark = 1024 * 1024;
1867 STACK_GROW_DIR_DETECTION;
1868
1869#ifdef STACKADDR_AVAILABLE
1870 if (get_stack(&base, &size) == 0) {
1871# ifdef __APPLE__
1872 if (pthread_equal(th->thread_id, native_main_thread.id)) {
1873 struct rlimit rlim;
1874 if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) {
1875 size = (size_t)rlim.rlim_cur;
1876 }
1877 }
1878# endif
1879 base = (char *)base + STACK_DIR_UPPER(+size, -size);
1880 }
1881 else
1882#endif
1883 if (th) {
1884 size = th->ec->machine.stack_maxsize;
1885 base = (char *)th->ec->machine.stack_start - STACK_DIR_UPPER(0, size);
1886 }
1887 else {
1888 return 0;
1889 }
1890 size /= RUBY_STACK_SPACE_RATIO;
1891 if (size > water_mark) size = water_mark;
1892 if (IS_STACK_DIR_UPPER()) {
1893 if (size > ~(size_t)base+1) size = ~(size_t)base+1;
1894 if (addr > base && addr <= (void *)((char *)base + size)) return 1;
1895 }
1896 else {
1897 if (size > (size_t)base) size = (size_t)base;
1898 if (addr > (void *)((char *)base - size) && addr <= base) return 1;
1899 }
1900 return 0;
1901}
1902#endif
1903
1904int
1905 rb_reserved_fd_p(int fd)
1906{
1907 /* no false-positive if out-of-FD at startup */
1908 if (fd < 0)
1909 return 0;
1910
1911#if UBF_TIMER == UBF_TIMER_PTHREAD
1912 if (fd == timer_pthread.low[0] || fd == timer_pthread.low[1])
1913 goto check_pid;
1914#endif
1915 if (fd == signal_self_pipe.normal[0] || fd == signal_self_pipe.normal[1])
1916 goto check_pid;
1917 if (fd == signal_self_pipe.ub_main[0] || fd == signal_self_pipe.ub_main[1])
1918 goto check_pid;
1919 return 0;
1920check_pid:
1921 if (signal_self_pipe.owner_process == getpid()) /* async-signal-safe */
1922 return 1;
1923 return 0;
1924}
1925
1926rb_nativethread_id_t
1927 rb_nativethread_self(void)
1928{
1929 return pthread_self();
1930}
1931
1932#if USE_MJIT
1933/* A function that wraps actual worker function, for pthread abstraction. */
1934static void *
1935 mjit_worker(void *arg)
1936{
1937 void (*worker_func)(void) = (void(*)(void))arg;
1938
1939#ifdef SET_CURRENT_THREAD_NAME
1940 SET_CURRENT_THREAD_NAME("ruby-mjitworker"); /* 16 byte including NUL */
1941#endif
1942 worker_func();
1943 return NULL;
1944}
1945
1946/* Launch MJIT thread. Returns FALSE if it fails to create thread. */
1947int
1948rb_thread_create_mjit_thread(void (*worker_func)(void))
1949{
1950 pthread_attr_t attr;
1951 pthread_t worker_pid;
1952 int ret = FALSE;
1953
1954 if (pthread_attr_init(&attr) != 0) return ret;
1955
1956 /* jit_worker thread is not to be joined */
1957 if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0
1958 && pthread_create(&worker_pid, &attr, mjit_worker, (void *)worker_func) == 0) {
1959 ret = TRUE;
1960 }
1961 pthread_attr_destroy(&attr);
1962 return ret;
1963}
1964#endif
1965
1966int
1967 rb_sigwait_fd_get(const rb_thread_t *th)
1968{
1969 if (signal_self_pipe.normal[0] >= 0) {
1970 VM_ASSERT(signal_self_pipe.owner_process == getpid());
1971 /*
1972 * no need to keep firing the timer if any thread is sleeping
1973 * on the signal self-pipe
1974 */
1975 ubf_timer_disarm();
1976
1977 if (ATOMIC_PTR_CAS(sigwait_th, THREAD_INVALID, th) == THREAD_INVALID) {
1978 return signal_self_pipe.normal[0];
1979 }
1980 }
1981 return -1; /* avoid thundering herd and work stealing/starvation */
1982}
1983
1984void
1985 rb_sigwait_fd_put(const rb_thread_t *th, int fd)
1986{
1987 const rb_thread_t *old;
1988
1989 VM_ASSERT(signal_self_pipe.normal[0] == fd);
1990 old = ATOMIC_PTR_EXCHANGE(sigwait_th, THREAD_INVALID);
1991 if (old != th) assert(old == th);
1992}
1993
1994#ifndef HAVE_PPOLL
1995/* TODO: don't ignore sigmask */
1996static int
1997ruby_ppoll(struct pollfd *fds, nfds_t nfds,
1998 const struct timespec *ts, const sigset_t *sigmask)
1999{
2000 int timeout_ms;
2001
2002 if (ts) {
2003 int tmp, tmp2;
2004
2005 if (ts->tv_sec > INT_MAX/1000)
2006 timeout_ms = INT_MAX;
2007 else {
2008 tmp = (int)(ts->tv_sec * 1000);
2009 /* round up 1ns to 1ms to avoid excessive wakeups for <1ms sleep */
2010 tmp2 = (int)((ts->tv_nsec + 999999L) / (1000L * 1000L));
2011 if (INT_MAX - tmp < tmp2)
2012 timeout_ms = INT_MAX;
2013 else
2014 timeout_ms = (int)(tmp + tmp2);
2015 }
2016 }
2017 else
2018 timeout_ms = -1;
2019
2020 return poll(fds, nfds, timeout_ms);
2021}
2022# define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask))
2023#endif
2024
2025void
2026 rb_sigwait_sleep(rb_thread_t *th, int sigwait_fd, const rb_hrtime_t *rel)
2027{
2028 struct pollfd pfd;
2029 struct timespec ts;
2030
2031 pfd.fd = sigwait_fd;
2032 pfd.events = POLLIN;
2033
2034 if (!BUSY_WAIT_SIGNALS && ubf_threads_empty()) {
2035 (void)ppoll(&pfd, 1, rb_hrtime2timespec(&ts, rel), 0);
2036 check_signals_nogvl(th, sigwait_fd);
2037 }
2038 else {
2039 rb_hrtime_t to = RB_HRTIME_MAX, end;
2040 int n = 0;
2041
2042 if (rel) {
2043 to = *rel;
2044 end = rb_hrtime_add(rb_hrtime_now(), to);
2045 }
2046 /*
2047 * tricky: this needs to return on spurious wakeup (no auto-retry).
2048 * But we also need to distinguish between periodic quantum
2049 * wakeups, so we care about the result of consume_communication_pipe
2050 *
2051 * We want to avoid spurious wakeup for Mutex#sleep compatibility
2052 * [ruby-core:88102]
2053 */
2054 for (;;) {
2055 const rb_hrtime_t *sto = sigwait_timeout(th, sigwait_fd, &to, &n);
2056
2057 if (n) return;
2058 n = ppoll(&pfd, 1, rb_hrtime2timespec(&ts, sto), 0);
2059 if (check_signals_nogvl(th, sigwait_fd))
2060 return;
2061 if (n || (th && RUBY_VM_INTERRUPTED(th->ec)))
2062 return;
2063 if (rel && hrtime_update_expire(&to, end))
2064 return;
2065 }
2066 }
2067}
2068
2069/*
2070 * we need to guarantee wakeups from native_ppoll_sleep because
2071 * ubf_select may not be going through ubf_list if other threads
2072 * are all sleeping.
2073 */
2074static void
2075ubf_ppoll_sleep(void *ignore)
2076{
2077 rb_thread_wakeup_timer_thread_fd(signal_self_pipe.ub_main[1]);
2078}
2079
2080/*
2081 * Single CPU setups benefit from explicit sched_yield() before ppoll(),
2082 * since threads may be too starved to enter the GVL waitqueue for
2083 * us to detect contention. Instead, we want to kick other threads
2084 * so they can run and possibly prevent us from entering slow paths
2085 * in ppoll() or similar syscalls.
2086 *
2087 * Confirmed on FreeBSD 11.2 and Linux 4.19.
2088 * [ruby-core:90417] [Bug #15398]
2089 */
2090#define GVL_UNLOCK_BEGIN_YIELD(th) do { \
2091 const native_thread_data_t *next; \
2092 rb_global_vm_lock_t *gvl = rb_ractor_gvl(th->ractor); \
2093 RB_GC_SAVE_MACHINE_CONTEXT(th); \
2094 rb_native_mutex_lock(&gvl->lock); \
2095 next = gvl_release_common(gvl); \
2096 rb_native_mutex_unlock(&gvl->lock); \
2097 if (!next && rb_ractor_living_thread_num(th->ractor) > 1) { \
2098 native_thread_yield(); \
2099 }
2100
2101/*
2102 * This function does not exclusively acquire sigwait_fd, so it
2103 * cannot safely read from it. However, it can be woken up in
2104 * 4 ways:
2105 *
2106 * 1) ubf_ppoll_sleep (from another thread)
2107 * 2) rb_thread_wakeup_timer_thread (from signal handler)
2108 * 3) any unmasked signal hitting the process
2109 * 4) periodic ubf timer wakeups (after 3)
2110 */
2111static void
2112native_ppoll_sleep(rb_thread_t *th, rb_hrtime_t *rel)
2113{
2114 rb_native_mutex_lock(&th->interrupt_lock);
2115 th->unblock.func = ubf_ppoll_sleep;
2116 rb_native_mutex_unlock(&th->interrupt_lock);
2117
2118 GVL_UNLOCK_BEGIN_YIELD(th);
2119
2120 if (!RUBY_VM_INTERRUPTED(th->ec)) {
2121 struct pollfd pfd[2];
2122 struct timespec ts;
2123
2124 pfd[0].fd = signal_self_pipe.normal[0]; /* sigwait_fd */
2125 pfd[1].fd = signal_self_pipe.ub_main[0];
2126 pfd[0].events = pfd[1].events = POLLIN;
2127 if (ppoll(pfd, 2, rb_hrtime2timespec(&ts, rel), 0) > 0) {
2128 if (pfd[1].revents & POLLIN) {
2129 (void)consume_communication_pipe(pfd[1].fd);
2130 }
2131 }
2132 /*
2133 * do not read the sigwait_fd, here, let uplevel callers
2134 * or other threads that, otherwise we may steal and starve
2135 * other threads
2136 */
2137 }
2138 unblock_function_clear(th);
2139 GVL_UNLOCK_END(th);
2140}
2141
2142static void
2143native_sleep(rb_thread_t *th, rb_hrtime_t *rel)
2144{
2145 int sigwait_fd = rb_sigwait_fd_get(th);
2146 rb_ractor_blocking_threads_inc(th->ractor, __FILE__, __LINE__);
2147
2148 if (sigwait_fd >= 0) {
2149 rb_native_mutex_lock(&th->interrupt_lock);
2150 th->unblock.func = ubf_sigwait;
2151 rb_native_mutex_unlock(&th->interrupt_lock);
2152
2153 GVL_UNLOCK_BEGIN_YIELD(th);
2154
2155 if (!RUBY_VM_INTERRUPTED(th->ec)) {
2156 rb_sigwait_sleep(th, sigwait_fd, rel);
2157 }
2158 else {
2159 check_signals_nogvl(th, sigwait_fd);
2160 }
2161 unblock_function_clear(th);
2162 GVL_UNLOCK_END(th);
2163 rb_sigwait_fd_put(th, sigwait_fd);
2164 rb_sigwait_fd_migrate(th->vm);
2165 }
2166 else if (th == th->vm->ractor.main_thread) { /* always able to handle signals */
2167 native_ppoll_sleep(th, rel);
2168 }
2169 else {
2170 native_cond_sleep(th, rel);
2171 }
2172
2173 rb_ractor_blocking_threads_dec(th->ractor, __FILE__, __LINE__);
2174}
2175
2176#if UBF_TIMER == UBF_TIMER_PTHREAD
2177static void *
2178timer_pthread_fn(void *p)
2179{
2180 rb_vm_t *vm = p;
2181 pthread_t main_thread_id = vm->ractor.main_thread->thread_id;
2182 struct pollfd pfd;
2183 int timeout = -1;
2184 int ccp;
2185
2186 pfd.fd = timer_pthread.low[0];
2187 pfd.events = POLLIN;
2188
2189 while (system_working > 0) {
2190 (void)poll(&pfd, 1, timeout);
2191 ccp = consume_communication_pipe(pfd.fd);
2192
2193 if (system_working > 0) {
2194 if (ATOMIC_CAS(timer_pthread.armed, 1, 1)) {
2195 pthread_kill(main_thread_id, SIGVTALRM);
2196
2197 if (rb_signal_buff_size() || !ubf_threads_empty()) {
2198 timeout = TIME_QUANTUM_MSEC;
2199 }
2200 else {
2201 ATOMIC_SET(timer_pthread.armed, 0);
2202 timeout = -1;
2203 }
2204 }
2205 else if (ccp) {
2206 pthread_kill(main_thread_id, SIGVTALRM);
2207 ATOMIC_SET(timer_pthread.armed, 0);
2208 timeout = -1;
2209 }
2210 }
2211 }
2212
2213 return 0;
2214}
2215#endif /* UBF_TIMER_PTHREAD */
2216
2217static VALUE
2218ubf_caller(void *ignore)
2219{
2220 rb_thread_sleep_forever();
2221
2222 return Qfalse;
2223}
2224
2225/*
2226 * Called if and only if one thread is running, and
2227 * the unblock function is NOT async-signal-safe
2228 * This assumes USE_THREAD_CACHE is true for performance reasons
2229 */
2230static VALUE
2231rb_thread_start_unblock_thread(void)
2232{
2233 return rb_thread_create(ubf_caller, 0);
2234}
2235#endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */
ptr
struct RIMemo * ptr
Definition: debug.c:88
assert
#define assert(x)
Definition: dlmalloc.c:1176
ATOMIC_PTR_CAS
#define ATOMIC_PTR_CAS(var, old, new)
Definition: dtoa.c:505
done
struct tab * done
Definition: enough.c:233
max
int max
Definition: enough.c:225
len
uint8_t len
Definition: escape.c:17
rb_eThreadError
VALUE rb_eThreadError
Definition: eval.c:953
EXIT_FAILURE
#define EXIT_FAILURE
Definition: eval_intern.h:32
strrchr
char * strrchr(const char *, const char)
RSTRING_PTR
#define RSTRING_PTR(string)
Definition: fbuffer.h:19
MAYBE_UNUSED
#define MAYBE_UNUSED
Definition: ffi_common.h:30
alloca
#define alloca
Definition: ffi_common.h:27
UNLIKELY
#define UNLIKELY(x)
Definition: ffi_common.h:126
memcpy
#define memcpy(d, s, n)
Definition: ffi_common.h:55
rb_gc_force_recycle
void rb_gc_force_recycle(VALUE obj)
Definition: gc.c:7968
STACK_DIR_UPPER
#define STACK_DIR_UPPER(a, b)
Definition: gc.h:108
STACK_UPPER
#define STACK_UPPER(x, a, b)
Definition: gc.h:92
STACK_GROW_DIR_DETECTION
#define STACK_GROW_DIR_DETECTION
Definition: gc.h:107
IS_STACK_DIR_UPPER
#define IS_STACK_DIR_UPPER()
Definition: gc.h:110
ruby_init_stack
void ruby_init_stack(volatile VALUE *)
rb_raise
void rb_raise(VALUE exc, const char *fmt,...)
Definition: error.c:2917
rb_eNotImpError
VALUE rb_eNotImpError
Definition: error.c:1067
rb_bug
void rb_bug(const char *fmt,...)
Definition: error.c:768
rb_bug_errno
void rb_bug_errno(const char *mesg, int errno_arg)
Definition: error.c:796
rb_warn
void rb_warn(const char *fmt,...)
Definition: error.c:408
rb_async_bug_errno
void rb_async_bug_errno(const char *mesg, int errno_arg)
Definition: error.c:817
rb_sys_fail
void rb_sys_fail(const char *mesg)
Definition: error.c:3041
WRITE_CONST
#define WRITE_CONST(fd, str)
Definition: error.c:814
RB_HRTIME_MAX
#define RB_HRTIME_MAX
Definition: hrtime.h:38
rb_hrtime_now
rb_hrtime_t rb_hrtime_now(void)
Definition: thread.c:1439
RB_HRTIME_PER_MSEC
#define RB_HRTIME_PER_MSEC
Definition: hrtime.h:36
rb_hrtime_t
uint64_t rb_hrtime_t
Definition: hrtime.h:47
RB_HRTIME_PER_SEC
#define RB_HRTIME_PER_SEC
Definition: hrtime.h:37
__attribute__
void *PTR64 __attribute__((mode(DI)))
Definition: ffi.c:41
rb_cloexec_pipe
int rb_cloexec_pipe(int fildes[2])
Definition: io.c:406
rb_update_max_fd
void rb_update_max_fd(int fd)
Definition: io.c:233
rb_reserved_fd_p
int rb_reserved_fd_p(int fd)
rb_thread_sleep_forever
void rb_thread_sleep_forever(void)
Definition: thread.c:1524
rb_thread_create
VALUE rb_thread_create(VALUE(*)(void *), void *)
Definition: thread.c:1119
rb_timespec_now
void rb_timespec_now(struct timespec *)
Definition: time.c:1889
strerror
char * strerror(int)
Definition: strerror.c:11
NUM2INT
#define NUM2INT
Definition: int.h:44
ACCESS_ONCE
#define ACCESS_ONCE(type, x)
Definition: internal.h:27
size
voidpf void uLong size
Definition: ioapi.h:138
int
typedef int(ZCALLBACK *close_file_func) OF((voidpf opaque
buf
voidpf void * buf
Definition: ioapi.h:138
mjit.h
mjit_worker
void mjit_worker(void)
Definition: mjit_worker.c:1418
id
const int id
Definition: nkf.c:209
name
const char * name
Definition: nkf.c:208
last
unsigned int last
Definition: nkf.c:4324
TRUE
#define TRUE
Definition: nkf.h:175
FALSE
#define FALSE
Definition: nkf.h:174
rb_fd_select
#define rb_fd_select
Definition: posix.h:43
rb_sigwait_fd_migrate
void rb_sigwait_fd_migrate(rb_vm_t *vm)
Definition: process.c:1125
rb_sigwait_fd_put
void rb_sigwait_fd_put(const rb_thread_t *, int fd)
rb_sigwait_fd_get
int rb_sigwait_fd_get(const rb_thread_t *)
rb_sigwait_sleep
void rb_sigwait_sleep(const rb_thread_t *, int fd, const rb_hrtime_t *)
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition: psych_emitter.c:7
rb_ractor_blocking_threads_dec
void rb_ractor_blocking_threads_dec(rb_ractor_t *cr, const char *file, int line)
Definition: ractor.c:1870
rb_ractor_gvl
rb_global_vm_lock_t * rb_ractor_gvl(rb_ractor_t *r)
Definition: ractor.c:1715
rb_gvl_init
void rb_gvl_init(rb_global_vm_lock_t *gvl)
rb_ractor_blocking_threads_inc
void rb_ractor_blocking_threads_inc(rb_ractor_t *cr, const char *file, int line)
Definition: ractor.c:1858
NULL
#define NULL
Definition: regenc.h:69
RSTRING_GETMEM
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Definition: rstring.h:211
ATOMIC_EXCHANGE
#define ATOMIC_EXCHANGE(var, val)
Definition: ruby_atomic.h:7
ATOMIC_PTR_EXCHANGE
#define ATOMIC_PTR_EXCHANGE(var, val)
Definition: ruby_atomic.h:13
ATOMIC_SET
#define ATOMIC_SET(var, val)
Definition: ruby_atomic.h:14
ATOMIC_CAS
#define ATOMIC_CAS(var, oldval, newval)
Definition: ruby_atomic.h:5
EWOULDBLOCK
#define EWOULDBLOCK
Definition: rubysocket.h:164
uint64_t
unsigned long long uint64_t
Definition: sha2.h:102
ruby_sigchld_handler
void ruby_sigchld_handler(rb_vm_t *vm)
Definition: signal.c:1090
rb_signal_buff_size
int rb_signal_buff_size(void)
Definition: signal.c:747
Qnil
#define Qnil
Definition: special_consts.h:51
Qfalse
#define Qfalse
Definition: special_consts.h:50
NIL_P
#define NIL_P
Definition: special_consts.h:46
native_thread_data_struct
Definition: thread_pthread.h:20
native_thread_data_struct::ubf
struct list_node ubf
Definition: thread_pthread.h:22
native_thread_data_struct::gvl
struct list_node gvl
Definition: thread_pthread.h:23
native_thread_data_struct::cond
struct native_thread_data_struct::@171 cond
native_thread_data_struct::intr
rb_nativethread_cond_t intr
Definition: thread_pthread.h:35
native_thread_data_struct::node
union native_thread_data_struct::@170 node
native_thread_data_struct::gvlq
rb_nativethread_cond_t gvlq
Definition: thread_pthread.h:36
rb_execution_context_struct
Definition: vm_core.h:854
rb_execution_context_struct::machine
struct rb_execution_context_struct::@200 machine
rb_execution_context_struct::stack_maxsize
size_t stack_maxsize
Definition: vm_core.h:903
rb_execution_context_struct::stack_start
VALUE * stack_start
Definition: vm_core.h:901
rb_fdset_t
Definition: largesize.h:69
rb_global_vm_lock_struct
Definition: thread_pthread.h:45
rb_global_vm_lock_struct::switch_wait_cond
rb_nativethread_cond_t switch_wait_cond
Definition: thread_pthread.h:67
rb_global_vm_lock_struct::waitq
struct list_head waitq
Definition: thread_pthread.h:61
rb_global_vm_lock_struct::need_yield
int need_yield
Definition: thread_pthread.h:68
rb_global_vm_lock_struct::switch_cond
rb_nativethread_cond_t switch_cond
Definition: thread_pthread.h:66
rb_global_vm_lock_struct::timer_err
int timer_err
Definition: thread_pthread.h:63
rb_global_vm_lock_struct::lock
rb_nativethread_lock_t lock
Definition: thread_pthread.h:48
rb_global_vm_lock_struct::owner
const struct rb_thread_struct * owner
Definition: thread_pthread.h:47
rb_global_vm_lock_struct::timer
const struct rb_thread_struct * timer
Definition: thread_pthread.h:62
rb_global_vm_lock_struct::wait_yield
int wait_yield
Definition: thread_pthread.h:69
rb_thread_cond_struct
Definition: thread_win32.h:24
rb_thread_struct
Definition: vm_core.h:935
rb_thread_struct::ec
rb_execution_context_t * ec
Definition: vm_core.h:941
rb_thread_struct::unblock
struct rb_unblock_callback unblock
Definition: vm_core.h:983
rb_thread_struct::native_thread_data
native_thread_data_t native_thread_data
Definition: vm_core.h:966
rb_thread_struct::vm
rb_vm_t * vm
Definition: vm_core.h:939
rb_thread_struct::ractor
rb_ractor_t * ractor
Definition: vm_core.h:938
rb_thread_struct::priority
int8_t priority
Definition: vm_core.h:963
rb_thread_struct::thread_id
rb_nativethread_id_t thread_id
Definition: vm_core.h:953
rb_thread_struct::name
VALUE name
Definition: vm_core.h:1019
rb_thread_struct::interrupt_lock
rb_nativethread_lock_t interrupt_lock
Definition: vm_core.h:982
rb_unblock_callback::arg
void * arg
Definition: vm_core.h:828
rb_unblock_callback::func
rb_unblock_function_t * func
Definition: vm_core.h:827
rb_vm_struct
Definition: vm_core.h:564
rb_vm_struct::ractor
struct rb_vm_struct::@194 ractor
rb_vm_struct::main_thread
struct rb_thread_struct * main_thread
Definition: vm_core.h:573
rb_vm_struct::ubf_async_safe
volatile int ubf_async_safe
Definition: vm_core.h:603
rb_vm_struct::default_params
struct rb_vm_struct::@196 default_params
rb_vm_struct::thread_vm_stack_size
size_t thread_vm_stack_size
Definition: vm_core.h:672
rb_vm_struct::thread_machine_stack_size
size_t thread_machine_stack_size
Definition: vm_core.h:673
state
Definition: blast.c:41
timespec
Definition: missing.h:62
timespec::tv_nsec
long tv_nsec
Definition: missing.h:64
timespec::tv_sec
time_t tv_sec
Definition: missing.h:63
timeval
Definition: missing.h:51
snprintf
#define snprintf
Definition: subst.h:14
PRI_THREAD_ID
#define PRI_THREAD_ID
Definition: thread.c:340
GVL_UNLOCK_BEGIN
#define GVL_UNLOCK_BEGIN(th)
Definition: thread.c:173
USE_EVENTFD
#define USE_EVENTFD
Definition: thread.c:385
thread_id_str
#define thread_id_str(th)
Definition: thread.c:339
GVL_UNLOCK_END
#define GVL_UNLOCK_END(th)
Definition: thread.c:177
fill_thread_id_str
#define fill_thread_id_str(th)
Definition: thread.c:338
thread_debug
#define thread_debug
Definition: thread.c:333
BUSY_WAIT_SIGNALS
#define BUSY_WAIT_SIGNALS
Definition: thread.c:381
rb_nativethread_self
rb_nativethread_id_t rb_nativethread_self()
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
rb_native_cond_initialize
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
rb_native_mutex_trylock
int rb_native_mutex_trylock(rb_nativethread_lock_t *lock)
rb_native_cond_broadcast
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
rb_native_mutex_initialize
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
rb_native_mutex_destroy
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
rb_native_cond_destroy
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
rb_native_cond_signal
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
rb_native_cond_wait
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
rb_native_cond_timedwait
void rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec)
RB_NATIVETHREAD_LOCK_INIT
#define RB_NATIVETHREAD_LOCK_INIT
Definition: thread_pthread.h:17
RB_THREAD_LOCAL_SPECIFIER
#define RB_THREAD_LOCAL_SPECIFIER
Definition: thread_pthread.h:74
ruby_current_ec_key
native_tls_key_t ruby_current_ec_key
Definition: vm.c:400
VALUE
unsigned long VALUE
Definition: value.h:38
RUBY_VM_INTERRUPTED
#define RUBY_VM_INTERRUPTED(ec)
Definition: vm_core.h:1881
RUBY_VM_SET_TIMER_INTERRUPT
#define RUBY_VM_SET_TIMER_INTERRUPT(ec)
Definition: vm_core.h:1875
Init_native_thread
void Init_native_thread(rb_thread_t *th)
RB_ALTSTACK_FREE
#define RB_ALTSTACK_FREE(var)
Definition: vm_core.h:143
RB_ALTSTACK_INIT
#define RB_ALTSTACK_INIT(var, altstack)
Definition: vm_core.h:142
VM_ASSERT
#define VM_ASSERT(expr)
Definition: vm_core.h:61
RUBY_VM_THREAD_VM_STACK_SIZE
#define RUBY_VM_THREAD_VM_STACK_SIZE
Definition: vm_core.h:685
rb_thread_wakeup_timer_thread
void rb_thread_wakeup_timer_thread(int)
RB_ALTSTACK
#define RB_ALTSTACK(var)
Definition: vm_core.h:144
RUBY_VM_SET_TRAP_INTERRUPT
#define RUBY_VM_SET_TRAP_INTERRUPT(ec)
Definition: vm_core.h:1878
err
int err
Definition: win32.c:142
O_NONBLOCK
#define O_NONBLOCK
Definition: win32.h:584
ETIMEDOUT
#define ETIMEDOUT
Definition: win32.h:543
CLOCK_MONOTONIC
#define CLOCK_MONOTONIC
Definition: win32.h:134
uintptr_t
unsigned int uintptr_t
Definition: win32.h:106
fcntl
int fcntl(int, int,...)
Definition: win32.c:4338
F_SETFL
#define F_SETFL
Definition: win32.h:581
write
int write(ozstream &zs, const T *x, Items items)
Definition: zstream.h:264