Ruby: transient_heap.c Source File

Ruby 3.0.5p211 (2022-11-24 revision ba5cf0f7c52d4d35cc6a173c89eda98ceffa2dcf)

transient_heap.c
Go to the documentation of this file.
1/**********************************************************************
2
3 transient_heap.c - implement transient_heap.
4
5 Copyright (C) 2018 Koichi Sasada
6
7**********************************************************************/
8
9#include "debug_counter.h"
10#include "gc.h"
11#include "internal.h"
12#include "internal/gc.h"
13#include "internal/hash.h"
14#include "internal/sanitizers.h"
15#include "internal/static_assert.h"
16#include "internal/struct.h"
17#include "internal/variable.h"
18#include "ruby/debug.h"
19#include "ruby/ruby.h"
20#include "ruby_assert.h"
21#include "transient_heap.h"
22#include "vm_debug.h"
23#include "vm_sync.h"
24
25#if USE_TRANSIENT_HEAP /* USE_TRANSIENT_HEAP */
26/*
27 * 1: enable assertions
28 * 2: enable verify all transient heaps
29 */
30#ifndef TRANSIENT_HEAP_CHECK_MODE
31#define TRANSIENT_HEAP_CHECK_MODE 0
32#endif
33#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
34
35/*
36 * 1: show events
37 * 2: show dump at events
38 * 3: show all operations
39 */
40#define TRANSIENT_HEAP_DEBUG 0
41
42/* For Debug: Provide blocks infinitely.
43 * This mode generates blocks unlimitedly
44 * and prohibit access free'ed blocks to check invalid access.
45 */
46#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
47
48#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
49#include <sys/mman.h>
50#include <errno.h>
51#endif
52
53/* For Debug: Prohibit promoting to malloc space.
54 */
55#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
56
57/* size configuration */
58#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
59
60 /* K M */
61#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */
62#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */
63#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */
64#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
65#define TRANSIENT_HEAP_USABLE_SIZE (TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header))
66
67#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
68#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
69
70#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
71#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
72
73enum transient_heap_status {
74 transient_heap_none,
75 transient_heap_marking,
76 transient_heap_escaping
77};
78
79struct transient_heap_block {
80 struct transient_heap_block_header {
81 int16_t index;
82 int16_t last_marked_index;
83 int16_t objects;
84 struct transient_heap_block *next_block;
85 } info;
86 char buff[TRANSIENT_HEAP_USABLE_SIZE];
87};
88
89struct transient_heap {
90 struct transient_heap_block *using_blocks;
91 struct transient_heap_block *marked_blocks;
92 struct transient_heap_block *free_blocks;
93 int total_objects;
94 int total_marked_objects;
95 int total_blocks;
96 enum transient_heap_status status;
97
98 VALUE *promoted_objects;
99 int promoted_objects_size;
100 int promoted_objects_index;
101
102 struct transient_heap_block *arena;
103 int arena_index; /* increment only */
104};
105
106struct transient_alloc_header {
107 uint16_t magic;
108 uint16_t size;
109 int16_t next_marked_index;
110 int16_t dummy;
111 VALUE obj;
112};
113
114static struct transient_heap global_transient_heap;
115
116static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj);
117static const void *transient_heap_ptr(VALUE obj, int error);
118static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr);
119
120#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
121
122static void
123transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block)
124{
125 int i=0, n=0;
126
127 while (i<block->info.index) {
128 void *ptr = &block->buff[i];
129 struct transient_alloc_header *header = ptr;
130 fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, ptr, header->size, header->next_marked_index, rb_obj_info(header->obj));
131 i += header->size;
132 n++;
133 }
134}
135
136static void
137transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
138{
139 while (block) {
140 fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
141 type_str, (void *)block, block->info.index, block->info.objects, block->info.last_marked_index, (void *)block->info.next_block);
142
143 transient_heap_block_dump(theap, block);
144 block = block->info.next_block;
145 }
146}
147
148static void
149transient_heap_dump(struct transient_heap* theap)
150{
151 fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);
152 transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks");
153 transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks");
154 transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks");
155}
156
157/* Debug: dump all transient_heap blocks */
158void
159rb_transient_heap_dump(void)
160{
161 transient_heap_dump(&global_transient_heap);
162}
163
164#if TRANSIENT_HEAP_CHECK_MODE >= 2
165 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void transient_heap_ptr_check(struct transient_heap *theap, VALUE obj));
166static void
167transient_heap_ptr_check(struct transient_heap *theap, VALUE obj)
168{
169 if (obj != Qundef) {
170 const void *ptr = transient_heap_ptr(obj, FALSE);
171 TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr));
172 }
173}
174
175 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static int transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block));
176static int
177transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)
178{
179 int i=0, n=0;
180 struct transient_alloc_header *header;
181
182 while (i<block->info.index) {
183 header = (void *)&block->buff[i];
184 TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
185 transient_heap_ptr_check(theap, header->obj);
186 n ++;
187 i += header->size;
188 }
189 TH_ASSERT(block->info.objects == n);
190
191 return n;
192}
193
194static int
195transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr)
196{
197 int n = 0;
198 struct transient_heap_block *block = blocks;
199 while (block) {
200 n += transient_heap_block_verify(theap, block);
201 *block_num_ptr += 1;
202 block = block->info.next_block;
203 }
204
205 return n;
206}
207#endif
208
209static void
210transient_heap_verify(struct transient_heap *theap)
211{
212#if TRANSIENT_HEAP_CHECK_MODE >= 2
213 int n=0, block_num=0;
214
215 n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num);
216 n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num);
217
218 TH_ASSERT(n == theap->total_objects);
219 TH_ASSERT(n >= theap->total_marked_objects);
220 TH_ASSERT(block_num == theap->total_blocks);
221#endif
222}
223
224/* Debug: check assertions for all transient_heap blocks */
225void
226 rb_transient_heap_verify(void)
227{
228 transient_heap_verify(&global_transient_heap);
229}
230
231static struct transient_heap*
232transient_heap_get(void)
233{
234 struct transient_heap* theap = &global_transient_heap;
235 transient_heap_verify(theap);
236 return theap;
237}
238
239static void
240reset_block(struct transient_heap_block *block)
241{
242 __msan_allocated_memory(block, sizeof block);
243 block->info.index = 0;
244 block->info.objects = 0;
245 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
246 block->info.next_block = NULL;
247 __asan_poison_memory_region(&block->buff, sizeof block->buff);
248}
249
250static void
251connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block)
252{
253 block->info.next_block = theap->free_blocks;
254 theap->free_blocks = block;
255}
256
257static void
258connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block)
259{
260 block->info.next_block = theap->using_blocks;
261 theap->using_blocks = block;
262}
263
264#if 0
265static void
266connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block)
267{
268 block->info.next_block = theap->marked_blocks;
269 theap->marked_blocks = block;
270}
271#endif
272
273static void
274append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks)
275{
276 if (theap->marked_blocks) {
277 struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL;
278 while (block) {
279 last_block = block;
280 block = block->info.next_block;
281 }
282
283 TH_ASSERT(last_block->info.next_block == NULL);
284 last_block->info.next_block = append_blocks;
285 }
286 else {
287 theap->marked_blocks = append_blocks;
288 }
289}
290
291static struct transient_heap_block *
292transient_heap_block_alloc(struct transient_heap* theap)
293{
294 struct transient_heap_block *block;
295#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
296 block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,
297 MAP_PRIVATE | MAP_ANONYMOUS,
298 -1, 0);
299 if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno);
300#else
301 if (theap->arena == NULL) {
302 theap->arena = rb_aligned_malloc(TRANSIENT_HEAP_BLOCK_SIZE, TRANSIENT_HEAP_TOTAL_SIZE);
303 if (theap->arena == NULL) {
304 rb_bug("transient_heap_block_alloc: failed\n");
305 }
306 }
307
308 TH_ASSERT(theap->arena_index < TRANSIENT_HEAP_BLOCK_NUM);
309 block = &theap->arena[theap->arena_index++];
310 TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0);
311#endif
312 reset_block(block);
313
314 TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0);
315 if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, (void *)block);
316 return block;
317}
318
319
320static struct transient_heap_block *
321transient_heap_allocatable_block(struct transient_heap* theap)
322{
323 struct transient_heap_block *block;
324
325#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
326 block = transient_heap_block_alloc(theap);
327 theap->total_blocks++;
328#else
329 /* get one block from free_blocks */
330 block = theap->free_blocks;
331 if (block) {
332 theap->free_blocks = block->info.next_block;
333 block->info.next_block = NULL;
334 theap->total_blocks++;
335 }
336#endif
337
338 return block;
339}
340
341static struct transient_alloc_header *
342transient_heap_allocatable_header(struct transient_heap* theap, size_t size)
343{
344 struct transient_heap_block *block = theap->using_blocks;
345
346 while (block) {
347 TH_ASSERT(block->info.index <= (int16_t)TRANSIENT_HEAP_USABLE_SIZE);
348
349 if (TRANSIENT_HEAP_USABLE_SIZE - block->info.index >= size) {
350 struct transient_alloc_header *header = (void *)&block->buff[block->info.index];
351 block->info.index += size;
352 block->info.objects++;
353 return header;
354 }
355 else {
356 block = transient_heap_allocatable_block(theap);
357 if (block) connect_to_using_blocks(theap, block);
358 }
359 }
360
361 return NULL;
362}
363
364void *
365 rb_transient_heap_alloc(VALUE obj, size_t req_size)
366{
367 // only on single main ractor
368 if (ruby_single_main_ractor == NULL) return NULL;
369
370 void *ret;
371 struct transient_heap* theap = transient_heap_get();
372 size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
373
374 TH_ASSERT(RB_TYPE_P(obj, T_ARRAY) ||
375 RB_TYPE_P(obj, T_OBJECT) ||
376 RB_TYPE_P(obj, T_STRUCT) ||
377 RB_TYPE_P(obj, T_HASH)); /* supported types */
378
379 if (size > TRANSIENT_HEAP_ALLOC_MAX) {
380 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj));
381 ret = NULL;
382 }
383#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
384 else if (RB_OBJ_PROMOTED_RAW(obj)) {
385 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj));
386 ret = NULL;
387 }
388#else
389 else if (RBASIC_CLASS(obj) == 0) {
390 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj));
391 ret = NULL;
392 }
393#endif
394 else {
395 struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size);
396 if (header) {
397 void *ptr;
398
399 /* header is poisoned to prevent buffer overflow, should
400 * unpoison first... */
401 asan_unpoison_memory_region(header, sizeof *header, true);
402
403 header->size = size;
404 header->magic = TRANSIENT_HEAP_ALLOC_MAGIC;
405 header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
406 header->obj = obj; /* TODO: can we eliminate it? */
407
408 /* header is fixed; shall poison again */
409 asan_poison_memory_region(header, sizeof *header);
410 ptr = header + 1;
411
412 theap->total_objects++; /* statistics */
413
414#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
415 if (RB_OBJ_PROMOTED_RAW(obj)) {
416 transient_heap_promote_add(theap, obj);
417 }
418#endif
419 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", (void *)header, ptr, (int)size, rb_obj_info(obj));
420
421 RB_DEBUG_COUNTER_INC(theap_alloc);
422
423 /* ptr is set up; OK to unpoison. */
424 asan_unpoison_memory_region(ptr, size - sizeof *header, true);
425 ret = ptr;
426 }
427 else {
428 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj));
429 RB_DEBUG_COUNTER_INC(theap_alloc_fail);
430 ret = NULL;
431 }
432 }
433
434 return ret;
435}
436
437void
438Init_TransientHeap(void)
439{
440 int i, block_num;
441 struct transient_heap* theap = transient_heap_get();
442
443#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
444 block_num = 0;
445#else
446 TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE);
447 block_num = TRANSIENT_HEAP_BLOCK_NUM;
448#endif
449 for (i=0; i<block_num; i++) {
450 connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
451 }
452 theap->using_blocks = transient_heap_allocatable_block(theap);
453
454 theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE;
455 theap->promoted_objects_index = 0;
456 /* should not use ALLOC_N to be free from GC */
457 theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size);
458 STATIC_ASSERT(
459 integer_overflow,
460 sizeof(VALUE) <= SIZE_MAX / TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE);
461 if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed.");
462}
463
464static struct transient_heap_block *
465blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header)
466{
467 struct transient_heap_block *block = blocks;
468
469 while (block) {
470 if (block->buff <= (char *)header && (char *)header < block->buff + TRANSIENT_HEAP_USABLE_SIZE) {
471 return block;
472 }
473 block = block->info.next_block;
474 }
475
476 return NULL;
477}
478
479static struct transient_heap_block *
480alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header)
481{
482 struct transient_heap_block *block;
483
484 if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) {
485 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n");
486 return block;
487 }
488 else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) {
489 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n");
490 return block;
491 }
492 else {
493 return NULL;
494 }
495}
496
497static struct transient_alloc_header *
498ptr_to_alloc_header(const void *ptr)
499{
500 struct transient_alloc_header *header = (void *)ptr;
501 header -= 1;
502 return header;
503}
504
505static int
506transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr)
507{
508 if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) {
509 return TRUE;
510 }
511 else {
512 return FALSE;
513 }
514}
515
516
517int
518rb_transient_heap_managed_ptr_p(const void *ptr)
519{
520 return transient_header_managed_ptr_p(transient_heap_get(), ptr);
521}
522
523static struct transient_heap_block *
524alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header)
525{
526 struct transient_heap_block *block;
527#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
528 block = alloc_header_to_block_verbose(theap, header);
529 if (block == NULL) {
530 transient_heap_dump(theap);
531 rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header);
532 }
533#else
534 block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1));
535 TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
536#endif
537 return block;
538}
539
540void
541 rb_transient_heap_mark(VALUE obj, const void *ptr)
542{
543 ASSERT_vm_locking();
544
545 struct transient_alloc_header *header = ptr_to_alloc_header(ptr);
546 asan_unpoison_memory_region(header, sizeof *header, false);
547 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("rb_transient_heap_mark: wrong header, %s (%p)", rb_obj_info(obj), ptr);
548 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr);
549
550#if TRANSIENT_HEAP_CHECK_MODE > 0
551 {
552 struct transient_heap* theap = transient_heap_get();
553 TH_ASSERT(theap->status == transient_heap_marking);
554 TH_ASSERT(transient_header_managed_ptr_p(theap, ptr));
555
556 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {
557 transient_heap_dump(theap);
558 rb_bug("rb_transient_heap_mark: magic is broken");
559 }
560 else if (header->obj != obj) {
561 // transient_heap_dump(theap);
562 rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
563 rb_obj_info(header->obj), rb_obj_info(obj));
564 }
565 }
566#endif
567
568 if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) {
569 /* already marked */
570 return;
571 }
572 else {
573 struct transient_heap* theap = transient_heap_get();
574 struct transient_heap_block *block = alloc_header_to_block(theap, header);
575 __asan_unpoison_memory_region(&block->info, sizeof block->info);
576 header->next_marked_index = block->info.last_marked_index;
577 block->info.last_marked_index = (int)((char *)header - block->buff);
578 theap->total_marked_objects++;
579
580 transient_heap_verify(theap);
581 }
582}
583
584 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error));
585static const void *
586transient_heap_ptr(VALUE obj, int error)
587{
588 const void *ptr = NULL;
589
590 switch (BUILTIN_TYPE(obj)) {
591 case T_ARRAY:
592 if (RARRAY_TRANSIENT_P(obj)) {
593 TH_ASSERT(!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG));
594 ptr = RARRAY(obj)->as.heap.ptr;
595 }
596 break;
597 case T_OBJECT:
598 if (ROBJ_TRANSIENT_P(obj)) {
599 ptr = ROBJECT_IVPTR(obj);
600 }
601 break;
602 case T_STRUCT:
603 if (RSTRUCT_TRANSIENT_P(obj)) {
604 ptr = rb_struct_const_heap_ptr(obj);
605 }
606 break;
607 case T_HASH:
608 if (RHASH_TRANSIENT_P(obj)) {
609 TH_ASSERT(RHASH_AR_TABLE_P(obj));
610 ptr = (VALUE *)(RHASH(obj)->as.ar);
611 }
612 else {
613 ptr = NULL;
614 }
615 break;
616 default:
617 if (error) {
618 rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj));
619 }
620 }
621
622 return ptr;
623}
624
625static void
626transient_heap_promote_add(struct transient_heap* theap, VALUE obj)
627{
628 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj));
629
630 if (TRANSIENT_HEAP_DEBUG_DONT_PROMOTE) {
631 /* duplicate check */
632 int i;
633 for (i=0; i<theap->promoted_objects_index; i++) {
634 if (theap->promoted_objects[i] == obj) return;
635 }
636 }
637
638 if (theap->promoted_objects_size <= theap->promoted_objects_index) {
639 theap->promoted_objects_size *= 2;
640 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);
641 if (UNLIKELY((size_t)theap->promoted_objects_size > SIZE_MAX / sizeof(VALUE))) {
642 /* realloc failure due to integer overflow */
643 theap->promoted_objects = NULL;
644 }
645 else {
646 theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE));
647 }
648 if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed");
649 }
650 theap->promoted_objects[theap->promoted_objects_index++] = obj;
651}
652
653void
654 rb_transient_heap_promote(VALUE obj)
655{
656 ASSERT_vm_locking();
657
658 if (transient_heap_ptr(obj, FALSE)) {
659 struct transient_heap* theap = transient_heap_get();
660 transient_heap_promote_add(theap, obj);
661 }
662 else {
663 /* ignore */
664 }
665}
666
667static struct transient_alloc_header *
668alloc_header(struct transient_heap_block* block, int index)
669{
670 return (void *)&block->buff[index];
671}
672
673static void
674transient_heap_reset(void)
675{
676 ASSERT_vm_locking();
677
678 struct transient_heap* theap = transient_heap_get();
679 struct transient_heap_block* block;
680
681 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n");
682
683 block = theap->marked_blocks;
684 while (block) {
685 struct transient_heap_block *next_block = block->info.next_block;
686 theap->total_objects -= block->info.objects;
687#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
688 if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {
689 rb_bug("madvise err:%d", errno);
690 }
691 if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {
692 rb_bug("mprotect err:%d", errno);
693 }
694#else
695 reset_block(block);
696 connect_to_free_blocks(theap, block);
697#endif
698 theap->total_blocks--;
699 block = next_block;
700 }
701
702 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks);
703
704 theap->marked_blocks = NULL;
705 theap->total_marked_objects = 0;
706}
707
708static void
709transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block)
710{
711 int marked_index = block->info.last_marked_index;
712 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
713
714 while (marked_index >= 0) {
715 struct transient_alloc_header *header = alloc_header(block, marked_index);
716 asan_unpoison_memory_region(header, sizeof *header, true);
717 VALUE obj = header->obj;
718 TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
719 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("transient_heap_block_evacuate: wrong header %p %s\n", (void *)header, rb_obj_info(obj));
720
721 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", (void *)header, rb_obj_info(obj));
722
723 if (obj != Qnil) {
724 RB_DEBUG_COUNTER_INC(theap_evacuate);
725
726 switch (BUILTIN_TYPE(obj)) {
727 case T_ARRAY:
728 rb_ary_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
729 break;
730 case T_OBJECT:
731 rb_obj_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
732 break;
733 case T_STRUCT:
734 rb_struct_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
735 break;
736 case T_HASH:
737 rb_hash_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
738 break;
739 default:
740 rb_bug("unsupported: %s\n", rb_obj_info(obj));
741 }
742 header->obj = Qundef; /* for debug */
743 }
744 marked_index = header->next_marked_index;
745 asan_poison_memory_region(header, sizeof *header);
746 }
747}
748
749#if USE_RUBY_DEBUG_LOG
750static const char *
751transient_heap_status_cstr(enum transient_heap_status status)
752{
753 switch (status) {
754 case transient_heap_none: return "none";
755 case transient_heap_marking: return "marking";
756 case transient_heap_escaping: return "escaping";
757 }
758 UNREACHABLE_RETURN(NULL);
759}
760#endif
761
762static void
763transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status)
764{
765 RUBY_DEBUG_LOG("%s -> %s",
766 transient_heap_status_cstr(theap->status),
767 transient_heap_status_cstr(status));
768
769 TH_ASSERT(theap->status != status);
770 theap->status = status;
771}
772
773static void
774transient_heap_evacuate(void *dmy)
775{
776 struct transient_heap* theap = transient_heap_get();
777
778 if (theap->total_marked_objects == 0) return;
779 if (ruby_single_main_ractor == NULL) rb_bug("not single ractor mode");
780 if (theap->status == transient_heap_marking) {
781 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n");
782 }
783 else {
784 VALUE gc_disabled = rb_gc_disable_no_rest();
785 {
786 struct transient_heap_block* block;
787
788 RUBY_DEBUG_LOG("start gc_disabled:%d", RTEST(gc_disabled));
789
790 if (TRANSIENT_HEAP_DEBUG >= 1) {
791 int i;
792 fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks);
793 if (TRANSIENT_HEAP_DEBUG >= 4) {
794 for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i]));
795 }
796 }
797 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
798
799 TH_ASSERT(theap->status == transient_heap_none);
800 transient_heap_update_status(theap, transient_heap_escaping);
801
802 /* evacuate from marked blocks */
803 block = theap->marked_blocks;
804 while (block) {
805 transient_heap_block_evacuate(theap, block);
806 block = block->info.next_block;
807 }
808
809 /* evacuate from using blocks
810 only affect incremental marking */
811 block = theap->using_blocks;
812 while (block) {
813 transient_heap_block_evacuate(theap, block);
814 block = block->info.next_block;
815 }
816
817 /* all objects in marked_objects are escaped. */
818 transient_heap_reset();
819
820 if (TRANSIENT_HEAP_DEBUG > 0) {
821 fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks);
822 }
823
824 transient_heap_verify(theap);
825 transient_heap_update_status(theap, transient_heap_none);
826 }
827 if (gc_disabled != Qtrue) rb_gc_enable();
828 RUBY_DEBUG_LOG("finish", 0);
829 }
830}
831
832void
833 rb_transient_heap_evacuate(void)
834{
835 transient_heap_evacuate(NULL);
836}
837
838static void
839clear_marked_index(struct transient_heap_block* block)
840{
841 int marked_index = block->info.last_marked_index;
842
843 while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {
844 struct transient_alloc_header *header = alloc_header(block, marked_index);
845 /* header is poisoned to prevent buffer overflow, should
846 * unpoison first... */
847 asan_unpoison_memory_region(header, sizeof *header, false);
848 TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE);
849 if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", (void *)block, marked_index);
850
851 marked_index = header->next_marked_index;
852 header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
853 }
854
855 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
856}
857
858static void
859blocks_clear_marked_index(struct transient_heap_block* block)
860{
861 while (block) {
862 clear_marked_index(block);
863 block = block->info.next_block;
864 }
865}
866
867static void
868transient_heap_block_update_refs(struct transient_heap* theap, struct transient_heap_block* block)
869{
870 int marked_index = block->info.last_marked_index;
871
872 while (marked_index >= 0) {
873 struct transient_alloc_header *header = alloc_header(block, marked_index);
874
875 asan_unpoison_memory_region(header, sizeof *header, false);
876
877 header->obj = rb_gc_location(header->obj);
878
879 marked_index = header->next_marked_index;
880 asan_poison_memory_region(header, sizeof *header);
881 }
882}
883
884static void
885transient_heap_blocks_update_refs(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
886{
887 while (block) {
888 transient_heap_block_update_refs(theap, block);
889 block = block->info.next_block;
890 }
891}
892
893void
894 rb_transient_heap_update_references(void)
895{
896 ASSERT_vm_locking();
897
898 struct transient_heap* theap = transient_heap_get();
899 int i;
900
901 transient_heap_blocks_update_refs(theap, theap->using_blocks, "using_blocks");
902 transient_heap_blocks_update_refs(theap, theap->marked_blocks, "marked_blocks");
903
904 for (i=0; i<theap->promoted_objects_index; i++) {
905 VALUE obj = theap->promoted_objects[i];
906 theap->promoted_objects[i] = rb_gc_location(obj);
907 }
908}
909
910void
911 rb_transient_heap_start_marking(int full_marking)
912{
913 ASSERT_vm_locking();
914 RUBY_DEBUG_LOG("full?:%d", full_marking);
915
916 struct transient_heap* theap = transient_heap_get();
917
918 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promoted:%d full_marking:%d\n",
919 theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking);
920 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
921
922 blocks_clear_marked_index(theap->marked_blocks);
923 blocks_clear_marked_index(theap->using_blocks);
924
925 if (theap->using_blocks) {
926 if (theap->using_blocks->info.objects > 0) {
927 append_to_marked_blocks(theap, theap->using_blocks);
928 theap->using_blocks = NULL;
929 }
930 else {
931 append_to_marked_blocks(theap, theap->using_blocks->info.next_block);
932 theap->using_blocks->info.next_block = NULL;
933 }
934 }
935
936 if (theap->using_blocks == NULL) {
937 theap->using_blocks = transient_heap_allocatable_block(theap);
938 }
939
940 TH_ASSERT(theap->status == transient_heap_none);
941 transient_heap_update_status(theap, transient_heap_marking);
942 theap->total_marked_objects = 0;
943
944 if (full_marking) {
945 theap->promoted_objects_index = 0;
946 }
947 else { /* mark promoted objects */
948 int i;
949 for (i=0; i<theap->promoted_objects_index; i++) {
950 VALUE obj = theap->promoted_objects[i];
951 const void *ptr = transient_heap_ptr(obj, TRUE);
952 if (ptr) {
953 rb_transient_heap_mark(obj, ptr);
954 }
955 }
956 }
957
958 transient_heap_verify(theap);
959}
960
961void
962 rb_transient_heap_finish_marking(void)
963{
964 ASSERT_vm_locking();
965 RUBY_DEBUG_LOG("", 0);
966
967 struct transient_heap* theap = transient_heap_get();
968
969 RUBY_DEBUG_LOG("objects:%d, marked:%d",
970 theap->total_objects,
971 theap->total_marked_objects);
972 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
973
974 TH_ASSERT(theap->total_objects >= theap->total_marked_objects);
975
976 TH_ASSERT(theap->status == transient_heap_marking);
977 transient_heap_update_status(theap, transient_heap_none);
978
979 if (theap->total_marked_objects > 0) {
980 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n");
981 rb_postponed_job_register_one(0, transient_heap_evacuate, NULL);
982 }
983 else {
984 transient_heap_reset();
985 }
986
987 transient_heap_verify(theap);
988}
989#endif /* USE_TRANSIENT_HEAP */
UNREACHABLE_RETURN
#define UNREACHABLE_RETURN
Definition: assume.h:31
ptr
struct RIMemo * ptr
Definition: debug.c:88
debug.h
rb_postponed_job_register_one
int rb_postponed_job_register_one(unsigned int flags, rb_postponed_job_func_t func, void *data)
Definition: vm_trace.c:1627
debug_counter.h
RB_DEBUG_COUNTER_INC
#define RB_DEBUG_COUNTER_INC(type)
Definition: debug_counter.h:444
UNLIKELY
#define UNLIKELY(x)
Definition: ffi_common.h:126
rb_gc_location
VALUE rb_gc_location(VALUE value)
Definition: gc.c:9003
rb_gc_enable
VALUE rb_gc_enable(void)
Definition: gc.c:9888
rb_aligned_malloc
void * rb_aligned_malloc(size_t alignment, size_t size)
Definition: gc.c:10344
rb_obj_info
const char * rb_obj_info(VALUE obj)
Definition: gc.c:12499
rb_gc_disable_no_rest
VALUE rb_gc_disable_no_rest(void)
Definition: gc.c:9910
FL_TEST_RAW
#define FL_TEST_RAW
Definition: fl_type.h:131
rb_bug
void rb_bug(const char *fmt,...)
Definition: error.c:768
gc.h
Internal header for GC.
hash.h
Internal header for Hash.
RHASH
#define RHASH(obj)
Definition: hash.h:57
STATIC_ASSERT
#define STATIC_ASSERT
Definition: static_assert.h:14
struct.h
Internal header for Struct.
internal.h
size
voidpf void uLong size
Definition: ioapi.h:138
int
typedef int(ZCALLBACK *close_file_func) OF((voidpf opaque
SIZE_MAX
#define SIZE_MAX
Definition: limits.h:71
TRUE
#define TRUE
Definition: nkf.h:175
FALSE
#define FALSE
Definition: nkf.h:174
ROUND_UP
#define ROUND_UP(v, a)
Definition: ffi.c:37
RARRAY_EMBED_FLAG
#define RARRAY_EMBED_FLAG
Definition: rarray.h:43
RARRAY
#define RARRAY(obj)
Definition: rarray.h:42
RBASIC_CLASS
#define RBASIC_CLASS
Definition: rbasic.h:35
NULL
#define NULL
Definition: regenc.h:69
ruby.h
ruby_assert.h
sanitizers.h
Internal header for ASAN / MSAN / etc.
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
#define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x)
Definition: sanitizers.h:34
Qundef
#define Qundef
Definition: special_consts.h:53
Qtrue
#define Qtrue
Definition: special_consts.h:52
RTEST
#define RTEST
Definition: special_consts.h:42
Qnil
#define Qnil
Definition: special_consts.h:51
realloc
#define realloc
Definition: st.c:172
malloc
#define malloc
Definition: st.c:170
transient_heap.h
rb_transient_heap_finish_marking
#define rb_transient_heap_finish_marking()
Definition: transient_heap.h:56
rb_obj_transient_heap_evacuate
#define rb_obj_transient_heap_evacuate(x, y)
Definition: transient_heap.h:60
rb_transient_heap_promote
#define rb_transient_heap_promote(obj)
Definition: transient_heap.h:52
rb_transient_heap_verify
#define rb_transient_heap_verify()
Definition: transient_heap.h:51
rb_ary_transient_heap_evacuate
#define rb_ary_transient_heap_evacuate(x, y)
Definition: transient_heap.h:59
rb_hash_transient_heap_evacuate
#define rb_hash_transient_heap_evacuate(x, y)
Definition: transient_heap.h:61
rb_struct_transient_heap_evacuate
#define rb_struct_transient_heap_evacuate(x, y)
Definition: transient_heap.h:62
rb_transient_heap_evacuate
#define rb_transient_heap_evacuate()
Definition: transient_heap.h:55
rb_transient_heap_alloc
#define rb_transient_heap_alloc(o, s)
Definition: transient_heap.h:50
rb_transient_heap_mark
#define rb_transient_heap_mark(obj, ptr)
Definition: transient_heap.h:57
rb_transient_heap_update_references
#define rb_transient_heap_update_references()
Definition: transient_heap.h:54
rb_transient_heap_start_marking
#define rb_transient_heap_start_marking(full_marking)
Definition: transient_heap.h:53
error
void error(const char *msg)
Definition: untgz.c:593
VALUE
unsigned long VALUE
Definition: value.h:38
T_STRUCT
#define T_STRUCT
Definition: value_type.h:78
T_HASH
#define T_HASH
Definition: value_type.h:64
T_ARRAY
#define T_ARRAY
Definition: value_type.h:55
T_OBJECT
#define T_OBJECT
Definition: value_type.h:74
BUILTIN_TYPE
#define BUILTIN_TYPE
Definition: value_type.h:84
ruby_single_main_ractor
rb_ractor_t * ruby_single_main_ractor
Definition: vm.c:381
vm_debug.h
RUBY_DEBUG_LOG
#define RUBY_DEBUG_LOG(fmt,...)
Definition: vm_debug.h:112
vm_sync.h
ASSERT_vm_locking
#define ASSERT_vm_locking()
Definition: vm_sync.h:134
intptr_t
int intptr_t
Definition: win32.h:90