Ruby: mjit_compile.c Source File

Ruby 3.0.5p211 (2022-11-24 revision ba5cf0f7c52d4d35cc6a173c89eda98ceffa2dcf)

mjit_compile.c
Go to the documentation of this file.
1/**********************************************************************
2
3 mjit_compile.c - MRI method JIT compiler
4
5 Copyright (C) 2017 Takashi Kokubun <takashikkbn@gmail.com>.
6
7**********************************************************************/
8
9// NOTE: All functions in this file are executed on MJIT worker. So don't
10// call Ruby methods (C functions that may call rb_funcall) or trigger
11// GC (using ZALLOC, xmalloc, xfree, etc.) in this file.
12
13#include "ruby/internal/config.h" // defines USE_MJIT
14
15#if USE_MJIT
16
17#include "internal.h"
18#include "internal/compile.h"
19#include "internal/hash.h"
20#include "internal/object.h"
21#include "internal/variable.h"
22#include "mjit.h"
23#include "vm_core.h"
24#include "vm_callinfo.h"
25#include "vm_exec.h"
26#include "vm_insnhelper.h"
27
28#include "builtin.h"
29#include "insns.inc"
30#include "insns_info.inc"
31
32// Macros to check if a position is already compiled using compile_status.stack_size_for_pos
33#define NOT_COMPILED_STACK_SIZE -1
34#define ALREADY_COMPILED_P(status, pos) (status->stack_size_for_pos[pos] != NOT_COMPILED_STACK_SIZE)
35
36// For propagating information needed for lazily pushing a frame.
37struct inlined_call_context {
38 int orig_argc; // ci->orig_argc
39 VALUE me; // vm_cc_cme(cc)
40 int param_size; // def_iseq_ptr(vm_cc_cme(cc)->def)->body->param.size
41 int local_size; // def_iseq_ptr(vm_cc_cme(cc)->def)->body->local_table_size
42};
43
44// Storage to keep compiler's status. This should have information
45// which is global during one `mjit_compile` call. Ones conditional
46// in each branch should be stored in `compile_branch`.
47struct compile_status {
48 bool success; // has true if compilation has had no issue
49 int *stack_size_for_pos; // stack_size_for_pos[pos] has stack size for the position (otherwise -1)
50 // If true, JIT-ed code will use local variables to store pushed values instead of
51 // using VM's stack and moving stack pointer.
52 bool local_stack_p;
53 // Safely-accessible ivar cache entries copied from main thread.
54 union iseq_inline_storage_entry *is_entries;
55 // Index of call cache entries captured to compiled_iseq to be marked on GC
56 int cc_entries_index;
57 // A pointer to root (i.e. not inlined) iseq being compiled.
58 const struct rb_iseq_constant_body *compiled_iseq;
59 int compiled_id; // Just a copy of compiled_iseq->jit_unit->id
60 // Mutated optimization levels
61 struct rb_mjit_compile_info *compile_info;
62 bool merge_ivar_guards_p; // If true, merge guards of ivar accesses
63 rb_serial_t ivar_serial; // ic_serial of IVC in is_entries (used only when merge_ivar_guards_p)
64 size_t max_ivar_index; // Max IVC index in is_entries (used only when merge_ivar_guards_p)
65 // If `inlined_iseqs[pos]` is not NULL, `mjit_compile_body` tries to inline ISeq there.
66 const struct rb_iseq_constant_body **inlined_iseqs;
67 struct inlined_call_context inline_context;
68};
69
70// Storage to keep data which is consistent in each conditional branch.
71// This is created and used for one `compile_insns` call and its values
72// should be copied for extra `compile_insns` call.
73struct compile_branch {
74 unsigned int stack_size; // this simulates sp (stack pointer) of YARV
75 bool finish_p; // if true, compilation in this branch should stop and let another branch to be compiled
76};
77
78struct case_dispatch_var {
79 FILE *f;
80 unsigned int base_pos;
81 VALUE last_value;
82};
83
84static size_t
85call_data_index(CALL_DATA cd, const struct rb_iseq_constant_body *body)
86{
87 return cd - body->call_data;
88}
89
90const struct rb_callcache ** mjit_iseq_cc_entries(const struct rb_iseq_constant_body *const body);
91
92// Using this function to refer to cc_entries allocated by `mjit_capture_cc_entries`
93// instead of storing cc_entries in status directly so that we always refer to a new address
94// returned by `realloc` inside it.
95static const struct rb_callcache **
96captured_cc_entries(const struct compile_status *status)
97{
98 VM_ASSERT(status->cc_entries_index != -1);
99 return mjit_iseq_cc_entries(status->compiled_iseq) + status->cc_entries_index;
100}
101
102// Returns true if call cache is still not obsoleted and vm_cc_cme(cc)->def->type is available.
103static bool
104has_valid_method_type(CALL_CACHE cc)
105{
106 return vm_cc_cme(cc) != NULL;
107}
108
109// Returns true if MJIT thinks this cc's opt_* insn may fallback to opt_send_without_block.
110static bool
111has_cache_for_send(CALL_CACHE cc, int insn)
112{
113 extern bool rb_vm_opt_cfunc_p(CALL_CACHE cc, int insn);
114 return has_valid_method_type(cc) &&
115 !(vm_cc_cme(cc)->def->type == VM_METHOD_TYPE_CFUNC && rb_vm_opt_cfunc_p(cc, insn));
116}
117
118// Returns true if iseq can use fastpath for setup, otherwise NULL. This becomes true in the same condition
119// as CC_SET_FASTPATH (in vm_callee_setup_arg) is called from vm_call_iseq_setup.
120static bool
121fastpath_applied_iseq_p(const CALL_INFO ci, const CALL_CACHE cc, const rb_iseq_t *iseq)
122{
123 extern bool rb_simple_iseq_p(const rb_iseq_t *iseq);
124 return iseq != NULL
125 && !(vm_ci_flag(ci) & VM_CALL_KW_SPLAT) && rb_simple_iseq_p(iseq) // Top of vm_callee_setup_arg. In this case, opt_pc is 0.
126 && vm_ci_argc(ci) == (unsigned int)iseq->body->param.lead_num // exclude argument_arity_error (assumption: `calling->argc == ci->orig_argc` in send insns)
127 && vm_call_iseq_optimizable_p(ci, cc); // CC_SET_FASTPATH condition
128}
129
130// Return true if an object of the klass may be a special const. See: rb_class_of
131static bool
132maybe_special_const_class_p(const VALUE klass)
133{
134 return klass == rb_cFalseClass
135 || klass == rb_cNilClass
136 || klass == rb_cTrueClass
137 || klass == rb_cInteger
138 || klass == rb_cSymbol
139 || klass == rb_cFloat;
140}
141
142static int
143compile_case_dispatch_each(VALUE key, VALUE value, VALUE arg)
144{
145 struct case_dispatch_var *var = (struct case_dispatch_var *)arg;
146 unsigned int offset;
147
148 if (var->last_value != value) {
149 offset = FIX2INT(value);
150 var->last_value = value;
151 fprintf(var->f, " case %d:\n", offset);
152 fprintf(var->f, " goto label_%d;\n", var->base_pos + offset);
153 fprintf(var->f, " break;\n");
154 }
155 return ST_CONTINUE;
156}
157
158// Calling rb_id2str in MJIT worker causes random SEGV. So this is disabled by default.
159static void
160comment_id(FILE *f, ID id)
161{
162#ifdef MJIT_COMMENT_ID
163 VALUE name = rb_id2str(id);
164 const char *p, *e;
165 char c, prev = '\0';
166
167 if (!name) return;
168 p = RSTRING_PTR(name);
169 e = RSTRING_END(name);
170 fputs("/* :\"", f);
171 for (; p < e; ++p) {
172 switch (c = *p) {
173 case '*': case '/': if (prev != (c ^ ('/' ^ '*'))) break;
174 case '\\': case '"': fputc('\\', f);
175 }
176 fputc(c, f);
177 prev = c;
178 }
179 fputs("\" */", f);
180#endif
181}
182
183static void compile_insns(FILE *f, const struct rb_iseq_constant_body *body, unsigned int stack_size,
184 unsigned int pos, struct compile_status *status);
185
186// Main function of JIT compilation, vm_exec_core counterpart for JIT. Compile one insn to `f`, may modify
187// b->stack_size and return next position.
188//
189// When you add a new instruction to insns.def, it would be nice to have JIT compilation support here but
190// it's optional. This JIT compiler just ignores ISeq which includes unknown instruction, and ISeq which
191// does not have it can be compiled as usual.
192static unsigned int
193compile_insn(FILE *f, const struct rb_iseq_constant_body *body, const int insn, const VALUE *operands,
194 const unsigned int pos, struct compile_status *status, struct compile_branch *b)
195{
196 unsigned int next_pos = pos + insn_len(insn);
197
198/*****************/
199 #include "mjit_compile.inc"
200/*****************/
201
202 // If next_pos is already compiled and this branch is not finished yet,
203 // next instruction won't be compiled in C code next and will need `goto`.
204 if (!b->finish_p && next_pos < body->iseq_size && ALREADY_COMPILED_P(status, next_pos)) {
205 fprintf(f, "goto label_%d;\n", next_pos);
206
207 // Verify stack size assumption is the same among multiple branches
208 if ((unsigned int)status->stack_size_for_pos[next_pos] != b->stack_size) {
209 if (mjit_opts.warnings || mjit_opts.verbose)
210 fprintf(stderr, "MJIT warning: JIT stack assumption is not the same between branches (%d != %u)\n",
211 status->stack_size_for_pos[next_pos], b->stack_size);
212 status->success = false;
213 }
214 }
215
216 return next_pos;
217}
218
219// Compile one conditional branch. If it has branchXXX insn, this should be
220// called multiple times for each branch.
221static void
222compile_insns(FILE *f, const struct rb_iseq_constant_body *body, unsigned int stack_size,
223 unsigned int pos, struct compile_status *status)
224{
225 int insn;
226 struct compile_branch branch;
227
228 branch.stack_size = stack_size;
229 branch.finish_p = false;
230
231 while (pos < body->iseq_size && !ALREADY_COMPILED_P(status, pos) && !branch.finish_p) {
232#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
233 insn = rb_vm_insn_addr2insn((void *)body->iseq_encoded[pos]);
234#else
235 insn = (int)body->iseq_encoded[pos];
236#endif
237 status->stack_size_for_pos[pos] = (int)branch.stack_size;
238
239 fprintf(f, "\nlabel_%d: /* %s */\n", pos, insn_name(insn));
240 pos = compile_insn(f, body, insn, body->iseq_encoded + (pos+1), pos, status, &branch);
241 if (status->success && branch.stack_size > body->stack_max) {
242 if (mjit_opts.warnings || mjit_opts.verbose)
243 fprintf(stderr, "MJIT warning: JIT stack size (%d) exceeded its max size (%d)\n", branch.stack_size, body->stack_max);
244 status->success = false;
245 }
246 if (!status->success)
247 break;
248 }
249}
250
251// Print the block to cancel inlined method call. It's supporting only `opt_send_without_block` for now.
252static void
253compile_inlined_cancel_handler(FILE *f, const struct rb_iseq_constant_body *body, struct inlined_call_context *inline_context)
254{
255 fprintf(f, "\ncancel:\n");
256 fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
257 fprintf(f, " rb_mjit_recompile_inlining(original_iseq);\n");
258
259 // Swap pc/sp set on cancel with original pc/sp.
260 fprintf(f, " const VALUE *current_pc = reg_cfp->pc;\n");
261 fprintf(f, " VALUE *current_sp = reg_cfp->sp;\n");
262 fprintf(f, " reg_cfp->pc = orig_pc;\n");
263 fprintf(f, " reg_cfp->sp = orig_sp;\n\n");
264
265 // Lazily push the current call frame.
266 fprintf(f, " struct rb_calling_info calling;\n");
267 fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); // assumes `opt_send_without_block`
268 fprintf(f, " calling.argc = %d;\n", inline_context->orig_argc);
269 fprintf(f, " calling.recv = reg_cfp->self;\n");
270 fprintf(f, " reg_cfp->self = orig_self;\n");
271 fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n\n",
272 inline_context->me, inline_context->param_size, inline_context->local_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE
273
274 // Start usual cancel from here.
275 fprintf(f, " reg_cfp = ec->cfp;\n"); // work on the new frame
276 fprintf(f, " reg_cfp->pc = current_pc;\n");
277 fprintf(f, " reg_cfp->sp = current_sp;\n");
278 for (unsigned int i = 0; i < body->stack_max; i++) { // should be always `status->local_stack_p`
279 fprintf(f, " *(vm_base_ptr(reg_cfp) + %d) = stack[%d];\n", i, i);
280 }
281 // We're not just returning Qundef here so that caller's normal cancel handler can
282 // push back `stack` to `cfp->sp`.
283 fprintf(f, " return vm_exec(ec, false);\n");
284}
285
286// Print the block to cancel JIT execution.
287static void
288compile_cancel_handler(FILE *f, const struct rb_iseq_constant_body *body, struct compile_status *status)
289{
290 if (status->inlined_iseqs == NULL) { // the current ISeq is being inlined
291 compile_inlined_cancel_handler(f, body, &status->inline_context);
292 return;
293 }
294
295 fprintf(f, "\nsend_cancel:\n");
296 fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_send_inline);\n");
297 fprintf(f, " rb_mjit_recompile_send(original_iseq);\n");
298 fprintf(f, " goto cancel;\n");
299
300 fprintf(f, "\nivar_cancel:\n");
301 fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_ivar_inline);\n");
302 fprintf(f, " rb_mjit_recompile_ivar(original_iseq);\n");
303 fprintf(f, " goto cancel;\n");
304
305 fprintf(f, "\nexivar_cancel:\n");
306 fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_exivar_inline);\n");
307 fprintf(f, " rb_mjit_recompile_exivar(original_iseq);\n");
308 fprintf(f, " goto cancel;\n");
309
310 fprintf(f, "\nconst_cancel:\n");
311 fprintf(f, " rb_mjit_recompile_const(original_iseq);\n");
312 fprintf(f, " goto cancel;\n");
313
314 fprintf(f, "\ncancel:\n");
315 fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
316 if (status->local_stack_p) {
317 for (unsigned int i = 0; i < body->stack_max; i++) {
318 fprintf(f, " *(vm_base_ptr(reg_cfp) + %d) = stack[%d];\n", i, i);
319 }
320 }
321 fprintf(f, " return Qundef;\n");
322}
323
324extern int
325 mjit_capture_cc_entries(const struct rb_iseq_constant_body *compiled_iseq, const struct rb_iseq_constant_body *captured_iseq);
326
327// Copy current is_entries and use it throughout the current compilation consistently.
328// While ic->entry has been immutable since https://github.com/ruby/ruby/pull/3662,
329// we still need this to avoid a race condition between entries and ivar_serial/max_ivar_index.
330static void
331mjit_capture_is_entries(const struct rb_iseq_constant_body *body, union iseq_inline_storage_entry *is_entries)
332{
333 if (is_entries == NULL)
334 return;
335 memcpy(is_entries, body->is_entries, sizeof(union iseq_inline_storage_entry) * body->is_size);
336}
337
338static bool
339mjit_compile_body(FILE *f, const rb_iseq_t *iseq, struct compile_status *status)
340{
341 const struct rb_iseq_constant_body *body = iseq->body;
342 status->success = true;
343 status->local_stack_p = !body->catch_except_p;
344
345 if (status->local_stack_p) {
346 fprintf(f, " VALUE stack[%d];\n", body->stack_max);
347 }
348 else {
349 fprintf(f, " VALUE *stack = reg_cfp->sp;\n");
350 }
351 if (status->inlined_iseqs != NULL) // i.e. compile root
352 fprintf(f, " static const rb_iseq_t *original_iseq = (const rb_iseq_t *)0x%"PRIxVALUE";\n", (VALUE)iseq);
353 fprintf(f, " static const VALUE *const original_body_iseq = (VALUE *)0x%"PRIxVALUE";\n",
354 (VALUE)body->iseq_encoded);
355 fprintf(f, " VALUE cfp_self = reg_cfp->self;\n"); // cache self across the method
356 fprintf(f, "#define GET_SELF() cfp_self\n");
357
358 // Generate merged ivar guards first if needed
359 if (!status->compile_info->disable_ivar_cache && status->merge_ivar_guards_p) {
360 fprintf(f, " if (UNLIKELY(!(RB_TYPE_P(GET_SELF(), T_OBJECT) && (rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(GET_SELF())->klass) &&", status->ivar_serial);
361 if (status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX) {
362 fprintf(f, "%"PRIuSIZE" < ROBJECT_NUMIV(GET_SELF())", status->max_ivar_index); // index < ROBJECT_NUMIV(obj) && !RB_FL_ANY_RAW(obj, ROBJECT_EMBED)
363 }
364 else {
365 fprintf(f, "ROBJECT_EMBED_LEN_MAX == ROBJECT_NUMIV(GET_SELF())"); // index < ROBJECT_NUMIV(obj) && RB_FL_ANY_RAW(obj, ROBJECT_EMBED)
366 }
367 fprintf(f, "))) {\n");
368 fprintf(f, " goto ivar_cancel;\n");
369 fprintf(f, " }\n");
370 }
371
372 // Simulate `opt_pc` in setup_parameters_complex. Other PCs which may be passed by catch tables
373 // are not considered since vm_exec doesn't call mjit_exec for catch tables.
374 if (body->param.flags.has_opt) {
375 int i;
376 fprintf(f, "\n");
377 fprintf(f, " switch (reg_cfp->pc - reg_cfp->iseq->body->iseq_encoded) {\n");
378 for (i = 0; i <= body->param.opt_num; i++) {
379 VALUE pc_offset = body->param.opt_table[i];
380 fprintf(f, " case %"PRIdVALUE":\n", pc_offset);
381 fprintf(f, " goto label_%"PRIdVALUE";\n", pc_offset);
382 }
383 fprintf(f, " }\n");
384 }
385
386 compile_insns(f, body, 0, 0, status);
387 compile_cancel_handler(f, body, status);
388 fprintf(f, "#undef GET_SELF");
389 return status->success;
390}
391
392// Return true if the ISeq can be inlined without pushing a new control frame.
393static bool
394inlinable_iseq_p(const struct rb_iseq_constant_body *body)
395{
396 // 1) If catch_except_p, caller frame should be preserved when callee catches an exception.
397 // Then we need to wrap `vm_exec()` but then we can't inline the call inside it.
398 //
399 // 2) If `body->catch_except_p` is false and `handles_sp?` of an insn is false,
400 // sp is not moved as we assume `status->local_stack_p = !body->catch_except_p`.
401 //
402 // 3) If `body->catch_except_p` is false and `always_leaf?` of an insn is true,
403 // pc is not moved.
404 if (body->catch_except_p)
405 return false;
406
407 unsigned int pos = 0;
408 while (pos < body->iseq_size) {
409#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
410 int insn = rb_vm_insn_addr2insn((void *)body->iseq_encoded[pos]);
411#else
412 int insn = (int)body->iseq_encoded[pos];
413#endif
414 // All insns in the ISeq except `leave` (to be overridden in the inlined code)
415 // should meet following strong assumptions:
416 // * Do not require `cfp->sp` motion
417 // * Do not move `cfp->pc`
418 // * Do not read any `cfp->pc`
419 if (insn == BIN(invokebuiltin) || insn == BIN(opt_invokebuiltin_delegate) || insn == BIN(opt_invokebuiltin_delegate_leave)) {
420 // builtin insn's inlinability is handled by `Primitive.attr! 'inline'` per iseq
421 if (!body->builtin_inline_p)
422 return false;
423 }
424 else if (insn != BIN(leave) && insn_may_depend_on_sp_or_pc(insn, body->iseq_encoded + (pos + 1)))
425 return false;
426 // At this moment, `cfp->ep` in an inlined method is not working.
427 switch (insn) {
428 case BIN(getlocal):
429 case BIN(getlocal_WC_0):
430 case BIN(getlocal_WC_1):
431 case BIN(setlocal):
432 case BIN(setlocal_WC_0):
433 case BIN(setlocal_WC_1):
434 case BIN(getblockparam):
435 case BIN(getblockparamproxy):
436 case BIN(setblockparam):
437 return false;
438 }
439 pos += insn_len(insn);
440 }
441 return true;
442}
443
444// Return an iseq pointer if cc has inlinable iseq.
445const rb_iseq_t *
446rb_mjit_inlinable_iseq(const struct rb_callinfo *ci, const struct rb_callcache *cc)
447{
448 const rb_iseq_t *iseq;
449 if (has_valid_method_type(cc) &&
450 !(vm_ci_flag(ci) & VM_CALL_TAILCALL) && // inlining only non-tailcall path
451 vm_cc_cme(cc)->def->type == VM_METHOD_TYPE_ISEQ &&
452 fastpath_applied_iseq_p(ci, cc, iseq = def_iseq_ptr(vm_cc_cme(cc)->def)) &&
453 // CC_SET_FASTPATH in vm_callee_setup_arg
454 inlinable_iseq_p(iseq->body)) {
455 return iseq;
456 }
457 return NULL;
458}
459
460static void
461init_ivar_compile_status(const struct rb_iseq_constant_body *body, struct compile_status *status)
462{
463 mjit_capture_is_entries(body, status->is_entries);
464
465 int num_ivars = 0;
466 unsigned int pos = 0;
467 status->max_ivar_index = 0;
468 status->ivar_serial = 0;
469
470 while (pos < body->iseq_size) {
471#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
472 int insn = rb_vm_insn_addr2insn((void *)body->iseq_encoded[pos]);
473#else
474 int insn = (int)body->iseq_encoded[pos];
475#endif
476 if (insn == BIN(getinstancevariable) || insn == BIN(setinstancevariable)) {
477 IVC ic = (IVC)body->iseq_encoded[pos+2];
478 IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache;
479 if (ic_copy->entry) { // Only initialized (ic_serial > 0) IVCs are optimized
480 num_ivars++;
481
482 if (status->max_ivar_index < ic_copy->entry->index) {
483 status->max_ivar_index = ic_copy->entry->index;
484 }
485
486 if (status->ivar_serial == 0) {
487 status->ivar_serial = ic_copy->entry->class_serial;
488 }
489 else if (status->ivar_serial != ic_copy->entry->class_serial) {
490 // Multiple classes have used this ISeq. Give up assuming one serial.
491 status->merge_ivar_guards_p = false;
492 return;
493 }
494 }
495 }
496 pos += insn_len(insn);
497 }
498 status->merge_ivar_guards_p = status->ivar_serial > 0 && num_ivars >= 2;
499}
500
501// This needs to be macro instead of a function because it's using `alloca`.
502#define INIT_COMPILE_STATUS(status, body, compile_root_p) do { \
503 status = (struct compile_status){ \
504 .stack_size_for_pos = (int *)alloca(sizeof(int) * body->iseq_size), \
505 .inlined_iseqs = compile_root_p ? \
506 alloca(sizeof(const struct rb_iseq_constant_body *) * body->iseq_size) : NULL, \
507 .is_entries = (body->is_size > 0) ? \
508 alloca(sizeof(union iseq_inline_storage_entry) * body->is_size) : NULL, \
509 .cc_entries_index = (body->ci_size > 0) ? \
510 mjit_capture_cc_entries(status.compiled_iseq, body) : -1, \
511 .compiled_id = status.compiled_id, \
512 .compiled_iseq = status.compiled_iseq, \
513 .compile_info = compile_root_p ? \
514 rb_mjit_iseq_compile_info(body) : alloca(sizeof(struct rb_mjit_compile_info)) \
515 }; \
516 memset(status.stack_size_for_pos, NOT_COMPILED_STACK_SIZE, sizeof(int) * body->iseq_size); \
517 if (compile_root_p) \
518 memset((void *)status.inlined_iseqs, 0, sizeof(const struct rb_iseq_constant_body *) * body->iseq_size); \
519 else \
520 memset(status.compile_info, 0, sizeof(struct rb_mjit_compile_info)); \
521} while (0)
522
523// Compile inlinable ISeqs to C code in `f`. It returns true if it succeeds to compile them.
524static bool
525precompile_inlinable_iseqs(FILE *f, const rb_iseq_t *iseq, struct compile_status *status)
526{
527 const struct rb_iseq_constant_body *body = iseq->body;
528 unsigned int pos = 0;
529 while (pos < body->iseq_size) {
530#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
531 int insn = rb_vm_insn_addr2insn((void *)body->iseq_encoded[pos]);
532#else
533 int insn = (int)body->iseq_encoded[pos];
534#endif
535 if (insn == BIN(opt_send_without_block)) { // `compile_inlined_cancel_handler` supports only `opt_send_without_block`
536 CALL_DATA cd = (CALL_DATA)body->iseq_encoded[pos + 1];
537 const struct rb_callinfo *ci = cd->ci;
538 const struct rb_callcache *cc = captured_cc_entries(status)[call_data_index(cd, body)]; // use copy to avoid race condition
539
540 extern bool rb_mjit_compiling_iseq_p(const rb_iseq_t *iseq);
541 const rb_iseq_t *child_iseq;
542 if ((child_iseq = rb_mjit_inlinable_iseq(ci, cc)) != NULL && rb_mjit_compiling_iseq_p(child_iseq)) {
543 status->inlined_iseqs[pos] = child_iseq->body;
544
545 if (mjit_opts.verbose >= 1) // print beforehand because ISeq may be GCed during copy job.
546 fprintf(stderr, "JIT inline: %s@%s:%d => %s@%s:%d\n",
547 RSTRING_PTR(child_iseq->body->location.label),
548 RSTRING_PTR(rb_iseq_path(child_iseq)), FIX2INT(child_iseq->body->location.first_lineno),
549 RSTRING_PTR(iseq->body->location.label),
550 RSTRING_PTR(rb_iseq_path(iseq)), FIX2INT(iseq->body->location.first_lineno));
551
552 struct compile_status child_status = { .compiled_iseq = status->compiled_iseq, .compiled_id = status->compiled_id };
553 INIT_COMPILE_STATUS(child_status, child_iseq->body, false);
554 child_status.inline_context = (struct inlined_call_context){
555 .orig_argc = vm_ci_argc(ci),
556 .me = (VALUE)vm_cc_cme(cc),
557 .param_size = child_iseq->body->param.size,
558 .local_size = child_iseq->body->local_table_size
559 };
560 if (child_iseq->body->ci_size > 0 && child_status.cc_entries_index == -1) {
561 return false;
562 }
563 init_ivar_compile_status(child_iseq->body, &child_status);
564
565 fprintf(f, "ALWAYS_INLINE(static VALUE _mjit%d_inlined_%d(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE orig_self, const rb_iseq_t *original_iseq));\n", status->compiled_id, pos);
566 fprintf(f, "static inline VALUE\n_mjit%d_inlined_%d(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE orig_self, const rb_iseq_t *original_iseq)\n{\n", status->compiled_id, pos);
567 fprintf(f, " const VALUE *orig_pc = reg_cfp->pc;\n");
568 fprintf(f, " VALUE *orig_sp = reg_cfp->sp;\n");
569 bool success = mjit_compile_body(f, child_iseq, &child_status);
570 fprintf(f, "\n} /* end of _mjit%d_inlined_%d */\n\n", status->compiled_id, pos);
571
572 if (!success)
573 return false;
574 }
575 }
576 pos += insn_len(insn);
577 }
578 return true;
579}
580
581// Compile ISeq to C code in `f`. It returns true if it succeeds to compile.
582bool
583mjit_compile(FILE *f, const rb_iseq_t *iseq, const char *funcname, int id)
584{
585 struct compile_status status = { .compiled_iseq = iseq->body, .compiled_id = id };
586 INIT_COMPILE_STATUS(status, iseq->body, true);
587 if (iseq->body->ci_size > 0 && status.cc_entries_index == -1) {
588 return false;
589 }
590 init_ivar_compile_status(iseq->body, &status);
591
592 if (!status.compile_info->disable_send_cache && !status.compile_info->disable_inlining) {
593 if (!precompile_inlinable_iseqs(f, iseq, &status))
594 return false;
595 }
596
597#ifdef _WIN32
598 fprintf(f, "__declspec(dllexport)\n");
599#endif
600 fprintf(f, "VALUE\n%s(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)\n{\n", funcname);
601 bool success = mjit_compile_body(f, iseq, &status);
602 fprintf(f, "\n} // end of %s\n", funcname);
603 return success;
604}
605
606#endif // USE_MJIT
compile.h
Internal header for the compiler.
rb_vm_insn_addr2insn
int rb_vm_insn_addr2insn(const void *)
Definition: iseq.c:3172
RSTRING_PTR
#define RSTRING_PTR(string)
Definition: fbuffer.h:19
memcpy
#define memcpy(d, s, n)
Definition: ffi_common.h:55
rb_cInteger
VALUE rb_cInteger
Definition: numeric.c:191
rb_cSymbol
VALUE rb_cSymbol
Definition: string.c:81
rb_cFloat
VALUE rb_cFloat
Definition: numeric.c:190
rb_cNilClass
VALUE rb_cNilClass
NilClass class.
Definition: object.c:53
rb_cFalseClass
VALUE rb_cFalseClass
FalseClass class.
Definition: object.c:55
rb_cTrueClass
VALUE rb_cTrueClass
TrueClass class.
Definition: object.c:54
config.h
Thin wrapper to ruby/config.h.
FIX2INT
#define FIX2INT
Definition: int.h:41
hash.h
Internal header for Hash.
object.h
Internal header for Object.
internal.h
PRIdVALUE
#define PRIdVALUE
Definition: inttypes.h:72
PRIuSIZE
#define PRIuSIZE
Definition: inttypes.h:127
PRIxVALUE
#define PRIxVALUE
Definition: inttypes.h:75
offset
voidpf uLong offset
Definition: ioapi.h:144
int
typedef int(ZCALLBACK *close_file_func) OF((voidpf opaque
rb_iseq_path
VALUE rb_iseq_path(const rb_iseq_t *iseq)
Definition: iseq.c:1087
VM_METHOD_TYPE_ISEQ
@ VM_METHOD_TYPE_ISEQ
Ruby method.
Definition: method.h:110
VM_METHOD_TYPE_CFUNC
@ VM_METHOD_TYPE_CFUNC
C method.
Definition: method.h:111
mjit.h
mjit_capture_cc_entries
int mjit_capture_cc_entries(const struct rb_iseq_constant_body *compiled_iseq, const struct rb_iseq_constant_body *captured_iseq)
Definition: mjit_worker.c:1283
mjit_opts
struct mjit_options mjit_opts
Definition: mjit_worker.c:199
mjit_iseq_cc_entries
const struct rb_callcache ** mjit_iseq_cc_entries(const struct rb_iseq_constant_body *const body)
Definition: mjit_worker.c:1270
rb_mjit_compiling_iseq_p
bool rb_mjit_compiling_iseq_p(const rb_iseq_t *iseq)
Definition: mjit_worker.c:770
name
const char * name
Definition: nkf.c:208
key
key
Definition: openssl_missing.h:185
NULL
#define NULL
Definition: regenc.h:69
ROBJECT_EMBED_LEN_MAX
#define ROBJECT_EMBED_LEN_MAX
Definition: robject.h:38
PRI_SERIALT_PREFIX
#define PRI_SERIALT_PREFIX
Definition: serial.h:21
rb_serial_t
unsigned LONG_LONG rb_serial_t
Definition: serial.h:19
f
#define f
ST_CONTINUE
@ ST_CONTINUE
Definition: st.h:99
__sFILE
Definition: vsnprintf.c:169
iseq_inline_iv_cache_entry
Definition: vm_core.h:235
iseq_inline_iv_cache_entry::entry
struct rb_iv_index_tbl_entry * entry
Definition: vm_core.h:236
rb_call_data
Definition: vm_callinfo.h:426
rb_call_data::ci
const struct rb_callinfo * ci
Definition: vm_callinfo.h:427
rb_callcache
Definition: vm_callinfo.h:274
rb_callcache::klass
const VALUE klass
Definition: vm_callinfo.h:278
rb_callinfo
Definition: vm_callinfo.h:61
rb_iseq_constant_body
Definition: vm_core.h:305
rb_iseq_constant_body::opt_table
const VALUE * opt_table
Definition: vm_core.h:368
rb_iseq_constant_body::size
unsigned int size
Definition: vm_core.h:359
rb_iseq_constant_body::ci_size
unsigned int ci_size
Definition: vm_core.h:426
rb_iseq_constant_body::catch_except_p
char catch_except_p
Definition: vm_core.h:429
rb_iseq_constant_body::local_table_size
unsigned int local_table_size
Definition: vm_core.h:424
rb_iseq_constant_body::builtin_inline_p
bool builtin_inline_p
Definition: vm_core.h:430
rb_iseq_constant_body::opt_num
int opt_num
Definition: vm_core.h:362
rb_iseq_constant_body::stack_max
unsigned int stack_max
Definition: vm_core.h:427
rb_iseq_constant_body::flags
struct rb_iseq_constant_body::@188::@190 flags
rb_iseq_constant_body::lead_num
int lead_num
Definition: vm_core.h:361
rb_iseq_constant_body::is_entries
union iseq_inline_storage_entry * is_entries
Definition: vm_core.h:414
rb_iseq_constant_body::iseq_encoded
VALUE * iseq_encoded
Definition: vm_core.h:319
rb_iseq_constant_body::iseq_size
unsigned int iseq_size
Definition: vm_core.h:318
rb_iseq_constant_body::location
rb_iseq_location_t location
Definition: vm_core.h:393
rb_iseq_constant_body::is_size
unsigned int is_size
Definition: vm_core.h:425
rb_iseq_constant_body::param
struct rb_iseq_constant_body::@188 param
parameter information
rb_iseq_constant_body::has_opt
unsigned int has_opt
Definition: vm_core.h:347
rb_iseq_constant_body::call_data
struct rb_call_data * call_data
Definition: vm_core.h:415
rb_iseq_location_struct::label
VALUE label
Definition: vm_core.h:269
rb_iseq_location_struct::first_lineno
VALUE first_lineno
Definition: vm_core.h:270
rb_iseq_struct
Definition: vm_core.h:444
rb_iseq_struct::body
struct rb_iseq_constant_body * body
Definition: vm_core.h:448
rb_iv_index_tbl_entry::index
uint32_t index
Definition: class.h:29
iseq_inline_storage_entry
Definition: vm_core.h:239
VALUE
unsigned long VALUE
Definition: value.h:38
ID
unsigned long ID
Definition: value.h:39
rb_id2str
#define rb_id2str(id)
Definition: vm_backtrace.c:30
vm_callinfo.h
VM_CALL_TAILCALL
#define VM_CALL_TAILCALL
Definition: vm_callinfo.h:39
VM_CALL_KW_SPLAT
#define VM_CALL_KW_SPLAT
Definition: vm_callinfo.h:38
vm_core.h
VM_ASSERT
#define VM_ASSERT(expr)
Definition: vm_core.h:61
CALL_DATA
struct rb_call_data * CALL_DATA
Definition: vm_core.h:1149
IVC
struct iseq_inline_iv_cache_entry * IVC
Definition: vm_core.h:1145
vm_exec.h
rb_vm_opt_cfunc_p
bool rb_vm_opt_cfunc_p(CALL_CACHE cc, int insn)
Definition: vm_insnhelper.c:5300
rb_simple_iseq_p
bool rb_simple_iseq_p(const rb_iseq_t *iseq)
Definition: vm_insnhelper.c:2152
vm_insnhelper.h
if
if((ID)(DISPID) nameid !=nameid)
Definition: win32ole.c:357
def
int def(FILE *source, FILE *dest, int level)
Definition: zpipe.c:36