/* -*- C -*- */ /* This is an auto-generated file and is a part of the programming language * Ruby. The person who created a program to generate this file (``I'' * hereafter) would like to refrain from defining licensing of this generated * source code. * * This file consists of many small parts of codes copyrighted by each author, * not only the ``I'' person. Those original authors agree with some * open-source license. I believe that the license we agree is the condition * mentioned in the file COPYING. It states "4. You may modify and include * the part of the software into any other software ...". But the problem is, * the license never makes it clear if such modified parts still remain in the * same license, or not. The fact that we agree with the source code's * licensing terms does not automatically define that of generated ones. This * is the reason why this file is under an unclear situation. All what I know * is that above provision guarantees this file to exist. * * Please let me hesitate to declare something about this nuanced contract. I * am not in the position to take over other authors' license to merge into my * one. Changing them to (say) GPLv3 is not doable by myself. Perhaps someday * it might turn out to be okay to say this file is under a license. I wish * the situation would become more clear in the future. */ /*******************************************************************/ /*******************************************************************/ /*******************************************************************/ /** This file is the main part of compile_insn() in mjit_compile.c. ---- This file is auto generated by insns2vm.rb DO NOT TOUCH! If you want to fix something, you must edit "tool/ruby_vm/views/mjit_compile.inc.erb" or tool/insns2vm.rb */ switch (insn) { case BIN(nop): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* none */\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_nop(); } fprintf(f, "}\n"); break; } case BIN(getlocal): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = *(vm_get_ep(GET_EP(), level) - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getlocal(idx, level) - 1); b->stack_size += attr_sp_inc_getlocal(idx, level); } fprintf(f, "}\n"); break; } case BIN(setlocal): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_env_write(vm_get_ep(GET_EP(), level), -(int)idx, val);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_set);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_set_dynamic, level > 0);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setlocal(idx, level); } fprintf(f, "}\n"); break; } case BIN(getblockparam): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " const VALUE *ep = vm_get_ep(GET_EP(), level);\n"); fprintf(f, " VM_ASSERT(VM_ENV_LOCAL_P(ep));\n"); fprintf(f, "\n"); fprintf(f, " if (!VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)) {\n"); fprintf(f, " val = rb_vm_bh_to_procval(ec, VM_ENV_BLOCK_HANDLER(ep));\n"); fprintf(f, " vm_env_write(ep, -(int)idx, val);\n"); fprintf(f, " VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM);\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = *(ep - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getblockparam(idx, level) - 1); b->stack_size += attr_sp_inc_getblockparam(idx, level); } fprintf(f, "}\n"); break; } case BIN(setblockparam): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " const VALUE *ep = vm_get_ep(GET_EP(), level);\n"); fprintf(f, " VM_ASSERT(VM_ENV_LOCAL_P(ep));\n"); fprintf(f, "\n"); fprintf(f, " vm_env_write(ep, -(int)idx, val);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_set);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_set_dynamic, level > 0);\n"); fprintf(f, "\n"); fprintf(f, " VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setblockparam(idx, level); } fprintf(f, "}\n"); break; } case BIN(getblockparamproxy): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " const VALUE *ep = vm_get_ep(GET_EP(), level);\n"); fprintf(f, " VM_ASSERT(VM_ENV_LOCAL_P(ep));\n"); fprintf(f, "\n"); fprintf(f, " if (!VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)) {\n"); fprintf(f, " VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep);\n"); fprintf(f, "\n"); fprintf(f, " if (block_handler) {\n"); fprintf(f, " switch (vm_block_handler_type(block_handler)) {\n"); fprintf(f, " case block_handler_type_iseq:\n"); fprintf(f, " case block_handler_type_ifunc:\n"); fprintf(f, " val = rb_block_param_proxy;\n"); fprintf(f, " break;\n"); fprintf(f, " case block_handler_type_symbol:\n"); fprintf(f, " val = rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));\n"); fprintf(f, " goto INSN_LABEL(set_%d);\n", pos); fprintf(f, " case block_handler_type_proc:\n"); fprintf(f, " val = VM_BH_TO_PROC(block_handler);\n"); fprintf(f, " goto INSN_LABEL(set_%d);\n", pos); fprintf(f, " default:\n"); fprintf(f, " VM_UNREACHABLE(getblockparamproxy);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = Qnil;\n"); fprintf(f, " INSN_LABEL(set_%d):\n", pos); fprintf(f, " vm_env_write(ep, -(int)idx, val);\n"); fprintf(f, " VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = *(ep - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getblockparamproxy(idx, level) - 1); b->stack_size += attr_sp_inc_getblockparamproxy(idx, level); } fprintf(f, "}\n"); break; } case BIN(getspecial): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t key) = (rb_num_t)operands[0]; MAYBE_UNUSED(rb_num_t type) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) key, type;\n"); fprintf(f, " key = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " type = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_getspecial(ec, GET_LEP(), key, type);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getspecial(key, type) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getspecial(key, type)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getspecial(key, type); } fprintf(f, "}\n"); break; } case BIN(setspecial): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t key) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) obj;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) key;\n"); fprintf(f, " key = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " lep_svar_set(ec, GET_LEP(), key, obj);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setspecial(key); } fprintf(f, "}\n"); break; } case BIN(getinstancevariable): { #if OPT_IC_FOR_IVAR { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(IVC ic) = (IVC)operands[1]; IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; if (!status->compile_info->disable_ivar_cache && ic_copy->entry) { // Only ic_copy is enabled. fprintf(f, "{\n"); fprintf(f, " VALUE obj = GET_SELF();\n"); fprintf(f, " const uint32_t index = %u;\n", (ic_copy->entry->index)); if (status->merge_ivar_guards_p) { fprintf(f, " VM_ASSERT(RB_TYPE_P(obj, T_OBJECT));\n"); fprintf(f, " VM_ASSERT((rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(obj)->klass));\n", ic_copy->entry->class_serial); fprintf(f, " VM_ASSERT(index < ROBJECT_NUMIV(obj));\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " if (LIKELY(%s && (val = ROBJECT(obj)->as.%s) != Qundef)) {\n", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]"); fprintf(f, " stack[%d] = val;\n", b->stack_size); fprintf(f, " }\n"); } else { fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); fprintf(f, " VALUE val;\n"); fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size); fprintf(f, " }\n"); } fprintf(f, " else {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto ivar_cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getinstancevariable(id, ic); fprintf(f, "}\n"); break; } else if (!status->compile_info->disable_exivar_cache && ic_copy->entry) { fprintf(f, "{\n"); fprintf(f, " VALUE obj = GET_SELF();\n"); fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); fprintf(f, " const uint32_t index = %u;\n", ic_copy->entry->index); fprintf(f, " struct gen_ivtbl *ivtbl;\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " if (LIKELY(FL_TEST_RAW(obj, FL_EXIVAR) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && rb_ivar_generic_ivtbl_lookup(obj, &ivtbl) && index < ivtbl->numiv && (val = ivtbl->ivptr[index]) != Qundef)) {\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto exivar_cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getinstancevariable(id, ic); fprintf(f, "}\n"); break; } } #endif // OPT_IC_FOR_IVAR fprintf(f, "{\n"); { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(IVC ic) = (IVC)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(IVC) ic;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " ic = (IVC)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_getinstancevariable(GET_ISEQ(), GET_SELF(), id, ic);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getinstancevariable(id, ic) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getinstancevariable(id, ic)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getinstancevariable(id, ic); } fprintf(f, "}\n"); break; } case BIN(setinstancevariable): { #if OPT_IC_FOR_IVAR { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(IVC ic) = (IVC)operands[1]; IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; if (!status->compile_info->disable_ivar_cache && ic_copy->entry) { // Only ic_copy is enabled. fprintf(f, "{\n"); fprintf(f, " VALUE obj = GET_SELF();\n"); fprintf(f, " const uint32_t index = %u;\n", (ic_copy->entry->index)); if (status->merge_ivar_guards_p) { fprintf(f, " VM_ASSERT(RB_TYPE_P(obj, T_OBJECT));\n"); fprintf(f, " VM_ASSERT((rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(obj)->klass));\n", ic_copy->entry->class_serial); fprintf(f, " VM_ASSERT(index < ROBJECT_NUMIV(obj));\n"); fprintf(f, " if (LIKELY(!RB_OBJ_FROZEN_RAW(obj) && %s)) {\n", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)"); fprintf(f, " RB_OBJ_WRITE(obj, &ROBJECT(obj)->as.%s, stack[%d]);\n", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]", b->stack_size - 1); fprintf(f, " }\n"); } else { fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN_RAW(obj))) {\n"); fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], stack[%d]);\n", b->stack_size - 1); fprintf(f, " }\n"); } fprintf(f, " else {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto ivar_cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setinstancevariable(id, ic); fprintf(f, "}\n"); break; } } #endif // OPT_IC_FOR_IVAR fprintf(f, "{\n"); { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(IVC ic) = (IVC)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(IVC) ic;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " ic = (IVC)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_setinstancevariable(GET_ISEQ(), GET_SELF(), id, val, ic);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_setinstancevariable(id, ic)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setinstancevariable(id, ic); } fprintf(f, "}\n"); break; } case BIN(getclassvariable): { fprintf(f, "{\n"); { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(ICVARC ic) = (ICVARC)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ICVARC) ic;\n"); fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " ic = (ICVARC)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " rb_control_frame_t *cfp = GET_CFP();\n"); fprintf(f, " val = vm_getclassvariable(GET_ISEQ(), cfp, id, ic);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getclassvariable(id, ic) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getclassvariable(id, ic)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getclassvariable(id, ic); } fprintf(f, "}\n"); break; } case BIN(setclassvariable): { fprintf(f, "{\n"); { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(ICVARC ic) = (ICVARC)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ICVARC) ic;\n"); fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " ic = (ICVARC)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_ensure_not_refinement_module(GET_SELF());\n"); fprintf(f, " vm_setclassvariable(GET_ISEQ(), GET_CFP(), id, val, ic);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_setclassvariable(id, ic)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setclassvariable(id, ic); } fprintf(f, "}\n"); break; } case BIN(getconstant): { fprintf(f, "{\n"); { MAYBE_UNUSED(ID id) = (ID)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) allow_nil, klass, val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " klass = stack[%d];\n", b->stack_size - 2); fprintf(f, " allow_nil = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_get_ev_const(ec, klass, id, allow_nil == Qtrue, 0);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getconstant(id) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getconstant(id)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getconstant(id); } fprintf(f, "}\n"); break; } case BIN(setconstant): { fprintf(f, "{\n"); { MAYBE_UNUSED(ID id) = (ID)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) cbase, val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 2); fprintf(f, " cbase = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_check_if_namespace(cbase);\n"); fprintf(f, " vm_ensure_not_refinement_module(GET_SELF());\n"); fprintf(f, " rb_const_set(cbase, id, val);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_setconstant(id)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setconstant(id); } fprintf(f, "}\n"); break; } case BIN(getglobal): { fprintf(f, "{\n"); { MAYBE_UNUSED(ID gid) = (ID)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) gid;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " gid = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_gvar_get(gid);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getglobal(gid) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getglobal(gid)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getglobal(gid); } fprintf(f, "}\n"); break; } case BIN(setglobal): { fprintf(f, "{\n"); { MAYBE_UNUSED(ID gid) = (ID)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) gid;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " gid = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " rb_gvar_set(gid, val);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_setglobal(gid)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setglobal(gid); } fprintf(f, "}\n"); break; } case BIN(putnil): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = Qnil;\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putnil() - 1); b->stack_size += attr_sp_inc_putnil(); } fprintf(f, "}\n"); break; } case BIN(putself): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = GET_SELF();\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putself() - 1); b->stack_size += attr_sp_inc_putself(); } fprintf(f, "}\n"); break; } case BIN(putobject): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE val) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putobject(val) - 1); b->stack_size += attr_sp_inc_putobject(val); } fprintf(f, "}\n"); break; } case BIN(putspecialobject): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t value_type) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) value_type;\n"); fprintf(f, " value_type = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " enum vm_special_object_type type;\n"); fprintf(f, "\n"); fprintf(f, " type = (enum vm_special_object_type)value_type;\n"); fprintf(f, " val = vm_get_special_object(GET_EP(), type);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putspecialobject(value_type) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_putspecialobject(value_type)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_putspecialobject(value_type); } fprintf(f, "}\n"); break; } case BIN(putstring): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE str) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) str, val;\n"); fprintf(f, " str = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_ec_str_resurrect(ec, str);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putstring(str) - 1); b->stack_size += attr_sp_inc_putstring(str); } fprintf(f, "}\n"); break; } case BIN(concatstrings): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_str_concat_literals(num, %s);\n", (status->local_stack_p ? "(stack + (stack_size - (num)))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_concatstrings(num) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_concatstrings(num)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_concatstrings(num); } fprintf(f, "}\n"); break; } case BIN(anytostring): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) str, val;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 2); fprintf(f, " str = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_obj_as_string_result(str, val);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_anytostring() - 1); b->stack_size += attr_sp_inc_anytostring(); } fprintf(f, "}\n"); break; } case BIN(toregexp): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t opt) = (rb_num_t)operands[0]; MAYBE_UNUSED(rb_num_t cnt) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) cnt, opt;\n"); fprintf(f, " opt = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " cnt = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " const VALUE ary = rb_ary_tmp_new_from_values(0, cnt, %s);\n", (status->local_stack_p ? "(stack + (stack_size - (cnt)))" : "STACK_ADDR_FROM_TOP(cnt)")); fprintf(f, " val = rb_reg_new_ary(ary, (int)opt);\n"); fprintf(f, " rb_ary_clear(ary);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_toregexp(opt, cnt) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_toregexp(opt, cnt)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_toregexp(opt, cnt); } fprintf(f, "}\n"); break; } case BIN(intern): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) str, sym;\n"); fprintf(f, " str = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " sym = rb_str_intern(str);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = sym;\n", b->stack_size + (int)attr_sp_inc_intern() - 1); b->stack_size += attr_sp_inc_intern(); } fprintf(f, "}\n"); break; } case BIN(newarray): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_ec_ary_new_from_values(ec, num, %s);\n", (status->local_stack_p ? "(stack + (stack_size - (num)))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_newarray(num) - 1); b->stack_size += attr_sp_inc_newarray(num); } fprintf(f, "}\n"); break; } case BIN(newarraykwsplat): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (RHASH_EMPTY_P(*%s)) {\n", (status->local_stack_p ? "(stack + (stack_size - (1)))" : "STACK_ADDR_FROM_TOP(1)")); fprintf(f, " val = rb_ary_new4(num-1, %s);\n", (status->local_stack_p ? "(stack + (stack_size - (num)))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = rb_ary_new4(num, %s);\n", (status->local_stack_p ? "(stack + (stack_size - (num)))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_newarraykwsplat(num) - 1); b->stack_size += attr_sp_inc_newarraykwsplat(num); } fprintf(f, "}\n"); break; } case BIN(duparray): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE ary) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ary, val;\n"); fprintf(f, " ary = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));\n"); fprintf(f, " val = rb_ary_resurrect(ary);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_duparray(ary) - 1); b->stack_size += attr_sp_inc_duparray(ary); } fprintf(f, "}\n"); break; } case BIN(duphash): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE hash) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) hash, val;\n"); fprintf(f, " hash = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " RUBY_DTRACE_CREATE_HOOK(HASH, RHASH_SIZE(hash) << 1);\n"); fprintf(f, " val = rb_hash_resurrect(hash);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_duphash(hash) - 1); b->stack_size += attr_sp_inc_duphash(hash); } fprintf(f, "}\n"); break; } case BIN(expandarray): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; MAYBE_UNUSED(rb_num_t flag) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ary;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) flag, num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " flag = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " ary = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_expandarray(%s, ary, num, (int)flag);\n", (status->local_stack_p ? "(stack + stack_size)" : "GET_SP()")); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_expandarray(num, flag)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_expandarray(num, flag); } fprintf(f, "}\n"); break; } case BIN(concatarray): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ary, ary1, ary2;\n"); fprintf(f, " ary1 = stack[%d];\n", b->stack_size - 2); fprintf(f, " ary2 = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " ary = vm_concat_array(ary1, ary2);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = ary;\n", b->stack_size + (int)attr_sp_inc_concatarray() - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_concatarray()); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_concatarray(); } fprintf(f, "}\n"); break; } case BIN(splatarray): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE flag) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ary, flag, obj;\n"); fprintf(f, " flag = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " ary = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " obj = vm_splat_array(flag, ary);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = obj;\n", b->stack_size + (int)attr_sp_inc_splatarray(flag) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_splatarray(flag)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_splatarray(flag); } fprintf(f, "}\n"); break; } case BIN(newhash): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " RUBY_DTRACE_CREATE_HOOK(HASH, num);\n"); fprintf(f, "\n"); fprintf(f, " if (num) {\n"); fprintf(f, " val = rb_hash_new_with_size(num / 2);\n"); fprintf(f, " rb_hash_bulk_insert(num, %s, val);\n", (status->local_stack_p ? "(stack + (stack_size - (num)))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = rb_hash_new();\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_newhash(num) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_newhash(num)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_newhash(num); } fprintf(f, "}\n"); break; } case BIN(newrange): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t flag) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) high, low, val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) flag;\n"); fprintf(f, " flag = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " low = stack[%d];\n", b->stack_size - 2); fprintf(f, " high = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_range_new(low, high, (int)flag);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_newrange(flag) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_newrange(flag)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_newrange(flag); } fprintf(f, "}\n"); break; } case BIN(pop): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " (void)val;\n"); fprintf(f, " /* none */\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_pop(); } fprintf(f, "}\n"); break; } case BIN(dup): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val, val1, val2;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val1 = val2 = val;\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val2;\n", b->stack_size + (int)attr_sp_inc_dup() - 1); fprintf(f, " stack[%d] = val1;\n", b->stack_size + (int)attr_sp_inc_dup() - 2); b->stack_size += attr_sp_inc_dup(); } fprintf(f, "}\n"); break; } case BIN(dupn): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " void *dst = %s;\n", (status->local_stack_p ? "(stack + stack_size)" : "GET_SP()")); fprintf(f, " void *src = %s;\n", (status->local_stack_p ? "(stack + (stack_size - (n)))" : "STACK_ADDR_FROM_TOP(n)")); fprintf(f, "\n"); fprintf(f, " MEMCPY(dst, src, VALUE, n);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_dupn(n); } fprintf(f, "}\n"); break; } case BIN(swap): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) obj, val;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* none */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_swap() - 1); fprintf(f, " stack[%d] = obj;\n", b->stack_size + (int)attr_sp_inc_swap() - 2); b->stack_size += attr_sp_inc_swap(); } fprintf(f, "}\n"); break; } case BIN(topn): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = %s;\n", (status->local_stack_p ? "*(stack + (stack_size - (n) - 1))" : "TOPN(n)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_topn(n) - 1); b->stack_size += attr_sp_inc_topn(n); } fprintf(f, "}\n"); break; } case BIN(setn): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " %s = val;\n", (status->local_stack_p ? "*(stack + (stack_size - (n) - 1))" : "TOPN(n)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_setn(n) - 1); b->stack_size += attr_sp_inc_setn(n); } fprintf(f, "}\n"); break; } case BIN(adjuststack): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* none */\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_adjuststack(n); } fprintf(f, "}\n"); break; } case BIN(defined): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t op_type) = (rb_num_t)operands[0]; MAYBE_UNUSED(VALUE obj) = (VALUE)operands[1]; MAYBE_UNUSED(VALUE pushval) = (VALUE)operands[2]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) obj, pushval, v, val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) op_type;\n"); fprintf(f, " op_type = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " obj = (VALUE)0x%"PRIxVALUE";", operands[1]); if (SYMBOL_P((VALUE)operands[1])) comment_id(f, SYM2ID((VALUE)operands[1])); fprintf(f, "\n"); fprintf(f, " pushval = (VALUE)0x%"PRIxVALUE";", operands[2]); if (SYMBOL_P((VALUE)operands[2])) comment_id(f, SYM2ID((VALUE)operands[2])); fprintf(f, "\n"); fprintf(f, " v = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = Qnil;\n"); fprintf(f, " if (vm_defined(ec, GET_CFP(), op_type, obj, v)) {\n"); fprintf(f, " val = pushval;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_defined(op_type, obj, pushval) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_defined(op_type, obj, pushval)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_defined(op_type, obj, pushval); } fprintf(f, "}\n"); break; } case BIN(checkmatch): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t flag) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) pattern, result, target;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) flag;\n"); fprintf(f, " flag = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " target = stack[%d];\n", b->stack_size - 2); fprintf(f, " pattern = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " result = vm_check_match(ec, target, pattern, flag);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = result;\n", b->stack_size + (int)attr_sp_inc_checkmatch(flag) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_checkmatch(flag)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_checkmatch(flag); } fprintf(f, "}\n"); break; } case BIN(checkkeyword): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t kw_bits_index) = (lindex_t)operands[0]; MAYBE_UNUSED(lindex_t keyword_index) = (lindex_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ret;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) keyword_index, kw_bits_index;\n"); fprintf(f, " kw_bits_index = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " keyword_index = (lindex_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " ret = vm_check_keyword(kw_bits_index, keyword_index, GET_EP());\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = ret;\n", b->stack_size + (int)attr_sp_inc_checkkeyword(kw_bits_index, keyword_index) - 1); b->stack_size += attr_sp_inc_checkkeyword(kw_bits_index, keyword_index); } fprintf(f, "}\n"); break; } case BIN(checktype): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t type) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ret, val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) type;\n"); fprintf(f, " type = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " ret = RBOOL(TYPE(val) == (int)type);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = ret;\n", b->stack_size + (int)attr_sp_inc_checktype(type) - 1); b->stack_size += attr_sp_inc_checktype(type); } fprintf(f, "}\n"); break; } case BIN(definemethod): { fprintf(f, "{\n"); { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(ISEQ iseq) = (ISEQ)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) iseq;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " iseq = (ISEQ)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_definemethod(id, iseq) + 0 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " vm_define_method(ec, Qnil, id, (VALUE)iseq, FALSE);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_definemethod(id, iseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_definemethod(id, iseq); } fprintf(f, "}\n"); break; } case BIN(definesmethod): { fprintf(f, "{\n"); { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(ISEQ iseq) = (ISEQ)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) iseq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " iseq = (ISEQ)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_definesmethod(id, iseq) + 0 - 1; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 1); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " vm_define_method(ec, obj, id, (VALUE)iseq, TRUE);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_definesmethod(id, iseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_definesmethod(id, iseq); } fprintf(f, "}\n"); break; } case BIN(send): { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; MAYBE_UNUSED(ISEQ blockiseq) = (ISEQ)operands[1]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_send(cd, blockiseq) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (const struct rb_callinfo *)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_send(cd, blockiseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_send(cd, blockiseq); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; MAYBE_UNUSED(ISEQ blockiseq) = (ISEQ)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) blockiseq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " blockiseq = (ISEQ)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_send(cd, blockiseq) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false);\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_send(cd, blockiseq) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_send(cd, blockiseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_send(cd, blockiseq); } fprintf(f, "}\n"); break; } case BIN(opt_send_without_block): { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } case BIN(objtostring): { fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_objtostring(GET_ISEQ(), recv, cd);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_objtostring(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_objtostring(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_objtostring(cd); } fprintf(f, "}\n"); break; } case BIN(opt_str_freeze): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE str) = (VALUE)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) str, val;\n"); fprintf(f, " str = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[1])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_str_freeze(str, BOP_FREEZE, idFreeze);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, " PUSH(rb_str_resurrect(str));\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_str_freeze(str, cd) - 1); b->stack_size += attr_sp_inc_opt_str_freeze(str, cd); } fprintf(f, "}\n"); break; } case BIN(opt_nil_p): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_nil_p))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_nil_p(GET_ISEQ(), cd, recv);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_nil_p(cd) - 1); b->stack_size += attr_sp_inc_opt_nil_p(cd); } fprintf(f, "}\n"); break; } case BIN(opt_str_uminus): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE str) = (VALUE)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) str, val;\n"); fprintf(f, " str = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[1])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_str_freeze(str, BOP_UMINUS, idUMinus);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, " PUSH(rb_str_resurrect(str));\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_str_uminus(str, cd) - 1); b->stack_size += attr_sp_inc_opt_str_uminus(str, cd); } fprintf(f, "}\n"); break; } case BIN(opt_newarray_max): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_newarray_max(ec, num, %s);\n", (status->local_stack_p ? "(stack + (stack_size - (num)))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_newarray_max(num) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_newarray_max(num)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_newarray_max(num); } fprintf(f, "}\n"); break; } case BIN(opt_newarray_min): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_newarray_min(ec, num, %s);\n", (status->local_stack_p ? "(stack + (stack_size - (num)))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_newarray_min(num) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_newarray_min(num)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_newarray_min(num); } fprintf(f, "}\n"); break; } case BIN(invokesuper): { fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; MAYBE_UNUSED(ISEQ blockiseq) = (ISEQ)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) blockiseq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " blockiseq = (ISEQ)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_invokesuper(cd, blockiseq) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, true);\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_super);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_invokesuper(cd, blockiseq) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_invokesuper(cd, blockiseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_invokesuper(cd, blockiseq); } fprintf(f, "}\n"); break; } case BIN(invokeblock): { fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_invokeblock(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_invokeblock(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_invokeblock(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_invokeblock(cd); } fprintf(f, "}\n"); break; } case BIN(leave): { if (b->stack_size != 1) { if (mjit_opts.warnings || mjit_opts.verbose) fprintf(stderr, "MJIT warning: Unexpected JIT stack_size on leave: %d\n", b->stack_size); status->success = false; } if (status->inlined_iseqs != NULL) { // the current ISeq is NOT being inlined fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); fprintf(f, " }\n"); fprintf(f, " ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);\n"); // vm_pop_frame } fprintf(f, " return stack[0];\n"); b->stack_size += attr_sp_inc_leave(); b->finish_p = TRUE; break; fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_leave() + 1 - 1; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 1); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " if (OPT_CHECKED_RUN) {\n"); fprintf(f, " const VALUE *const bp = vm_base_ptr(GET_CFP());\n"); fprintf(f, " if (GET_SP() != bp) {\n"); fprintf(f, " vm_stack_consistency_error(ec, GET_CFP(), bp);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, "\n"); fprintf(f, " if (vm_pop_frame(ec, GET_CFP(), GET_EP())) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_leave() - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_leave()); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_leave(); } fprintf(f, "}\n"); b->finish_p = TRUE; break; } case BIN(throw): { fprintf(f, "{\n"); { MAYBE_UNUSED(rb_num_t throw_state) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) throwobj, val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) throw_state;\n"); fprintf(f, " throw_state = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " throwobj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_throw(ec, GET_CFP(), throw_state, throwobj);\n"); fprintf(f, " THROW_EXCEPTION(val);\n"); fprintf(f, " /* unreachable */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_throw(throw_state) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_throw(throw_state)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_throw(throw_state); } fprintf(f, "}\n"); b->finish_p = TRUE; break; } case BIN(jump): { fprintf(f, "{\n"); { MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_jump(dst); } fprintf(f, "}\n"); break; } case BIN(branchif): { fprintf(f, "{\n"); { MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (RTEST(val)) {\n"); fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_branchif(dst); } fprintf(f, "}\n"); if (ALREADY_COMPILED_P(status, pos + insn_len(insn))) { fprintf(f, "goto label_%d;\n", pos + insn_len(insn)); } else { compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); } break; } case BIN(branchunless): { fprintf(f, "{\n"); { MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (!RTEST(val)) {\n"); fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_branchunless(dst); } fprintf(f, "}\n"); if (ALREADY_COMPILED_P(status, pos + insn_len(insn))) { fprintf(f, "goto label_%d;\n", pos + insn_len(insn)); } else { compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); } break; } case BIN(branchnil): { fprintf(f, "{\n"); { MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (NIL_P(val)) {\n"); fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_branchnil(dst); } fprintf(f, "}\n"); if (ALREADY_COMPILED_P(status, pos + insn_len(insn))) { fprintf(f, "goto label_%d;\n", pos + insn_len(insn)); } else { compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); } break; } case BIN(opt_getinlinecache): { OFFSET dst = (OFFSET)operands[0]; IC ic = (IC)operands[1]; struct iseq_inline_constant_cache_entry *ice = ic->entry; if (ice != NULL && !status->compile_info->disable_const_cache) { fprintf(f, " if (vm_inlined_ic_hit_p(0x%"PRIxVALUE", 0x%"PRIxVALUE", (const rb_cref_t *)0x%"PRIxVALUE", reg_cfp->ep)) {", ice->flags, ice->value, (VALUE)ice->ic_cref); fprintf(f, " stack[%d] = 0x%"PRIxVALUE";\n", b->stack_size, ice->value); fprintf(f, " goto label_%d;\n", pos + insn_len(insn) + (int)dst); fprintf(f, " }"); fprintf(f, " else {"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " goto const_cancel;\n"); fprintf(f, " }"); b->stack_size += attr_sp_inc_opt_getinlinecache(dst, ic); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; MAYBE_UNUSED(IC ic) = (IC)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(IC) ic;\n"); fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " ic = (IC)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " struct iseq_inline_constant_cache_entry *ice = ic->entry;\n"); fprintf(f, "\n"); fprintf(f, " // If there isn't an entry, then we're going to walk through the ISEQ\n"); fprintf(f, " // starting at this instruction until we get to the associated\n"); fprintf(f, " // opt_setinlinecache and associate this inline cache with every getconstant\n"); fprintf(f, " // listed in between. We're doing this here instead of when the instructions\n"); fprintf(f, " // are first compiled because it's possible to turn off inline caches and we\n"); fprintf(f, " // want this to work in either case.\n"); fprintf(f, " if (!ice) {\n"); fprintf(f, " vm_ic_compile(GET_CFP(), ic);\n"); fprintf(f, " }\n"); fprintf(f, "\n"); fprintf(f, " if (ice && vm_ic_hit_p(ice, GET_EP())) {\n"); fprintf(f, " val = ice->value;\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_getinlinecache(dst, ic) - 1); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = Qnil;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_getinlinecache(dst, ic) - 1); b->stack_size += attr_sp_inc_opt_getinlinecache(dst, ic); } fprintf(f, "}\n"); if (ALREADY_COMPILED_P(status, pos + insn_len(insn))) { fprintf(f, "goto label_%d;\n", pos + insn_len(insn)); } else { compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); } break; } case BIN(opt_setinlinecache): { fprintf(f, "{\n"); { MAYBE_UNUSED(IC ic) = (IC)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(IC) ic;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " ic = (IC)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_ic_update(GET_ISEQ(), ic, val, GET_EP());\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_setinlinecache(ic) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_setinlinecache(ic)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_setinlinecache(ic); } fprintf(f, "}\n"); break; } case BIN(once): { fprintf(f, "{\n"); { MAYBE_UNUSED(ISEQ iseq) = (ISEQ)operands[0]; MAYBE_UNUSED(ISE ise) = (ISE)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ISE) ise;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) iseq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " iseq = (ISEQ)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " ise = (ISE)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_once(iseq, ise) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " val = vm_once_dispatch(ec, iseq, ise);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_once(iseq, ise) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_once(iseq, ise)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_once(iseq, ise); } fprintf(f, "}\n"); break; } case BIN(opt_case_dispatch): { fprintf(f, "{\n"); { MAYBE_UNUSED(CDHASH hash) = (CDHASH)operands[0]; MAYBE_UNUSED(OFFSET else_offset) = (OFFSET)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CDHASH) hash;\n"); fprintf(f, " MAYBE_UNUSED(OFFSET) else_offset;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) key;\n"); fprintf(f, " hash = (CDHASH)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " else_offset = (OFFSET)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " key = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " OFFSET dst = vm_case_dispatch(hash, else_offset, key);\n"); fprintf(f, "\n"); fprintf(f, " if (dst) {\n"); { struct case_dispatch_var arg; arg.f = f; arg.base_pos = pos + insn_len(insn); arg.last_value = Qundef; fprintf(f, " switch (dst) {\n"); st_foreach(RHASH_TBL_RAW(hash), compile_case_dispatch_each, (VALUE)&arg); fprintf(f, " case %lu:\n", else_offset); fprintf(f, " goto label_%lu;\n", arg.base_pos + else_offset); fprintf(f, " }\n"); } fprintf(f, " }\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_case_dispatch(hash, else_offset); } fprintf(f, "}\n"); if (ALREADY_COMPILED_P(status, pos + insn_len(insn))) { fprintf(f, "goto label_%d;\n", pos + insn_len(insn)); } else { compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); } break; } case BIN(opt_plus): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_plus))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_plus(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_plus(cd) - 1); b->stack_size += attr_sp_inc_opt_plus(cd); } fprintf(f, "}\n"); break; } case BIN(opt_minus): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_minus))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_minus(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_minus(cd) - 1); b->stack_size += attr_sp_inc_opt_minus(cd); } fprintf(f, "}\n"); break; } case BIN(opt_mult): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_mult))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_mult(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_mult(cd) - 1); b->stack_size += attr_sp_inc_opt_mult(cd); } fprintf(f, "}\n"); break; } case BIN(opt_div): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_div))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_div(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_div(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_div(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_div(cd); } fprintf(f, "}\n"); break; } case BIN(opt_mod): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_mod))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_mod(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_mod(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_mod(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_mod(cd); } fprintf(f, "}\n"); break; } case BIN(opt_eq): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_eq))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = opt_equality(GET_ISEQ(), recv, obj, cd);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_eq(cd) - 1); b->stack_size += attr_sp_inc_opt_eq(cd); } fprintf(f, "}\n"); break; } case BIN(opt_neq): { fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd_eq) = (CALL_DATA)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd, cd_eq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd_eq = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[1])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_neq(GET_ISEQ(), cd, cd_eq, recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_neq(cd_eq, cd) - 1); b->stack_size += attr_sp_inc_opt_neq(cd_eq, cd); } fprintf(f, "}\n"); break; } case BIN(opt_lt): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_lt))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_lt(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_lt(cd) - 1); b->stack_size += attr_sp_inc_opt_lt(cd); } fprintf(f, "}\n"); break; } case BIN(opt_le): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_le))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_le(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_le(cd) - 1); b->stack_size += attr_sp_inc_opt_le(cd); } fprintf(f, "}\n"); break; } case BIN(opt_gt): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_gt))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_gt(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_gt(cd) - 1); b->stack_size += attr_sp_inc_opt_gt(cd); } fprintf(f, "}\n"); break; } case BIN(opt_ge): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_ge))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_ge(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_ge(cd) - 1); b->stack_size += attr_sp_inc_opt_ge(cd); } fprintf(f, "}\n"); break; } case BIN(opt_ltlt): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_ltlt))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_ltlt(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_ltlt(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_ltlt(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_ltlt(cd); } fprintf(f, "}\n"); break; } case BIN(opt_and): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_and))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_and(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_and(cd) - 1); b->stack_size += attr_sp_inc_opt_and(cd); } fprintf(f, "}\n"); break; } case BIN(opt_or): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_or))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_or(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_or(cd) - 1); b->stack_size += attr_sp_inc_opt_or(cd); } fprintf(f, "}\n"); break; } case BIN(opt_aref): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_aref))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_aref(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_aref(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_aref(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_aref(cd); } fprintf(f, "}\n"); break; } case BIN(opt_aset): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_aset))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, set, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 3); fprintf(f, " obj = stack[%d];\n", b->stack_size - 2); fprintf(f, " set = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_aset(recv, obj, set);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_aset(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_aset(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_aset(cd); } fprintf(f, "}\n"); break; } case BIN(opt_aset_with): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE key) = (VALUE)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) key, recv, val;\n"); fprintf(f, " key = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[1])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " VALUE tmp = vm_opt_aset_with(recv, key, val);\n"); fprintf(f, "\n"); fprintf(f, " if (tmp != Qundef) {\n"); fprintf(f, " val = tmp;\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, "#ifndef MJIT_HEADER\n"); fprintf(f, " %s = rb_str_resurrect(key);\n", (status->local_stack_p ? "*(stack + (stack_size - (0) - 1))" : "TOPN(0)")); fprintf(f, " PUSH(val);\n"); fprintf(f, "#endif\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_aset_with(key, cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_aset_with(key, cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_aset_with(key, cd); } fprintf(f, "}\n"); break; } case BIN(opt_aref_with): { fprintf(f, "{\n"); { MAYBE_UNUSED(VALUE key) = (VALUE)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) key, recv, val;\n"); fprintf(f, " key = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[1])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_aref_with(recv, key);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#ifndef MJIT_HEADER\n"); fprintf(f, " PUSH(rb_str_resurrect(key));\n"); fprintf(f, "#endif\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_aref_with(key, cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_aref_with(key, cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_aref_with(key, cd); } fprintf(f, "}\n"); break; } case BIN(opt_length): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_length))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_length(recv, BOP_LENGTH);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_length(cd) - 1); b->stack_size += attr_sp_inc_opt_length(cd); } fprintf(f, "}\n"); break; } case BIN(opt_size): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_size))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_length(recv, BOP_SIZE);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_size(cd) - 1); b->stack_size += attr_sp_inc_opt_size(cd); } fprintf(f, "}\n"); break; } case BIN(opt_empty_p): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_empty_p))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_empty_p(recv);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_empty_p(cd) - 1); b->stack_size += attr_sp_inc_opt_empty_p(cd); } fprintf(f, "}\n"); break; } case BIN(opt_succ): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_succ))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_succ(recv);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_succ(cd) - 1); b->stack_size += attr_sp_inc_opt_succ(cd); } fprintf(f, "}\n"); break; } case BIN(opt_not): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_not))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_not(GET_ISEQ(), cd, recv);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_not(cd) - 1); b->stack_size += attr_sp_inc_opt_not(cd); } fprintf(f, "}\n"); break; } case BIN(opt_regexpmatch2): { if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[0], body)], BIN(opt_regexpmatch2))) { { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; size_t cd_index = call_data_index(cd, body); const struct rb_callcache **cc_entries = captured_cc_entries(status); const struct rb_callcache *captured_cc = cc_entries[cd_index]; const rb_iseq_t *iseq = NULL; const CALL_INFO ci = cd->ci; int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC && !rb_splat_or_kwargs_p(ci) && !kw_splat) || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) )) { const bool cfunc_debug = false; // Set true when you want to see inlined cfunc if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); int sp_inc = (int)sp_inc_of_sendish(ci); fprintf(f, "{\n"); bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " VALUE val;\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.kw_splat = %d;\n", kw_splat); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. fprintf(f, " calling.cc = cc;"); fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); } else { // VM_METHOD_TYPE_ISEQ fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); if (ISEQ_BODY(iseq)->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " val = vm_exec(ec, true);\n"); } else { fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " val = vm_exec(ec, false);\n"); fprintf(f, " }\n"); } } fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj1, obj2, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, vm_ci_mid(((CALL_DATA)operands[0])->ci)); fprintf(f, "\n"); fprintf(f, " obj2 = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj1 = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_regexpmatch2(obj2, obj1);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_regexpmatch2(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_regexpmatch2(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_regexpmatch2(cd); } fprintf(f, "}\n"); break; } case BIN(invokebuiltin): { RB_BUILTIN bf = (RB_BUILTIN)operands[0]; rb_snum_t sp_inc = attr_sp_inc_invokebuiltin(bf); unsigned sp = b->stack_size + (unsigned)sp_inc; VM_ASSERT(b->stack_size > -sp_inc); VM_ASSERT(sp_inc < UINT_MAX - b->stack_size); if (bf->compiler) { fprintf(f, "{\n"); fprintf(f, " VALUE val;\n"); bf->compiler(f, -1, b->stack_size, body->builtin_inline_p); fprintf(f, " stack[%u] = val;\n", sp - 1); fprintf(f, "}\n"); b->stack_size = sp; break; } fprintf(f, "{\n"); { MAYBE_UNUSED(RB_BUILTIN bf) = (RB_BUILTIN)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(RB_BUILTIN) bf;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " bf = (RB_BUILTIN)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_invoke_builtin(ec, reg_cfp, bf, %s);\n", (status->local_stack_p ? "(stack + (stack_size - (bf->argc)))" : "STACK_ADDR_FROM_TOP(bf->argc)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_invokebuiltin(bf) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_invokebuiltin(bf)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_invokebuiltin(bf); } fprintf(f, "}\n"); break; } case BIN(opt_invokebuiltin_delegate): { RB_BUILTIN bf = (RB_BUILTIN)operands[0]; rb_num_t index = (rb_num_t)operands[1]; rb_snum_t sp_inc = attr_sp_inc_opt_invokebuiltin_delegate(bf, index); unsigned sp = b->stack_size + (unsigned)sp_inc; VM_ASSERT(b->stack_size > -sp_inc); VM_ASSERT(sp_inc < UINT_MAX - b->stack_size); if (bf->compiler) { fprintf(f, "{\n"); fprintf(f, " VALUE val;\n"); bf->compiler(f, (rb_num_t)operands[1], b->stack_size, body->builtin_inline_p); fprintf(f, " stack[%u] = val;\n", sp - 1); fprintf(f, "}\n"); b->stack_size = sp; break; } fprintf(f, "{\n"); { MAYBE_UNUSED(RB_BUILTIN bf) = (RB_BUILTIN)operands[0]; MAYBE_UNUSED(rb_num_t index) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(RB_BUILTIN) bf;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) index;\n"); fprintf(f, " bf = (RB_BUILTIN)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " index = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_invoke_builtin_delegate(ec, reg_cfp, bf, (unsigned int)index);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_invokebuiltin_delegate(bf, index) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_invokebuiltin_delegate(bf, index)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_invokebuiltin_delegate(bf, index); } fprintf(f, "}\n"); break; } case BIN(opt_invokebuiltin_delegate_leave): { RB_BUILTIN bf = (RB_BUILTIN)operands[0]; rb_num_t index = (rb_num_t)operands[1]; rb_snum_t sp_inc = attr_sp_inc_opt_invokebuiltin_delegate_leave(bf, index); unsigned sp = b->stack_size + (unsigned)sp_inc; VM_ASSERT(b->stack_size > -sp_inc); VM_ASSERT(sp_inc < UINT_MAX - b->stack_size); if (bf->compiler) { fprintf(f, "{\n"); fprintf(f, " VALUE val;\n"); bf->compiler(f, (rb_num_t)operands[1], b->stack_size, body->builtin_inline_p); fprintf(f, " stack[%u] = val;\n", sp - 1); fprintf(f, "}\n"); } if (status->inlined_iseqs != NULL) { // the current ISeq is NOT being inlined fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); fprintf(f, " }\n"); fprintf(f, " ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);\n"); // vm_pop_frame } fprintf(f, " return stack[0];\n"); b->stack_size += attr_sp_inc_opt_invokebuiltin_delegate_leave(bf, index); b->finish_p = TRUE; break; fprintf(f, "{\n"); { MAYBE_UNUSED(RB_BUILTIN bf) = (RB_BUILTIN)operands[0]; MAYBE_UNUSED(rb_num_t index) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(RB_BUILTIN) bf;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) index;\n"); fprintf(f, " bf = (RB_BUILTIN)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " index = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (true) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_invoke_builtin_delegate(ec, reg_cfp, bf, (unsigned int)index);\n"); fprintf(f, "\n"); fprintf(f, " /* leave fastpath */\n"); fprintf(f, " /* TracePoint/return fallbacks this insn to opt_invokebuiltin_delegate */\n"); fprintf(f, " if (vm_pop_frame(ec, GET_CFP(), GET_EP())) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_invokebuiltin_delegate_leave(bf, index) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_invokebuiltin_delegate_leave(bf, index)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_invokebuiltin_delegate_leave(bf, index); } fprintf(f, "}\n"); b->finish_p = TRUE; break; } case BIN(getlocal_WC_0): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " level = 0;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = *(vm_get_ep(GET_EP(), level) - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getlocal_WC_0(idx) - 1); b->stack_size += attr_sp_inc_getlocal_WC_0(idx); } fprintf(f, "}\n"); break; } case BIN(getlocal_WC_1): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " level = 1;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = *(vm_get_ep(GET_EP(), level) - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getlocal_WC_1(idx) - 1); b->stack_size += attr_sp_inc_getlocal_WC_1(idx); } fprintf(f, "}\n"); break; } case BIN(setlocal_WC_0): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " level = 0;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_env_write(vm_get_ep(GET_EP(), level), -(int)idx, val);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_set);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_set_dynamic, level > 0);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setlocal_WC_0(idx); } fprintf(f, "}\n"); break; } case BIN(setlocal_WC_1): { fprintf(f, "{\n"); { MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " level = 1;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_env_write(vm_get_ep(GET_EP(), level), -(int)idx, val);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_set);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_set_dynamic, level > 0);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setlocal_WC_1(idx); } fprintf(f, "}\n"); break; } case BIN(putobject_INT2FIX_0_): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = INT2FIX(0);\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putobject_INT2FIX_0_() - 1); b->stack_size += attr_sp_inc_putobject_INT2FIX_0_(); } fprintf(f, "}\n"); break; } case BIN(putobject_INT2FIX_1_): { fprintf(f, "{\n"); { if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = INT2FIX(1);\n"); MAYBE_UNUSED(bool pc_moved_p) = false; if (false) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = true; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putobject_INT2FIX_1_() - 1); b->stack_size += attr_sp_inc_putobject_INT2FIX_1_(); } fprintf(f, "}\n"); break; } default: if (mjit_opts.warnings || mjit_opts.verbose) fprintf(stderr, "MJIT warning: Skipped to compile unsupported instruction: %s\n", insn_name(insn)); status->success = false; break; }