/* -*- C -*- */ /* This is an auto-generated file and is a part of the programming language * Ruby. The person who created a program to generate this file (``I'' * hereafter) would like to refrain from defining licensing of this generated * source code. * * This file consists of many small parts of codes copyrighted by each author, * not only the ``I'' person. Those original authors agree with some * open-source license. I believe that the license we agree is the condition * mentioned in the file COPYING. It states "4. You may modify and include * the part of the software into any other software ...". But the problem is, * the license never makes it clear if such modified parts still remain in the * same license, or not. The fact that we agree with the source code's * licensing terms does not automatically define that of generated ones. This * is the reason why this file is under an unclear situation. All what I know * is that above provision guarantees this file to exist. * * Please let me hesitate to declare something about this nuanced contract. I * am not in the position to take over other authors' license to merge into my * one. Changing them to (say) GPLv3 is not doable by myself. Perhaps someday * it might turn out to be okay to say this file is under a license. I wish * the situation would become more clear in the future. */ /*******************************************************************/ /*******************************************************************/ /*******************************************************************/ /** This file is the main part of compile_insn() in mjit_compile.c. ---- This file is auto generated by insns2vm.rb DO NOT TOUCH! If you want to fix something, you must edit "tool/ruby_vm/views/mjit_compile.inc.erb" or tool/insns2vm.rb */ switch (insn) { case BIN(nop): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* none */\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_nop(); } fprintf(f, "}\n"); break; case BIN(getlocal): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = *(vm_get_ep(GET_EP(), level) - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getlocal(idx, level) - 1); b->stack_size += attr_sp_inc_getlocal(idx, level); } fprintf(f, "}\n"); break; case BIN(setlocal): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_env_write(vm_get_ep(GET_EP(), level), -(int)idx, val);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_set);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_set_dynamic, level > 0);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setlocal(idx, level); } fprintf(f, "}\n"); break; case BIN(getblockparam): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " const VALUE *ep = vm_get_ep(GET_EP(), level);\n"); fprintf(f, " VM_ASSERT(VM_ENV_LOCAL_P(ep));\n"); fprintf(f, "\n"); fprintf(f, " if (!VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)) {\n"); fprintf(f, " val = rb_vm_bh_to_procval(ec, VM_ENV_BLOCK_HANDLER(ep));\n"); fprintf(f, " vm_env_write(ep, -(int)idx, val);\n"); fprintf(f, " VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM);\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = *(ep - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getblockparam(idx, level) - 1); b->stack_size += attr_sp_inc_getblockparam(idx, level); } fprintf(f, "}\n"); break; case BIN(setblockparam): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " const VALUE *ep = vm_get_ep(GET_EP(), level);\n"); fprintf(f, " VM_ASSERT(VM_ENV_LOCAL_P(ep));\n"); fprintf(f, "\n"); fprintf(f, " vm_env_write(ep, -(int)idx, val);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_set);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_set_dynamic, level > 0);\n"); fprintf(f, "\n"); fprintf(f, " VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setblockparam(idx, level); } fprintf(f, "}\n"); break; case BIN(getblockparamproxy): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; MAYBE_UNUSED(rb_num_t level) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " level = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " const VALUE *ep = vm_get_ep(GET_EP(), level);\n"); fprintf(f, " VM_ASSERT(VM_ENV_LOCAL_P(ep));\n"); fprintf(f, "\n"); fprintf(f, " if (!VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)) {\n"); fprintf(f, " VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep);\n"); fprintf(f, "\n"); fprintf(f, " if (block_handler) {\n"); fprintf(f, " switch (vm_block_handler_type(block_handler)) {\n"); fprintf(f, " case block_handler_type_iseq:\n"); fprintf(f, " case block_handler_type_ifunc:\n"); fprintf(f, " val = rb_block_param_proxy;\n"); fprintf(f, " break;\n"); fprintf(f, " case block_handler_type_symbol:\n"); fprintf(f, " val = rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));\n"); fprintf(f, " goto INSN_LABEL(set_%d);\n", pos); fprintf(f, " case block_handler_type_proc:\n"); fprintf(f, " val = VM_BH_TO_PROC(block_handler);\n"); fprintf(f, " goto INSN_LABEL(set_%d);\n", pos); fprintf(f, " default:\n"); fprintf(f, " VM_UNREACHABLE(getblockparamproxy);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = Qnil;\n"); fprintf(f, " INSN_LABEL(set_%d):\n", pos); fprintf(f, " vm_env_write(ep, -(int)idx, val);\n"); fprintf(f, " VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = *(ep - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getblockparamproxy(idx, level) - 1); b->stack_size += attr_sp_inc_getblockparamproxy(idx, level); } fprintf(f, "}\n"); break; case BIN(getspecial): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t key) = (rb_num_t)operands[0]; MAYBE_UNUSED(rb_num_t type) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) key, type;\n"); fprintf(f, " key = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " type = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_getspecial(ec, GET_LEP(), key, type);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getspecial(key, type) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getspecial(key, type)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getspecial(key, type); } fprintf(f, "}\n"); break; case BIN(setspecial): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t key) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) obj;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) key;\n"); fprintf(f, " key = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " lep_svar_set(ec, GET_LEP(), key, obj);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setspecial(key); } fprintf(f, "}\n"); break; case BIN(getinstancevariable): #if OPT_IC_FOR_IVAR { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(IVC ic) = (IVC)operands[1]; IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; if (!status->compile_info->disable_ivar_cache && ic_copy->ic_serial) { fprintf(f, "{\n"); fprintf(f, " VALUE obj = GET_SELF();\n"); fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->ic_serial); fprintf(f, " const st_index_t index = %"PRIuSIZE";\n", ic_copy->index); fprintf(f, " VALUE val;\n"); fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto ivar_cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getinstancevariable(id, ic); fprintf(f, "}\n"); break; } } #endif // OPT_IC_FOR_IVAR fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(IVC ic) = (IVC)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(IVC) ic;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " ic = (IVC)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_getinstancevariable(GET_SELF(), id, ic);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getinstancevariable(id, ic) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getinstancevariable(id, ic)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getinstancevariable(id, ic); } fprintf(f, "}\n"); break; case BIN(setinstancevariable): #if OPT_IC_FOR_IVAR { MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(IVC ic) = (IVC)operands[1]; IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; if (!status->compile_info->disable_ivar_cache && ic_copy->ic_serial) { fprintf(f, "{\n"); fprintf(f, " VALUE obj = GET_SELF();\n"); fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->ic_serial); fprintf(f, " const st_index_t index = %"PRIuSIZE";\n", ic_copy->index); fprintf(f, " VALUE val = stack[%d];\n", b->stack_size - 1); fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN(obj))) {\n"); fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], val);\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto ivar_cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setinstancevariable(id, ic); fprintf(f, "}\n"); break; } } #endif // OPT_IC_FOR_IVAR fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(IVC ic) = (IVC)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(IVC) ic;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " ic = (IVC)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_setinstancevariable(GET_SELF(), id, val, ic);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_setinstancevariable(id, ic)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setinstancevariable(id, ic); } fprintf(f, "}\n"); break; case BIN(getclassvariable): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ID id) = (ID)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_cvar_get(vm_get_cvar_base(vm_get_cref(GET_EP()), GET_CFP()), id);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getclassvariable(id) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getclassvariable(id)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getclassvariable(id); } fprintf(f, "}\n"); break; case BIN(setclassvariable): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ID id) = (ID)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_ensure_not_refinement_module(GET_SELF());\n"); fprintf(f, " rb_cvar_set(vm_get_cvar_base(vm_get_cref(GET_EP()), GET_CFP()), id, val);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_setclassvariable(id)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setclassvariable(id); } fprintf(f, "}\n"); break; case BIN(getconstant): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ID id) = (ID)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) allow_nil, klass, val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " klass = stack[%d];\n", b->stack_size - 2); fprintf(f, " allow_nil = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_get_ev_const(ec, klass, id, allow_nil == Qtrue, 0);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getconstant(id) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getconstant(id)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getconstant(id); } fprintf(f, "}\n"); break; case BIN(setconstant): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ID id) = (ID)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) cbase, val;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 2); fprintf(f, " cbase = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_check_if_namespace(cbase);\n"); fprintf(f, " vm_ensure_not_refinement_module(GET_SELF());\n"); fprintf(f, " rb_const_set(cbase, id, val);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_setconstant(id)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setconstant(id); } fprintf(f, "}\n"); break; case BIN(getglobal): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(GENTRY entry) = (GENTRY)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(GENTRY) entry;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " entry = (GENTRY)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " struct rb_global_entry *gentry = (void *)entry;\n"); fprintf(f, " val = rb_gvar_get(gentry);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getglobal(entry) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_getglobal(entry)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_getglobal(entry); } fprintf(f, "}\n"); break; case BIN(setglobal): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(GENTRY entry) = (GENTRY)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(GENTRY) entry;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " entry = (GENTRY)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " struct rb_global_entry *gentry = (void *)entry;\n"); fprintf(f, " rb_gvar_set(gentry, val);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_setglobal(entry)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setglobal(entry); } fprintf(f, "}\n"); break; case BIN(putnil): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = Qnil;\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putnil() - 1); b->stack_size += attr_sp_inc_putnil(); } fprintf(f, "}\n"); break; case BIN(putself): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = GET_SELF();\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putself() - 1); b->stack_size += attr_sp_inc_putself(); } fprintf(f, "}\n"); break; case BIN(putobject): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE val) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putobject(val) - 1); b->stack_size += attr_sp_inc_putobject(val); } fprintf(f, "}\n"); break; case BIN(putspecialobject): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t value_type) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) value_type;\n"); fprintf(f, " value_type = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " enum vm_special_object_type type;\n"); fprintf(f, "\n"); fprintf(f, " type = (enum vm_special_object_type)value_type;\n"); fprintf(f, " val = vm_get_special_object(GET_EP(), type);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putspecialobject(value_type) - 1); b->stack_size += attr_sp_inc_putspecialobject(value_type); } fprintf(f, "}\n"); break; case BIN(putstring): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE str) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) str, val;\n"); fprintf(f, " str = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_str_resurrect(str);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putstring(str) - 1); b->stack_size += attr_sp_inc_putstring(str); } fprintf(f, "}\n"); break; case BIN(concatstrings): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_str_concat_literals(num, %s);\n", (status->local_stack_p ? "stack + (stack_size - (num))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_concatstrings(num) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_concatstrings(num)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_concatstrings(num); } fprintf(f, "}\n"); break; case BIN(tostring): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) str, val;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 2); fprintf(f, " str = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_obj_as_string_result(str, val);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_tostring() - 1); b->stack_size += attr_sp_inc_tostring(); } fprintf(f, "}\n"); break; case BIN(freezestring): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE debug_info) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) debug_info, str;\n"); fprintf(f, " debug_info = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " str = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_freezestring(str, debug_info);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = str;\n", b->stack_size + (int)attr_sp_inc_freezestring(debug_info) - 1); b->stack_size += attr_sp_inc_freezestring(debug_info); } fprintf(f, "}\n"); break; case BIN(toregexp): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t opt) = (rb_num_t)operands[0]; MAYBE_UNUSED(rb_num_t cnt) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) cnt, opt;\n"); fprintf(f, " opt = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " cnt = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " const VALUE ary = rb_ary_tmp_new_from_values(0, cnt, %s);\n", (status->local_stack_p ? "stack + (stack_size - (cnt))" : "STACK_ADDR_FROM_TOP(cnt)")); fprintf(f, " val = rb_reg_new_ary(ary, (int)opt);\n"); fprintf(f, " rb_ary_clear(ary);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_toregexp(opt, cnt) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_toregexp(opt, cnt)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_toregexp(opt, cnt); } fprintf(f, "}\n"); break; case BIN(intern): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) str, sym;\n"); fprintf(f, " str = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " sym = rb_str_intern(str);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = sym;\n", b->stack_size + (int)attr_sp_inc_intern() - 1); b->stack_size += attr_sp_inc_intern(); } fprintf(f, "}\n"); break; case BIN(newarray): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_ary_new4(num, %s);\n", (status->local_stack_p ? "stack + (stack_size - (num))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_newarray(num) - 1); b->stack_size += attr_sp_inc_newarray(num); } fprintf(f, "}\n"); break; case BIN(newarraykwsplat): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (RHASH_EMPTY_P(*%s)) {\n", (status->local_stack_p ? "stack + (stack_size - (1))" : "STACK_ADDR_FROM_TOP(1)")); fprintf(f, " val = rb_ary_new4(num-1, %s);\n", (status->local_stack_p ? "stack + (stack_size - (num))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = rb_ary_new4(num, %s);\n", (status->local_stack_p ? "stack + (stack_size - (num))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_newarraykwsplat(num) - 1); b->stack_size += attr_sp_inc_newarraykwsplat(num); } fprintf(f, "}\n"); break; case BIN(duparray): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE ary) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ary, val;\n"); fprintf(f, " ary = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));\n"); fprintf(f, " val = rb_ary_resurrect(ary);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_duparray(ary) - 1); b->stack_size += attr_sp_inc_duparray(ary); } fprintf(f, "}\n"); break; case BIN(duphash): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE hash) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) hash, val;\n"); fprintf(f, " hash = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " RUBY_DTRACE_CREATE_HOOK(HASH, RHASH_SIZE(hash) << 1);\n"); fprintf(f, " val = rb_hash_resurrect(hash);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_duphash(hash) - 1); b->stack_size += attr_sp_inc_duphash(hash); } fprintf(f, "}\n"); break; case BIN(expandarray): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; MAYBE_UNUSED(rb_num_t flag) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ary;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) flag, num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " flag = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " ary = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_expandarray(%s, ary, num, (int)flag);\n", (status->local_stack_p ? "(stack + stack_size)" : "GET_SP()")); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_expandarray(num, flag)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_expandarray(num, flag); } fprintf(f, "}\n"); break; case BIN(concatarray): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ary, ary1, ary2;\n"); fprintf(f, " ary1 = stack[%d];\n", b->stack_size - 2); fprintf(f, " ary2 = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " ary = vm_concat_array(ary1, ary2);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = ary;\n", b->stack_size + (int)attr_sp_inc_concatarray() - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_concatarray()); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_concatarray(); } fprintf(f, "}\n"); break; case BIN(splatarray): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE flag) = (VALUE)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ary, flag, obj;\n"); fprintf(f, " flag = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " ary = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " obj = vm_splat_array(flag, ary);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = obj;\n", b->stack_size + (int)attr_sp_inc_splatarray(flag) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_splatarray(flag)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_splatarray(flag); } fprintf(f, "}\n"); break; case BIN(newhash): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " RUBY_DTRACE_CREATE_HOOK(HASH, num);\n"); fprintf(f, "\n"); fprintf(f, " val = rb_hash_new_with_size(num / 2);\n"); fprintf(f, "\n"); fprintf(f, " if (num) {\n"); fprintf(f, " rb_hash_bulk_insert(num, %s, val);\n", (status->local_stack_p ? "stack + (stack_size - (num))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_newhash(num) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_newhash(num)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_newhash(num); } fprintf(f, "}\n"); break; case BIN(newrange): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t flag) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) high, low, val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) flag;\n"); fprintf(f, " flag = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " low = stack[%d];\n", b->stack_size - 2); fprintf(f, " high = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = rb_range_new(low, high, (int)flag);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_newrange(flag) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_newrange(flag)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_newrange(flag); } fprintf(f, "}\n"); break; case BIN(pop): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " (void)val;\n"); fprintf(f, " /* none */\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_pop(); } fprintf(f, "}\n"); break; case BIN(dup): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val, val1, val2;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val1 = val2 = val;\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val2;\n", b->stack_size + (int)attr_sp_inc_dup() - 1); fprintf(f, " stack[%d] = val1;\n", b->stack_size + (int)attr_sp_inc_dup() - 2); b->stack_size += attr_sp_inc_dup(); } fprintf(f, "}\n"); break; case BIN(dupn): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " void *dst = %s;\n", (status->local_stack_p ? "(stack + stack_size)" : "GET_SP()")); fprintf(f, " void *src = %s;\n", (status->local_stack_p ? "stack + (stack_size - (n))" : "STACK_ADDR_FROM_TOP(n)")); fprintf(f, "\n"); fprintf(f, " MEMCPY(dst, src, VALUE, n);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_dupn(n); } fprintf(f, "}\n"); break; case BIN(swap): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) obj, val;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* none */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_swap() - 1); fprintf(f, " stack[%d] = obj;\n", b->stack_size + (int)attr_sp_inc_swap() - 2); b->stack_size += attr_sp_inc_swap(); } fprintf(f, "}\n"); break; case BIN(reverse): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " rb_num_t i;\n"); fprintf(f, " VALUE *sp = %s;\n", (status->local_stack_p ? "stack + (stack_size - (n))" : "STACK_ADDR_FROM_TOP(n)")); fprintf(f, "\n"); fprintf(f, " for (i=0; ilocal_stack_p ? "*(stack + (stack_size - (i) - 1))" : "TOPN(i)")); fprintf(f, " sp[i] = v1;\n"); fprintf(f, " %s = v0;\n", (status->local_stack_p ? "*(stack + (stack_size - (i) - 1))" : "TOPN(i)")); fprintf(f, " }\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_reverse(n); } fprintf(f, "}\n"); break; case BIN(topn): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = %s;\n", (status->local_stack_p ? "*(stack + (stack_size - (n) - 1))" : "TOPN(n)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_topn(n) - 1); b->stack_size += attr_sp_inc_topn(n); } fprintf(f, "}\n"); break; case BIN(setn): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " %s = val;\n", (status->local_stack_p ? "*(stack + (stack_size - (n) - 1))" : "TOPN(n)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_setn(n) - 1); b->stack_size += attr_sp_inc_setn(n); } fprintf(f, "}\n"); break; case BIN(adjuststack): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t n) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(rb_num_t) n;\n"); fprintf(f, " n = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* none */\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_adjuststack(n); } fprintf(f, "}\n"); break; case BIN(defined): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t op_type) = (rb_num_t)operands[0]; MAYBE_UNUSED(VALUE obj) = (VALUE)operands[1]; MAYBE_UNUSED(VALUE needstr) = (VALUE)operands[2]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) needstr, obj, v, val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) op_type;\n"); fprintf(f, " op_type = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " obj = (VALUE)0x%"PRIxVALUE";", operands[1]); if (SYMBOL_P((VALUE)operands[1])) comment_id(f, SYM2ID((VALUE)operands[1])); fprintf(f, "\n"); fprintf(f, " needstr = (VALUE)0x%"PRIxVALUE";", operands[2]); if (SYMBOL_P((VALUE)operands[2])) comment_id(f, SYM2ID((VALUE)operands[2])); fprintf(f, "\n"); fprintf(f, " v = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_defined(ec, GET_CFP(), op_type, obj, needstr, v);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_defined(op_type, obj, needstr) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_defined(op_type, obj, needstr)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_defined(op_type, obj, needstr); } fprintf(f, "}\n"); break; case BIN(checkmatch): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t flag) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) pattern, result, target;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) flag;\n"); fprintf(f, " flag = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " target = stack[%d];\n", b->stack_size - 2); fprintf(f, " pattern = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " result = vm_check_match(ec, target, pattern, flag);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = result;\n", b->stack_size + (int)attr_sp_inc_checkmatch(flag) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_checkmatch(flag)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_checkmatch(flag); } fprintf(f, "}\n"); break; case BIN(checkkeyword): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t kw_bits_index) = (lindex_t)operands[0]; MAYBE_UNUSED(lindex_t keyword_index) = (lindex_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ret;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) keyword_index, kw_bits_index;\n"); fprintf(f, " kw_bits_index = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " keyword_index = (lindex_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " ret = vm_check_keyword(kw_bits_index, keyword_index, GET_EP());\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = ret;\n", b->stack_size + (int)attr_sp_inc_checkkeyword(kw_bits_index, keyword_index) - 1); b->stack_size += attr_sp_inc_checkkeyword(kw_bits_index, keyword_index); } fprintf(f, "}\n"); break; case BIN(checktype): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t type) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) ret, val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) type;\n"); fprintf(f, " type = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " ret = (TYPE(val) == (int)type) ? Qtrue : Qfalse;\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = ret;\n", b->stack_size + (int)attr_sp_inc_checktype(type) - 1); b->stack_size += attr_sp_inc_checktype(type); } fprintf(f, "}\n"); break; case BIN(definemethod): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(ISEQ iseq) = (ISEQ)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) iseq;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " iseq = (ISEQ)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_definemethod(id, iseq) + 0 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " vm_define_method(ec, Qnil, id, (VALUE)iseq, FALSE);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_definemethod(id, iseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_definemethod(id, iseq); } fprintf(f, "}\n"); break; case BIN(definesmethod): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ID id) = (ID)operands[0]; MAYBE_UNUSED(ISEQ iseq) = (ISEQ)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ID) id;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) iseq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj;\n"); fprintf(f, " id = (ID)0x%"PRIxVALUE";", operands[0]); comment_id(f, (ID)operands[0]); fprintf(f, "\n"); fprintf(f, " iseq = (ISEQ)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_definesmethod(id, iseq) + 0 - 1; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 1); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " vm_define_method(ec, obj, id, (VALUE)iseq, TRUE);\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_definesmethod(id, iseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_definesmethod(id, iseq); } fprintf(f, "}\n"); break; case BIN(send): { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; MAYBE_UNUSED(ISEQ blockiseq) = (ISEQ)operands[1]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. argc += ((ci->flag & VM_CALL_ARGS_BLOCKARG) ? 1 : 0); // simulate `vm_caller_setup_arg_block`'s `--reg_cfp->sp` if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_send(cd, blockiseq) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_send(cd, blockiseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_send(cd, blockiseq); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; MAYBE_UNUSED(ISEQ blockiseq) = (ISEQ)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) blockiseq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " blockiseq = (ISEQ)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_send(cd, blockiseq) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), &cd->ci, blockiseq, false);\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_send(cd, blockiseq) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_send(cd, blockiseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_send(cd, blockiseq); } fprintf(f, "}\n"); break; case BIN(opt_send_without_block): { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; case BIN(opt_str_freeze): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE str) = (VALUE)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) str, val;\n"); fprintf(f, " str = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, ((CALL_DATA)operands[1])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_str_freeze(str, BOP_FREEZE, idFreeze);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, " PUSH(rb_str_resurrect(str));\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_str_freeze(str, cd) - 1); b->stack_size += attr_sp_inc_opt_str_freeze(str, cd); } fprintf(f, "}\n"); break; case BIN(opt_nil_p): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_nil_p(cd, recv);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_nil_p(cd) - 1); b->stack_size += attr_sp_inc_opt_nil_p(cd); } fprintf(f, "}\n"); break; case BIN(opt_str_uminus): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE str) = (VALUE)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) str, val;\n"); fprintf(f, " str = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, ((CALL_DATA)operands[1])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_str_freeze(str, BOP_UMINUS, idUMinus);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, " PUSH(rb_str_resurrect(str));\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_str_uminus(str, cd) - 1); b->stack_size += attr_sp_inc_opt_str_uminus(str, cd); } fprintf(f, "}\n"); break; case BIN(opt_newarray_max): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_newarray_max(num, %s);\n", (status->local_stack_p ? "stack + (stack_size - (num))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_newarray_max(num) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_newarray_max(num)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_newarray_max(num); } fprintf(f, "}\n"); break; case BIN(opt_newarray_min): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t num) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) num;\n"); fprintf(f, " num = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_newarray_min(num, %s);\n", (status->local_stack_p ? "stack + (stack_size - (num))" : "STACK_ADDR_FROM_TOP(num)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_newarray_min(num) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_newarray_min(num)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_newarray_min(num); } fprintf(f, "}\n"); break; case BIN(invokesuper): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; MAYBE_UNUSED(ISEQ blockiseq) = (ISEQ)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) blockiseq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " blockiseq = (ISEQ)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_invokesuper(cd, blockiseq) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), &cd->ci, blockiseq, true);\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_super_method);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_invokesuper(cd, blockiseq) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_invokesuper(cd, blockiseq)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_invokesuper(cd, blockiseq); } fprintf(f, "}\n"); break; case BIN(invokeblock): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_invokeblock(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " if (UNLIKELY(cd->cc.call != vm_invokeblock_i)) {\n"); fprintf(f, " cd->cc.call = vm_invokeblock_i; // check before setting to avoid CoW\n"); fprintf(f, " }\n"); fprintf(f, "\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_invokeblock);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_invokeblock(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_invokeblock(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_invokeblock(cd); } fprintf(f, "}\n"); break; case BIN(leave): if (b->stack_size != 1) { if (mjit_opts.warnings || mjit_opts.verbose) fprintf(stderr, "MJIT warning: Unexpected JIT stack_size on leave: %d\n", b->stack_size); status->success = false; } if (status->inlined_iseqs == NULL) { // the current ISeq is being inlined fprintf(f, " return stack[0];\n"); b->stack_size += attr_sp_inc_leave(); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_leave() + 1 - 1; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 1); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " if (OPT_CHECKED_RUN) {\n"); fprintf(f, " const VALUE *const bp = vm_base_ptr(GET_CFP());\n"); fprintf(f, " if (GET_SP() != bp) {\n"); fprintf(f, " vm_stack_consistency_error(ec, GET_CFP(), bp);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, "\n"); fprintf(f, " if (vm_pop_frame(ec, GET_CFP(), GET_EP())) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_leave() - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_leave()); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_leave(); } fprintf(f, "}\n"); b->finish_p = TRUE; break; case BIN(throw): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(rb_num_t throw_state) = (rb_num_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) throwobj, val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) throw_state;\n"); fprintf(f, " throw_state = (rb_num_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " throwobj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_throw(ec, GET_CFP(), throw_state, throwobj);\n"); fprintf(f, " THROW_EXCEPTION(val);\n"); fprintf(f, " /* unreachable */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_throw(throw_state) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_throw(throw_state)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_throw(throw_state); } fprintf(f, "}\n"); b->finish_p = TRUE; break; case BIN(jump): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " RUBY_VM_CHECK_INTS(ec);\n"); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_jump(dst)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_jump(dst); } fprintf(f, "}\n"); break; case BIN(branchif): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (RTEST(val)) {\n"); fprintf(f, " RUBY_VM_CHECK_INTS(ec);\n"); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_branchif(dst)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_branchif(dst); } fprintf(f, "}\n"); compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); break; case BIN(branchunless): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (!RTEST(val)) {\n"); fprintf(f, " RUBY_VM_CHECK_INTS(ec);\n"); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_branchunless(dst)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_branchunless(dst); } fprintf(f, "}\n"); compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); break; case BIN(branchnil): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (NIL_P(val)) {\n"); fprintf(f, " RUBY_VM_CHECK_INTS(ec);\n"); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_branchnil(dst)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_branchnil(dst); } fprintf(f, "}\n"); compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); break; case BIN(opt_getinlinecache): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(OFFSET dst) = (OFFSET)operands[0]; MAYBE_UNUSED(IC ic) = (IC)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(IC) ic;\n"); fprintf(f, " MAYBE_UNUSED(OFFSET) dst;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " dst = (OFFSET)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " ic = (IC)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " if (vm_ic_hit_p(ic, GET_EP())) {\n"); fprintf(f, " val = ic->value;\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_getinlinecache(dst, ic) - 1); next_pos = pos + insn_len(insn) + (unsigned int)dst; fprintf(f, " goto label_%d;\n", next_pos); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, " val = Qnil;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_getinlinecache(dst, ic) - 1); b->stack_size += attr_sp_inc_opt_getinlinecache(dst, ic); } fprintf(f, "}\n"); compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); break; case BIN(opt_setinlinecache): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(IC ic) = (IC)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(IC) ic;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " ic = (IC)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_ic_update(ic, val, GET_EP());\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_setinlinecache(ic) - 1); b->stack_size += attr_sp_inc_opt_setinlinecache(ic); } fprintf(f, "}\n"); break; case BIN(once): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(ISEQ iseq) = (ISEQ)operands[0]; MAYBE_UNUSED(ISE ise) = (ISE)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(ISE) ise;\n"); fprintf(f, " MAYBE_UNUSED(ISEQ) iseq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " iseq = (ISEQ)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " ise = (ISE)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_once(iseq, ise) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " val = vm_once_dispatch(ec, iseq, ise);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_once(iseq, ise) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_once(iseq, ise)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_once(iseq, ise); } fprintf(f, "}\n"); break; case BIN(opt_case_dispatch): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CDHASH hash) = (CDHASH)operands[0]; MAYBE_UNUSED(OFFSET else_offset) = (OFFSET)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CDHASH) hash;\n"); fprintf(f, " MAYBE_UNUSED(OFFSET) else_offset;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) key;\n"); fprintf(f, " hash = (CDHASH)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " else_offset = (OFFSET)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); fprintf(f, " key = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " OFFSET dst = vm_case_dispatch(hash, else_offset, key);\n"); fprintf(f, "\n"); fprintf(f, " if (dst) {\n"); { struct case_dispatch_var arg; arg.f = f; arg.base_pos = pos + insn_len(insn); arg.last_value = Qundef; fprintf(f, " switch (dst) {\n"); st_foreach(RHASH_TBL_RAW(hash), compile_case_dispatch_each, (VALUE)&arg); fprintf(f, " case %lu:\n", else_offset); fprintf(f, " goto label_%lu;\n", arg.base_pos + else_offset); fprintf(f, " }\n"); } fprintf(f, " }\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_case_dispatch(hash, else_offset); } fprintf(f, "}\n"); compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); break; case BIN(opt_plus): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_plus(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_plus(cd) - 1); b->stack_size += attr_sp_inc_opt_plus(cd); } fprintf(f, "}\n"); break; case BIN(opt_minus): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_minus(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_minus(cd) - 1); b->stack_size += attr_sp_inc_opt_minus(cd); } fprintf(f, "}\n"); break; case BIN(opt_mult): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_mult(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_mult(cd) - 1); b->stack_size += attr_sp_inc_opt_mult(cd); } fprintf(f, "}\n"); break; case BIN(opt_div): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_div(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_div(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_div(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_div(cd); } fprintf(f, "}\n"); break; case BIN(opt_mod): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_mod(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_mod(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_mod(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_mod(cd); } fprintf(f, "}\n"); break; case BIN(opt_eq): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = opt_eq_func(recv, obj, cd);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_eq(cd) - 1); b->stack_size += attr_sp_inc_opt_eq(cd); } fprintf(f, "}\n"); break; case BIN(opt_neq): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd_eq) = (CALL_DATA)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd, cd_eq;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd_eq = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, ((CALL_DATA)operands[1])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_neq(cd, cd_eq, recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_neq(cd_eq, cd) - 1); b->stack_size += attr_sp_inc_opt_neq(cd_eq, cd); } fprintf(f, "}\n"); break; case BIN(opt_lt): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_lt(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_lt(cd) - 1); b->stack_size += attr_sp_inc_opt_lt(cd); } fprintf(f, "}\n"); break; case BIN(opt_le): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_le(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_le(cd) - 1); b->stack_size += attr_sp_inc_opt_le(cd); } fprintf(f, "}\n"); break; case BIN(opt_gt): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_gt(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_gt(cd) - 1); b->stack_size += attr_sp_inc_opt_gt(cd); } fprintf(f, "}\n"); break; case BIN(opt_ge): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_ge(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_ge(cd) - 1); b->stack_size += attr_sp_inc_opt_ge(cd); } fprintf(f, "}\n"); break; case BIN(opt_ltlt): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_ltlt(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_ltlt(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_ltlt(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_ltlt(cd); } fprintf(f, "}\n"); break; case BIN(opt_and): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_and(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_and(cd) - 1); b->stack_size += attr_sp_inc_opt_and(cd); } fprintf(f, "}\n"); break; case BIN(opt_or): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_or(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_or(cd) - 1); b->stack_size += attr_sp_inc_opt_or(cd); } fprintf(f, "}\n"); break; case BIN(opt_aref): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_aref(recv, obj);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_aref(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_aref(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_aref(cd); } fprintf(f, "}\n"); break; case BIN(opt_aset): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj, recv, set, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 3); fprintf(f, " obj = stack[%d];\n", b->stack_size - 2); fprintf(f, " set = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_aset(recv, obj, set);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_aset(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_aset(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_aset(cd); } fprintf(f, "}\n"); break; case BIN(opt_aset_with): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE key) = (VALUE)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) key, recv, val;\n"); fprintf(f, " key = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, ((CALL_DATA)operands[1])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 2); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " VALUE tmp = vm_opt_aset_with(recv, key, val);\n"); fprintf(f, "\n"); fprintf(f, " if (tmp != Qundef) {\n"); fprintf(f, " val = tmp;\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, "#ifndef MJIT_HEADER\n"); fprintf(f, " %s = rb_str_resurrect(key);\n", (status->local_stack_p ? "*(stack + (stack_size - (0) - 1))" : "TOPN(0)")); fprintf(f, " PUSH(val);\n"); fprintf(f, "#endif\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_aset_with(key, cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_aset_with(key, cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_aset_with(key, cd); } fprintf(f, "}\n"); break; case BIN(opt_aref_with): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(VALUE key) = (VALUE)operands[0]; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) key, recv, val;\n"); fprintf(f, " key = (VALUE)0x%"PRIxVALUE";", operands[0]); if (SYMBOL_P((VALUE)operands[0])) comment_id(f, SYM2ID((VALUE)operands[0])); fprintf(f, "\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[1]); comment_id(f, ((CALL_DATA)operands[1])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_aref_with(recv, key);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#ifndef MJIT_HEADER\n"); fprintf(f, " PUSH(rb_str_resurrect(key));\n"); fprintf(f, "#endif\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_aref_with(key, cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_aref_with(key, cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_aref_with(key, cd); } fprintf(f, "}\n"); break; case BIN(opt_length): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_length(recv, BOP_LENGTH);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_length(cd) - 1); b->stack_size += attr_sp_inc_opt_length(cd); } fprintf(f, "}\n"); break; case BIN(opt_size): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_length(recv, BOP_SIZE);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_size(cd) - 1); b->stack_size += attr_sp_inc_opt_size(cd); } fprintf(f, "}\n"); break; case BIN(opt_empty_p): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_empty_p(recv);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_empty_p(cd) - 1); b->stack_size += attr_sp_inc_opt_empty_p(cd); } fprintf(f, "}\n"); break; case BIN(opt_succ): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_succ(recv);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_succ(cd) - 1); b->stack_size += attr_sp_inc_opt_succ(cd); } fprintf(f, "}\n"); break; case BIN(opt_not): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) recv, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " recv = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_not(cd, recv);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_not(cd) - 1); b->stack_size += attr_sp_inc_opt_not(cd); } fprintf(f, "}\n"); break; case BIN(opt_regexpmatch2): if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[0], body))) { { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg int param_size = iseq->body->param.size; fprintf(f, "{\n"); if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial[0]); fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); fprintf(f, " goto send_cancel;\n"); fprintf(f, " }\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { fprintf(f, " {\n"); fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); fprintf(f, " reg_cfp->self = orig_self;\n"); fprintf(f, " }\n"); } else { fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.argc = %d;\n", ci->orig_argc); fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); fprintf(f, " {\n"); fprintf(f, " VALUE v;\n"); fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE if (iseq->body->catch_except_p) { fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); fprintf(f, " v = vm_exec(ec, TRUE);\n"); } else { fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup fprintf(f, " v = vm_exec(ec, FALSE);\n"); fprintf(f, " }\n"); } fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); } b->stack_size += attr_sp_inc_opt_send_without_block(cd); fprintf(f, "}\n"); break; } } } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { { rb_snum_t i, push_size; push_size = -attr_sp_inc_opt_send_without_block(cd) + 1 - 0; fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ for (i = 0; i < push_size; i++) { fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); } } } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - 0); /* POPN(INSN_ATTR(popn)); */ } fprintf(f, " {\n"); fprintf(f, " VALUE bh = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " UNREACHABLE_RETURN(Qundef);\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_send_without_block(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_send_without_block(cd); } fprintf(f, "}\n"); break; } fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(CALL_DATA cd) = (CALL_DATA)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(CALL_DATA) cd;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) obj1, obj2, val;\n"); fprintf(f, " cd = (CALL_DATA)0x%"PRIxVALUE";", operands[0]); comment_id(f, ((CALL_DATA)operands[0])->ci.mid); fprintf(f, "\n"); fprintf(f, " obj2 = stack[%d];\n", b->stack_size - 2); fprintf(f, " obj1 = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_opt_regexpmatch2(obj2, obj1);\n"); fprintf(f, "\n"); fprintf(f, " if (val == Qundef) {\n"); if (status->local_stack_p) { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_regexpmatch2(cd) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_regexpmatch2(cd)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_regexpmatch2(cd); } fprintf(f, "}\n"); break; case BIN(invokebuiltin): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(RB_BUILTIN bf) = (RB_BUILTIN)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(RB_BUILTIN) bf;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) ret;\n"); fprintf(f, " bf = (RB_BUILTIN)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " ret = vm_invoke_builtin(ec, reg_cfp, bf, %s);\n", (status->local_stack_p ? "stack + (stack_size - (bf->argc))" : "STACK_ADDR_FROM_TOP(bf->argc)")); fprintf(f, " }\n"); fprintf(f, " stack[%d] = ret;\n", b->stack_size + (int)attr_sp_inc_invokebuiltin(bf) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_invokebuiltin(bf)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_invokebuiltin(bf); } fprintf(f, "}\n"); break; case BIN(opt_invokebuiltin_delegate): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(RB_BUILTIN bf) = (RB_BUILTIN)operands[0]; MAYBE_UNUSED(rb_num_t index) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(RB_BUILTIN) bf;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) ret;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) index;\n"); fprintf(f, " bf = (RB_BUILTIN)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " index = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " ret = vm_invoke_builtin_delegate(ec, reg_cfp, bf, (unsigned int)index);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = ret;\n", b->stack_size + (int)attr_sp_inc_opt_invokebuiltin_delegate(bf, index) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_invokebuiltin_delegate(bf, index)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_invokebuiltin_delegate(bf, index); } fprintf(f, "}\n"); break; case BIN(opt_invokebuiltin_delegate_leave): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(RB_BUILTIN bf) = (RB_BUILTIN)operands[0]; MAYBE_UNUSED(rb_num_t index) = (rb_num_t)operands[1]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(RB_BUILTIN) bf;\n"); fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) index;\n"); fprintf(f, " bf = (RB_BUILTIN)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " index = (rb_num_t)0x%"PRIxVALUE";", operands[1]); fprintf(f, "\n"); if (body->catch_except_p || TRUE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = vm_invoke_builtin_delegate(ec, reg_cfp, bf, (unsigned int)index);\n"); fprintf(f, "\n"); fprintf(f, " /* leave fastpath */\n"); fprintf(f, " /* TracePoint/return should fallback this insn to opt_invokebuiltin_delegate */\n"); fprintf(f, " if (vm_pop_frame(ec, GET_CFP(), GET_EP())) {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " }\n"); fprintf(f, " else {\n"); fprintf(f, "#if OPT_CALL_THREADED_CODE\n"); fprintf(f, " rb_ec_thread_ptr(ec)->retval = val;\n"); fprintf(f, " return 0;\n"); fprintf(f, "#else\n"); fprintf(f, " return val;\n"); fprintf(f, "#endif\n"); fprintf(f, " }\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_opt_invokebuiltin_delegate_leave(bf, index) - 1); fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)attr_sp_inc_opt_invokebuiltin_delegate_leave(bf, index)); if (!pc_moved_p) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); } fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); fprintf(f, " goto cancel;\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_opt_invokebuiltin_delegate_leave(bf, index); } fprintf(f, "}\n"); b->finish_p = TRUE; break; case BIN(getlocal_WC_0): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " level = 0;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = *(vm_get_ep(GET_EP(), level) - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getlocal_WC_0(idx) - 1); b->stack_size += attr_sp_inc_getlocal_WC_0(idx); } fprintf(f, "}\n"); break; case BIN(getlocal_WC_1): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " level = 1;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " val = *(vm_get_ep(GET_EP(), level) - idx);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_get);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0);\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_getlocal_WC_1(idx) - 1); b->stack_size += attr_sp_inc_getlocal_WC_1(idx); } fprintf(f, "}\n"); break; case BIN(setlocal_WC_0): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " level = 0;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_env_write(vm_get_ep(GET_EP(), level), -(int)idx, val);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_set);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_set_dynamic, level > 0);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setlocal_WC_0(idx); } fprintf(f, "}\n"); break; case BIN(setlocal_WC_1): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; MAYBE_UNUSED(lindex_t idx) = (lindex_t)operands[0]; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " MAYBE_UNUSED(lindex_t) idx;\n"); fprintf(f, " MAYBE_UNUSED(rb_num_t) level;\n"); fprintf(f, " level = 1;\n"); fprintf(f, " idx = (lindex_t)0x%"PRIxVALUE";", operands[0]); fprintf(f, "\n"); fprintf(f, " val = stack[%d];\n", b->stack_size - 1); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " vm_env_write(vm_get_ep(GET_EP(), level), -(int)idx, val);\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(lvar_set);\n"); fprintf(f, " (void)RB_DEBUG_COUNTER_INC_IF(lvar_set_dynamic, level > 0);\n"); fprintf(f, " }\n"); b->stack_size += attr_sp_inc_setlocal_WC_1(idx); } fprintf(f, "}\n"); break; case BIN(putobject_INT2FIX_0_): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = INT2FIX(0);\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putobject_INT2FIX_0_() - 1); b->stack_size += attr_sp_inc_putobject_INT2FIX_0_(); } fprintf(f, "}\n"); break; case BIN(putobject_INT2FIX_1_): fprintf(f, "{\n"); { MAYBE_UNUSED(int pc_moved_p) = FALSE; if (status->local_stack_p) { fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); } fprintf(f, " MAYBE_UNUSED(VALUE) val;\n"); fprintf(f, " val = INT2FIX(1);\n"); if (body->catch_except_p || FALSE) { fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ pc_moved_p = TRUE; } if (status->local_stack_p) { } else { fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); } fprintf(f, " {\n"); fprintf(f, " /* */\n"); fprintf(f, " }\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size + (int)attr_sp_inc_putobject_INT2FIX_1_() - 1); b->stack_size += attr_sp_inc_putobject_INT2FIX_1_(); } fprintf(f, "}\n"); break; default: if (mjit_opts.warnings || mjit_opts.verbose) fprintf(stderr, "MJIT warning: Skipped to compile unsupported instruction: %s\n", insn_name(insn)); status->success = false; break; }