From 0fed37ad7f22e64a89e749fba4e665d13dc3ef0d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 10 Oct 2024 13:05:10 +0300 Subject: [PATCH 01/10] Better trace coverage (JIT trampoline calls) --- ext/opcache/jit/zend_jit_internal.h | 14 +++++-- ext/opcache/jit/zend_jit_ir.c | 24 ++++++++---- ext/opcache/jit/zend_jit_trace.c | 54 +++++++++++++-------------- ext/opcache/jit/zend_jit_vm_helpers.c | 37 +++++++++--------- 4 files changed, 70 insertions(+), 59 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 4fe07222d8f72..9a09be49f1bd5 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -248,8 +248,11 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key); _(RECURSIVE_CALL, "recursive call") \ _(RECURSIVE_RET, "recursive return") \ _(RETURN, "return") \ - _(INTERPRETER, "exit to VM interpreter") \ _(LINK, "link to another trace") \ + _(INTERPRETER, "exit to VM interpreter") \ + _(TRAMPOLINE, "trampoline call") \ + _(PROP_HOOK_CALL, "property hook call") \ + _(BAD_FUNC, "bad function call") \ /* compilation and linking successful */ \ _(COMPILED, "compiled") \ _(ALREADY_DONE, "already prcessed") \ @@ -267,9 +270,6 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key); _(BLACK_LIST, "trace blacklisted") \ _(INNER_LOOP, "inner loop") /* trace it */ \ _(COMPILED_LOOP, "compiled loop") \ - _(TRAMPOLINE, "trampoline call") \ - _(PROP_HOOK_CALL, "property hook call") \ - _(BAD_FUNC, "bad function call") \ _(COMPILER_ERROR, "JIT compilation error") \ /* no recoverable error (blacklist immediately) */ \ _(NO_SHM, "insufficient shared memory") \ @@ -380,6 +380,12 @@ typedef enum _zend_jit_trace_op { #define ZEND_JIT_TRACE_FAKE_INFO(level) \ (((level) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) | ZEND_JIT_TRACE_FAKE_INIT_CALL) +#define ZEND_JIT_TRACE_NUM_ARGS_INFO(count) \ + ((count) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) + +#define ZEND_JIT_TRACE_NUM_ARGS(info) \ + (((info) & ZEND_JIT_TRACE_FAKE_LEVEL_MASK) >> ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) + #define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \ _info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \ } while (0) diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index e4a66a6743daf..4ff052c909ad3 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -9064,6 +9064,14 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, jit->delayed_call_level = call_level; } + if (trace + && trace->op == ZEND_JIT_TRACE_END + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { + if (!zend_jit_set_ip(jit, opline + 1)) { + return 0; + } + } + return 1; } @@ -9324,7 +9332,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, if (trace && trace->op == ZEND_JIT_TRACE_END - && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { if (!zend_jit_set_ip(jit, opline + 1)) { return 0; } @@ -9933,7 +9941,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (trace && !func) { if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { - ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); + ZEND_ASSERT(!trace->func || trace->func->type == ZEND_INTERNAL_FUNCTION); #ifndef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? func = trace->func; @@ -10115,7 +10123,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (call_num_args <= func->op_array.num_args) { if (!trace || (trace->op == ZEND_JIT_TRACE_END - && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) { uint32_t num_args; if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { @@ -10149,7 +10157,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen } } else { if (!trace || (trace->op == ZEND_JIT_TRACE_END - && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) { ir_ref ip; if (zend_accel_in_shm(func->op_array.opcodes)) { @@ -10275,7 +10283,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen ir_ref observer_handler; ir_ref rx = jit_FP(jit); struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref); - if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { + if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); jit_SET_EX_OPLINE(jit, trace[1].opline); } else if (GCC_GLOBAL_REGS) { @@ -10568,7 +10576,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen jit_LOAD_IP_ADDR(jit, opline + 1); } else if (trace && trace->op == ZEND_JIT_TRACE_END - && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { jit_LOAD_IP_ADDR(jit, opline + 1); } } @@ -16908,7 +16916,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { if (trace->op != ZEND_JIT_TRACE_END || (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && - trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { + trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { /* this check may be handled by the following OPLINE guard or jmp [IP] */ ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); @@ -16926,7 +16934,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr } if (trace->op != ZEND_JIT_TRACE_END || (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && - trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { + trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { const zend_op *next_opline = trace->opline; const zend_op *exit_opline = NULL; diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 02d2e62ccf4ff..dc3f3f7328bb7 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -427,25 +427,25 @@ static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa #define CHECK_OP1_DATA_TRACE_TYPE() \ CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type) -static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array) +static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array, uint32_t num_args) { if (op_array && op_array->type == ZEND_USER_FUNCTION) { return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack))); } else if (op_array) { return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack))); } else { - return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack)); + return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(num_args * sizeof(zend_jit_trace_stack))); } } -static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array, uint32_t num_args) { - return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array)); + return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array, num_args)); } static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) { - return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array)); + return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array, 0)); } static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type) @@ -1323,7 +1323,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin * Calculate size of abstract stack; * Construct regular SSA for involved op_array */ op_array = trace_buffer->op_array; - stack_top = stack_size = zend_jit_trace_frame_size(op_array); + stack_top = stack_size = zend_jit_trace_frame_size(op_array, 0); stack_bottom = 0; p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; ssa_ops_count = 0; @@ -1363,7 +1363,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa_ops_count += zend_jit_trace_op_len(p->opline); } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { call_level++; - stack_top += zend_jit_trace_frame_size(p->op_array); + stack_top += zend_jit_trace_frame_size(p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info)); if (stack_top > stack_size) { stack_size = stack_top; } @@ -1377,7 +1377,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; } } - frame_size = zend_jit_trace_frame_size(p->op_array); + frame_size = zend_jit_trace_frame_size(p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info)); if (call_level == 0) { if (stack_top + frame_size > stack_size) { stack_size = stack_top + frame_size; @@ -1389,7 +1389,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_ENTER) { op_array = p->op_array; if (call_level == 0) { - stack_top += zend_jit_trace_frame_size(op_array); + stack_top += zend_jit_trace_frame_size(op_array, 0); if (stack_top > stack_size) { stack_size = stack_top; } @@ -1414,7 +1414,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } else if (p->op == ZEND_JIT_TRACE_BACK) { if (level == 0) { - stack_bottom += zend_jit_trace_frame_size(p->op_array); + stack_bottom += zend_jit_trace_frame_size(p->op_array, 0); jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; @@ -1431,7 +1431,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa = zend_jit_trace_build_ssa(op_array, script); } } else { - stack_top -= zend_jit_trace_frame_size(op_array); + stack_top -= zend_jit_trace_frame_size(op_array, 0); level--; } op_array = p->op_array; @@ -1534,7 +1534,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin len--; } } else if (p->op == ZEND_JIT_TRACE_ENTER) { - frame = zend_jit_trace_call_frame(frame, op_array); + frame = zend_jit_trace_call_frame(frame, op_array, 0); stack = frame->stack; op_array = p->op_array; level++; @@ -1754,7 +1754,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } frame = JIT_G(current_frame); - top = zend_jit_trace_call_frame(frame, op_array); + top = zend_jit_trace_call_frame(frame, op_array, 0); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); frame->used_stack = 0; @@ -2448,7 +2448,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call = top; TRACE_FRAME_INIT(call, op_array, 0, 0); call->used_stack = 0; - top = zend_jit_trace_call_frame(top, op_array); + top = zend_jit_trace_call_frame(top, op_array, 0); } else { ZEND_ASSERT(&call->func->op_array == op_array); } @@ -2583,7 +2583,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call->prev = frame->call; call->used_stack = 0; frame->call = call; - top = zend_jit_trace_call_frame(top, p->op_array); + top = zend_jit_trace_call_frame(top, p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info)); if (p->func && p->func->type == ZEND_USER_FUNCTION) { for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { SET_STACK_INFO(call->stack, i, -1); @@ -2626,6 +2626,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin if (idx > 0 && ssa_ops[idx-1].result_def >= 0 + && p->func && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) { ZEND_ASSERT(ssa_opcodes[idx-1] == opline); @@ -3156,7 +3157,7 @@ static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *t } } - frame = zend_jit_trace_call_frame(frame, op_array); + frame = zend_jit_trace_call_frame(frame, op_array, 0); frame->prev = prev_frame; frame->func = (const zend_function*)p->op_array; stack = frame->stack; @@ -3306,8 +3307,7 @@ static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *t } phi = phi->next; } - } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK - || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) { for (i = 0; i < op_array->last_var + op_array->T; i++) { int var = STACK_VAR(stack, i); if (var >= 0 && RA_HAS_REG(var) @@ -4123,7 +4123,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); op_array = p->op_array; frame = JIT_G(current_frame); - top = zend_jit_trace_call_frame(frame, op_array); + top = zend_jit_trace_call_frame(frame, op_array, 0); TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); frame->used_stack = checked_stack = peek_checked_stack = 0; stack = frame->stack; @@ -7006,7 +7006,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } frame->call = call; - top = zend_jit_trace_call_frame(top, p->op_array); + top = zend_jit_trace_call_frame(top, p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info)); if (p->func) { if (p->func->type == ZEND_USER_FUNCTION) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { @@ -7192,8 +7192,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par zend_jit_trace_end_loop(&ctx, jit->trace_loop_ref, timeout_exit_addr); /* jump back to start of the trace loop */ } - } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK - || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) { if (ra && (p-1)->op != ZEND_JIT_TRACE_ENTER && (p-1)->op != ZEND_JIT_TRACE_BACK @@ -7303,8 +7302,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } } - } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK - || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) { if (opline && (opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL @@ -7929,7 +7927,7 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa level, ' ', (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "", (p->func && p->func->common.scope) ? "::" : "", - p->func ? ZSTR_VAL(p->func->common.function_name) : "???"); + (p->func && p->func->common.function_name) ? ZSTR_VAL(p->func->common.function_name) : "???"); } else { fprintf(stderr, " %*c>skip\n", level, ' '); @@ -7938,9 +7936,9 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa if (p->func != (zend_function*)&zend_pass_function) { fprintf(stderr, " %*c>call %s%s%s\n", level, ' ', - p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "", - p->func->common.scope ? "::" : "", - ZSTR_VAL(p->func->common.function_name)); + (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "", + (p->func && p->func->common.scope) ? "::" : "", + (p->func && p->func->common.function_name) ? ZSTR_VAL(p->func->common.function_name) : "???"); } else { fprintf(stderr, " %*c>skip\n", level, ' '); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index d42c3c6366d4a..607d382bd52ee 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -914,11 +914,19 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) { - if (EX(call)->func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) { + zend_function *func = EX(call)->func; + uint32_t info = 0; + + if (func->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { + func = NULL; + info = ZEND_JIT_TRACE_NUM_ARGS_INFO(ZEND_CALL_NUM_ARGS(EX(call))); +//??? stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; +//??? break; + } else if (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) { stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; break; } - TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func); + TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, info, func); } } else if (opline->opcode == ZEND_INCLUDE_OR_EVAL || opline->opcode == ZEND_CALLABLE_CONVERT) { @@ -1100,21 +1108,9 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, if (EX(call) && EX(call)->prev_execute_data == prev_call) { zend_function *func; + uint32_t info = 0; zend_jit_op_array_trace_extension *jit_extension; - if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { - /* TODO: Can we continue recording ??? */ - stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; - break; - } else if (EX(call)->func->common.fn_flags & ZEND_ACC_NEVER_CACHE) { - /* TODO: Can we continue recording ??? */ - stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; - break; - } else if (EX(call)->func->common.prop_info) { - /* TODO: Can we continue recording ??? */ - stop = ZEND_JIT_TRACE_STOP_PROP_HOOK_CALL; - break; - } func = EX(call)->func; if (func->type == ZEND_INTERNAL_FUNCTION && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) { @@ -1139,18 +1135,21 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, opline = EX(opline); #endif - if (JIT_G(max_polymorphic_calls) == 0 + if (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { + /* continue recording */ + func = NULL; + info = ZEND_JIT_TRACE_NUM_ARGS_INFO(ZEND_CALL_NUM_ARGS(EX(call))); + } else if (JIT_G(max_polymorphic_calls) == 0 && zend_jit_may_be_polymorphic_call(opline - 1)) { func = NULL; + ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC); } else if ((is_megamorphic == ZEND_JIT_EXIT_METHOD_CALL || is_megamorphic == ZEND_JIT_EXIT_CLOSURE_CALL) && trace_buffer[1].opline == opline - 1) { func = NULL; - } - if (!func) { ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC); } - TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func); + TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, info, func); } prev_call = EX(call); } From 8243338503a31a1580b6db7bcab50d9d63a5564c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Oct 2024 12:50:24 +0300 Subject: [PATCH 02/10] clenup trampoline by zend_jit_free_trampoline() --- ext/opcache/jit/zend_jit_ir.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 4ff052c909ad3..204a667732078 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -8431,13 +8431,21 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co used_stack_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + bool may_be_trampoline = !func && (opline->opcode == ZEND_INIT_METHOD_CALL); + int32_t exit_point = zend_jit_trace_get_exit_point(opline, + may_be_trampoline ? + (ZEND_JIT_EXIT_TO_VM | ZEND_JIT_EXIT_METHOD_CALL) : ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } + if (may_be_trampoline) { + jit->trace->exit_info[exit_point].poly_func_ref = func_ref; + jit->trace->exit_info[exit_point].poly_this_ref = this_ref; + } + ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { if_enough_stack = ir_IF(ref); From 995cf5dbae151eff7609dd6978722f4495c420d5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Oct 2024 12:51:43 +0300 Subject: [PATCH 03/10] Fix ZEND_JIT_TRACE_INIT_CALL/ZEND_JIT_TRACE_DO_ICALL num_args mismatch It may be caused by SEND_UNPACK/SEND_ARRAY --- ext/opcache/jit/zend_jit_trace.c | 40 +++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index dc3f3f7328bb7..a3abbc1217900 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -1297,6 +1297,39 @@ typedef struct _zend_tssa { static const zend_op _nop_opcode = {0}; +static uint32_t find_trampoline_num_args(zend_jit_trace_rec *start, zend_jit_trace_rec *p) +{ + int inline_level = 0, call_level = 0; + + p--; + while (p != start) { + if (p->op == ZEND_JIT_TRACE_INIT_CALL) { + if (inline_level == 0) { + if (call_level == 0) { + ZEND_ASSERT(!p->op_array); + return ZEND_JIT_TRACE_NUM_ARGS(p->info); + } else { + call_level--; + } + } + } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + if (inline_level == 0) { + call_level++; + } + } else if (p->op == ZEND_JIT_TRACE_ENTER) { + if (inline_level) { + inline_level--; + } else { + return 0; + } + } else if (p->op == ZEND_JIT_TRACE_BACK) { + inline_level++; + } + p--; + } + return 0; +} + static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr) { zend_ssa *tssa; @@ -1368,6 +1401,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin stack_size = stack_top; } } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + uint32_t num_args = 0; if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) { if (p->func && p->func != (zend_function*)&zend_pass_function @@ -1377,7 +1411,11 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; } } - frame_size = zend_jit_trace_frame_size(p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info)); + if (!p->func) { + /* Find num_args in the corresponding ZEND_JIT_TRACE_INIT_CALL record */ + num_args = find_trampoline_num_args(trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE, p); + } + frame_size = zend_jit_trace_frame_size(p->op_array, num_args); if (call_level == 0) { if (stack_top + frame_size > stack_size) { stack_size = stack_top + frame_size; From c6dd5aeaa82fe0ccbf0ef0d504a86eb8bdb94f0f Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Oct 2024 12:54:14 +0300 Subject: [PATCH 04/10] cleanup --- ext/opcache/jit/zend_jit_vm_helpers.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 607d382bd52ee..f3d84676ecf8e 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -918,10 +918,8 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, uint32_t info = 0; if (func->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { + /* continue recording */ func = NULL; - info = ZEND_JIT_TRACE_NUM_ARGS_INFO(ZEND_CALL_NUM_ARGS(EX(call))); -//??? stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; -//??? break; } else if (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) { stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; break; @@ -1138,7 +1136,6 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, if (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { /* continue recording */ func = NULL; - info = ZEND_JIT_TRACE_NUM_ARGS_INFO(ZEND_CALL_NUM_ARGS(EX(call))); } else if (JIT_G(max_polymorphic_calls) == 0 && zend_jit_may_be_polymorphic_call(opline - 1)) { func = NULL; @@ -1149,6 +1146,9 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, func = NULL; ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC); } + if (!func) { + info = ZEND_JIT_TRACE_NUM_ARGS_INFO(ZEND_CALL_NUM_ARGS(EX(call))); + } TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, info, func); } prev_call = EX(call); From cab8472a23ce228eae7d3b2c18e12b2c71338f09 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Oct 2024 12:59:25 +0300 Subject: [PATCH 05/10] cleanup --- ext/opcache/jit/zend_jit_vm_helpers.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index f3d84676ecf8e..cff6932fe7615 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -915,7 +915,6 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) { zend_function *func = EX(call)->func; - uint32_t info = 0; if (func->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { /* continue recording */ @@ -924,7 +923,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; break; } - TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, info, func); + TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, func); } } else if (opline->opcode == ZEND_INCLUDE_OR_EVAL || opline->opcode == ZEND_CALLABLE_CONVERT) { From ceffc8545203bd55daec5e2a5c4d7509fec5ac67 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Oct 2024 13:33:11 +0300 Subject: [PATCH 06/10] Don't record function that may be temporary --- ext/opcache/jit/zend_jit_vm_helpers.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index cff6932fe7615..ef610813e52e8 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -916,7 +916,9 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) { zend_function *func = EX(call)->func; - if (func->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { + if ((func->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) + || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE) + || func->common.prop_info) { /* continue recording */ func = NULL; } else if (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) { @@ -1132,7 +1134,9 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, opline = EX(opline); #endif - if (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { + if ((func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) + || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE) + || func->common.prop_info) { /* continue recording */ func = NULL; } else if (JIT_G(max_polymorphic_calls) == 0 From 8112c606da36df222ce42e3aeb3bdd6dfe9e4970 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 11 Oct 2024 13:34:00 +0300 Subject: [PATCH 07/10] cleanup --- ext/opcache/jit/zend_jit_vm_helpers.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index ef610813e52e8..3ea23ab4f4b40 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -989,13 +989,11 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { - /* TODO: Can we continue recording ??? */ stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; break; } if (EX(func)->op_array.prop_info) { - /* TODO: Can we continue recording ??? */ stop = ZEND_JIT_TRACE_STOP_PROP_HOOK_CALL; break; } From 6f0b4a8df3f03a80683e26c7ffc44fb7c241f492 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 14 Oct 2024 12:35:51 +0300 Subject: [PATCH 08/10] Prevent invalid run_time_cache allocation for "bad" internal functions --- ext/opcache/jit/zend_jit_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 140f69dee062c..3eec4df0ecc7e 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -42,7 +42,7 @@ static zend_never_inline zend_op_array* ZEND_FASTCALL zend_jit_init_func_run_tim { void **run_time_cache; - if (!RUN_TIME_CACHE(op_array)) { + if (op_array->type == ZEND_USER_FUNCTION && !RUN_TIME_CACHE(op_array)) { run_time_cache = zend_arena_alloc(&CG(arena), op_array->cache_size); memset(run_time_cache, 0, op_array->cache_size); ZEND_MAP_PTR_SET(op_array->run_time_cache, run_time_cache); From b2cf43a5bb2ce47e0208ade1d499f2a6010ca3f5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 15 Oct 2024 20:53:44 +0300 Subject: [PATCH 09/10] Update zend_jit_trace_record_fake_init_call_ex() accordingly --- ext/opcache/jit/zend_jit_vm_helpers.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 3ea23ab4f4b40..a6bac35155a9f 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -508,15 +508,6 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend } func = call->func; - if (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE|ZEND_ACC_NEVER_CACHE)) { - /* TODO: Can we continue recording ??? */ - return -1; - } - /* Function is a property hook. */ - if (func->common.prop_info) { - /* TODO: Can we continue recording ??? */ - return -1; - } if (func->type == ZEND_INTERNAL_FUNCTION && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) { return -1; @@ -524,7 +515,7 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend if (func->type == ZEND_USER_FUNCTION) { jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); - if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) + if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) { return -1; @@ -533,7 +524,13 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend func = (zend_function*)jit_extension->op_array; } } - if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM + + if ((func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) + || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE) + || func->common.prop_info) { + /* continue recording */ + func = NULL; + } else if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM /* TODO: use more accurate check ??? */ && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC) || func->common.scope)) { From 271f553ece552791cdd0d9638ba38e9deff84e4c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 15 Oct 2024 21:52:04 +0300 Subject: [PATCH 10/10] Better handling of "bad" functions and fake closures --- ext/opcache/jit/zend_jit_vm_helpers.c | 31 ++++++++++++--------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index a6bac35155a9f..70122eb9736e9 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -510,22 +510,21 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend func = call->func; if (func->type == ZEND_INTERNAL_FUNCTION && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) { - return -1; - } - if (func->type == ZEND_USER_FUNCTION) { + func = NULL; + } else if (func->type == ZEND_USER_FUNCTION) { jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) { - return -1; - } - if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) { + func = NULL; + } else if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) { func = (zend_function*)jit_extension->op_array; } } - if ((func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) + if (!func + || (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE) || func->common.prop_info) { /* continue recording */ @@ -961,7 +960,8 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); if (UNEXPECTED(!jit_extension) - || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) { + || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) + || (op_array->fn_flags & ZEND_ACC_FAKE_CLOSURE)) { stop = ZEND_JIT_TRACE_STOP_INTERPRETER; break; } @@ -1108,19 +1108,15 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, func = EX(call)->func; if (func->type == ZEND_INTERNAL_FUNCTION && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) { - stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; - break; - } - if (func->type == ZEND_USER_FUNCTION) { + func = NULL; + } else if (func->type == ZEND_USER_FUNCTION) { jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) { - stop = ZEND_JIT_TRACE_STOP_INTERPRETER; - break; - } - if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) { + func = NULL; + } else if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) { func = (zend_function*)jit_extension->op_array; } } @@ -1129,7 +1125,8 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, opline = EX(opline); #endif - if ((func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) + if (!func + || (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE) || func->common.prop_info) { /* continue recording */