Author: Arnaud Le Blanc (arnaud-lb)
Date: 2025-05-14T12:30:36+02:00
Commit: https://github.com/php/php-src/commit/1de16c7f15f3f927bf7e7c26b3a6b1bd5803b1cc
Raw diff: https://github.com/php/php-src/commit/1de16c7f15f3f927bf7e7c26b3a6b1bd5803b1cc.diff
Merge branch 'PHP-8.4'
* PHP-8.4:
Snapshotted poly_func / poly_this may be spilled
Changed paths:
M ext/opcache/jit/zend_jit_internal.h
M ext/opcache/jit/zend_jit_ir.c
M ext/opcache/jit/zend_jit_trace.c
Diff:
diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index 06249d70e6d2..3623689f05aa 100644
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -437,16 +437,22 @@ struct _zend_jit_trace_rec {
#define ZEND_JIT_TRACE_START_REC_SIZE 2
+typedef struct _zend_jit_ref_snapshot {
+ union {
+ int32_t ref; /* While generating code: The ir_ref to snapshot */
+ int32_t offset; /* After compilation / during deopt: C stack offset if 'reg' is
spilled */
+ };
+ int8_t reg; /* Set after compilation by zend_jit_snapshot_handler() */
+} zend_jit_ref_snapshot;
+
typedef struct _zend_jit_trace_exit_info {
- const zend_op *opline; /* opline where VM should continue execution */
- const zend_op_array *op_array;
- uint32_t flags; /* set of ZEND_JIT_EXIT_... */
- uint32_t stack_size;
- uint32_t stack_offset;
- int32_t poly_func_ref;
- int32_t poly_this_ref;
- int8_t poly_func_reg;
- int8_t poly_this_reg;
+ const zend_op *opline; /* opline where VM should continue execution */
+ const zend_op_array *op_array;
+ uint32_t flags; /* set of ZEND_JIT_EXIT_... */
+ uint32_t stack_size;
+ uint32_t stack_offset;
+ zend_jit_ref_snapshot poly_func;
+ zend_jit_ref_snapshot poly_this;
} zend_jit_trace_exit_info;
typedef struct _zend_jit_trace_stack {
diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c
index a6140bc8706f..2a3b9baf28e6 100644
--- a/ext/opcache/jit/zend_jit_ir.c
+++ b/ext/opcache/jit/zend_jit_ir.c
@@ -277,6 +277,8 @@ typedef struct _zend_jit_ctx {
ir_ref tls;
#endif
ir_ref fp;
+ ir_ref poly_func_ref; /* restored from parent trace snapshot */
+ ir_ref poly_this_ref; /* restored from parent trace snapshot */
ir_ref trace_loop_ref;
ir_ref return_inputs;
const zend_op_array *op_array;
@@ -624,12 +626,12 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
uint32_t exit_point = 0, n = 0;
if (addr < 0) {
- if (t->exit_count > 0
- && jit->ctx.ir_base[addr].val.u64 ==
(uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) {
- exit_point = t->exit_count - 1;
- if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
- n = 2;
- }
+ /* addr is not always the address of the *last* exit point,
+ * so we can not optimize this to 'exit_point = t->exit_count-1' */
+ exit_point = zend_jit_exit_point_by_addr(ptr);
+ ZEND_ASSERT(exit_point != -1);
+ if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
+ n = 2;
}
}
@@ -660,8 +662,8 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);
}
if (n) {
- ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref);
- ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref);
+ ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func.ref);
+ ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this.ref);
}
}
}
@@ -710,6 +712,31 @@ uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t,
uint
return new_exit_point;
}
+static void zend_jit_resolve_ref_snapshot(zend_jit_ref_snapshot *dest, ir_ctx *ctx, ir_ref
snapshot_ref, ir_insn *snapshot, int op)
+{
+ int8_t *reg_ops = ctx->regs[snapshot_ref];
+ ZEND_ASSERT(reg_ops[op] != ZREG_NONE);
+
+ int8_t reg = reg_ops[op];
+ int32_t offset;
+
+ if (IR_REG_SPILLED(reg)) {
+ reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD;
+ offset = ir_get_spill_slot_offset(ctx, ir_insn_op(snapshot, op));
+ } else {
+ offset = 0;
+ }
+
+ dest->reg = reg;
+ dest->offset = offset;
+}
+
+static bool zend_jit_ref_snapshot_equals(const zend_jit_ref_snapshot *a, const
zend_jit_ref_snapshot *b)
+{
+ return a->reg == b->reg
+ && (!IR_REG_SPILLED(a->reg) || (a->offset == b->offset));
+}
+
void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)
{
zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
@@ -722,18 +749,19 @@ void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn
*snaps
exit_flags = t->exit_info[exit_point].flags;
if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {
- int8_t *reg_ops = ctx->regs[snapshot_ref];
+ zend_jit_ref_snapshot func, this;
+ zend_jit_resolve_ref_snapshot(&func, ctx, snapshot_ref, snapshot, n - 1);
+ zend_jit_resolve_ref_snapshot(&this, ctx, snapshot_ref, snapshot, n);
- ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1);
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
- && (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1]
- || t->exit_info[exit_point].poly_this_reg != reg_ops[n])) {
+ && (!zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_func, &func)
+ || !zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_this, &this))) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
- t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1];
- t->exit_info[exit_point].poly_this_reg = reg_ops[n];
+ t->exit_info[exit_point].poly_func = func;
+ t->exit_info[exit_point].poly_this = this;
n -= 2;
}
@@ -2709,6 +2737,8 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
jit->tls = IR_UNUSED;
#endif
jit->fp = IR_UNUSED;
+ jit->poly_func_ref = IR_UNUSED;
+ jit->poly_this_ref = IR_UNUSED;
jit->trace_loop_ref = IR_UNUSED;
jit->return_inputs = IR_UNUSED;
jit->bb_start_ref = NULL;
@@ -4386,6 +4416,18 @@ static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t
reg)
return ir_RLOAD(type, reg);
}
+/* Same as zend_jit_deopt_rload(), but 'reg' may be spilled on C stack */
+static ir_ref zend_jit_deopt_rload_spilled(zend_jit_ctx *jit, ir_type type, int8_t reg, int32_t
offset)
+{
+ ZEND_ASSERT(reg >= 0);
+
+ if (IR_REG_SPILLED(reg)) {
+ return ir_LOAD(type, ir_ADD_OFFSET(zend_jit_deopt_rload(jit, type, IR_REG_NUM(reg)), offset));
+ } else {
+ return zend_jit_deopt_rload(jit, type, reg);
+ }
+}
+
static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
@@ -8440,10 +8482,9 @@ static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline,
uint32
return 1;
}
-static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg)
+static int zend_jit_free_trampoline(zend_jit_ctx *jit, ir_ref func)
{
// JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
- ir_ref func = ir_RLOAD_A(func_reg);
ir_ref if_trampoline = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))),
ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE)));
@@ -8537,8 +8578,8 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline,
co
}
if (may_be_trampoline) {
- jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
- jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
+ jit->trace->exit_info[exit_point].poly_func.ref = func_ref;
+ jit->trace->exit_info[exit_point].poly_this.ref = this_ref;
}
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
@@ -8936,15 +8977,15 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
zend_class_entry *trace_ce,
zend_jit_trace_rec *trace,
int checked_stack,
- int8_t func_reg,
- int8_t this_reg,
+ ir_ref func_ref,
+ ir_ref this_ref,
bool polymorphic_side_trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
zval *function_name;
- ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL;
+ ir_ref if_static = IR_UNUSED, cold_path;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
@@ -8962,10 +9003,8 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
}
if (polymorphic_side_trace) {
- /* function is passed in r0 from parent_trace */
- ZEND_ASSERT(func_reg >= 0 && this_reg >= 0);
- func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
- this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
+ /* function is passed from parent snapshot */
+ ZEND_ASSERT(func_ref != IR_UNUSED && this_ref != IR_UNUSED);
} else {
ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2;
@@ -9111,8 +9150,8 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
return 0;
}
- jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
- jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
+ jit->trace->exit_info[exit_point].poly_func.ref = func_ref;
+ jit->trace->exit_info[exit_point].poly_this.ref = this_ref;
func = (zend_function*)trace->func;
@@ -17319,9 +17358,13 @@ static int zend_jit_trace_start(zend_jit_ctx *jit,
}
if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
- ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 &&
parent->exit_info[exit_num].poly_this_reg >= 0);
- ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg);
- ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg);
+ ZEND_ASSERT(parent->exit_info[exit_num].poly_func.reg >= 0 &&
parent->exit_info[exit_num].poly_this.reg >= 0);
+ if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_func.reg)) {
+ ir_RLOAD_A(parent->exit_info[exit_num].poly_func.reg);
+ }
+ if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_this.reg)) {
+ ir_RLOAD_A(parent->exit_info[exit_num].poly_this.reg);
+ }
}
ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num));
diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index f3011dff99dd..aeec674d60f3 100644
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -199,10 +199,8 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline,
uint32_t
t->exit_info[exit_point].flags = flags;
t->exit_info[exit_point].stack_size = stack_size;
t->exit_info[exit_point].stack_offset = stack_offset;
- t->exit_info[exit_point].poly_func_ref = 0;
- t->exit_info[exit_point].poly_this_ref = 0;
- t->exit_info[exit_point].poly_func_reg = ZREG_NONE;
- t->exit_info[exit_point].poly_this_reg = ZREG_NONE;
+ t->exit_info[exit_point].poly_func = (zend_jit_ref_snapshot){.reg = ZREG_NONE};
+ t->exit_info[exit_point].poly_this = (zend_jit_ref_snapshot){.reg = ZREG_NONE};
}
return exit_point;
@@ -3514,17 +3512,18 @@ static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,
uint32_t
}
static int zend_jit_trace_deoptimization(
- zend_jit_ctx *jit,
- uint32_t flags,
- const zend_op *opline,
- zend_jit_trace_stack *parent_stack,
- int parent_vars_count,
- zend_ssa *ssa,
- zend_jit_trace_stack *stack,
- zend_jit_exit_const *constants,
- int8_t func_reg,
- bool polymorphic_side_trace)
+ zend_jit_ctx *jit,
+ const zend_jit_trace_exit_info *exit_info,
+ zend_jit_trace_stack *parent_stack,
+ int parent_vars_count,
+ zend_ssa *ssa,
+ zend_jit_trace_stack *stack,
+ zend_jit_exit_const *constants,
+ bool polymorphic_side_trace)
{
+ uint32_t flags = exit_info->flags;
+ const zend_op *opline = exit_info->opline;
+
int i;
int check2 = -1;
@@ -3668,9 +3667,16 @@ static int zend_jit_trace_deoptimization(
zend_jit_check_exception(jit);
}
- if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
- if (!zend_jit_free_trampoline(jit, func_reg)) {
- return 0;
+ if (flags & ZEND_JIT_EXIT_METHOD_CALL) {
+ jit->poly_func_ref = zend_jit_deopt_rload_spilled(jit, IR_ADDR,
+ exit_info->poly_func.reg, exit_info->poly_func.offset);
+ jit->poly_this_ref = zend_jit_deopt_rload_spilled(jit, IR_ADDR,
+ exit_info->poly_this.reg, exit_info->poly_this.offset);
+
+ if (!polymorphic_side_trace) {
+ if (!zend_jit_free_trampoline(jit, jit->poly_func_ref)) {
+ return 0;
+ }
}
}
@@ -4265,11 +4271,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t
par
if (parent_trace) {
/* Deoptimization */
if (!zend_jit_trace_deoptimization(&ctx,
- zend_jit_traces[parent_trace].exit_info[exit_num].flags,
- zend_jit_traces[parent_trace].exit_info[exit_num].opline,
+ &zend_jit_traces[parent_trace].exit_info[exit_num],
parent_stack, parent_vars_count, ssa, stack,
zend_jit_traces[parent_trace].constants,
- zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg,
polymorphic_side_trace)) {
goto jit_failure;
}
@@ -6381,8 +6385,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t
par
op_array, ssa, ssa_op, frame->call_level,
op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
p + 1, peek_checked_stack - checked_stack,
- polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg :
-1,
- polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg :
-1,
+ polymorphic_side_trace ? jit->poly_func_ref : -1,
+ polymorphic_side_trace ? jit->poly_this_ref : -1,
polymorphic_side_trace)) {
goto jit_failure;
}
@@ -7419,11 +7423,9 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t
exit_n
NULL;
if (!zend_jit_trace_deoptimization(&ctx,
- zend_jit_traces[trace_num].exit_info[exit_num].flags,
- zend_jit_traces[trace_num].exit_info[exit_num].opline,
+ &zend_jit_traces[trace_num].exit_info[exit_num],
stack, stack_size, NULL, NULL,
zend_jit_traces[trace_num].constants,
- zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg,
0)) {
goto jit_failure;
}
@@ -7973,6 +7975,17 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa
*tssa
}
}
+static void zend_jit_dump_ref_snapshot(zend_jit_ref_snapshot *rs)
+{
+ if (rs->reg == ZREG_NONE) {
+ fprintf(stderr, "?");
+ } else if (!IR_REG_SPILLED(rs->reg)) {
+ fprintf(stderr, "%s", zend_reg_name(rs->reg));
+ } else {
+ fprintf(stderr, "0x%x(%s)", rs->offset, zend_reg_name(IR_REG_NUM(rs->reg)));
+ }
+}
+
static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
{
int i, j;
@@ -8003,9 +8016,11 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
if (t->exit_info[i].flags &
(ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
fprintf(stderr, "/POLY");
if (t->exit_info[i].flags & ZEND_JIT_EXIT_METHOD_CALL) {
- fprintf(stderr, "(%s, %s)",
- t->exit_info[i].poly_func_reg != ZREG_NONE ?
zend_reg_name(t->exit_info[i].poly_func_reg) : "?",
- t->exit_info[i].poly_this_reg != ZREG_NONE ?
zend_reg_name(t->exit_info[i].poly_this_reg) : "?");
+ fprintf(stderr, "(");
+ zend_jit_dump_ref_snapshot(&t->exit_info[i].poly_func);
+ fprintf(stderr, ", ");
+ zend_jit_dump_ref_snapshot(&t->exit_info[i].poly_this);
+ fprintf(stderr, ")");
}
}
if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
@@ -8737,9 +8752,15 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num,
zend_jit_registers_buf
}
}
if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
- ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0);
- zend_function *func = (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg];
+ zend_jit_ref_snapshot *func_snapshot = &t->exit_info[exit_num].poly_func;
+ ZEND_ASSERT(func_snapshot->reg >= 0);
+ zend_function *func;
+ if (IR_REG_SPILLED(func_snapshot->reg)) {
+ func = *(zend_function**)(regs->gpr[IR_REG_NUM(func_snapshot->reg)] +
func_snapshot->offset);
+ } else {
+ func = (zend_function*)regs->gpr[func_snapshot->reg];
+ }
if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
zend_string_release_ex(func->common.function_name, 0);
zend_free_trampoline(func);