diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index 74a9789f35ad..cb047d8a5136 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=30a40633b0b8f3e8e66da0a25941c4199d69d38a +revision=43d1d1023a6ea811b11b39ed738f5e18696a1254 git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" @@ -32,6 +32,7 @@ rm ir_load.c rm ir_load_llvm.c rm ir_main.c rm ir_mem2ssa.c +rm ir_ra_fast.c # add extra files git restore README diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 734e9086648f..6cfd2a0007f6 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -119,7 +119,20 @@ static void ir_print_double(double v, FILE *f) char buf[128]; if (isnan(v)) { - fprintf(f, "nan"); + union { + double d; + long long i; + } u1, u2, u3; + u1.d = v; + u2.d = NAN; + u3.d = -NAN; + if (u1.i == u2.i) { + fprintf(f, "nan"); + } else if (u1.i == u3.i) { + fprintf(f, "-nan"); + } else { + fprintf(f, "nan(0x%llx)", u1.i); + } } else { snprintf(buf, sizeof(buf), "%g", v); if (strtod(buf, NULL) != v) { @@ -137,7 +150,20 @@ static void ir_print_float(float v, FILE *f) char buf[128]; if (isnan(v)) { - fprintf(f, "nan"); + union { + float f; + int i; + } u1, u2, u3; + u1.f = v; + u2.f = NAN; + u3.f = -NAN; + if (u1.i == u2.i) { + fprintf(f, "nan"); + } else if (u1.i == u3.i) { + fprintf(f, "-nan"); + } else { + fprintf(f, "nan(0x%x)", u1.i); + } } else { snprintf(buf, sizeof(buf), "%g", v); if (strtod(buf, NULL) != v) { diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 1d7bd07b3815..74481865e0f8 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -968,8 +968,9 @@ int ir_coalesce(ir_ctx *ctx); int ir_compute_dessa_moves(ir_ctx *ctx); int ir_reg_alloc(ir_ctx *ctx); int ir_reg_alloc_simple(ir_ctx *ctx); +int ir_reg_alloc_fast(ir_ctx *ctx); -#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_simple(_ctx) +#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_fast(_ctx) int ir_regs_number(void); bool ir_reg_is_int(int32_t reg); diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 2bc28e59e33f..965b3e9f7455 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -1413,8 +1413,9 @@ binop_fp: if (IR_IS_TYPE_VECTOR(insn->type)) { return IR_VECTOR_OP; } - IR_FALLTHROUGH; #endif + IR_ASSERT(IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)); + IR_FALLTHROUGH; case IR_BSWAP: case IR_CTLZ: case IR_CTTZ: @@ -3487,29 +3488,41 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) } if (ir_type_size[type] == 8) { - | cmp Rx(op1_reg), #0 - if (op2_reg != IR_REG_NONE) { - | add Rx(def_reg), Rx(op1_reg), Rx(op2_reg) + if (shift == 63) { + | movz Rx(def_reg), #0x8000, lsl #48 + | cmp Rx(def_reg), Rx(op1_reg) + | cset Rx(def_reg), eq } else { - | add Rx(def_reg), Rx(op1_reg), #offset + | cmp Rx(op1_reg), #0 + if (op2_reg != IR_REG_NONE) { + | add Rx(def_reg), Rx(op1_reg), Rx(op2_reg) + } else { + | add Rx(def_reg), Rx(op1_reg), #offset + } + | csel Rx(def_reg), Rx(def_reg), Rx(op1_reg), lt + | asr Rx(def_reg), Rx(def_reg), #shift } - | csel Rx(def_reg), Rx(def_reg), Rx(op1_reg), lt - | asr Rx(def_reg), Rx(def_reg), #shift } else { - | cmp Rw(op1_reg), #0 - if (op2_reg != IR_REG_NONE) { - | add Rw(def_reg), Rw(op1_reg), Rw(op2_reg) - } else { - | add Rw(def_reg), Rw(op1_reg), #offset - } - | csel Rw(def_reg), Rw(def_reg), Rw(op1_reg), lt - if (ir_type_size[type] == 4) { - | asr Rw(def_reg), Rw(def_reg), #shift - } else if (ir_type_size[type] == 2) { - | ubfx Rw(def_reg), Rw(def_reg), #shift, #16 + if (shift == 31) { + | movz Rw(def_reg), #0x8000, lsl #16 + | cmp Rw(def_reg), Rw(op1_reg) + | cset Rw(def_reg), eq } else { - IR_ASSERT(ir_type_size[type] == 1); - | ubfx Rw(def_reg), Rw(def_reg), #shift, #8 + | cmp Rw(op1_reg), #0 + if (op2_reg != IR_REG_NONE) { + | add Rw(def_reg), Rw(op1_reg), Rw(op2_reg) + } else { + | add Rw(def_reg), Rw(op1_reg), #offset + } + | csel Rw(def_reg), Rw(def_reg), Rw(op1_reg), lt + if (ir_type_size[type] == 4) { + | asr Rw(def_reg), Rw(def_reg), #shift + } else if (ir_type_size[type] == 2) { + | ubfx Rw(def_reg), Rw(def_reg), #shift, #16 + } else { + IR_ASSERT(ir_type_size[type] == 1); + | ubfx Rw(def_reg), Rw(def_reg), #shift, #8 + } } } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -3965,10 +3978,10 @@ static void ir_emit_binop_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_FP_REG_REG_REG_OP fdiv, type, def_reg, op1_reg, op2_reg break; case IR_MIN: - | ASM_FP_REG_REG_REG_OP fmin, type, def_reg, op1_reg, op2_reg + | ASM_FP_REG_REG_REG_OP fminnm, type, def_reg, op1_reg, op2_reg break; case IR_MAX: - | ASM_FP_REG_REG_REG_OP fmax, type, def_reg, op1_reg, op2_reg + | ASM_FP_REG_REG_REG_OP fmaxnm, type, def_reg, op1_reg, op2_reg break; } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -9272,10 +9285,10 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { IR_ASSERT(ival->stack_spill_pos == -1); ival->stack_spill_pos = stack_start + stack_offset; - if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { - /* Remap VAR to PARAM stack slot */ - ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); - } + } + if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { + /* Remap VAR to PARAM stack slot */ + ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); } } if (sizeof(void*) == 8) { diff --git a/ext/opcache/jit/ir/ir_cfg.c b/ext/opcache/jit/ir/ir_cfg.c index 4e4db166d37e..621f02a2372d 100644 --- a/ext/opcache/jit/ir/ir_cfg.c +++ b/ext/opcache/jit/ir/ir_cfg.c @@ -1106,6 +1106,34 @@ static IR_NEVER_INLINE uint32_t ir_collect_irreducible_loops(ir_ctx *ctx, uint32 return loops; } +static IR_NEVER_INLINE void ir_check_loop_entry_in_preds(ir_ctx *ctx, uint32_t hdr, ir_worklist *work) +{ + ir_block *blocks = ctx->cfg_blocks; + uint32_t *edges = ctx->cfg_edges; + + ir_worklist_clear(work); + ir_bitset_incl(work->visited, blocks[hdr].idom); + ir_worklist_push(work, hdr); + while (ir_worklist_len(work)) { + uint32_t b = ir_worklist_pop(work); + ir_block *bb = &blocks[b]; + + if (bb->flags & IR_BB_ENTRY) { + blocks[hdr].flags |= IR_BB_LOOP_WITH_ENTRY_IN_PREDS; + break; + } + + uint32_t *p = &edges[bb->predecessors]; + uint32_t count = bb->predecessors_count; + + for (; count > 0; p++, count--) { + if (!ir_dominates(blocks, hdr, *p)) { + ir_worklist_push(work, *p); + } + } + } +} + int ir_find_loops(ir_ctx *ctx) { uint32_t b, j, n; @@ -1295,6 +1323,10 @@ int ir_find_loops(ir_ctx *ctx) b = loops; do { bb = &blocks[b]; + if (ctx->entries_count + && (bb->predecessors_count > 2 || (bb->flags & IR_BB_IRREDUCIBLE_LOOP))) { + ir_check_loop_entry_in_preds(ctx, b, &work); + } b = bb->next_loop; IR_ASSERT(bb->flags & IR_BB_LOOP_HEADER); bb->loop_depth = (bb->loop_header) ? blocks[bb->loop_header].loop_depth + 1 : 1; diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index bc3ae5859be3..de6b53141a67 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -541,7 +541,9 @@ static IR_NEVER_INLINE void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_reg, ir_reg tmp_fp_reg) { int i; - int8_t *pred, *loc, *types; + int8_t *loc; /* array of the current locations of the original source registers: initially loc[from] = from */ + int8_t *pred; /* array of original source registers that must be copied to destinations: pred[to] = from */ + int8_t *types; /* array of the types of the orginal source registers: types[from] = typeof(from) */ ir_reg to, from; ir_type type; ir_regset todo, ready, srcs; @@ -625,18 +627,19 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ IR_ASSERT(tmp_reg == IR_REG_NONE || !IR_REGSET_IN(srcs, tmp_reg) || pred[loc[tmp_reg]] == tmp_reg); IR_ASSERT(tmp_fp_reg == IR_REG_NONE || !IR_REGSET_IN(srcs, tmp_fp_reg) || pred[loc[tmp_fp_reg]] == tmp_fp_reg); - /* now we resolve all "windmill axles" - cycles (this reuires temporary registers) */ + /* now we resolve all "windmill axles" - cycles (this requires temporary registers) */ while (todo != IR_REGSET_EMPTY) { - to = ir_regset_pop_first(&todo); + to = ir_regset_pop_first(&todo); /* select a register to break the cycle */ from = pred[to]; IR_ASSERT(to != loc[from]); - type = types[from]; + type = types[to]; /* type of the register that needs to be moved into a temporary register */ if (IR_IS_TYPE_INT(type)) { #ifdef IR_HAVE_SWAP_INT if (pred[from] == to) { - if (ir_type_size[types[to]] > ir_type_size[type]) { - type = types[to]; + if (ir_type_size[types[from]] > ir_type_size[type]) { + type = types[from]; } + /* resolve the cycle with 2 registers using SWAP */ ir_emit_swap(ctx, type, to, from); IR_REGSET_EXCL(todo, from); loc[to] = from; @@ -646,11 +649,13 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ #endif IR_ASSERT(tmp_reg != IR_REG_NONE); IR_ASSERT(tmp_reg >= IR_REG_GP_FIRST && tmp_reg <= IR_REG_GP_LAST); + /* break cycle, moving one of its register into a temporary register */ ir_emit_mov(ctx, type, tmp_reg, to); loc[to] = tmp_reg; } else { #ifdef IR_HAVE_SWAP_FP - if (pred[from] == to && types[to] == type) { + if (pred[from] == to && types[from] == type) { + /* resolve the cycle with 2 registers using SWAP */ ir_emit_swap_fp(ctx, type, to, from); IR_REGSET_EXCL(todo, from); loc[to] = from; @@ -660,6 +665,7 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ #endif IR_ASSERT(tmp_fp_reg != IR_REG_NONE); IR_ASSERT(tmp_fp_reg >= IR_REG_FP_FIRST && tmp_fp_reg <= IR_REG_FP_LAST); + /* break cycle, moving one of its register into a temporary register */ ir_emit_fp_mov(ctx, type, tmp_fp_reg, to); loc[to] = tmp_fp_reg; } diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 16dc6f536b8c..1debe2fa25a0 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -30,7 +30,7 @@ IR_FOLD(EQ(C_DOUBLE, C_DOUBLE)) IR_FOLD(EQ(C_FLOAT, C_FLOAT)) { - IR_FOLD_BOOL(op1_insn->val.d == op2_insn->val.d); + IR_FOLD_BOOL(op1_insn->val.f == op2_insn->val.f); } IR_FOLD(NE(C_BOOL, C_BOOL)) @@ -1001,7 +1001,8 @@ IR_FOLD(MUL_OV(C_I64, C_I64)) int64_t res; IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); res = op1_insn->val.u64 * op2_insn->val.u64; - if ((op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) { + if ((op1_insn->val.i64 == -1 && res == INT64_MIN) + || (op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) { IR_FOLD_NEXT; } IR_FOLD_CONST_U(res); diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c index 712411774a47..54c894f27cba 100644 --- a/ext/opcache/jit/ir/ir_gcm.c +++ b/ext/opcache/jit/ir/ir_gcm.c @@ -100,7 +100,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca) #endif flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags; - if ((flags & IR_BB_LOOP_WITH_ENTRY) + if ((flags & (IR_BB_LOOP_WITH_ENTRY|IR_BB_LOOP_WITH_ENTRY_IN_PREDS)) && !(ctx->binding && ir_binding_find(ctx, ref))) { /* Don't move loop invariant code across an OSR ENTRY if we can't restore it */ return lca; @@ -135,7 +135,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca) break; } flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags; - if ((flags & IR_BB_LOOP_WITH_ENTRY) + if ((flags & (IR_BB_LOOP_WITH_ENTRY|IR_BB_LOOP_WITH_ENTRY_IN_PREDS)) && !(ctx->binding && ir_binding_find(ctx, ref))) { break; } diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index 6fa8efa9cf71..9f91fbfba9f8 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -1204,6 +1204,7 @@ void ir_iter_cleanup(ir_ctx *ctx); #define IR_BB_PREV_EMPTY_ENTRY (1<<8) #define IR_BB_OSR_ENTRY_LOADS (1<<9) /* OSR Entry-point with register LOADs */ #define IR_BB_LOOP_WITH_ENTRY (1<<10) /* set together with LOOP_HEADER if there is an ENTRY in the loop */ +#define IR_BB_LOOP_WITH_ENTRY_IN_PREDS (1<<18) /* set together with LOOP_HEADER if an ENTRY reaches the loop bypassing its immediate dominator */ /* The following flags are set by GCM */ #define IR_BB_HAS_PHI (1<<11) diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index 53c3cbd8a047..f8d418d06140 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -812,6 +812,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) IR_ASSERT(use_insn->op3 == ref); /* skip VSTORE (VAR is going to be remapped to PARAM) */ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + if (ctx->use_lists[ref].count != 1) { + param_ival->flags &= ~IR_LIVE_INTERVAL_MEM_PARAM; + } param_ival->use_pos = param_ival->use_pos->next; ctx->rules[use] = IR_SKIPPED | IR_NOP; } @@ -1530,6 +1533,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) IR_ASSERT(use_insn->op3 == ref); /* skip VSTORE (VAR is going to be remapped to PARAM) */ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + if (ctx->use_lists[ref].count != 1) { + param_ival->flags &= ~IR_LIVE_INTERVAL_MEM_PARAM; + } param_ival->use_pos = param_ival->use_pos->next; ctx->rules[use] = IR_SKIPPED | IR_NOP; } diff --git a/ext/opcache/jit/ir/ir_ra_simple.c b/ext/opcache/jit/ir/ir_ra_simple.c index 4ecdd79986b0..7a78801f4d71 100644 --- a/ext/opcache/jit/ir/ir_ra_simple.c +++ b/ext/opcache/jit/ir/ir_ra_simple.c @@ -1,6 +1,6 @@ /* * IR - Lightweight JIT Compilation Framework - * Simple Local Register Allocation (used for -O0) + * Simple Local Register Allocation (not used anymore) * This file is part of the IR Project distributed under the MIT-style LICENSE. * Authors: Dmitry Stogov * @@ -131,6 +131,15 @@ static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data return reg; } +/* The following checks for the safety of VAR slot reuse are incomplete. + * Some of the unsupported edge cases are demonstrated at https://github.com/dstogov/ir/pull/232 + * and https://github.com/dstogov/ir/pull/230 (there are may be other problems). + * Unfortunatly, the attempts to fix the problems break the simplicity of this allocator and make it ridiculous. + * The complete removing of VAR slot reuse makes the already bad generated code significanly worse. + * + * A better appoach is implemented in ir_ra_fast.c + */ + static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load) { ir_use_list *use_list = &ctx->use_lists[load]; diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 3ad6997b071e..2ca202add5e8 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -2673,16 +2673,16 @@ static bool ir_is_zero(const ir_ctx *ctx, ir_ref ref) && ctx->ir_base[ref].val.u32 == 0; } -static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) +static bool ir_is_min_max_const(const ir_ctx *ctx, const ir_insn *cond, ir_ref ref) { if (cond->op == IR_ULE) { /* (x <= 3 ? 4 : x) => (x < 4 ? 4 : x) => max(x, 4) */ /* (x <= 3 ? x : 4) => (x < 4 ? x : 4) => min(x, 4) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.u64 != 0 && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 - 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_UGE) { @@ -2690,9 +2690,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x >= 3 ? x : 2) => (x > 2 ? x : 2) => max(x, 2) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.u64 != UINT64_MAX && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 + 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_LE) { @@ -2700,9 +2700,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x <= 3 ? x : 4) => (x < 4 ? x : 4) => min(x, 4) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) - && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 - 1 + && ctx->ir_base[ref].val.i64 != INT64_MIN + && ctx->ir_base[cond->op2].val.i64 == ctx->ir_base[ref].val.i64 - 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_GE) { @@ -2710,9 +2710,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x >= 3 ? x : 2) => (x > 2 ? x : 2) => max(x, 2) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.i64 != INT64_MAX && ctx->ir_base[cond->op2].val.i64 == ctx->ir_base[ref].val.i64 + 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } @@ -2758,18 +2758,19 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re } if (is_cmp + && !IR_IS_TYPE_FP(type) && ((insn->op2 == cond->op1 && (insn->op3 == cond->op2 || (IR_IS_CONST_REF(cond->op2) && (IR_IS_CONST_REF(insn->op3) && IR_IS_TYPE_INT(insn->type) - && ir_fix_min_max_const(ctx, cond, insn->op3))))) + && ir_is_min_max_const(ctx, cond, insn->op3))))) || (insn->op3 == cond->op1 && (insn->op2 == cond->op2 || (IR_IS_CONST_REF(cond->op2) && (IR_IS_CONST_REF(insn->op2) && IR_IS_TYPE_INT(insn->type) - && ir_fix_min_max_const(ctx, cond, insn->op2))))))) { + && ir_is_min_max_const(ctx, cond, insn->op2))))))) { /* MAX/MIN * * prev prev @@ -2804,11 +2805,16 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re ir_remove_unused_vars(ctx, start2_ref, end2_ref); } - insn->op = ( - (is_less ? cond->op1 : cond->op2) - == - ((start1->op == IR_IF_TRUE) ? insn->op2 : insn->op3) - ) ? IR_MIN : IR_MAX; + if (start1->op != IR_IF_TRUE) { + IR_ASSERT(start1->op == IR_IF_FALSE); + is_less = !is_less; + } + if (cond->op1 != insn->op2) { + IR_ASSERT(cond->op1 == insn->op3); + is_less = !is_less; + } + + insn->op = is_less ? IR_MIN : IR_MAX; insn->inputs_count = 2; if (insn->op2 > insn->op3) { insn->op1 = insn->op2; @@ -2939,7 +2945,12 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re } return 1; - } else if (insn->op2 <= root_ref && insn->op3 <= root_ref + } else if (((insn->op2 <= root_ref && insn->op3 <= root_ref) + || ((cond->op >= IR_EQ && cond->op <= IR_UNORDERED) + && ((insn->op2 == cond->op1 && (insn->op3 == cond->op2 || insn->op3 <= root_ref)) + || (insn->op2 == cond->op2 && (insn->op3 == cond->op1 || insn->op3 <= root_ref)) + || (insn->op3 == cond->op1 && insn->op2 <= root_ref) + || (insn->op3 == cond->op2 && insn->op2 <= root_ref)))) && cond->op != IR_OVERFLOW // TODO: temporary disable IF-conversion for RLOAD. // We don't track anti-dependencies in GCM and Local Scheduling. diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index f33c89f574f5..463be9e750d3 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -1622,6 +1622,7 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = { _(BINOP_INT) \ _(BINOP_SSE2) \ _(BINOP_AVX) \ + _(MIN_MAX_FP) \ _(SHIFT) \ _(SHIFT_CONST) \ _(COPY_INT) \ @@ -1636,6 +1637,7 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = { _(COND_TEST_BIT) \ _(COND_CMP_INT) \ _(COND_CMP_FP) \ + _(COND_MIN_MAX_FP) \ _(GUARD_CMP_INT) \ _(GUARD_CMP_FP) \ _(GUARD_TEST_INT) \ @@ -1998,6 +2000,28 @@ op2_const: case IR_COND_CMP_FP: flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG; break; + case IR_COND_MIN_MAX_FP: + flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG; + if (!(ctx->mflags & IR_X86_AVX)) { + insn = &ctx->ir_base[ref]; + if (!IR_IS_CONST_REF(insn->op3) || ctx->vregs[insn->op3] > 0) { + /* def_reg can't be the same as op3_reg */ + flags |= IR_DEF_CONFLICTS_WITH_INPUT_REGS; + } + } + break; + case IR_MIN_MAX_FP: + flags = IR_USE_MUST_BE_IN_REG | IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_DEF_CONFLICTS_WITH_INPUT_REGS; + insn = &ctx->ir_base[ref]; + constraints->tmp_regs[0] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n = 1; + if (IR_IS_CONST_REF(insn->op1)) { + n = ir_add_const_tmp_reg(ctx, insn->op1, 1, n, constraints); + } + if (IR_IS_CONST_REF(insn->op2)) { + n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); + } + break; case IR_VSTORE_INT: flags = IR_OP3_MUST_BE_IN_REG; insn = &ctx->ir_base[ref]; @@ -4496,7 +4520,7 @@ binop_fp: goto binop_vector; #endif } else { - goto binop_fp; + return IR_MIN_MAX_FP; } break; case IR_COPY: @@ -5047,6 +5071,11 @@ store_int: return IR_TWO_REGS | IR_COND_I64_CMP_FP; } #endif + if ((op1_insn->op >= IR_LT && op1_insn->op <= IR_GT) && + ((op1_insn->op1 == insn->op2 && op1_insn->op2 == insn->op3) || + (op1_insn->op1 == insn->op3 && op1_insn->op2 == insn->op2))) { + return IR_COND_MIN_MAX_FP; + } return IR_COND_CMP_FP; } #if IR_X86_I64 @@ -7543,6 +7572,30 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) | add Rd(def_reg), Rd(op1_reg) |.if X64 || } +|.endif + | ASM_REG_IMM_OP sar, type, def_reg, shift + } else if (shift == (uint32_t)ir_type_size[type] * 8 - 1) { +|.if X64 +|| if (ir_type_size[type] == 8) { + | mov64 Rq(def_reg), 0x8000000000000000 + | cmp Rq(def_reg), Rq(op1_reg) + | sete Rb(def_reg) + | movzx Rd(def_reg), Rb(def_reg) +|| } else { +|.endif + | xor Rd(def_reg), Rd(def_reg) + | cmp Rd(op1_reg), 0x80000000 +|.if X64 + | sete Rb(def_reg) +|| } +|.else + if (def_reg < IR_REG_R3) { + | sete Rb(def_reg) + } else { + | jne >1 + | mov Rd(def_reg), 1 + |1: + } |.endif } else { |.if X64 @@ -7564,8 +7617,8 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) |.endif | ASM_REG_REG_OP test, type, op1_reg, op1_reg | ASM_REG_REG_OP2 cmovns, type, def_reg, op1_reg + | ASM_REG_IMM_OP sar, type, def_reg, shift } - | ASM_REG_IMM_OP sar, type, def_reg, shift if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); @@ -8614,12 +8667,6 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_REG_OP divs, type, def_reg, op2_reg break; - case IR_MIN: - | ASM_SSE2_REG_REG_OP mins, type, def_reg, op2_reg - break; - case IR_MAX: - | ASM_SSE2_REG_REG_OP maxs, type, def_reg, op2_reg - break; } } else if (IR_IS_CONST_REF(op2)) { int label = ir_get_const_label(ctx, op2); @@ -8639,12 +8686,6 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_TXT_OP divs, type, def_reg, [=>label] break; - case IR_MIN: - | ASM_SSE2_REG_TXT_OP mins, type, def_reg, [=>label] - break; - case IR_MAX: - | ASM_SSE2_REG_TXT_OP maxs, type, def_reg, [=>label] - break; } } else { ir_mem mem; @@ -8669,14 +8710,88 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_MEM_OP divs, type, def_reg, mem break; - case IR_MIN: - | ASM_SSE2_REG_MEM_OP mins, type, def_reg, mem - break; - case IR_MAX: - | ASM_SSE2_REG_MEM_OP maxs, type, def_reg, mem - break; } } + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, insn->type, def, def_reg); + } +} + +static void ir_emit_min_max_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = insn->type; + ir_ref op1 = insn->op1; + ir_ref op2 = insn->op2; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op1_reg = ctx->regs[def][1]; + ir_reg op2_reg = ctx->regs[def][2]; + ir_reg tmp_reg = ctx->regs[def][3]; + + IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); + + if (IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + ir_emit_load(ctx, type, op1_reg, op1); + } + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + if (op1 != op2) { + ir_emit_load(ctx, type, op2_reg, op2); + } + } + + if (!(ctx->mflags & IR_X86_AVX)) { + ir_emit_fp_mov(ctx, type, def_reg, op1_reg); + } + + if (type == IR_DOUBLE) { + if (ctx->mflags & IR_X86_AVX) { + if (insn->op == IR_MAX) { + | vmaxsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | vminsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | vcmpsd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), 7 + | vblendvpd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } else { + if (insn->op == IR_MAX) { + | maxsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | minsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | movapd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + | cmpsd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST), 7 + | andpd xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + | andnpd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST) + | orpd xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } + } else { + IR_ASSERT(type == IR_FLOAT); + if (ctx->mflags & IR_X86_AVX) { + if (insn->op == IR_MAX) { + | vmaxss xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | vminss xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | vcmpss xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), 7 + | vblendvps xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } else { + if (insn->op == IR_MAX) { + | maxss xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | minss xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | movaps xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + | cmpss xmm(tmp_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST), 7 + | andps xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + | andnps xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST) + | orps xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } + } + if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } @@ -9469,6 +9584,20 @@ static void ir_emit_jcc(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint } } +static bool ir_insns_may_change_cmp_flags(ir_ctx *ctx, ir_ref from, ir_ref to) +{ + ir_ref ref; + + for (ref = ctx->prev_ref[to]; ref != from; ref = ctx->prev_ref[ref]) { + if (!(ctx->rules[ref] & (IR_FUSED|IR_SKIPPED)) + && ctx->ir_base[ref].op != IR_VAR + && ctx->ir_base[ref].op != IR_PHI) { + return 1; + } + } + return 0; +} + static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next_block) { ir_insn *cmp_insn = &ctx->ir_base[insn->op2]; @@ -9518,7 +9647,9 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i if (ir_rule(ctx, prev_insn->op1) == IR_CMP_AND_BRANCH_INT) { prev_insn = &ctx->ir_base[prev_insn->op1]; prev_insn = &ctx->ir_base[prev_insn->op2]; - if (prev_insn->op1 == cmp_insn->op1 && prev_insn->op2 == cmp_insn->op2) { + if (prev_insn->op1 == cmp_insn->op1 + && prev_insn->op2 == cmp_insn->op2 + && !ir_insns_may_change_cmp_flags(ctx, insn->op1, def)) { same_comparison = true; } } @@ -9666,7 +9797,7 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, op3); if (op1 == op3) { - op1_reg = op2_reg; + op1_reg = op3_reg; } } if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { @@ -10295,6 +10426,137 @@ static void ir_emit_cond_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } +static void ir_emit_cond_min_max_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = insn->type; + ir_insn *cmp_insn = &ctx->ir_base[insn->op1]; + ir_ref op2 = insn->op2; + ir_ref op3 = insn->op3; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op2_reg = ctx->regs[def][2]; + ir_reg op3_reg = ctx->regs[def][3]; + ir_reg cmp_op1_reg, cmp_op2_reg; + ir_op op; + + IR_ASSERT(def_reg != IR_REG_NONE); + + if (UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { + cmp_op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); + cmp_op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); + } else { + cmp_op1_reg = ctx->regs[insn->op1][1]; + cmp_op2_reg = ctx->regs[insn->op1][2]; + } + if (cmp_insn->op1 == op2) { + IR_ASSERT(cmp_insn->op2 == op3); + op = (cmp_insn->op == IR_LT || cmp_insn->op == IR_LE) ? IR_MIN : IR_MAX; + if (op2_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op1_reg) == IR_REG_NUM(op2_reg)); + op2_reg |= cmp_op1_reg; /* merge spill load flag */ + } + if (op3_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op2_reg) == IR_REG_NUM(op3_reg)); + op3_reg |= cmp_op2_reg; /* merge spill load flag */ + } + } else { + IR_ASSERT(cmp_insn->op1 == op3 && cmp_insn->op2 == op2); + op = (cmp_insn->op == IR_LT || cmp_insn->op == IR_LE) ? IR_MAX : IR_MIN; + if (op2_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op2_reg) == IR_REG_NUM(op2_reg)); + op2_reg |= cmp_op2_reg; /* merge spill load flag */ + } + if (op3_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op1_reg) == IR_REG_NUM(op3_reg)); + op3_reg |= cmp_op1_reg; /* merge spill load flag */ + } + } + + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, type, op2_reg, op2); + } + } + if (op3_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op3_reg)) { + op3_reg = IR_REG_NUM(op3_reg); + if (op2 != op3) { + ir_emit_load(ctx, type, op3_reg, op3); + } + } + } + + IR_ASSERT((ctx->mflags & IR_X86_AVX) || def_reg != op3_reg); + + if (op2_reg != IR_REG_NONE) { + if (!(ctx->mflags & IR_X86_AVX) && def_reg != op2_reg) { + ir_emit_fp_mov(ctx, type, def_reg, op2_reg); + } + } else { + ir_emit_load(ctx, type, def_reg, op2); + op2_reg = def_reg; + } + + if (op3_reg != IR_REG_NONE) { + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_REG_OP vmins, type, def_reg, op2_reg, op3_reg + } else { + | ASM_AVX_REG_REG_REG_OP vmaxs, type, def_reg, op2_reg, op3_reg + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_REG_OP mins, type, def_reg, op3_reg + } else { + | ASM_SSE2_REG_REG_OP maxs, type, def_reg, op3_reg + } + } + } else if (IR_IS_CONST_REF(op3)) { + int label = ir_get_const_label(ctx, op3); + + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_TXT_OP vmins, type, def_reg, op2_reg, [=>label] + } else { + | ASM_AVX_REG_REG_TXT_OP vmaxs, type, def_reg, op2_reg, [=>label] + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_TXT_OP mins, type, def_reg, [=>label] + } else { + | ASM_SSE2_REG_TXT_OP maxs, type, def_reg, [=>label] + } + } + } else { + ir_mem mem; + + if (ir_rule(ctx, op3) & IR_FUSED) { + mem = ir_fuse_load(ctx, def, op3); + } else { + mem = ir_ref_spill_slot(ctx, op3); + } + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_MEM_OP vmins, type, def_reg, op2_reg, mem + } else { + | ASM_AVX_REG_REG_MEM_OP vmaxs, type, def_reg, op2_reg, mem + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_MEM_OP mins, type, def_reg, mem + } else { + | ASM_SSE2_REG_MEM_OP maxs, type, def_reg, mem + } + } + } + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, type, def, def_reg); + } +} + static void ir_emit_return_void(ir_ctx *ctx) { ir_backend_data *data = ctx->data; @@ -15009,7 +15271,9 @@ static void ir_emit_cmp_and_branch_i64(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i if (ir_rule(ctx, prev_insn->op1) == IR_CMP_AND_BRANCH_INT) { prev_insn = &ctx->ir_base[prev_insn->op1]; prev_insn = &ctx->ir_base[prev_insn->op2]; - if (prev_insn->op1 == cmp_insn->op1 && prev_insn->op2 == cmp_insn->op2) { + if (prev_insn->op1 == cmp_insn->op1 + && prev_insn->op2 == cmp_insn->op2 + && !ir_insns_may_change_cmp_flags(ctx, insn->op1, def)) { same_comparison = true; } } @@ -17070,7 +17334,7 @@ static void ir_emit_cond_cmp_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(tmp_reg != IR_REG_NONE); - if (!IR_IS_CONST_REF(insn->op2) && UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { + if (!IR_IS_CONST_REF(insn->op1) && UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); } else { @@ -17232,7 +17496,7 @@ static void ir_emit_cond_i64_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(tmp_reg != IR_REG_NONE); - if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { + if (UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); } else { @@ -26183,6 +26447,12 @@ static void ir_emit_load_params(ir_ctx *ctx) ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_start + stack_offset); } if (dst_reg != IR_REG_NONE && IR_REG_SPILLED(ctx->regs[use][0])) { +#if IR_X86_I64 + if (insn->type == IR_I64 || insn->type == IR_U64) { + ir_emit_store_i64_lo(ctx, use, IR_REG_I64_LO(dst_reg)); + ir_emit_store_i64_hi(ctx, use, IR_REG_I64_HI(dst_reg)); + } else +#endif ir_emit_store(ctx, insn->type, use, dst_reg); } } @@ -26283,10 +26553,10 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { IR_ASSERT(ival->stack_spill_pos == -1); ival->stack_spill_pos = stack_start + stack_offset; - if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { - /* Remap VAR to PARAM stack slot */ - ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); - } + } + if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { + /* Remap VAR to PARAM stack slot */ + ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); } } if (sizeof(void*) == 8) { @@ -26636,6 +26906,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_BINOP_AVX: ir_emit_binop_avx(ctx, i, insn); break; + case IR_MIN_MAX_FP: + ir_emit_min_max_fp(ctx, i, insn); + break; case IR_MUL_INT: case IR_DIV_INT: case IR_MOD_INT: @@ -26759,6 +27032,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_COND_CMP_FP: ir_emit_cond_cmp_fp(ctx, i, insn); break; + case IR_COND_MIN_MAX_FP: + ir_emit_cond_min_max_fp(ctx, i, insn); + break; case IR_SWITCH: ir_emit_switch(ctx, b, i, insn); break;