diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index 74a9789f35ad..9a1f8faa1be1 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=30a40633b0b8f3e8e66da0a25941c4199d69d38a +revision=4a1208ccbfc8ca32a5103baf9820ad3cbb6ff06e git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" @@ -32,6 +32,8 @@ rm ir_load.c rm ir_load_llvm.c rm ir_main.c rm ir_mem2ssa.c +rm ir_ra_fast.c +rm ir_ra_simple.c # add extra files git restore README diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 224e9e54a2e4..2a1ac4e332fc 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -48,7 +48,6 @@ AS_VAR_IF([PHP_OPCACHE_JIT], [yes], [ jit/ir/ir_patch.c jit/ir/ir_perf.c jit/ir/ir_ra.c - jit/ir/ir_ra_simple.c jit/ir/ir_save.c jit/ir/ir_sccp.c jit/ir/ir_strtab.c diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index f92b02e59671..cd22b363db0b 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -25,7 +25,7 @@ if (PHP_OPCACHE_JIT == "yes") { } else if (CHECK_HEADER("ir/ir.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) { var dasm_flags = (X64 ? "-D X64=1" : "") + (X64 ? " -D X64WIN=1" : "") + " -D WIN=1"; var ir_target = (X64 ? "IR_TARGET_X64" : "IR_TARGET_X86"); - var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_ra_simple.c ir_save.c \ + var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_save.c \ ir_dump.c ir_check.c ir_patch.c"; DEFINE("IR_TARGET", ir_target); diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32 index b5273a1b86ea..7caf81458870 100644 --- a/ext/opcache/jit/Makefile.frag.w32 +++ b/ext/opcache/jit/Makefile.frag.w32 @@ -19,11 +19,6 @@ $(BUILD_DIR)\ext\opcache\jit\ir\ir_ra.obj: \ ext\opcache\jit\ir\ir_private.h \ ext\opcache\jit\ir\ir_x86.h -$(BUILD_DIR)\ext\opcache\jit\ir\ir_ra_simple.obj: \ - ext\opcache\jit\ir\ir.h \ - ext\opcache\jit\ir\ir_private.h \ - ext\opcache\jit\ir\ir_x86.h - $(BUILD_DIR)\ext\opcache\jit\ir\ir_emit.obj: \ ext\opcache\jit\ir\ir.h \ ext\opcache\jit\ir\ir_private.h \ diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 734e9086648f..6cfd2a0007f6 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -119,7 +119,20 @@ static void ir_print_double(double v, FILE *f) char buf[128]; if (isnan(v)) { - fprintf(f, "nan"); + union { + double d; + long long i; + } u1, u2, u3; + u1.d = v; + u2.d = NAN; + u3.d = -NAN; + if (u1.i == u2.i) { + fprintf(f, "nan"); + } else if (u1.i == u3.i) { + fprintf(f, "-nan"); + } else { + fprintf(f, "nan(0x%llx)", u1.i); + } } else { snprintf(buf, sizeof(buf), "%g", v); if (strtod(buf, NULL) != v) { @@ -137,7 +150,20 @@ static void ir_print_float(float v, FILE *f) char buf[128]; if (isnan(v)) { - fprintf(f, "nan"); + union { + float f; + int i; + } u1, u2, u3; + u1.f = v; + u2.f = NAN; + u3.f = -NAN; + if (u1.i == u2.i) { + fprintf(f, "nan"); + } else if (u1.i == u3.i) { + fprintf(f, "-nan"); + } else { + fprintf(f, "nan(0x%x)", u1.i); + } } else { snprintf(buf, sizeof(buf), "%g", v); if (strtod(buf, NULL) != v) { diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 1d7bd07b3815..74481865e0f8 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -968,8 +968,9 @@ int ir_coalesce(ir_ctx *ctx); int ir_compute_dessa_moves(ir_ctx *ctx); int ir_reg_alloc(ir_ctx *ctx); int ir_reg_alloc_simple(ir_ctx *ctx); +int ir_reg_alloc_fast(ir_ctx *ctx); -#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_simple(_ctx) +#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_fast(_ctx) int ir_regs_number(void); bool ir_reg_is_int(int32_t reg); diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 2bc28e59e33f..965b3e9f7455 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -1413,8 +1413,9 @@ binop_fp: if (IR_IS_TYPE_VECTOR(insn->type)) { return IR_VECTOR_OP; } - IR_FALLTHROUGH; #endif + IR_ASSERT(IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)); + IR_FALLTHROUGH; case IR_BSWAP: case IR_CTLZ: case IR_CTTZ: @@ -3487,29 +3488,41 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) } if (ir_type_size[type] == 8) { - | cmp Rx(op1_reg), #0 - if (op2_reg != IR_REG_NONE) { - | add Rx(def_reg), Rx(op1_reg), Rx(op2_reg) + if (shift == 63) { + | movz Rx(def_reg), #0x8000, lsl #48 + | cmp Rx(def_reg), Rx(op1_reg) + | cset Rx(def_reg), eq } else { - | add Rx(def_reg), Rx(op1_reg), #offset + | cmp Rx(op1_reg), #0 + if (op2_reg != IR_REG_NONE) { + | add Rx(def_reg), Rx(op1_reg), Rx(op2_reg) + } else { + | add Rx(def_reg), Rx(op1_reg), #offset + } + | csel Rx(def_reg), Rx(def_reg), Rx(op1_reg), lt + | asr Rx(def_reg), Rx(def_reg), #shift } - | csel Rx(def_reg), Rx(def_reg), Rx(op1_reg), lt - | asr Rx(def_reg), Rx(def_reg), #shift } else { - | cmp Rw(op1_reg), #0 - if (op2_reg != IR_REG_NONE) { - | add Rw(def_reg), Rw(op1_reg), Rw(op2_reg) - } else { - | add Rw(def_reg), Rw(op1_reg), #offset - } - | csel Rw(def_reg), Rw(def_reg), Rw(op1_reg), lt - if (ir_type_size[type] == 4) { - | asr Rw(def_reg), Rw(def_reg), #shift - } else if (ir_type_size[type] == 2) { - | ubfx Rw(def_reg), Rw(def_reg), #shift, #16 + if (shift == 31) { + | movz Rw(def_reg), #0x8000, lsl #16 + | cmp Rw(def_reg), Rw(op1_reg) + | cset Rw(def_reg), eq } else { - IR_ASSERT(ir_type_size[type] == 1); - | ubfx Rw(def_reg), Rw(def_reg), #shift, #8 + | cmp Rw(op1_reg), #0 + if (op2_reg != IR_REG_NONE) { + | add Rw(def_reg), Rw(op1_reg), Rw(op2_reg) + } else { + | add Rw(def_reg), Rw(op1_reg), #offset + } + | csel Rw(def_reg), Rw(def_reg), Rw(op1_reg), lt + if (ir_type_size[type] == 4) { + | asr Rw(def_reg), Rw(def_reg), #shift + } else if (ir_type_size[type] == 2) { + | ubfx Rw(def_reg), Rw(def_reg), #shift, #16 + } else { + IR_ASSERT(ir_type_size[type] == 1); + | ubfx Rw(def_reg), Rw(def_reg), #shift, #8 + } } } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -3965,10 +3978,10 @@ static void ir_emit_binop_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_FP_REG_REG_REG_OP fdiv, type, def_reg, op1_reg, op2_reg break; case IR_MIN: - | ASM_FP_REG_REG_REG_OP fmin, type, def_reg, op1_reg, op2_reg + | ASM_FP_REG_REG_REG_OP fminnm, type, def_reg, op1_reg, op2_reg break; case IR_MAX: - | ASM_FP_REG_REG_REG_OP fmax, type, def_reg, op1_reg, op2_reg + | ASM_FP_REG_REG_REG_OP fmaxnm, type, def_reg, op1_reg, op2_reg break; } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -9272,10 +9285,10 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { IR_ASSERT(ival->stack_spill_pos == -1); ival->stack_spill_pos = stack_start + stack_offset; - if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { - /* Remap VAR to PARAM stack slot */ - ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); - } + } + if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { + /* Remap VAR to PARAM stack slot */ + ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); } } if (sizeof(void*) == 8) { diff --git a/ext/opcache/jit/ir/ir_cfg.c b/ext/opcache/jit/ir/ir_cfg.c index 4e4db166d37e..621f02a2372d 100644 --- a/ext/opcache/jit/ir/ir_cfg.c +++ b/ext/opcache/jit/ir/ir_cfg.c @@ -1106,6 +1106,34 @@ static IR_NEVER_INLINE uint32_t ir_collect_irreducible_loops(ir_ctx *ctx, uint32 return loops; } +static IR_NEVER_INLINE void ir_check_loop_entry_in_preds(ir_ctx *ctx, uint32_t hdr, ir_worklist *work) +{ + ir_block *blocks = ctx->cfg_blocks; + uint32_t *edges = ctx->cfg_edges; + + ir_worklist_clear(work); + ir_bitset_incl(work->visited, blocks[hdr].idom); + ir_worklist_push(work, hdr); + while (ir_worklist_len(work)) { + uint32_t b = ir_worklist_pop(work); + ir_block *bb = &blocks[b]; + + if (bb->flags & IR_BB_ENTRY) { + blocks[hdr].flags |= IR_BB_LOOP_WITH_ENTRY_IN_PREDS; + break; + } + + uint32_t *p = &edges[bb->predecessors]; + uint32_t count = bb->predecessors_count; + + for (; count > 0; p++, count--) { + if (!ir_dominates(blocks, hdr, *p)) { + ir_worklist_push(work, *p); + } + } + } +} + int ir_find_loops(ir_ctx *ctx) { uint32_t b, j, n; @@ -1295,6 +1323,10 @@ int ir_find_loops(ir_ctx *ctx) b = loops; do { bb = &blocks[b]; + if (ctx->entries_count + && (bb->predecessors_count > 2 || (bb->flags & IR_BB_IRREDUCIBLE_LOOP))) { + ir_check_loop_entry_in_preds(ctx, b, &work); + } b = bb->next_loop; IR_ASSERT(bb->flags & IR_BB_LOOP_HEADER); bb->loop_depth = (bb->loop_header) ? blocks[bb->loop_header].loop_depth + 1 : 1; diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index bc3ae5859be3..de6b53141a67 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -541,7 +541,9 @@ static IR_NEVER_INLINE void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_reg, ir_reg tmp_fp_reg) { int i; - int8_t *pred, *loc, *types; + int8_t *loc; /* array of the current locations of the original source registers: initially loc[from] = from */ + int8_t *pred; /* array of original source registers that must be copied to destinations: pred[to] = from */ + int8_t *types; /* array of the types of the orginal source registers: types[from] = typeof(from) */ ir_reg to, from; ir_type type; ir_regset todo, ready, srcs; @@ -625,18 +627,19 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ IR_ASSERT(tmp_reg == IR_REG_NONE || !IR_REGSET_IN(srcs, tmp_reg) || pred[loc[tmp_reg]] == tmp_reg); IR_ASSERT(tmp_fp_reg == IR_REG_NONE || !IR_REGSET_IN(srcs, tmp_fp_reg) || pred[loc[tmp_fp_reg]] == tmp_fp_reg); - /* now we resolve all "windmill axles" - cycles (this reuires temporary registers) */ + /* now we resolve all "windmill axles" - cycles (this requires temporary registers) */ while (todo != IR_REGSET_EMPTY) { - to = ir_regset_pop_first(&todo); + to = ir_regset_pop_first(&todo); /* select a register to break the cycle */ from = pred[to]; IR_ASSERT(to != loc[from]); - type = types[from]; + type = types[to]; /* type of the register that needs to be moved into a temporary register */ if (IR_IS_TYPE_INT(type)) { #ifdef IR_HAVE_SWAP_INT if (pred[from] == to) { - if (ir_type_size[types[to]] > ir_type_size[type]) { - type = types[to]; + if (ir_type_size[types[from]] > ir_type_size[type]) { + type = types[from]; } + /* resolve the cycle with 2 registers using SWAP */ ir_emit_swap(ctx, type, to, from); IR_REGSET_EXCL(todo, from); loc[to] = from; @@ -646,11 +649,13 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ #endif IR_ASSERT(tmp_reg != IR_REG_NONE); IR_ASSERT(tmp_reg >= IR_REG_GP_FIRST && tmp_reg <= IR_REG_GP_LAST); + /* break cycle, moving one of its register into a temporary register */ ir_emit_mov(ctx, type, tmp_reg, to); loc[to] = tmp_reg; } else { #ifdef IR_HAVE_SWAP_FP - if (pred[from] == to && types[to] == type) { + if (pred[from] == to && types[from] == type) { + /* resolve the cycle with 2 registers using SWAP */ ir_emit_swap_fp(ctx, type, to, from); IR_REGSET_EXCL(todo, from); loc[to] = from; @@ -660,6 +665,7 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ #endif IR_ASSERT(tmp_fp_reg != IR_REG_NONE); IR_ASSERT(tmp_fp_reg >= IR_REG_FP_FIRST && tmp_fp_reg <= IR_REG_FP_LAST); + /* break cycle, moving one of its register into a temporary register */ ir_emit_fp_mov(ctx, type, tmp_fp_reg, to); loc[to] = tmp_fp_reg; } diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 16dc6f536b8c..c8292191b5e8 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -30,7 +30,7 @@ IR_FOLD(EQ(C_DOUBLE, C_DOUBLE)) IR_FOLD(EQ(C_FLOAT, C_FLOAT)) { - IR_FOLD_BOOL(op1_insn->val.d == op2_insn->val.d); + IR_FOLD_BOOL(op1_insn->val.f == op2_insn->val.f); } IR_FOLD(NE(C_BOOL, C_BOOL)) @@ -1001,7 +1001,8 @@ IR_FOLD(MUL_OV(C_I64, C_I64)) int64_t res; IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); res = op1_insn->val.u64 * op2_insn->val.u64; - if ((op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) { + if ((op1_insn->val.i64 == -1 && res == INT64_MIN) + || (op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) { IR_FOLD_NEXT; } IR_FOLD_CONST_U(res); @@ -2669,10 +2670,6 @@ IR_FOLD(MUL(NEG, C_I8)) IR_FOLD(MUL(NEG, C_I16)) IR_FOLD(MUL(NEG, C_I32)) IR_FOLD(MUL(NEG, C_I64)) -IR_FOLD(DIV(NEG, C_I8)) -IR_FOLD(DIV(NEG, C_I16)) -IR_FOLD(DIV(NEG, C_I32)) -IR_FOLD(DIV(NEG, C_I64)) { op1 = op1_insn->op1; val.u64 = (0 - op2_insn->val.u64); @@ -2868,13 +2865,22 @@ IR_FOLD(DIV(_, C_FLOAT)) } IR_FOLD(MUL(NEG, NEG)) -IR_FOLD(DIV(NEG, NEG)) { op1 = op1_insn->op1; op2 = op2_insn->op1; IR_FOLD_RESTART; } +IR_FOLD(DIV(NEG, NEG)) +{ + if (IR_IS_TYPE_FP(IR_OPT_TYPE(opt))) { + op1 = op1_insn->op1; + op2 = op2_insn->op1; + IR_FOLD_RESTART; + } + IR_FOLD_NEXT; +} + IR_FOLD(AND(_, C_BOOL)) { IR_FOLD_COPY(op2_insn->val.b ? op1 : op2); diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c index 712411774a47..54c894f27cba 100644 --- a/ext/opcache/jit/ir/ir_gcm.c +++ b/ext/opcache/jit/ir/ir_gcm.c @@ -100,7 +100,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca) #endif flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags; - if ((flags & IR_BB_LOOP_WITH_ENTRY) + if ((flags & (IR_BB_LOOP_WITH_ENTRY|IR_BB_LOOP_WITH_ENTRY_IN_PREDS)) && !(ctx->binding && ir_binding_find(ctx, ref))) { /* Don't move loop invariant code across an OSR ENTRY if we can't restore it */ return lca; @@ -135,7 +135,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca) break; } flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags; - if ((flags & IR_BB_LOOP_WITH_ENTRY) + if ((flags & (IR_BB_LOOP_WITH_ENTRY|IR_BB_LOOP_WITH_ENTRY_IN_PREDS)) && !(ctx->binding && ir_binding_find(ctx, ref))) { break; } diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index 6fa8efa9cf71..9f91fbfba9f8 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -1204,6 +1204,7 @@ void ir_iter_cleanup(ir_ctx *ctx); #define IR_BB_PREV_EMPTY_ENTRY (1<<8) #define IR_BB_OSR_ENTRY_LOADS (1<<9) /* OSR Entry-point with register LOADs */ #define IR_BB_LOOP_WITH_ENTRY (1<<10) /* set together with LOOP_HEADER if there is an ENTRY in the loop */ +#define IR_BB_LOOP_WITH_ENTRY_IN_PREDS (1<<18) /* set together with LOOP_HEADER if an ENTRY reaches the loop bypassing its immediate dominator */ /* The following flags are set by GCM */ #define IR_BB_HAS_PHI (1<<11) diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index 53c3cbd8a047..f8d418d06140 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -812,6 +812,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) IR_ASSERT(use_insn->op3 == ref); /* skip VSTORE (VAR is going to be remapped to PARAM) */ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + if (ctx->use_lists[ref].count != 1) { + param_ival->flags &= ~IR_LIVE_INTERVAL_MEM_PARAM; + } param_ival->use_pos = param_ival->use_pos->next; ctx->rules[use] = IR_SKIPPED | IR_NOP; } @@ -1530,6 +1533,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) IR_ASSERT(use_insn->op3 == ref); /* skip VSTORE (VAR is going to be remapped to PARAM) */ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + if (ctx->use_lists[ref].count != 1) { + param_ival->flags &= ~IR_LIVE_INTERVAL_MEM_PARAM; + } param_ival->use_pos = param_ival->use_pos->next; ctx->rules[use] = IR_SKIPPED | IR_NOP; } diff --git a/ext/opcache/jit/ir/ir_ra_simple.c b/ext/opcache/jit/ir/ir_ra_simple.c deleted file mode 100644 index 4ecdd79986b0..000000000000 --- a/ext/opcache/jit/ir/ir_ra_simple.c +++ /dev/null @@ -1,662 +0,0 @@ -/* - * IR - Lightweight JIT Compilation Framework - * Simple Local Register Allocation (used for -O0) - * This file is part of the IR Project distributed under the MIT-style LICENSE. - * Authors: Dmitry Stogov - * - * Simple RA assign registers for each IR instruction separately and always pass - * values between instrutios in stack spill slots. - */ - -#ifndef _GNU_SOURCE -# define _GNU_SOURCE -#endif - -#include "ir.h" - -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) -# include "ir_x86.h" -#elif defined(IR_TARGET_AARCH64) -# include "ir_aarch64.h" -#else -# error "Unknown IR target" -#endif - -#include "ir_private.h" - -typedef struct { - ir_reg_alloc_data ra_data; - int32_t num; - ir_regset preserved_regs; - ir_regset clobbered[IR_SUB_REFS_COUNT]; - struct { - uint8_t type; - int8_t start; - int8_t end; - int8_t hint; - int8_t flags; - ir_ref root; - ir_ref ref; - ir_ref op; - } regs[32]; -} ir_reg_alloc_simple_data; - -static void _add_scratch(ir_reg_alloc_simple_data *data, ir_reg reg, int8_t start, int8_t end) -{ - int8_t j; - - if (start < 0) start = 0; // TODO: ARGVAL support ??? - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT); - if (reg >= IR_REG_NUM) { - for (j = start; j < end; j++) { - data->clobbered[j] = IR_REGSET_UNION(data->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]); - } - } else { - for (j = start; j < end; j++) { - IR_REGSET_INCL(data->clobbered[j], reg); - } - } -} - -static void _add_reg(ir_reg_alloc_simple_data *data, ir_type type, - int8_t start, int8_t end, ir_reg hint, int8_t flags, - ir_ref root, ir_ref ref, ir_ref op) -{ - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32); - data->regs[data->num].type = type; - data->regs[data->num].start = start; - data->regs[data->num].end = end; - data->regs[data->num].hint = hint; - data->regs[data->num].flags = flags; - data->regs[data->num].root = root; - data->regs[data->num].ref = ref; - data->regs[data->num].op = op; - data->num++; -} - -static ir_reg _get_free_reg(ir_type type, ir_regset available) -{ - if (IR_IS_TYPE_INT(type)) { - available = IR_REGSET_INTERSECTION(available, IR_REGSET_GP); - } else { - IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); - available = IR_REGSET_INTERSECTION(available, IR_REGSET_FP); - } - if (!IR_REGSET_IS_EMPTY(available)) { - return IR_REGSET_FIRST(available); - } else { - return IR_REG_NONE; - } -} - -static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data *data, int j) -{ - int n; - ir_regset available; - ir_reg reg; - - if (IR_IS_TYPE_INT(type)) { - available = IR_REGSET_GP; - if (ctx->flags & IR_USE_FRAME_POINTER) { - IR_REGSET_EXCL(available, IR_REG_FRAME_POINTER); - } - -#if defined(IR_TARGET_X86) - if (ir_type_size[type] == 1) { - /* TODO: if no registers avialivle, we may use of one this register for already allocated interval ??? */ - IR_REGSET_EXCL(available, IR_REG_RBP); - IR_REGSET_EXCL(available, IR_REG_RSI); - IR_REGSET_EXCL(available, IR_REG_RDI); - } -#endif - } else { - IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); - available = IR_REGSET_FP; - } - for (n = data->regs[j].start; n < data->regs[j].end; n++) { - available = IR_REGSET_DIFFERENCE(available, data->clobbered[n]); - } - if (IR_REGSET_IS_EMPTY(available)) { - fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n"); - IR_ASSERT(0); - exit(-1); - } - - reg = IR_REGSET_FIRST(available); - if (IR_REGSET_IN(data->preserved_regs, reg)) { - ir_regset tmp = (ir_regset)ctx->used_preserved_regs; - IR_REGSET_INCL(tmp, reg); - ctx->used_preserved_regs = tmp; - } - return reg; -} - -static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load) -{ - ir_use_list *use_list = &ctx->use_lists[load]; - ir_ref *p, use, i, n = use_list->count; - ir_ref last_use = IR_UNUSED; - ir_insn *insn; - int address_taken = -1; /* unknown */ - - if (n) { - /* Check if all uses of the result of VLOAD are in the same basic block (after VLOAD) */ - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - use = *p; - if (use < load || use > bb->end) return 0; - if (use > last_use) last_use = use; - } - /* Check if the result of VLOAD can not be clobbered between VLOAD and the last use */ - for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) { - if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) { - return 0; - } else if (address_taken - && (insn->op == IR_STORE - || insn->op == IR_STORE_v - || insn->op == IR_CALL)) { - if (address_taken == -1) { - ir_use_list *use_list = &ctx->use_lists[var]; - ir_ref *p, n = use_list->count; - - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - if (ctx->ir_base[*p].op == IR_VADDR) { - return 0; - } - } - address_taken = 0; - } - } - n = ir_insn_len(insn); - i += n; - insn += n; - } - } - return 1; -} - -static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref store, ir_ref val) -{ - ir_ref i, n; - ir_insn *insn; - int address_taken = -1; /* unknown */ - - /* Check if the value stored by VSTORE is produced in the same basic block (before VSTORE) */ - if (val < bb->start || val > store || store > bb->end) return 0; - - /* Check if the instructions between the "producer" and VSTORE can not access the same VAR */ - for (i = val, insn = &ctx->ir_base[i]; i < store;) { - if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) - && insn->op2 == var) { - return 0; - } else if (address_taken - && (insn->op == IR_LOAD - || insn->op == IR_LOAD_v - || insn->op == IR_STORE - || insn->op == IR_STORE_v - || insn->op == IR_CALL)) { - if (address_taken == -1) { - ir_use_list *use_list = &ctx->use_lists[var]; - ir_ref *p, n = use_list->count; - - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - if (ctx->ir_base[*p].op == IR_VADDR) { - return 0; - } - } - address_taken = 0; - } - } - n = ir_insn_len(insn); - i += n; - insn += n; - } - return 1; -} - -static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_simple_data *data) -{ - ir_ref stack[4]; - int stack_pos = 0; - ir_target_constraints constraints; - ir_insn *insn; - uint32_t j, n, flags, def_flags; - ir_ref *p, child; - - while (1) { - IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED); - - if (!(ctx->rules[input] & IR_SIMPLE)) { - def_flags = ir_get_target_constraints(ctx, input, &constraints); - n = constraints.tmps_count; - while (n > 0) { - n--; - if (constraints.tmp_regs[n].type) { - ir_reg flags = 0; - ir_ref op = constraints.tmp_regs[n].num; - - if (op > 0 && op <= ctx->ir_base[input].inputs_count) { - ir_ref *ops = ctx->ir_base[input].ops; - - if (IR_IS_CONST_REF(ops[op])) { - /* rematerialization */ - flags = IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { - /* local address rematerialization */ - flags = IR_REG_SPILL_LOAD; - } - } - _add_reg(data, constraints.tmp_regs[n].type, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags, - IR_UNUSED, input, op); - } else { - _add_scratch(data, constraints.tmp_regs[n].reg, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); - } - } - } else { - def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; - constraints.hints_count = 0; - } - - insn = &ctx->ir_base[input]; - flags = ir_op_flags[insn->op]; - n = IR_INPUT_EDGES_COUNT(flags); - j = 1; - p = insn->ops + j; - if (flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) { - j++; - p++; - } - for (; j <= n; j++, p++) { - IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); - child = *p; - if (child > 0) { - if (ctx->vregs[child] > 0) { - if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) { - ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; - int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF; - - _add_reg(data, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD, - ref, input, j); - } - } else if (ctx->rules[child] & IR_FUSED) { - IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos))); - stack[stack_pos++] = child; - } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) { - ctx->regs[input][j] = ctx->ir_base[child].op2; - } - } - } - if (!stack_pos) { - break; - } - input = stack[--stack_pos]; - } -} - -static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn) -{ - size_t size = ir_get_type_size(var_insn->type); - size_t align = var_insn->op3; - int32_t offset; - -#if IR_SIMD - if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) { - size = 4; - align = IR_MAX(align, 4); - } -#endif - - offset = ir_allocate_spill_slot(ctx, size, align); - ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); -} - -int ir_reg_alloc_simple(ir_ctx *ctx) -{ - uint32_t b; - ir_block *bb; - ir_insn *insn; - ir_ref i, n, j, *p; - uint32_t *rule, insn_flags; - ir_target_constraints constraints; - uint32_t def_flags; - ir_reg reg; - ir_regset scratch; - ir_reg_alloc_simple_data data; - int32_t offset; - - ir_compute_dessa_moves(ctx); - - memset(&data.ra_data, 0, sizeof(data.ra_data)); - data.ra_data.cc = ir_get_call_conv_dsc(ctx->flags); - ctx->data = &data.ra_data; - - ctx->stack_frame_size = 0; - ctx->call_stack_size = 0; - ctx->used_preserved_regs = 0; - ctx->used_preserved_regs = ctx->fixed_save_regset; - - scratch = ir_scratch_regset[data.ra_data.cc->scratch_reg - IR_REG_NUM]; - data.preserved_regs = IR_REGSET_DIFFERENCE(data.ra_data.cc->preserved_regs, ctx->fixed_save_regset); - - ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count); - memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count); - - /* vregs + tmp + fixed + SRATCH + ALL */ - ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*)); - - if (!ctx->arena) { - ctx->arena = ir_arena_create(16 * 1024); - } - - for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) { - IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE)); - for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) { - if (*rule & (IR_FUSED|IR_SKIPPED)) { - if ((*rule & IR_RULE_MASK) == IR_ALLOCA) { - if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) { - /* spill slot already allocated */ - } else if (insn->op == IR_VAR) { - if (ctx->use_lists[i].count > 0) { - ir_allocate_var_spill_slot(ctx, i, insn); - } - } else if (insn->op == IR_ALLOCA) { - if (ctx->use_lists[i].count > 0) { - ir_insn *val = &ctx->ir_base[insn->op2]; - - IR_ASSERT(IR_IS_CONST_REF(insn->op2)); - IR_ASSERT(IR_IS_TYPE_INT(val->type)); - IR_ASSERT(!IR_IS_SYM_CONST(val->op)); - IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); - IR_ASSERT(val->val.i64 < 0x7fffffff); - offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); - ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset); - } - } else if (insn->op == IR_VADDR) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) { - ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]); - } - ctx->vregs[i] = ctx->vregs[insn->op1]; - } - } - } else { - data.num = 0; - for (j = 0; j < IR_SUB_REFS_COUNT; j++) { - data.clobbered[j] = IR_REGSET_EMPTY; - } - - def_flags = ir_get_target_constraints(ctx, i, &constraints); - n = constraints.tmps_count; - while (n) { - n--; - - IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT); - if (constraints.tmp_regs[n].type) { - ir_reg flags = 0; - ir_ref op = constraints.tmp_regs[n].num; - - if (op > 0 && op <= insn->inputs_count) { - ir_ref *ops = insn->ops; - - if (IR_IS_CONST_REF(ops[op])) { - /* rematerialization */ - flags = IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { - /* local address rematerialization */ - flags = IR_REG_SPILL_LOAD; - } - } - _add_reg(&data, constraints.tmp_regs[n].type, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags, - IR_UNUSED, i, op); - } else { - _add_scratch(&data, constraints.tmp_regs[n].reg, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); - } - } - - if (ctx->vregs[i] > 0) { - reg = constraints.def_reg; - if (!ctx->live_intervals[ctx->vregs[i]]) { - do { - ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); - memset(ival, 0, sizeof(ir_live_interval)); - ctx->live_intervals[ctx->vregs[i]] = ival; - ival->type = insn->type; - ival->reg = IR_REG_NONE; - ival->vreg = ctx->vregs[i]; - ival->stack_spill_pos = -1; - - if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { - ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); - } - if (ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) { - ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]); - reg = IR_REG_NONE; - def_flags &= ~IR_USE_MUST_BE_IN_REG; - break; - } - } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) { - /* We may reuse parameter stack slot for spilling */ - ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - if (ctx->use_lists[i].count > 0) { - ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; - if (use < bb->end) { - ir_insn *use_insn = &ctx->ir_base[use]; - if (use_insn->op == IR_VSTORE - && !(ctx->rules[use] & IR_SKIPPED) - /* first use of VAR is VSTORE(_, VAR, PARAM) */ - && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) { - IR_ASSERT(use_insn->op3 == i); - /* skip VSTORE (VAR is going to be remapped to PARAM) */ - ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; - ival->use_pos = NULL; - ctx->rules[use] = IR_SKIPPED | IR_NOP; - } - } - } - break; - } - - if (ctx->use_lists[i].count == 1) { - ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; - ir_insn *use_insn = &ctx->ir_base[use]; - - if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) - && use_insn->op3 == i - && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[use_insn->op2])) { - ir_allocate_var_spill_slot(ctx, use_insn->op2, &ctx->ir_base[use_insn->op2]); - } - ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[use_insn->op2]); - break; - } - } - - size_t size = ir_get_type_size(ival->type); -#if IR_SIMD - if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { - size = 4; - } -#endif - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); - } while (0); - } - - if (def_flags & IR_USE_MUST_BE_IN_REG) { - ir_live_pos def_pos; - - if (reg != IR_REG_NONE) { - def_pos = IR_SAVE_SUB_REF; - } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - def_pos = IR_USE_SUB_REF; - } else { - def_pos = IR_LOAD_SUB_REF; - } - } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - def_pos = IR_LOAD_SUB_REF; - } else { - def_pos = IR_DEF_SUB_REF; - } - - _add_reg(&data, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE, - IR_UNUSED, i, 0); - } - } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { - ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); - } - } - - n = insn->inputs_count; - insn_flags = ir_op_flags[insn->op]; - j = 1; - p = insn->ops + 1; - if (insn_flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) { - j++; - p++; - } - for (; j <= n; j++, p++) { - ir_ref input = *p; - ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; - ir_live_pos use_pos; - uint32_t use_flags = IR_USE_FLAGS(def_flags, j); - - if (reg != IR_REG_NONE) { -#if IR_X86_I64 - if (use_flags & IR_HINT_TWO_REGS) { - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); - } else -#endif - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg); - } - - if (input > 0) { - if (ctx->vregs[input] > 0) { - use_pos = IR_USE_SUB_REF; - if (reg != IR_REG_NONE) { - use_pos = IR_LOAD_SUB_REF; - } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (j == 1) { - if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - use_pos = IR_USE_SUB_REF; - } else { - use_pos = IR_LOAD_SUB_REF; - } - } else if (input == insn->op1) { - /* Input is the same as "op1" */ - use_pos = IR_LOAD_SUB_REF; - } - } - if (use_flags & IR_USE_MUST_BE_IN_REG) { - _add_reg(&data, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD, - IR_UNUSED, i, j); - } - } else { - if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) { - ir_add_fusion_data(ctx, i, input, &data); - } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { - ctx->regs[i][j] = ctx->ir_base[input].op2; - } - } - } - } - - for (j = 0; j < data.num; j++) { - ir_regset available = scratch; -#if IR_X86_I64 - ir_reg reg2; -#endif - - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - available = IR_REGSET_DIFFERENCE(available, data.clobbered[n]); - } - reg = data.regs[j].hint; -#if IR_X86_I64 - reg2 = IR_REG_NONE; - if (reg != IR_REG_NONE && (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64)) { - reg2 = IR_REG_I64_HI(reg); - reg = IR_REG_I64_LO(reg); - } -#endif - if (reg == IR_REG_NONE || !IR_REGSET_IN(available, reg)) { - reg = _get_free_reg(data.regs[j].type, available); - if (UNEXPECTED(reg == IR_REG_NONE)) { - reg = _get_free_reg2(ctx, data.regs[j].type, &data, j); - } - } - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - IR_REGSET_INCL(data.clobbered[n], reg); - } -#if IR_X86_I64 - if (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64) { - IR_REGSET_EXCL(available, reg); - if (reg2 == IR_REG_NONE || !IR_REGSET_IN(available, reg2)) { - reg2 = _get_free_reg(data.regs[j].type, available); - if (UNEXPECTED(reg2 == IR_REG_NONE)) { - reg2 = _get_free_reg2(ctx, data.regs[j].type, &data, j); - } - } - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - IR_REGSET_INCL(data.clobbered[n], reg2); - } - if (reg > reg2) { - SWAP_REGS(reg, reg2); - } - reg = IR_REG_I64_PAIR(reg, reg2); - } -#endif - reg = reg | data.regs[j].flags; - if (data.regs[j].op == 4 && insn->inputs_count < 4) { - if (!ctx->tmp_regs) { - ctx->tmp_regs = ir_mem_malloc(ctx->insns_count); - memset(ctx->tmp_regs, -1, ctx->insns_count); - } - ctx->tmp_regs[data.regs[j].ref] = reg; - } else if (!data.regs[j].root || ctx->regs[data.regs[j].ref][data.regs[j].op] == IR_REG_NONE) { - ctx->regs[data.regs[j].ref][data.regs[j].op] = reg; - } else if (ctx->regs[data.regs[j].ref][data.regs[j].op] != reg) { - ctx->rules[data.regs[j].ref] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg); - } - } - } - - n = ir_insn_len(insn); - i += n; - insn += n; - rule += n; - } - if (bb->flags & IR_BB_DESSA_MOVES) { -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) - ctx->regs[bb->end][0] = IR_REG_RAX; - ctx->regs[bb->end][1] = IR_REG_XMM0; -#elif defined(IR_TARGET_AARCH64) - ctx->regs[bb->end][0] = IR_REG_X0; - ctx->regs[bb->end][1] = IR_REG_V0; -#else -# error "Unknown IR target" -#endif - } - } - -#ifdef IR_TARGET_X86 - if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); - } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) - && data.ra_data.cc->fp_ret_reg == IR_REG_NONE) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); - } else { - ctx->ret_slot = -1; - } -#endif - - ctx->flags |= IR_NO_STACK_COMBINE; - ir_fix_stack_frame(ctx); - ctx->data = NULL; - - return 1; -} diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 3ad6997b071e..2ca202add5e8 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -2673,16 +2673,16 @@ static bool ir_is_zero(const ir_ctx *ctx, ir_ref ref) && ctx->ir_base[ref].val.u32 == 0; } -static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) +static bool ir_is_min_max_const(const ir_ctx *ctx, const ir_insn *cond, ir_ref ref) { if (cond->op == IR_ULE) { /* (x <= 3 ? 4 : x) => (x < 4 ? 4 : x) => max(x, 4) */ /* (x <= 3 ? x : 4) => (x < 4 ? x : 4) => min(x, 4) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.u64 != 0 && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 - 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_UGE) { @@ -2690,9 +2690,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x >= 3 ? x : 2) => (x > 2 ? x : 2) => max(x, 2) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.u64 != UINT64_MAX && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 + 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_LE) { @@ -2700,9 +2700,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x <= 3 ? x : 4) => (x < 4 ? x : 4) => min(x, 4) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) - && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 - 1 + && ctx->ir_base[ref].val.i64 != INT64_MIN + && ctx->ir_base[cond->op2].val.i64 == ctx->ir_base[ref].val.i64 - 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_GE) { @@ -2710,9 +2710,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x >= 3 ? x : 2) => (x > 2 ? x : 2) => max(x, 2) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.i64 != INT64_MAX && ctx->ir_base[cond->op2].val.i64 == ctx->ir_base[ref].val.i64 + 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } @@ -2758,18 +2758,19 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re } if (is_cmp + && !IR_IS_TYPE_FP(type) && ((insn->op2 == cond->op1 && (insn->op3 == cond->op2 || (IR_IS_CONST_REF(cond->op2) && (IR_IS_CONST_REF(insn->op3) && IR_IS_TYPE_INT(insn->type) - && ir_fix_min_max_const(ctx, cond, insn->op3))))) + && ir_is_min_max_const(ctx, cond, insn->op3))))) || (insn->op3 == cond->op1 && (insn->op2 == cond->op2 || (IR_IS_CONST_REF(cond->op2) && (IR_IS_CONST_REF(insn->op2) && IR_IS_TYPE_INT(insn->type) - && ir_fix_min_max_const(ctx, cond, insn->op2))))))) { + && ir_is_min_max_const(ctx, cond, insn->op2))))))) { /* MAX/MIN * * prev prev @@ -2804,11 +2805,16 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re ir_remove_unused_vars(ctx, start2_ref, end2_ref); } - insn->op = ( - (is_less ? cond->op1 : cond->op2) - == - ((start1->op == IR_IF_TRUE) ? insn->op2 : insn->op3) - ) ? IR_MIN : IR_MAX; + if (start1->op != IR_IF_TRUE) { + IR_ASSERT(start1->op == IR_IF_FALSE); + is_less = !is_less; + } + if (cond->op1 != insn->op2) { + IR_ASSERT(cond->op1 == insn->op3); + is_less = !is_less; + } + + insn->op = is_less ? IR_MIN : IR_MAX; insn->inputs_count = 2; if (insn->op2 > insn->op3) { insn->op1 = insn->op2; @@ -2939,7 +2945,12 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re } return 1; - } else if (insn->op2 <= root_ref && insn->op3 <= root_ref + } else if (((insn->op2 <= root_ref && insn->op3 <= root_ref) + || ((cond->op >= IR_EQ && cond->op <= IR_UNORDERED) + && ((insn->op2 == cond->op1 && (insn->op3 == cond->op2 || insn->op3 <= root_ref)) + || (insn->op2 == cond->op2 && (insn->op3 == cond->op1 || insn->op3 <= root_ref)) + || (insn->op3 == cond->op1 && insn->op2 <= root_ref) + || (insn->op3 == cond->op2 && insn->op2 <= root_ref)))) && cond->op != IR_OVERFLOW // TODO: temporary disable IF-conversion for RLOAD. // We don't track anti-dependencies in GCM and Local Scheduling. diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index f33c89f574f5..463be9e750d3 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -1622,6 +1622,7 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = { _(BINOP_INT) \ _(BINOP_SSE2) \ _(BINOP_AVX) \ + _(MIN_MAX_FP) \ _(SHIFT) \ _(SHIFT_CONST) \ _(COPY_INT) \ @@ -1636,6 +1637,7 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = { _(COND_TEST_BIT) \ _(COND_CMP_INT) \ _(COND_CMP_FP) \ + _(COND_MIN_MAX_FP) \ _(GUARD_CMP_INT) \ _(GUARD_CMP_FP) \ _(GUARD_TEST_INT) \ @@ -1998,6 +2000,28 @@ op2_const: case IR_COND_CMP_FP: flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG; break; + case IR_COND_MIN_MAX_FP: + flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG; + if (!(ctx->mflags & IR_X86_AVX)) { + insn = &ctx->ir_base[ref]; + if (!IR_IS_CONST_REF(insn->op3) || ctx->vregs[insn->op3] > 0) { + /* def_reg can't be the same as op3_reg */ + flags |= IR_DEF_CONFLICTS_WITH_INPUT_REGS; + } + } + break; + case IR_MIN_MAX_FP: + flags = IR_USE_MUST_BE_IN_REG | IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_DEF_CONFLICTS_WITH_INPUT_REGS; + insn = &ctx->ir_base[ref]; + constraints->tmp_regs[0] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n = 1; + if (IR_IS_CONST_REF(insn->op1)) { + n = ir_add_const_tmp_reg(ctx, insn->op1, 1, n, constraints); + } + if (IR_IS_CONST_REF(insn->op2)) { + n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); + } + break; case IR_VSTORE_INT: flags = IR_OP3_MUST_BE_IN_REG; insn = &ctx->ir_base[ref]; @@ -4496,7 +4520,7 @@ binop_fp: goto binop_vector; #endif } else { - goto binop_fp; + return IR_MIN_MAX_FP; } break; case IR_COPY: @@ -5047,6 +5071,11 @@ store_int: return IR_TWO_REGS | IR_COND_I64_CMP_FP; } #endif + if ((op1_insn->op >= IR_LT && op1_insn->op <= IR_GT) && + ((op1_insn->op1 == insn->op2 && op1_insn->op2 == insn->op3) || + (op1_insn->op1 == insn->op3 && op1_insn->op2 == insn->op2))) { + return IR_COND_MIN_MAX_FP; + } return IR_COND_CMP_FP; } #if IR_X86_I64 @@ -7543,6 +7572,30 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) | add Rd(def_reg), Rd(op1_reg) |.if X64 || } +|.endif + | ASM_REG_IMM_OP sar, type, def_reg, shift + } else if (shift == (uint32_t)ir_type_size[type] * 8 - 1) { +|.if X64 +|| if (ir_type_size[type] == 8) { + | mov64 Rq(def_reg), 0x8000000000000000 + | cmp Rq(def_reg), Rq(op1_reg) + | sete Rb(def_reg) + | movzx Rd(def_reg), Rb(def_reg) +|| } else { +|.endif + | xor Rd(def_reg), Rd(def_reg) + | cmp Rd(op1_reg), 0x80000000 +|.if X64 + | sete Rb(def_reg) +|| } +|.else + if (def_reg < IR_REG_R3) { + | sete Rb(def_reg) + } else { + | jne >1 + | mov Rd(def_reg), 1 + |1: + } |.endif } else { |.if X64 @@ -7564,8 +7617,8 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) |.endif | ASM_REG_REG_OP test, type, op1_reg, op1_reg | ASM_REG_REG_OP2 cmovns, type, def_reg, op1_reg + | ASM_REG_IMM_OP sar, type, def_reg, shift } - | ASM_REG_IMM_OP sar, type, def_reg, shift if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); @@ -8614,12 +8667,6 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_REG_OP divs, type, def_reg, op2_reg break; - case IR_MIN: - | ASM_SSE2_REG_REG_OP mins, type, def_reg, op2_reg - break; - case IR_MAX: - | ASM_SSE2_REG_REG_OP maxs, type, def_reg, op2_reg - break; } } else if (IR_IS_CONST_REF(op2)) { int label = ir_get_const_label(ctx, op2); @@ -8639,12 +8686,6 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_TXT_OP divs, type, def_reg, [=>label] break; - case IR_MIN: - | ASM_SSE2_REG_TXT_OP mins, type, def_reg, [=>label] - break; - case IR_MAX: - | ASM_SSE2_REG_TXT_OP maxs, type, def_reg, [=>label] - break; } } else { ir_mem mem; @@ -8669,14 +8710,88 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_MEM_OP divs, type, def_reg, mem break; - case IR_MIN: - | ASM_SSE2_REG_MEM_OP mins, type, def_reg, mem - break; - case IR_MAX: - | ASM_SSE2_REG_MEM_OP maxs, type, def_reg, mem - break; } } + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, insn->type, def, def_reg); + } +} + +static void ir_emit_min_max_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = insn->type; + ir_ref op1 = insn->op1; + ir_ref op2 = insn->op2; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op1_reg = ctx->regs[def][1]; + ir_reg op2_reg = ctx->regs[def][2]; + ir_reg tmp_reg = ctx->regs[def][3]; + + IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); + + if (IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + ir_emit_load(ctx, type, op1_reg, op1); + } + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + if (op1 != op2) { + ir_emit_load(ctx, type, op2_reg, op2); + } + } + + if (!(ctx->mflags & IR_X86_AVX)) { + ir_emit_fp_mov(ctx, type, def_reg, op1_reg); + } + + if (type == IR_DOUBLE) { + if (ctx->mflags & IR_X86_AVX) { + if (insn->op == IR_MAX) { + | vmaxsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | vminsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | vcmpsd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), 7 + | vblendvpd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } else { + if (insn->op == IR_MAX) { + | maxsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | minsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | movapd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + | cmpsd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST), 7 + | andpd xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + | andnpd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST) + | orpd xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } + } else { + IR_ASSERT(type == IR_FLOAT); + if (ctx->mflags & IR_X86_AVX) { + if (insn->op == IR_MAX) { + | vmaxss xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | vminss xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | vcmpss xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), 7 + | vblendvps xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } else { + if (insn->op == IR_MAX) { + | maxss xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | minss xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | movaps xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + | cmpss xmm(tmp_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST), 7 + | andps xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + | andnps xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST) + | orps xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } + } + if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } @@ -9469,6 +9584,20 @@ static void ir_emit_jcc(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint } } +static bool ir_insns_may_change_cmp_flags(ir_ctx *ctx, ir_ref from, ir_ref to) +{ + ir_ref ref; + + for (ref = ctx->prev_ref[to]; ref != from; ref = ctx->prev_ref[ref]) { + if (!(ctx->rules[ref] & (IR_FUSED|IR_SKIPPED)) + && ctx->ir_base[ref].op != IR_VAR + && ctx->ir_base[ref].op != IR_PHI) { + return 1; + } + } + return 0; +} + static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next_block) { ir_insn *cmp_insn = &ctx->ir_base[insn->op2]; @@ -9518,7 +9647,9 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i if (ir_rule(ctx, prev_insn->op1) == IR_CMP_AND_BRANCH_INT) { prev_insn = &ctx->ir_base[prev_insn->op1]; prev_insn = &ctx->ir_base[prev_insn->op2]; - if (prev_insn->op1 == cmp_insn->op1 && prev_insn->op2 == cmp_insn->op2) { + if (prev_insn->op1 == cmp_insn->op1 + && prev_insn->op2 == cmp_insn->op2 + && !ir_insns_may_change_cmp_flags(ctx, insn->op1, def)) { same_comparison = true; } } @@ -9666,7 +9797,7 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, op3); if (op1 == op3) { - op1_reg = op2_reg; + op1_reg = op3_reg; } } if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { @@ -10295,6 +10426,137 @@ static void ir_emit_cond_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } +static void ir_emit_cond_min_max_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = insn->type; + ir_insn *cmp_insn = &ctx->ir_base[insn->op1]; + ir_ref op2 = insn->op2; + ir_ref op3 = insn->op3; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op2_reg = ctx->regs[def][2]; + ir_reg op3_reg = ctx->regs[def][3]; + ir_reg cmp_op1_reg, cmp_op2_reg; + ir_op op; + + IR_ASSERT(def_reg != IR_REG_NONE); + + if (UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { + cmp_op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); + cmp_op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); + } else { + cmp_op1_reg = ctx->regs[insn->op1][1]; + cmp_op2_reg = ctx->regs[insn->op1][2]; + } + if (cmp_insn->op1 == op2) { + IR_ASSERT(cmp_insn->op2 == op3); + op = (cmp_insn->op == IR_LT || cmp_insn->op == IR_LE) ? IR_MIN : IR_MAX; + if (op2_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op1_reg) == IR_REG_NUM(op2_reg)); + op2_reg |= cmp_op1_reg; /* merge spill load flag */ + } + if (op3_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op2_reg) == IR_REG_NUM(op3_reg)); + op3_reg |= cmp_op2_reg; /* merge spill load flag */ + } + } else { + IR_ASSERT(cmp_insn->op1 == op3 && cmp_insn->op2 == op2); + op = (cmp_insn->op == IR_LT || cmp_insn->op == IR_LE) ? IR_MAX : IR_MIN; + if (op2_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op2_reg) == IR_REG_NUM(op2_reg)); + op2_reg |= cmp_op2_reg; /* merge spill load flag */ + } + if (op3_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op1_reg) == IR_REG_NUM(op3_reg)); + op3_reg |= cmp_op1_reg; /* merge spill load flag */ + } + } + + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, type, op2_reg, op2); + } + } + if (op3_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op3_reg)) { + op3_reg = IR_REG_NUM(op3_reg); + if (op2 != op3) { + ir_emit_load(ctx, type, op3_reg, op3); + } + } + } + + IR_ASSERT((ctx->mflags & IR_X86_AVX) || def_reg != op3_reg); + + if (op2_reg != IR_REG_NONE) { + if (!(ctx->mflags & IR_X86_AVX) && def_reg != op2_reg) { + ir_emit_fp_mov(ctx, type, def_reg, op2_reg); + } + } else { + ir_emit_load(ctx, type, def_reg, op2); + op2_reg = def_reg; + } + + if (op3_reg != IR_REG_NONE) { + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_REG_OP vmins, type, def_reg, op2_reg, op3_reg + } else { + | ASM_AVX_REG_REG_REG_OP vmaxs, type, def_reg, op2_reg, op3_reg + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_REG_OP mins, type, def_reg, op3_reg + } else { + | ASM_SSE2_REG_REG_OP maxs, type, def_reg, op3_reg + } + } + } else if (IR_IS_CONST_REF(op3)) { + int label = ir_get_const_label(ctx, op3); + + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_TXT_OP vmins, type, def_reg, op2_reg, [=>label] + } else { + | ASM_AVX_REG_REG_TXT_OP vmaxs, type, def_reg, op2_reg, [=>label] + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_TXT_OP mins, type, def_reg, [=>label] + } else { + | ASM_SSE2_REG_TXT_OP maxs, type, def_reg, [=>label] + } + } + } else { + ir_mem mem; + + if (ir_rule(ctx, op3) & IR_FUSED) { + mem = ir_fuse_load(ctx, def, op3); + } else { + mem = ir_ref_spill_slot(ctx, op3); + } + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_MEM_OP vmins, type, def_reg, op2_reg, mem + } else { + | ASM_AVX_REG_REG_MEM_OP vmaxs, type, def_reg, op2_reg, mem + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_MEM_OP mins, type, def_reg, mem + } else { + | ASM_SSE2_REG_MEM_OP maxs, type, def_reg, mem + } + } + } + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, type, def, def_reg); + } +} + static void ir_emit_return_void(ir_ctx *ctx) { ir_backend_data *data = ctx->data; @@ -15009,7 +15271,9 @@ static void ir_emit_cmp_and_branch_i64(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i if (ir_rule(ctx, prev_insn->op1) == IR_CMP_AND_BRANCH_INT) { prev_insn = &ctx->ir_base[prev_insn->op1]; prev_insn = &ctx->ir_base[prev_insn->op2]; - if (prev_insn->op1 == cmp_insn->op1 && prev_insn->op2 == cmp_insn->op2) { + if (prev_insn->op1 == cmp_insn->op1 + && prev_insn->op2 == cmp_insn->op2 + && !ir_insns_may_change_cmp_flags(ctx, insn->op1, def)) { same_comparison = true; } } @@ -17070,7 +17334,7 @@ static void ir_emit_cond_cmp_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(tmp_reg != IR_REG_NONE); - if (!IR_IS_CONST_REF(insn->op2) && UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { + if (!IR_IS_CONST_REF(insn->op1) && UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); } else { @@ -17232,7 +17496,7 @@ static void ir_emit_cond_i64_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(tmp_reg != IR_REG_NONE); - if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { + if (UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); } else { @@ -26183,6 +26447,12 @@ static void ir_emit_load_params(ir_ctx *ctx) ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_start + stack_offset); } if (dst_reg != IR_REG_NONE && IR_REG_SPILLED(ctx->regs[use][0])) { +#if IR_X86_I64 + if (insn->type == IR_I64 || insn->type == IR_U64) { + ir_emit_store_i64_lo(ctx, use, IR_REG_I64_LO(dst_reg)); + ir_emit_store_i64_hi(ctx, use, IR_REG_I64_HI(dst_reg)); + } else +#endif ir_emit_store(ctx, insn->type, use, dst_reg); } } @@ -26283,10 +26553,10 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { IR_ASSERT(ival->stack_spill_pos == -1); ival->stack_spill_pos = stack_start + stack_offset; - if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { - /* Remap VAR to PARAM stack slot */ - ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); - } + } + if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { + /* Remap VAR to PARAM stack slot */ + ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); } } if (sizeof(void*) == 8) { @@ -26636,6 +26906,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_BINOP_AVX: ir_emit_binop_avx(ctx, i, insn); break; + case IR_MIN_MAX_FP: + ir_emit_min_max_fp(ctx, i, insn); + break; case IR_MUL_INT: case IR_DIV_INT: case IR_MOD_INT: @@ -26759,6 +27032,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_COND_CMP_FP: ir_emit_cond_cmp_fp(ctx, i, insn); break; + case IR_COND_MIN_MAX_FP: + ir_emit_cond_min_max_fp(ctx, i, insn); + break; case IR_SWITCH: ir_emit_switch(ctx, b, i, insn); break;