From c2b689b9ef7b7295b335a6b849a0d6d854ea3dc9 Mon Sep 17 00:00:00 2001 From: Jorg Sowa Date: Wed, 4 Nov 2026 00:31:25 +0100 Subject: [PATCH 1/2] Update IR IR commit: 43d1d1023a6ea811b11b39ed738f5e18696a1254 --- .github/scripts/download-bundled/jit-ir.sh | 2 +- ext/opcache/config.m4 | 1 + ext/opcache/config.w32 | 2 +- ext/opcache/jit/Makefile.frag.w32 | 5 + ext/opcache/jit/ir/ir.c | 30 +- ext/opcache/jit/ir/ir.h | 3 +- ext/opcache/jit/ir/ir_aarch64.dasc | 65 +- ext/opcache/jit/ir/ir_cfg.c | 32 + ext/opcache/jit/ir/ir_emit.c | 20 +- ext/opcache/jit/ir/ir_fold.h | 5 +- ext/opcache/jit/ir/ir_gcm.c | 4 +- ext/opcache/jit/ir/ir_private.h | 1 + ext/opcache/jit/ir/ir_ra.c | 6 + ext/opcache/jit/ir/ir_ra_fast.c | 1481 ++++++++++++++++++++ ext/opcache/jit/ir/ir_ra_simple.c | 11 +- ext/opcache/jit/ir/ir_sccp.c | 39 +- ext/opcache/jit/ir/ir_x86.dasc | 334 ++++- 17 files changed, 1955 insertions(+), 86 deletions(-) create mode 100644 ext/opcache/jit/ir/ir_ra_fast.c diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index 74a9789f35ad..ee9d31b41b7d 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=30a40633b0b8f3e8e66da0a25941c4199d69d38a +revision=43d1d1023a6ea811b11b39ed738f5e18696a1254 git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 224e9e54a2e4..89c1fa488e97 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -48,6 +48,7 @@ AS_VAR_IF([PHP_OPCACHE_JIT], [yes], [ jit/ir/ir_patch.c jit/ir/ir_perf.c jit/ir/ir_ra.c + jit/ir/ir_ra_fast.c jit/ir/ir_ra_simple.c jit/ir/ir_save.c jit/ir/ir_sccp.c diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index f92b02e59671..d3d3415fd01d 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -25,7 +25,7 @@ if (PHP_OPCACHE_JIT == "yes") { } else if (CHECK_HEADER("ir/ir.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) { var dasm_flags = (X64 ? "-D X64=1" : "") + (X64 ? " -D X64WIN=1" : "") + " -D WIN=1"; var ir_target = (X64 ? "IR_TARGET_X64" : "IR_TARGET_X86"); - var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_ra_simple.c ir_save.c \ + var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_ra_simple.c ir_ra_fast.c ir_save.c \ ir_dump.c ir_check.c ir_patch.c"; DEFINE("IR_TARGET", ir_target); diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32 index b5273a1b86ea..e38ed60aedda 100644 --- a/ext/opcache/jit/Makefile.frag.w32 +++ b/ext/opcache/jit/Makefile.frag.w32 @@ -24,6 +24,11 @@ $(BUILD_DIR)\ext\opcache\jit\ir\ir_ra_simple.obj: \ ext\opcache\jit\ir\ir_private.h \ ext\opcache\jit\ir\ir_x86.h +$(BUILD_DIR)\ext\opcache\jit\ir\ir_ra_fast.obj: \ + ext\opcache\jit\ir\ir.h \ + ext\opcache\jit\ir\ir_private.h \ + ext\opcache\jit\ir\ir_x86.h + $(BUILD_DIR)\ext\opcache\jit\ir\ir_emit.obj: \ ext\opcache\jit\ir\ir.h \ ext\opcache\jit\ir\ir_private.h \ diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 734e9086648f..6cfd2a0007f6 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -119,7 +119,20 @@ static void ir_print_double(double v, FILE *f) char buf[128]; if (isnan(v)) { - fprintf(f, "nan"); + union { + double d; + long long i; + } u1, u2, u3; + u1.d = v; + u2.d = NAN; + u3.d = -NAN; + if (u1.i == u2.i) { + fprintf(f, "nan"); + } else if (u1.i == u3.i) { + fprintf(f, "-nan"); + } else { + fprintf(f, "nan(0x%llx)", u1.i); + } } else { snprintf(buf, sizeof(buf), "%g", v); if (strtod(buf, NULL) != v) { @@ -137,7 +150,20 @@ static void ir_print_float(float v, FILE *f) char buf[128]; if (isnan(v)) { - fprintf(f, "nan"); + union { + float f; + int i; + } u1, u2, u3; + u1.f = v; + u2.f = NAN; + u3.f = -NAN; + if (u1.i == u2.i) { + fprintf(f, "nan"); + } else if (u1.i == u3.i) { + fprintf(f, "-nan"); + } else { + fprintf(f, "nan(0x%x)", u1.i); + } } else { snprintf(buf, sizeof(buf), "%g", v); if (strtod(buf, NULL) != v) { diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 1d7bd07b3815..74481865e0f8 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -968,8 +968,9 @@ int ir_coalesce(ir_ctx *ctx); int ir_compute_dessa_moves(ir_ctx *ctx); int ir_reg_alloc(ir_ctx *ctx); int ir_reg_alloc_simple(ir_ctx *ctx); +int ir_reg_alloc_fast(ir_ctx *ctx); -#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_simple(_ctx) +#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_fast(_ctx) int ir_regs_number(void); bool ir_reg_is_int(int32_t reg); diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 2bc28e59e33f..965b3e9f7455 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -1413,8 +1413,9 @@ binop_fp: if (IR_IS_TYPE_VECTOR(insn->type)) { return IR_VECTOR_OP; } - IR_FALLTHROUGH; #endif + IR_ASSERT(IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)); + IR_FALLTHROUGH; case IR_BSWAP: case IR_CTLZ: case IR_CTTZ: @@ -3487,29 +3488,41 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) } if (ir_type_size[type] == 8) { - | cmp Rx(op1_reg), #0 - if (op2_reg != IR_REG_NONE) { - | add Rx(def_reg), Rx(op1_reg), Rx(op2_reg) + if (shift == 63) { + | movz Rx(def_reg), #0x8000, lsl #48 + | cmp Rx(def_reg), Rx(op1_reg) + | cset Rx(def_reg), eq } else { - | add Rx(def_reg), Rx(op1_reg), #offset + | cmp Rx(op1_reg), #0 + if (op2_reg != IR_REG_NONE) { + | add Rx(def_reg), Rx(op1_reg), Rx(op2_reg) + } else { + | add Rx(def_reg), Rx(op1_reg), #offset + } + | csel Rx(def_reg), Rx(def_reg), Rx(op1_reg), lt + | asr Rx(def_reg), Rx(def_reg), #shift } - | csel Rx(def_reg), Rx(def_reg), Rx(op1_reg), lt - | asr Rx(def_reg), Rx(def_reg), #shift } else { - | cmp Rw(op1_reg), #0 - if (op2_reg != IR_REG_NONE) { - | add Rw(def_reg), Rw(op1_reg), Rw(op2_reg) - } else { - | add Rw(def_reg), Rw(op1_reg), #offset - } - | csel Rw(def_reg), Rw(def_reg), Rw(op1_reg), lt - if (ir_type_size[type] == 4) { - | asr Rw(def_reg), Rw(def_reg), #shift - } else if (ir_type_size[type] == 2) { - | ubfx Rw(def_reg), Rw(def_reg), #shift, #16 + if (shift == 31) { + | movz Rw(def_reg), #0x8000, lsl #16 + | cmp Rw(def_reg), Rw(op1_reg) + | cset Rw(def_reg), eq } else { - IR_ASSERT(ir_type_size[type] == 1); - | ubfx Rw(def_reg), Rw(def_reg), #shift, #8 + | cmp Rw(op1_reg), #0 + if (op2_reg != IR_REG_NONE) { + | add Rw(def_reg), Rw(op1_reg), Rw(op2_reg) + } else { + | add Rw(def_reg), Rw(op1_reg), #offset + } + | csel Rw(def_reg), Rw(def_reg), Rw(op1_reg), lt + if (ir_type_size[type] == 4) { + | asr Rw(def_reg), Rw(def_reg), #shift + } else if (ir_type_size[type] == 2) { + | ubfx Rw(def_reg), Rw(def_reg), #shift, #16 + } else { + IR_ASSERT(ir_type_size[type] == 1); + | ubfx Rw(def_reg), Rw(def_reg), #shift, #8 + } } } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -3965,10 +3978,10 @@ static void ir_emit_binop_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_FP_REG_REG_REG_OP fdiv, type, def_reg, op1_reg, op2_reg break; case IR_MIN: - | ASM_FP_REG_REG_REG_OP fmin, type, def_reg, op1_reg, op2_reg + | ASM_FP_REG_REG_REG_OP fminnm, type, def_reg, op1_reg, op2_reg break; case IR_MAX: - | ASM_FP_REG_REG_REG_OP fmax, type, def_reg, op1_reg, op2_reg + | ASM_FP_REG_REG_REG_OP fmaxnm, type, def_reg, op1_reg, op2_reg break; } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -9272,10 +9285,10 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { IR_ASSERT(ival->stack_spill_pos == -1); ival->stack_spill_pos = stack_start + stack_offset; - if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { - /* Remap VAR to PARAM stack slot */ - ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); - } + } + if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { + /* Remap VAR to PARAM stack slot */ + ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); } } if (sizeof(void*) == 8) { diff --git a/ext/opcache/jit/ir/ir_cfg.c b/ext/opcache/jit/ir/ir_cfg.c index 4e4db166d37e..621f02a2372d 100644 --- a/ext/opcache/jit/ir/ir_cfg.c +++ b/ext/opcache/jit/ir/ir_cfg.c @@ -1106,6 +1106,34 @@ static IR_NEVER_INLINE uint32_t ir_collect_irreducible_loops(ir_ctx *ctx, uint32 return loops; } +static IR_NEVER_INLINE void ir_check_loop_entry_in_preds(ir_ctx *ctx, uint32_t hdr, ir_worklist *work) +{ + ir_block *blocks = ctx->cfg_blocks; + uint32_t *edges = ctx->cfg_edges; + + ir_worklist_clear(work); + ir_bitset_incl(work->visited, blocks[hdr].idom); + ir_worklist_push(work, hdr); + while (ir_worklist_len(work)) { + uint32_t b = ir_worklist_pop(work); + ir_block *bb = &blocks[b]; + + if (bb->flags & IR_BB_ENTRY) { + blocks[hdr].flags |= IR_BB_LOOP_WITH_ENTRY_IN_PREDS; + break; + } + + uint32_t *p = &edges[bb->predecessors]; + uint32_t count = bb->predecessors_count; + + for (; count > 0; p++, count--) { + if (!ir_dominates(blocks, hdr, *p)) { + ir_worklist_push(work, *p); + } + } + } +} + int ir_find_loops(ir_ctx *ctx) { uint32_t b, j, n; @@ -1295,6 +1323,10 @@ int ir_find_loops(ir_ctx *ctx) b = loops; do { bb = &blocks[b]; + if (ctx->entries_count + && (bb->predecessors_count > 2 || (bb->flags & IR_BB_IRREDUCIBLE_LOOP))) { + ir_check_loop_entry_in_preds(ctx, b, &work); + } b = bb->next_loop; IR_ASSERT(bb->flags & IR_BB_LOOP_HEADER); bb->loop_depth = (bb->loop_header) ? blocks[bb->loop_header].loop_depth + 1 : 1; diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index bc3ae5859be3..de6b53141a67 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -541,7 +541,9 @@ static IR_NEVER_INLINE void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_reg, ir_reg tmp_fp_reg) { int i; - int8_t *pred, *loc, *types; + int8_t *loc; /* array of the current locations of the original source registers: initially loc[from] = from */ + int8_t *pred; /* array of original source registers that must be copied to destinations: pred[to] = from */ + int8_t *types; /* array of the types of the orginal source registers: types[from] = typeof(from) */ ir_reg to, from; ir_type type; ir_regset todo, ready, srcs; @@ -625,18 +627,19 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ IR_ASSERT(tmp_reg == IR_REG_NONE || !IR_REGSET_IN(srcs, tmp_reg) || pred[loc[tmp_reg]] == tmp_reg); IR_ASSERT(tmp_fp_reg == IR_REG_NONE || !IR_REGSET_IN(srcs, tmp_fp_reg) || pred[loc[tmp_fp_reg]] == tmp_fp_reg); - /* now we resolve all "windmill axles" - cycles (this reuires temporary registers) */ + /* now we resolve all "windmill axles" - cycles (this requires temporary registers) */ while (todo != IR_REGSET_EMPTY) { - to = ir_regset_pop_first(&todo); + to = ir_regset_pop_first(&todo); /* select a register to break the cycle */ from = pred[to]; IR_ASSERT(to != loc[from]); - type = types[from]; + type = types[to]; /* type of the register that needs to be moved into a temporary register */ if (IR_IS_TYPE_INT(type)) { #ifdef IR_HAVE_SWAP_INT if (pred[from] == to) { - if (ir_type_size[types[to]] > ir_type_size[type]) { - type = types[to]; + if (ir_type_size[types[from]] > ir_type_size[type]) { + type = types[from]; } + /* resolve the cycle with 2 registers using SWAP */ ir_emit_swap(ctx, type, to, from); IR_REGSET_EXCL(todo, from); loc[to] = from; @@ -646,11 +649,13 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ #endif IR_ASSERT(tmp_reg != IR_REG_NONE); IR_ASSERT(tmp_reg >= IR_REG_GP_FIRST && tmp_reg <= IR_REG_GP_LAST); + /* break cycle, moving one of its register into a temporary register */ ir_emit_mov(ctx, type, tmp_reg, to); loc[to] = tmp_reg; } else { #ifdef IR_HAVE_SWAP_FP - if (pred[from] == to && types[to] == type) { + if (pred[from] == to && types[from] == type) { + /* resolve the cycle with 2 registers using SWAP */ ir_emit_swap_fp(ctx, type, to, from); IR_REGSET_EXCL(todo, from); loc[to] = from; @@ -660,6 +665,7 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_ #endif IR_ASSERT(tmp_fp_reg != IR_REG_NONE); IR_ASSERT(tmp_fp_reg >= IR_REG_FP_FIRST && tmp_fp_reg <= IR_REG_FP_LAST); + /* break cycle, moving one of its register into a temporary register */ ir_emit_fp_mov(ctx, type, tmp_fp_reg, to); loc[to] = tmp_fp_reg; } diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 16dc6f536b8c..1debe2fa25a0 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -30,7 +30,7 @@ IR_FOLD(EQ(C_DOUBLE, C_DOUBLE)) IR_FOLD(EQ(C_FLOAT, C_FLOAT)) { - IR_FOLD_BOOL(op1_insn->val.d == op2_insn->val.d); + IR_FOLD_BOOL(op1_insn->val.f == op2_insn->val.f); } IR_FOLD(NE(C_BOOL, C_BOOL)) @@ -1001,7 +1001,8 @@ IR_FOLD(MUL_OV(C_I64, C_I64)) int64_t res; IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); res = op1_insn->val.u64 * op2_insn->val.u64; - if ((op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) { + if ((op1_insn->val.i64 == -1 && res == INT64_MIN) + || (op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) { IR_FOLD_NEXT; } IR_FOLD_CONST_U(res); diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c index 712411774a47..54c894f27cba 100644 --- a/ext/opcache/jit/ir/ir_gcm.c +++ b/ext/opcache/jit/ir/ir_gcm.c @@ -100,7 +100,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca) #endif flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags; - if ((flags & IR_BB_LOOP_WITH_ENTRY) + if ((flags & (IR_BB_LOOP_WITH_ENTRY|IR_BB_LOOP_WITH_ENTRY_IN_PREDS)) && !(ctx->binding && ir_binding_find(ctx, ref))) { /* Don't move loop invariant code across an OSR ENTRY if we can't restore it */ return lca; @@ -135,7 +135,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca) break; } flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags; - if ((flags & IR_BB_LOOP_WITH_ENTRY) + if ((flags & (IR_BB_LOOP_WITH_ENTRY|IR_BB_LOOP_WITH_ENTRY_IN_PREDS)) && !(ctx->binding && ir_binding_find(ctx, ref))) { break; } diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index 6fa8efa9cf71..9f91fbfba9f8 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -1204,6 +1204,7 @@ void ir_iter_cleanup(ir_ctx *ctx); #define IR_BB_PREV_EMPTY_ENTRY (1<<8) #define IR_BB_OSR_ENTRY_LOADS (1<<9) /* OSR Entry-point with register LOADs */ #define IR_BB_LOOP_WITH_ENTRY (1<<10) /* set together with LOOP_HEADER if there is an ENTRY in the loop */ +#define IR_BB_LOOP_WITH_ENTRY_IN_PREDS (1<<18) /* set together with LOOP_HEADER if an ENTRY reaches the loop bypassing its immediate dominator */ /* The following flags are set by GCM */ #define IR_BB_HAS_PHI (1<<11) diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index 53c3cbd8a047..f8d418d06140 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -812,6 +812,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) IR_ASSERT(use_insn->op3 == ref); /* skip VSTORE (VAR is going to be remapped to PARAM) */ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + if (ctx->use_lists[ref].count != 1) { + param_ival->flags &= ~IR_LIVE_INTERVAL_MEM_PARAM; + } param_ival->use_pos = param_ival->use_pos->next; ctx->rules[use] = IR_SKIPPED | IR_NOP; } @@ -1530,6 +1533,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) IR_ASSERT(use_insn->op3 == ref); /* skip VSTORE (VAR is going to be remapped to PARAM) */ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + if (ctx->use_lists[ref].count != 1) { + param_ival->flags &= ~IR_LIVE_INTERVAL_MEM_PARAM; + } param_ival->use_pos = param_ival->use_pos->next; ctx->rules[use] = IR_SKIPPED | IR_NOP; } diff --git a/ext/opcache/jit/ir/ir_ra_fast.c b/ext/opcache/jit/ir/ir_ra_fast.c new file mode 100644 index 000000000000..a3c893167561 --- /dev/null +++ b/ext/opcache/jit/ir/ir_ra_fast.c @@ -0,0 +1,1481 @@ +/* + * IR - Lightweight JIT Compilation Framework + * Fast Local Register Allocation (used for -O0) + * This file is part of the IR Project distributed under the MIT-style LICENSE. + * Authors: Dmitry Stogov + */ + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif + +#include "ir.h" + +#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) +# include "ir_x86.h" +#elif defined(IR_TARGET_AARCH64) +# include "ir_aarch64.h" +#else +# error "Unknown IR target" +#endif + +#include "ir_private.h" + +typedef struct { + ir_reg_alloc_data ra_data; + ir_block *bb; + int32_t num; + int32_t tmp_num; + ir_regset scratch; + ir_regset preserved; + ir_regset fixed; + ir_regset allocated; + ir_regset clobbered_args; + ir_regset clobbered[IR_SUB_REFS_COUNT]; + struct { + ir_ref def; + ir_ref root; + ir_ref use; + ir_ref op; + } state[IR_REG_NUM]; + struct { + uint8_t type; + int8_t start; + int8_t end; + int8_t hint; + int8_t flags; + ir_ref root; + ir_ref ref; + ir_ref op; + ir_ref def; + } regs[32], tmp_regs[32]; +} ir_reg_alloc_fast_data; + +#ifdef IR_DEBUG +# define IR_DEBUG_SPILLS 1 +# if IR_X86_I64 +# define IR_REG_NAME_FMT "%s%s%s" +# define IR_REG_NAME_VAL(_reg, _type) (((_type) == IR_I64 || (_type) == IR_U64) ? \ + ir_reg_name(IR_REG_I64_LO(IR_REG_NUM(_reg)), IR_U32) : \ + ir_reg_name(IR_REG_NUM(_reg), _type)), \ + (((_type) == IR_I64 || (_type) == IR_U64) ? " and " : ""), \ + (((_type) == IR_I64 || (_type) == IR_U64) ? \ + ir_reg_name(IR_REG_I64_HI(IR_REG_NUM(_reg)), IR_U32) : "") +# else +# define IR_REG_NAME_FMT "%s" +# define IR_REG_NAME_VAL(_reg, _type) ir_reg_name(IR_REG_NUM(_reg), _type) +# endif +# define IR_LOG(comment) do { \ + if (ctx->flags & IR_DEBUG_RA) { \ + fprintf(stderr, "---- %s\n", comment); \ + } \ + } while (0) +# define IR_LOG_ASSIGN(_ref, _pos, _x, _y, _reg) do { \ + if (ctx->flags & IR_DEBUG_RA) { \ + fprintf(stderr, " ---- Assign R%d (d_%d) to " IR_REG_NAME_FMT "%s at %d (%d.%d)\n", \ + ctx->vregs[_ref], _ref, \ + IR_REG_NAME_VAL(_reg, ctx->ir_base[_ref].type), \ + (((_reg) & IR_REG_SPILL_LOAD) ? " (spill)" : ""), \ + _pos, _x, _y); \ + } \ + } while (0) +# define IR_LOG_ASSIGN_TMP(_type, _pos, _x, _y, _reg) do { \ + if (ctx->flags & IR_DEBUG_RA) { \ + fprintf(stderr, " ---- Assign TMP to " IR_REG_NAME_FMT "%s at %d (%d.%d)\n", \ + IR_REG_NAME_VAL(_reg, _type), \ + (((_reg) & IR_REG_SPILL_LOAD) ? " (spill)" : ""), \ + _pos, _x, _y); \ + } \ + } while (0) +# define IR_LOG_ASSIGN_VAR(_var, _spill) do { \ + if (IR_DEBUG_SPILLS && (ctx->flags & IR_DEBUG_RA)) { \ + fprintf(stderr, " ---- Spill VAR d_%d to 0x%x\n", \ + _var, _spill); \ + } \ + } while (0) +# define IR_LOG_ASSIGN_SPILL(_vreg, _spill) do { \ + if (IR_DEBUG_SPILLS && (ctx->flags & IR_DEBUG_RA)) { \ + fprintf(stderr, " ---- Spill R%d to 0x%x\n", \ + _vreg, _spill); \ + } \ + } while (0) +# define IR_LOG_FREE(_ref, _reg) do { \ + if (ctx->flags & IR_DEBUG_RA) { \ + fprintf(stderr, " ---- Free R%d(d_%d) assigned to " IR_REG_NAME_FMT "\n", \ + ctx->vregs[_ref], _ref, \ + IR_REG_NAME_VAL(_reg, ctx->ir_base[_ref].type)); \ + } \ + } while (0) +# define IR_LOG_EVICT(_ref, _reg) do { \ + if (ctx->flags & IR_DEBUG_RA) { \ + fprintf(stderr, " ---- Evict R%d(d_%d) assigned to " IR_REG_NAME_FMT "\n", \ + ctx->vregs[_ref], _ref, \ + IR_REG_NAME_VAL(_reg, ctx->ir_base[_ref].type)); \ + } \ + } while (0) +# define IR_LOG_SPILL_LOAD(_ref, _pos, _x, _y, _reg) do { \ + if (ctx->flags & IR_DEBUG_RA) { \ + fprintf(stderr, " ---- Add Spill Load R%d (d_%d) to " IR_REG_NAME_FMT " at %d (%d.%d)\n", \ + ctx->vregs[_ref], _ref, \ + IR_REG_NAME_VAL(_reg, ctx->ir_base[_ref].type), \ + _pos, _x, _y); \ + } \ + } while (0) +#else +# define IR_LOG(comment) +# define IR_LOG_ASSIGN(_ref, _pos, _x, _y, _reg) +# define IR_LOG_ASSIGN_TMP(_type, _pos, _x, _y, _reg) +# define IR_LOG_ASSIGN_VAR(_vreg, _spill) +# define IR_LOG_ASSIGN_SPILL(_vreg, _spill) +# define IR_LOG_FREE(_ref, _reg) +# define IR_LOG_EVICT(_ref, _reg) +# define IR_LOG_SPILL_LOAD(_ref, _pos, _x, _y, _reg) +#endif + +static void _add_scratch(ir_reg_alloc_fast_data *data, ir_reg reg, int8_t start, int8_t end) +{ + int8_t j; + + if (start < 0) { + /* ARGVAL support */ + if (reg >= IR_REG_NUM) { + data->clobbered_args = IR_REGSET_UNION(data->clobbered_args, ir_scratch_regset[reg - IR_REG_NUM]); + } else { + IR_REGSET_INCL(data->clobbered_args, reg); + } + start = 0; + } + IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT); + if (reg >= IR_REG_NUM) { + for (j = start; j < end; j++) { + data->clobbered[j] = IR_REGSET_UNION(data->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]); + } + } else { + for (j = start; j < end; j++) { + IR_REGSET_INCL(data->clobbered[j], reg); + } + } +} + +static void _add_tmp_reg(ir_reg_alloc_fast_data *data, ir_type type, + int8_t start, int8_t end, uint8_t flags, + ir_ref root, ir_ref ref, ir_ref op) +{ + uintptr_t n; + + IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32); + n = data->tmp_num++; + data->tmp_regs[n].type = type; + data->tmp_regs[n].start = start; + data->tmp_regs[n].end = end; + data->tmp_regs[n].flags = flags; + data->tmp_regs[n].root = root; + data->tmp_regs[n].ref = ref; + data->tmp_regs[n].op = op; +} + +static void _add_reg(ir_reg_alloc_fast_data *data, ir_type type, + int8_t start, int8_t end, ir_reg hint, int8_t flags, + ir_ref root, ir_ref ref, ir_ref op, ir_ref def) +{ + uintptr_t n; + + IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32); + n = data->num++; + data->regs[n].type = type; + data->regs[n].start = start; + data->regs[n].end = end; + data->regs[n].hint = hint; + data->regs[n].flags = flags; + data->regs[n].root = root; + data->regs[n].ref = ref; + data->regs[n].op = op; + data->regs[n].def = def; +} + +static void _pick_spill(ir_ctx *ctx, ir_live_interval *ival) +{ + size_t size = ir_get_type_size(ival->type); + + IR_ASSERT(ival->stack_spill_pos == -1); +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { + size = 4; + } +#endif + ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); + IR_LOG_ASSIGN_SPILL(ival->vreg, ival->stack_spill_pos); +} + +static void _evict_reg(ir_ctx *ctx, ir_reg_alloc_fast_data *data, ir_reg reg) +{ + ir_live_interval *ival; + ir_ref ref = data->state[reg].def; + + IR_ASSERT(ctx->vregs[ref] > 0); + ival = ctx->live_intervals[ctx->vregs[ref]]; + + /* allocate spill slot */ + if (ival->stack_spill_pos == -1) { + _pick_spill(ctx, ival); + } + + /* free reg */ +#if IR_X86_I64 + if (ival->type == IR_I64 || ival->type == IR_U64) { + if (ival->reg == IR_REG_NONE && ival->reg_hi == IR_REG_NONE) { + /* already evicted */ + return; + } + + IR_ASSERT(ival->reg != IR_REG_NONE && ival->reg_hi != IR_REG_NONE && (ival->reg == reg || ival->reg_hi == reg)); + + ir_reg pair = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); + IR_LOG_EVICT(ref, pair); + IR_REGSET_EXCL(data->allocated, ival->reg); + IR_REGSET_EXCL(data->allocated, ival->reg_hi); + ival->reg = ival->reg_hi = IR_REG_NONE; + + if (data->state[reg].use) { + /* add spill load for the previous use */ + if (!data->state[reg].root + || ctx->ir_base[data->state[reg].use].op == IR_ARGVAL) { + ir_set_alocated_reg(ctx, data->state[reg].use, data->state[reg].op, pair | IR_REG_SPILL_LOAD); + } else { + ctx->rules[data->state[reg].use] |= IR_FUSED_REG; + ir_set_fused_reg(ctx, data->state[reg].root, data->state[reg].use, data->state[reg].op, + pair | IR_REG_SPILL_LOAD); + } + IR_LOG_SPILL_LOAD(ref, data->state[reg].root, data->state[reg].use, data->state[reg].op, pair); + } + return; + } +#endif + IR_ASSERT(ival->reg == reg); + IR_LOG_EVICT(ref, reg); + IR_REGSET_EXCL(data->allocated, reg); + ival->reg = IR_REG_NONE; + + if (data->state[reg].use) { + /* add spill load for the previous use */ + if (!data->state[reg].root + || ctx->ir_base[data->state[reg].use].op == IR_ARGVAL) { + ir_set_alocated_reg(ctx, data->state[reg].use, data->state[reg].op, reg | IR_REG_SPILL_LOAD); + } else { + ctx->rules[data->state[reg].use] |= IR_FUSED_REG; + ir_set_fused_reg(ctx, data->state[reg].root, data->state[reg].use, data->state[reg].op, + reg | IR_REG_SPILL_LOAD); + } + IR_LOG_SPILL_LOAD(ref, data->state[reg].root, data->state[reg].use, data->state[reg].op, reg); + } +} + +static ir_regset _preferred_regset(ir_ctx *ctx, ir_reg_alloc_fast_data *data, ir_ref use, ir_ref def) +{ + ir_ref ref; + + if (!use || def < data->bb->start || def >= use) { + return data->scratch; + } + for (ref = ctx->prev_ref[use]; ref != def; ref = ctx->prev_ref[ref]) { + if (ctx->ir_base[ref].op == IR_CALL) { + return data->preserved; + } + } + return data->scratch; +} + +static ir_reg _pick_free_reg(ir_ctx *ctx, ir_reg_alloc_fast_data *data, ir_type type, ir_ref use, ir_ref def) +{ + ir_regset available; + ir_ref reg; + + if (IR_IS_TYPE_INT(type)) { + available = IR_REGSET_GP; +#if defined(IR_TARGET_X86) + if (ir_type_size[type] == 1) { + IR_REGSET_EXCL(available, IR_REG_RBP); + IR_REGSET_EXCL(available, IR_REG_RSI); + IR_REGSET_EXCL(available, IR_REG_RDI); + } +#endif + } else { + IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); + available = IR_REGSET_FP; + } + available = IR_REGSET_DIFFERENCE(available, data->allocated); + if (!IR_REGSET_IS_EMPTY(available)) { + ir_regset preferred = IR_REGSET_INTERSECTION(_preferred_regset(ctx, data, use, def), available); + + if (!IR_REGSET_IS_EMPTY(preferred)) { + reg = IR_REGSET_FIRST(preferred); + if (IR_REGSET_IN(data->preserved, reg)) { + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; + IR_REGSET_INCL(tmp, reg); + ctx->used_preserved_regs = tmp; + } + return reg; + } + reg = IR_REGSET_FIRST(available); + if (IR_REGSET_IN(data->preserved, reg)) { + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; + IR_REGSET_INCL(tmp, reg); + ctx->used_preserved_regs = tmp; + } + return reg; + } + return IR_REG_NONE; +} + +static ir_reg _pick_reg(ir_ctx *ctx, ir_reg_alloc_fast_data *data, ir_type type, ir_regset used, uint8_t flags, ir_ref use, ir_ref def) +{ + ir_regset available; + ir_reg reg, best_reg; + ir_ref best_cost; + + if (IR_IS_TYPE_INT(type)) { + available = IR_REGSET_GP; +#if defined(IR_TARGET_X86) + if (ir_type_size[type] == 1) { + IR_REGSET_EXCL(available, IR_REG_RBP); + IR_REGSET_EXCL(available, IR_REG_RSI); + IR_REGSET_EXCL(available, IR_REG_RDI); + } +#endif + } else { + IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); + available = IR_REGSET_FP; + } + available = IR_REGSET_DIFFERENCE(available, used); + if (!IR_REGSET_IS_EMPTY(available)) { + ir_regset available_unused = IR_REGSET_DIFFERENCE(available, data->allocated); + ir_regset preferred = IR_REGSET_INTERSECTION(_preferred_regset(ctx, data, use, def), available); + + if (!IR_REGSET_IS_EMPTY(available_unused)) { + ir_regset available_preferred = IR_REGSET_INTERSECTION(available_unused, preferred); + + if (!IR_REGSET_IS_EMPTY(available_preferred)) { + reg = IR_REGSET_FIRST(available_preferred); + if (IR_REGSET_IN(data->preserved, reg)) { + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; + IR_REGSET_INCL(tmp, reg); + ctx->used_preserved_regs = tmp; + } + return reg; + } + reg = IR_REGSET_FIRST(available_unused); + if (IR_REGSET_IN(data->preserved, reg)) { + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; + IR_REGSET_INCL(tmp, reg); + ctx->used_preserved_regs = tmp; + } + return reg; + } + + if (!(flags & IR_USE_MUST_BE_IN_REG)) { + return IR_REG_NONE; + } + + /* select best reg to evict */ + available = IR_REGSET_INTERSECTION(available, data->allocated); + best_cost = 0x7fffffff; + best_reg = IR_REG_NONE; + IR_REGSET_FOREACH(available, reg) { + if (use) { + /* skip registers allocated for the same instruction */ + if (data->state[reg].root) { + if (data->state[reg].root == use) continue; + } else { + if (data->state[reg].use == use) continue; + } + } + if (data->state[reg].def < best_cost) { + best_cost = data->state[reg].def; + best_reg = reg; + } + } IR_REGSET_FOREACH_END(); + + _evict_reg(ctx, data, best_reg); + return best_reg; + } + + if (!(flags & IR_USE_MUST_BE_IN_REG)) { + return IR_REG_NONE; + } + + fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n"); + IR_ASSERT(0); + exit(-1); +} + +static ir_live_interval *_create_ival(ir_ctx *ctx, uint32_t vreg, ir_type type) +{ + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + + memset(ival, 0, sizeof(ir_live_interval)); + ctx->live_intervals[vreg] = ival; + ival->type = type; + ival->reg = IR_REG_NONE; + ival->vreg = vreg; + ival->stack_spill_pos = -1; + return ival; +} + +IR_ALWAYS_INLINE ir_live_interval *_get_ival(ir_ctx *ctx, ir_ref ref) +{ + int32_t vreg = ctx->vregs[ref]; + ir_live_interval *ival; + + IR_ASSERT(vreg > 0); + ival = ctx->live_intervals[vreg]; + if (!ival) { + return _create_ival(ctx, vreg, ctx->ir_base[ref].type); + } + return ival; +} + +static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn) +{ + size_t size = ir_get_type_size(var_insn->type); + size_t align = var_insn->op3; + int32_t offset; + +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) { + size = 4; + align = IR_MAX(align, 4); + } +#endif + + offset = ir_allocate_spill_slot(ctx, size, align); + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); + IR_LOG_ASSIGN_VAR(var, offset); +} + +static bool _may_coalesce(ir_ctx *ctx, ir_ref input, ir_ref def) +{ + /* Coalescing of values connected trough PHI requires global live range analyzes */ + return 0; +} + +static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_fast_data *data, ir_ref start) +{ + ir_ref stack[4]; + int stack_pos = 0; + ir_target_constraints constraints; + ir_insn *insn; + uint32_t j, n, def_flags; + ir_ref *p, child; + + while (1) { + IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED); + + if (!(ctx->rules[input] & IR_SIMPLE)) { + def_flags = ir_get_target_constraints(ctx, input, &constraints); + n = constraints.tmps_count; + while (n > 0) { + n--; + if (constraints.tmp_regs[n].type) { + uint8_t flags = 0; + ir_ref op = constraints.tmp_regs[n].num; + + if (op > 0 && op <= ctx->ir_base[input].inputs_count) { + ir_ref *ops = ctx->ir_base[input].ops; + + if (IR_IS_CONST_REF(ops[op])) { + /* rematerialization */ + flags = IR_REG_SPILL_LOAD; + } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { + /* local address rematerialization */ + flags = IR_REG_SPILL_LOAD; + } + } + _add_tmp_reg(data, constraints.tmp_regs[n].type, + constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, flags, + ref, input, op); + } else { + _add_scratch(data, constraints.tmp_regs[n].reg, + constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); + } + } + } else { + def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; + constraints.hints_count = 0; + } + + insn = &ctx->ir_base[input]; + n = insn->inputs_count; + j = 1; + p = insn->ops + j; + if (ir_op_flags[insn->op] & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) { + j++; + p++; + } + for (; j <= n; j++, p++) { + IR_ASSERT(IR_OPND_KIND(ir_op_flags[insn->op], j) == IR_OPND_DATA); + child = *p; + if (child > 0) { + if (ctx->vregs[child] > 0) { + uint32_t use_flags = IR_USE_FLAGS(def_flags, j); + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[child]]; + + if (!ival) { + ival = _create_ival(ctx, ctx->vregs[child], ctx->ir_base[child].type); + } + if (child < start || child >= ref) { + /* live before BB start - spill it */ + if (ival->stack_spill_pos == -1) { + _pick_spill(ctx, ival); + } + if (!(use_flags & IR_USE_MUST_BE_IN_REG)) { + ctx->rules[input] |= IR_FUSED_REG; + continue; + } + } + + ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; + int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF; + + _add_reg(data, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, use_flags, + ref, input, j, child); + } else if (ctx->rules[child] & IR_FUSED) { + IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos))); + stack[stack_pos++] = child; + } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) { + ir_set_alocated_reg(ctx, input, j, ctx->ir_base[child].op2); + } + } + } + if (!stack_pos) { + break; + } + input = stack[--stack_pos]; + } +} + +int ir_reg_alloc_fast(ir_ctx *ctx) +{ + uint32_t b; + ir_block *bb; + ir_insn *insn; + ir_ref i, n, j, *p; + ir_target_constraints constraints; + uint32_t def_flags; + ir_reg reg; + ir_regset param_regs; + ir_reg_alloc_fast_data data; + int32_t offset; + + ir_compute_dessa_moves(ctx); + + IR_LOG("Start Fast Local RA"); + + memset(&data.ra_data, 0, sizeof(data.ra_data)); + data.ra_data.cc = ir_get_call_conv_dsc(ctx->flags); + ctx->data = &data.ra_data; + + ctx->stack_frame_size = 0; + ctx->call_stack_size = 0; + ctx->used_preserved_regs = ctx->fixed_save_regset; + + data.fixed = (ir_regset)ctx->fixed_regset; + if (ctx->flags & IR_USE_FRAME_POINTER) { + IR_REGSET_INCL(data.fixed, IR_REG_FRAME_POINTER); + } + + data.scratch = IR_REGSET_DIFFERENCE(ir_scratch_regset[data.ra_data.cc->scratch_reg - IR_REG_NUM], data.fixed); + data.preserved = IR_REGSET_DIFFERENCE(data.ra_data.cc->preserved_regs, data.fixed); + param_regs = IR_REGSET_EMPTY; + + ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count); + memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count); + + /* vregs + tmp + fixed + SRATCH + ALL */ + ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*)); + + if (!ctx->arena) { + ctx->arena = ir_arena_create(16 * 1024); + } + + /* For all basic blocks in reverse order */ + for (b = ctx->cfg_blocks_count, bb = ctx->cfg_blocks + b; b > 0; bb--, b--) { + IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE)); + + if (bb->flags & IR_BB_EMPTY) { + continue; + } + + data.allocated = IR_REGSET_EMPTY; + data.bb = bb; + + if (bb->flags & IR_BB_DESSA_MOVES) { + ir_block *succ_bb; + ir_use_list *use_list; + ir_ref k, *p, n; + bool needs_tmp_gp = 0; + bool needs_tmp_fp = 0; + + /* For all PHI functions in the successor of this block */ + IR_ASSERT(bb->successors_count == 1); + succ_bb = &ctx->cfg_blocks[ctx->cfg_edges[bb->successors]]; + k = ir_phi_input_number(ctx, succ_bb, b); + use_list = &ctx->use_lists[succ_bb->start]; + n = use_list->count; + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + ir_ref use = *p; + ir_insn *use_insn = &ctx->ir_base[use]; + + if (use_insn->op == IR_PHI && ctx->vregs[use] > 0) { + ir_ref input = ir_insn_op(use_insn, k); + + /* Require tmp register for potential MEM->MEM DESSA move */ + if (use < bb->start || ctx->regs[use][0] == IR_REG_NONE) { + if (IR_IS_TYPE_INT(use_insn->type)) { + needs_tmp_gp = 1; + } else { + needs_tmp_fp = 1; + } + } + + if (input > 0 && ctx->vregs[input] > 0) { + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[input]]; + + if (!ival) { + ival = _create_ival(ctx, ctx->vregs[input], ctx->ir_base[input].type); + } + + /* If input of PHI, coming from this block, is defined in this block + * (otherwise it's treated as "global" and spilled). + */ + if (input > bb->start && input < bb->end) { +#if IR_X86_I64 + if (use_insn->type == IR_I64 || use_insn->type == IR_U64) { + ir_reg reg_hi, reg_lo; + + if (ival->reg != IR_REG_NONE) { + /* reuse previously allocated registers */ + reg = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); + ir_set_alocated_reg(ctx, use, k, reg); + IR_LOG_ASSIGN(input, bb->end, use, k, reg); + continue; + } else if (ctx->regs[use][0] != IR_REG_NONE + && !IR_REG_SPILLED(ctx->regs[use][0]) + && !IR_REGSET_IN(data.allocated, IR_REG_I64_LO(ctx->regs[use][0])) + && !IR_REGSET_IN(data.allocated, IR_REG_I64_HI(ctx->regs[use][0]))) { + /* lightweight register coalescing */ + reg = ctx->regs[use][0]; + reg_lo = IR_REG_I64_LO(reg); + reg_hi = IR_REG_I64_HI(reg); + } else { + reg_lo = _pick_free_reg(ctx, &data, ival->type, bb->end, input); + if (reg_lo == IR_REG_NONE) { + reg = IR_REG_NONE; + } else { + IR_REGSET_INCL(data.allocated, reg_lo); + reg_hi = _pick_free_reg(ctx, &data, ival->type, bb->end, input); + if (reg_hi == IR_REG_NONE) { + IR_REGSET_EXCL(data.allocated, reg_lo); + reg = IR_REG_NONE; + } else { + IR_ASSERT(reg_lo != reg_hi); + if (reg_lo > reg_hi) { + SWAP_REGS(reg_lo, reg_hi); + } + reg = IR_REG_I64_PAIR(reg_lo, reg_hi); + } + } + } + if (reg != IR_REG_NONE) { + data.state[reg_hi].def = input; + data.state[reg_hi].root = IR_UNUSED; + data.state[reg_hi].use = use; + data.state[reg_hi].op = k; + IR_REGSET_INCL(data.allocated, reg_hi); + ival->reg_hi = reg_hi; + data.state[reg_lo].def = input; + data.state[reg_lo].root = IR_UNUSED; + data.state[reg_lo].use = use; + data.state[reg_lo].op = k; + IR_REGSET_INCL(data.allocated, reg_lo); + ival->reg = reg_lo; + ir_set_alocated_reg(ctx, use, k, reg); + IR_LOG_ASSIGN(input, bb->end, use, k, reg); + continue; + } + } else +#endif + { + if (ival->reg != IR_REG_NONE) { + /* reuse previously allocated register */ + reg = ival->reg; + ir_set_alocated_reg(ctx, use, k, reg); + IR_LOG_ASSIGN(input, bb->end, use, k, reg); + continue; + } else if (ctx->regs[use][0] != IR_REG_NONE + && !IR_REG_SPILLED(ctx->regs[use][0]) + && !IR_REGSET_IN(data.allocated, ctx->regs[use][0])) { + /* lightweight register coalescing */ + reg = ctx->regs[use][0]; + } else { + reg = _pick_free_reg(ctx, &data, ival->type, bb->end, input); + } + if (reg != IR_REG_NONE) { + data.state[reg].def = input; + data.state[reg].root = IR_UNUSED; + data.state[reg].use = use; + data.state[reg].op = k; + IR_REGSET_INCL(data.allocated, reg); + ival->reg = reg; + ir_set_alocated_reg(ctx, use, k, reg); + IR_LOG_ASSIGN(input, bb->end, use, k, reg); + continue; + } + } + } + + if (ival->stack_spill_pos == -1) { + if (ctx->vregs[use] > 0 + && ctx->live_intervals[ctx->vregs[use]] + && ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos != -1 + && _may_coalesce(ctx, input, use)) { + /* lightweight spill slot coalescing */ + ival->stack_spill_pos = ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos; + IR_LOG_ASSIGN_SPILL(ival->vreg, ival->stack_spill_pos); + } else { + _pick_spill(ctx, ival); + } + } + } + } + } + + /* Add tmp registers for MEM->MEM DESSA moves */ + if (needs_tmp_gp) { + ctx->regs[bb->end][0] = _pick_reg(ctx, &data, IR_I32, IR_REGSET_EMPTY, IR_USE_MUST_BE_IN_REG, 0, 0); + } + if (needs_tmp_fp) { + ctx->regs[bb->end][1] = _pick_reg(ctx, &data, IR_DOUBLE, IR_REGSET_EMPTY, IR_USE_MUST_BE_IN_REG, 0, 0); + } + } + + /* For all basic block's instructions in reverse order */ + for (i = bb->end; i != bb->start; i = ctx->prev_ref[i]) { + uint32_t rule = ctx->rules[i]; + + if (rule & (IR_FUSED|IR_SKIPPED)) { + if ((rule & IR_RULE_MASK) == IR_ALLOCA) { + insn = ctx->ir_base + i; + + /* assign "static" spill slots for VARs and ALLOCAs */ + if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) { + /* spill slot already allocated */ + } else if (insn->op == IR_VAR) { + if (ctx->use_lists[i].count > 0) { + ir_allocate_var_spill_slot(ctx, i, insn); + } + } else if (insn->op == IR_ALLOCA) { + if (ctx->use_lists[i].count > 0) { + ir_insn *val = &ctx->ir_base[insn->op2]; + + IR_ASSERT(IR_IS_CONST_REF(insn->op2)); + IR_ASSERT(IR_IS_TYPE_INT(val->type)); + IR_ASSERT(!IR_IS_SYM_CONST(val->op)); + IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); + IR_ASSERT(val->val.i64 < 0x7fffffff); + offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); + ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset); + IR_LOG_ASSIGN_VAR(i, offset); + } + } else if (insn->op == IR_VADDR) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) { + ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]); + } + ctx->vregs[i] = ctx->vregs[insn->op1]; + } + } + /* registers for fused instructions are allocated together with their "root"s */ + continue; + } + + insn = ctx->ir_base + i; + if (ctx->vregs[i] > 0) { + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[i]]; + + if (!ival || (ival->reg == IR_REG_NONE && ival->stack_spill_pos == -1)) { + if (IR_IS_FOLDABLE_OP(insn->op) || (ir_op_flags[insn->op] & IR_OP_FLAG_NO_SIDE_EFFECT)) { + /* lightweigh dead code elimination */ + ctx->rules[i] = IR_SKIPPED | IR_NOP; + ctx->vregs[i] = 0; + continue; + } else if (insn->op == IR_PARAM) { + /* unused PARAM */ + ctx->rules[i] = IR_SKIPPED | IR_PARAM; + ctx->vregs[i] = 0; + continue; + } else if (insn->op == IR_CALL) { + /* unused CALL result */ + ctx->vregs[i] = 0; + } + + /* Handle PHI specially (inputs come from other blocks) */ + } else if (insn->op == IR_PHI) { + IR_ASSERT(ival); + if (ival->reg != IR_REG_NONE) { + /* use the previously allocated register */ +#if IR_X86_I64 + if (ival->type == IR_I64 || ival->type == IR_U64) { + IR_REGSET_EXCL(data.allocated, ival->reg); + IR_REGSET_EXCL(data.allocated, ival->reg_hi); + reg = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); + if (ival->stack_spill_pos != -1) { +#if 1 + /* Spilled PHI var is passed through memory */ + _evict_reg(ctx, &data, ival->reg); + reg = IR_REG_NONE; +#else + + reg |= IR_REG_SPILL_STORE; +#endif + } + ival->reg = ival->reg_hi = IR_REG_NONE; + ir_set_alocated_reg(ctx, i, 0, reg); + if (reg != IR_REG_NONE) { + IR_LOG_ASSIGN(i, i, i, 0, reg); + } + continue; + } +#endif + reg = ival->reg; + IR_REGSET_EXCL(data.allocated, reg); + if (ival->stack_spill_pos != -1) { +#if 1 + /* Spilled PHI var is passed through memory */ + _evict_reg(ctx, &data, reg); + reg = IR_REG_NONE; +#else + reg |= IR_REG_SPILL_STORE; +#endif + } + ival->reg = IR_REG_NONE; + ir_set_alocated_reg(ctx, i, 0, reg); + if (reg != IR_REG_NONE) { + IR_LOG_ASSIGN(i, i, i, 0, reg); + } + } else { + IR_ASSERT(ival->stack_spill_pos != -1); + } + continue; + + /* An attempt to swap instruction operands for better op1 register reuse */ + } else if ((rule & IR_MAY_SWAP) + && insn->op2 > 0 + && ctx->vregs[insn->op2] > 0 + && insn->op2 > bb->start + && (!ctx->live_intervals[ctx->vregs[insn->op2]] + || ctx->live_intervals[ctx->vregs[insn->op2]]->reg == IR_REG_NONE) + && !(insn->op1 > 0 + && ctx->vregs[insn->op1] > 0 + && insn->op1 > bb->start + && (!ctx->live_intervals[ctx->vregs[insn->op1]] + || ctx->live_intervals[ctx->vregs[insn->op1]]->reg == IR_REG_NONE))) { + SWAP_REFS(insn->op1, insn->op2); + } + } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { + ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); + } + if (b == 1 + && insn->op == IR_VSTORE + && !IR_IS_CONST_REF(insn->op3) + && ctx->ir_base[insn->op3].op == IR_PARAM + /* first use of VAR and PARAM is VSTORE(_, VAR, PARAM) */ + && ctx->use_edges[ctx->use_lists[insn->op2].refs] == i + && ctx->use_edges[ctx->use_lists[insn->op3].refs] == i) { + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op3]]; + + if (!ival) { + ival = _create_ival(ctx, ctx->vregs[insn->op3], ctx->ir_base[insn->op3].type); + } + if (ival->stack_spill_pos == -1) { + ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]); + IR_LOG_ASSIGN_SPILL(ival->vreg, ival->stack_spill_pos); + continue; + } + } + } + + /* Reset information for the current instruction */ + data.num = data.tmp_num = 0; + data.clobbered_args = data.fixed; + for (j = 0; j < IR_SUB_REFS_COUNT; j++) { + data.clobbered[j] = IR_REGSET_EMPTY; + } + + /* Get infirmation about target constraints */ + def_flags = ir_get_target_constraints(ctx, i, &constraints); + + /* Collect information about scratch and temporary registers */ + n = constraints.tmps_count; + while (n) { + n--; + IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT); + if (constraints.tmp_regs[n].type) { + ir_reg flags = 0; + ir_ref op = constraints.tmp_regs[n].num; + + if (op > 0 && op <= insn->inputs_count) { + ir_ref *ops = insn->ops; + + if (IR_IS_CONST_REF(ops[op])) { + /* rematerialization */ + flags = IR_REG_SPILL_LOAD; + } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { + /* local address rematerialization */ + flags = IR_REG_SPILL_LOAD; + } + } + _add_tmp_reg(&data, constraints.tmp_regs[n].type, + constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, flags, + IR_UNUSED, i, op); + } else { + _add_scratch(&data, constraints.tmp_regs[n].reg, + constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); + } + } + + /* Collect information about necessary input registers and related constaints */ + n = insn->inputs_count; + j = 1; + p = insn->ops + 1; + if (ir_op_flags[insn->op] & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) { + j++; + p++; + } + for (; j <= n; j++, p++) { + ir_ref input = *p; + ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; + ir_live_pos use_pos; + uint32_t use_flags = IR_USE_FLAGS(def_flags, j); + + if (reg != IR_REG_NONE) { +#if IR_X86_I64 + if (use_flags & IR_HINT_TWO_REGS) { + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); + } else +#endif + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg); + } + + if (input > 0) { + if (ctx->vregs[input] > 0) { + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[input]]; + + if (!ival) { + ival = _create_ival(ctx, ctx->vregs[input], ctx->ir_base[input].type); + } + if (input < bb->start || input >= i) { + /* live before BB start - spill it */ + if (ival->stack_spill_pos == -1) { + _pick_spill(ctx, ival); + } + if (!(use_flags & IR_USE_MUST_BE_IN_REG)) { + continue; + } + } + + use_pos = IR_USE_SUB_REF; + if (reg != IR_REG_NONE) { + use_pos = IR_LOAD_SUB_REF; +#if IR_X86_I64 + if (use_flags & IR_HINT_TWO_REGS) { + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); + } else +#endif + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg); + } else if (def_flags & IR_DEF_REUSES_OP1_REG) { + if (j == 1) { + if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { + use_pos = IR_USE_SUB_REF; + } else { + use_pos = IR_LOAD_SUB_REF; + } + } else if (input == insn->op1) { + /* Input is the same as "op1" */ + use_pos = IR_LOAD_SUB_REF; + } + } + _add_reg(&data, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, use_flags, + IR_UNUSED, i, j, input); + } else { + if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) { + ir_add_fusion_data(ctx, i, input, &data, bb->start); + } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { + ir_set_alocated_reg(ctx, i, j, ctx->ir_base[input].op2); + } + } + } + } + + /* allocate output register */ + if (ctx->vregs[i] > 0) { + ir_live_interval *ival; + ir_live_pos def_pos; + ir_regset used; + + if (constraints.def_reg != IR_REG_NONE) { + def_pos = IR_SAVE_SUB_REF; + if (insn->op == IR_PARAM) { + /* parameter register must not be used before its loaded */ + IR_REGSET_INCL(param_regs, constraints.def_reg); + } + } else if (def_flags & IR_DEF_REUSES_OP1_REG) { + if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { + def_pos = IR_USE_SUB_REF; + } else { + def_pos = IR_LOAD_SUB_REF; + } + } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { + def_pos = IR_LOAD_SUB_REF; + } else { + if (insn->op == IR_PARAM) { + /* We may reuse parameter stack slot for spilling */ + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[i]]; + + do { + if (ctx->use_lists[i].count > 0) { + ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; + if (use < bb->end) { + ir_insn *use_insn = &ctx->ir_base[use]; + if (use_insn->op == IR_VSTORE + && !(ctx->rules[use] & IR_SKIPPED) + /* first use of VAR is VSTORE(_, VAR, PARAM) */ + && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) { + IR_ASSERT(use_insn->op3 == i); + /* skip VSTORE (VAR is going to be remapped to PARAM) */ + ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + ival->use_pos = NULL; + ctx->rules[use] = IR_SKIPPED | IR_NOP; + if (ctx->use_lists[i].count != 1) { + break; + } + } + } + } + ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; + ival->stack_spill_pos = -1; // ??? + IR_LOG_ASSIGN_SPILL(ival->vreg, ival->stack_spill_pos); + } while (0); + } + def_pos = IR_DEF_SUB_REF; + } + + used = data.fixed; + for (n = def_pos; n <= IR_SAVE_SUB_REF; n++) { + used = IR_REGSET_UNION(used, data.clobbered[n]); + } + + ival = ctx->live_intervals[ctx->vregs[i]]; + if (!ival) { + ival = _create_ival(ctx, ctx->vregs[i], insn->type); + } + + +#if IR_X86_I64 + if (insn->type == IR_I64 || insn->type == IR_U64) { + /* allocate output register pair */ + ir_reg reg_lo, reg_hi; + + do { + if (ival->reg != IR_REG_NONE) { + if (!IR_REGSET_IN(used, ival->reg) && !IR_REGSET_IN(used, ival->reg_hi)) { + /* use the previously allocated register */ + reg_lo = ival->reg; + reg_hi = ival->reg_hi; + reg = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); + IR_REGSET_EXCL(data.allocated, ival->reg); + IR_REGSET_EXCL(data.allocated, ival->reg_hi); + ival->reg = ival->reg_hi = IR_REG_NONE; + IR_LOG_FREE(i, reg); + break; + } else { + _evict_reg(ctx, &data, ival->reg); + } + } + + if (def_flags & IR_USE_MUST_BE_IN_REG) { + reg = constraints.def_reg; + + if (reg != IR_REG_NONE + && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), IR_REG_I64_LO(reg)) + && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), IR_REG_I64_HI(reg))) { + /* use fixed register hint */ + reg_hi = IR_REG_I64_HI(reg); + reg_lo = IR_REG_I64_LO(reg); + } else { + reg_lo = _pick_reg(ctx, &data, insn->type, used, IR_USE_MUST_BE_IN_REG, 0, 0); + IR_REGSET_INCL(used, reg_lo); + reg_hi = _pick_reg(ctx, &data, insn->type, used, IR_USE_MUST_BE_IN_REG, 0, 0); + reg = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); + IR_ASSERT(reg_lo != reg_hi); + if (reg_lo > reg_hi) { + SWAP_REGS(reg_lo, reg_hi); + } + reg = IR_REG_I64_PAIR(reg_lo, reg_hi); + } + } else { + reg = IR_REG_NONE; + } + } while (0); + + if (reg != IR_REG_NONE) { + for (n = def_pos; n <= IR_SAVE_SUB_REF; n++) { + IR_REGSET_INCL(data.clobbered[n], reg_lo); + IR_REGSET_INCL(data.clobbered[n], reg_hi); + } + if (ival && ival->stack_spill_pos != -1) { + reg |= IR_REG_SPILL_STORE; + } + ir_set_alocated_reg(ctx, i, 0, reg); + IR_LOG_ASSIGN(i, i, i, 0, reg); + } + } else +#endif + do { + /* allocate output register */ + do { + if (ival->reg != IR_REG_NONE) { + if (!IR_REGSET_IN(used, ival->reg)) { + /* use the previously allocated register */ + reg = ival->reg; + IR_REGSET_EXCL(data.allocated, reg); + ival->reg = IR_REG_NONE; + IR_LOG_FREE(i, reg); + break; + } else { + _evict_reg(ctx, &data, ival->reg); + } + } + if (def_flags & IR_USE_MUST_BE_IN_REG) { + reg = constraints.def_reg; + if (reg != IR_REG_NONE && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg)) { + /* use fixed register hint */ + } else { + reg = _pick_reg(ctx, &data, insn->type, used, IR_USE_MUST_BE_IN_REG, 0, 0); + } + } else { + reg = IR_REG_NONE; + } + } while (0); + + if (reg != IR_REG_NONE) { + for (n = def_pos; n <= IR_SAVE_SUB_REF; n++) { + IR_REGSET_INCL(data.clobbered[n], reg); + } + if (ival && ival->stack_spill_pos != -1) { + reg |= IR_REG_SPILL_STORE; + } + ir_set_alocated_reg(ctx, i, 0, reg); + IR_LOG_ASSIGN(i, i, i, 0, reg); + } + } while (0); + } + + /* evict clobbered registers */ + if (data.allocated != IR_REGSET_EMPTY) { + ir_regset used = param_regs; + + for (n = 0; n < IR_SUB_REFS_COUNT; n++) { + used = IR_REGSET_UNION(used, data.clobbered[n]); + } + used = IR_REGSET_INTERSECTION(used, data.allocated); + if (used != IR_REGSET_EMPTY) { + IR_REGSET_FOREACH(used, reg) { + _evict_reg(ctx, &data, reg); + } IR_REGSET_FOREACH_END(); + } + } + + /* allocate temporary registers */ + for (j = 0; j < data.tmp_num; j++) { + ir_regset used = data.fixed; + + for (n = data.tmp_regs[j].start; n < data.tmp_regs[j].end; n++) { + used = IR_REGSET_UNION(used, data.clobbered[n]); + } + + reg = _pick_reg(ctx, &data, data.tmp_regs[j].type, used, IR_USE_MUST_BE_IN_REG, 0, 0); + for (n = data.tmp_regs[j].start; n < data.tmp_regs[j].end; n++) { + IR_REGSET_INCL(data.clobbered[n], reg); + } +#if IR_X86_I64 + if (data.tmp_regs[j].type == IR_I64 || data.tmp_regs[j].type == IR_U64) { + ir_reg reg_hi; + + IR_REGSET_INCL(used, reg); + reg_hi = _pick_reg(ctx, &data, data.tmp_regs[j].type, used, IR_USE_MUST_BE_IN_REG, 0, 0); + for (n = data.tmp_regs[j].start; n < data.tmp_regs[j].end; n++) { + IR_REGSET_INCL(data.clobbered[n], reg_hi); + } + IR_ASSERT(reg != reg_hi); + if (reg > reg_hi) { + SWAP_REGS(reg, reg_hi); + } + reg = IR_REG_I64_PAIR(reg, reg_hi); + } +#endif + reg = reg | data.tmp_regs[j].flags; + if (data.tmp_regs[j].op == 4 && insn->inputs_count < 4) { + if (!ctx->tmp_regs) { + ctx->tmp_regs = ir_mem_malloc(ctx->insns_count); + memset(ctx->tmp_regs, -1, ctx->insns_count); + } + ctx->tmp_regs[data.tmp_regs[j].ref] = reg; + } else if (!data.tmp_regs[j].root + || ir_get_alocated_reg(ctx, data.tmp_regs[j].ref, data.tmp_regs[j].op) == IR_REG_NONE) { + ir_set_alocated_reg(ctx, data.tmp_regs[j].ref, data.tmp_regs[j].op, reg); + } else if (ir_get_alocated_reg(ctx, data.tmp_regs[j].ref, data.tmp_regs[j].op) != reg) { + ctx->rules[data.tmp_regs[j].ref] |= IR_FUSED_REG; + ir_set_fused_reg(ctx, data.tmp_regs[j].root, data.tmp_regs[j].ref, data.tmp_regs[j].op, reg); + } + IR_LOG_ASSIGN_TMP(data.tmp_regs[j].type, i, data.tmp_regs[j].ref, data.tmp_regs[j].op, reg); + } + + /* allocate input registers */ + for (j = 0; j < data.num; j++) { + ir_regset used = data.clobbered_args; + ir_live_interval *ival; + + for (n = data.regs[j].start; n < data.regs[j].end; n++) { + used = IR_REGSET_UNION(used, data.clobbered[n]); + } + +#if IR_X86_I64 + if (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64) { + ir_reg reg_lo, reg_hi; + + /* allocate input register pair */ + do { + IR_ASSERT(data.regs[j].def > 0 && ctx->vregs[data.regs[j].def] > 0); + ival = ctx->live_intervals[ctx->vregs[data.regs[j].def]]; + reg_lo = ival->reg; + reg_hi = ival->reg_hi; + + /* reuse previously allocated registers if possible */ + if (reg_lo != IR_REG_NONE && reg_hi != IR_REG_NONE) { + IR_ASSERT(data.state[reg_lo].def == data.regs[j].def); + IR_ASSERT(data.state[reg_hi].def == data.regs[j].def); + if ((data.state[reg_lo].root == 0 && data.state[reg_lo].use == i) + || data.state[reg_lo].root == i) { + /* keep previous operand of the same instraction as "last use" */ + break; + } else { + /* update "last use" */ + IR_ASSERT(!IR_REGSET_IN(used, reg_lo) && !IR_REGSET_IN(used, ival->reg_hi)); + data.state[reg_lo].root = data.regs[j].root; + data.state[reg_lo].use = data.regs[j].ref; + data.state[reg_lo].op = data.regs[j].op; + data.state[reg_hi].root = data.regs[j].root; + data.state[reg_hi].use = data.regs[j].ref; + data.state[reg_hi].op = data.regs[j].op; + reg = IR_REG_I64_PAIR(reg_lo, reg_hi); + break; + } + } + + /* try fixed register hint */ + reg = data.regs[j].hint; + reg_hi = IR_REG_NONE; + reg_lo = IR_REG_NONE; + if (reg != IR_REG_NONE) { + if (data.regs[j].flags & IR_HINT_TWO_REGS) { + reg_hi = IR_REG_I64_HI(reg); + reg_lo = IR_REG_I64_LO(reg); + } else { + reg_lo = reg; + } + } + + if (reg_lo != IR_REG_NONE + && reg_hi != IR_REG_NONE + && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg_lo) + && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg_hi)) { + /* use fixed register hint */ + } else if ((def_flags & IR_DEF_REUSES_OP1_REG) + && data.regs[j].op == 1 + && data.regs[j].root == IR_UNUSED + && ctx->regs[i][0] != IR_REG_NONE + && (insn->type == IR_I64 || insn->type == IR_U64) + && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), IR_REG_I64_LO(IR_REG_NUM(ctx->regs[i][0]))) + && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), IR_REG_I64_HI(IR_REG_NUM(ctx->regs[i][0]))) + && (ival->reg == IR_REG_NONE && ival->stack_spill_pos == -1)) { + /* reuse for op1 the register allocated for instruction result (lightweigh coalescing) */ + reg = IR_REG_NUM(ctx->regs[i][0]); + reg_hi = IR_REG_I64_HI(reg); + reg_lo = IR_REG_I64_LO(reg); + } else { + reg_lo = _pick_reg(ctx, &data, data.regs[j].type, used, data.regs[j].flags, i, data.regs[j].def); + if (reg_lo == IR_REG_NONE) { + reg = IR_REG_NONE; + break; + } + IR_REGSET_INCL(used, reg_lo); + reg_hi = _pick_reg(ctx, &data, data.regs[j].type, used, data.regs[j].flags, i, data.regs[j].def); + IR_REGSET_EXCL(used, reg_lo); + if (reg_hi == IR_REG_NONE) { + reg = IR_REG_NONE; + break; + } + IR_ASSERT(reg_lo != reg_hi); + if (reg_lo > reg_hi) { + SWAP_REGS(reg_lo, reg_hi); + } + reg = IR_REG_I64_PAIR(reg_lo, reg_hi); + } + + data.state[reg_lo].def = data.regs[j].def; + data.state[reg_lo].root = data.regs[j].root; + data.state[reg_lo].use = data.regs[j].ref; + data.state[reg_lo].op = data.regs[j].op; + data.state[reg_hi].def = data.regs[j].def; + data.state[reg_hi].root = data.regs[j].root; + data.state[reg_hi].use = data.regs[j].ref; + data.state[reg_hi].op = data.regs[j].op; + IR_REGSET_INCL(data.allocated, reg_lo); + IR_REGSET_INCL(data.allocated, reg_hi); + for (n = data.regs[j].start; n < data.regs[j].end; n++) { + IR_REGSET_INCL(data.clobbered[n], reg_lo); + IR_REGSET_INCL(data.clobbered[n], reg_hi); + } + } while (0); + + if (reg != IR_REG_NONE) { + ival->reg = reg_lo; + ival->reg_hi = reg_hi; + if (!data.regs[j].root + || (!(ctx->rules[data.regs[j].ref] & IR_FUSED_REG) + && ir_get_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op) == IR_REG_NONE)) { + ir_set_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op, reg); + } else if (ir_get_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op) != reg) { + ctx->rules[data.regs[j].ref] |= IR_FUSED_REG; + ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg); + } + IR_LOG_ASSIGN(data.regs[j].def, i, data.regs[j].ref, data.regs[j].op, reg); + } else if (ival->stack_spill_pos == -1) { + _pick_spill(ctx, ival); + } + + continue; + } +#endif + + /* allocate input register */ + do { + IR_ASSERT(data.regs[j].def > 0 && ctx->vregs[data.regs[j].def] > 0); + ival = ctx->live_intervals[ctx->vregs[data.regs[j].def]]; + reg = ival->reg; + + /* reuse previously allocated registers if possible */ + if (reg != IR_REG_NONE) { + IR_ASSERT(data.state[reg].def == data.regs[j].def); + if ((data.state[reg].root == 0 && data.state[reg].use == i) + || data.state[reg].root == i) { + /* keep previous operand of the same instraction as "last use" */ + break; + } else { + /* update "last use" */ + IR_ASSERT(!IR_REGSET_IN(used, reg)); + data.state[reg].root = data.regs[j].root; + data.state[reg].use = data.regs[j].ref; + data.state[reg].op = data.regs[j].op; + break; + } + } + + reg = data.regs[j].hint; + if (reg != IR_REG_NONE && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg)) { + /* use fixed register hint */ + } else if ((def_flags & IR_DEF_REUSES_OP1_REG) + && data.regs[j].op == 1 + && data.regs[j].root == IR_UNUSED + && ctx->regs[i][0] != IR_REG_NONE + && IR_IS_TYPE_INT(insn->type) == IR_IS_TYPE_INT(data.regs[j].type) + && (ival->reg == IR_REG_NONE && ival->stack_spill_pos == -1)) { + /* reuse for op1 the register allocated for instruction result (lightweigh coalescing) */ + reg = IR_REG_NUM(ctx->regs[i][0]); +#if IR_X86_I64 + if (insn->type == IR_I64 || insn->type == IR_U64) { + reg = IR_REG_I64_LO(reg); + } +#endif + if (IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg)) { + goto pick; + } +#if defined(IR_TARGET_X86) + if (IR_IS_TYPE_INT(data.regs[j].type) + && ir_type_size[data.regs[j].type] == 1 + && (reg == IR_REG_RBP || reg == IR_REG_RSI || reg == IR_REG_RDI)) { + goto pick; + } +#endif + } else { +pick: + reg = _pick_reg(ctx, &data, data.regs[j].type, used, data.regs[j].flags, i, data.regs[j].def); + if (reg == IR_REG_NONE) { + break; + } + } + data.state[reg].def = data.regs[j].def; + data.state[reg].root = data.regs[j].root; + data.state[reg].use = data.regs[j].ref; + data.state[reg].op = data.regs[j].op; + IR_REGSET_INCL(data.allocated, reg); + for (n = data.regs[j].start; n < data.regs[j].end; n++) { + IR_REGSET_INCL(data.clobbered[n], reg); + } + } while (0); + + if (reg != IR_REG_NONE) { + ival->reg = reg; + if (!data.regs[j].root + || (!(ctx->rules[data.regs[j].ref] & IR_FUSED_REG) + && ir_get_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op) == IR_REG_NONE)) { + ir_set_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op, reg); + } else if (ir_get_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op) != reg) { + ctx->rules[data.regs[j].ref] |= IR_FUSED_REG; + ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg); + } + IR_LOG_ASSIGN(data.regs[j].def, i, data.regs[j].ref, data.regs[j].op, reg); + } else if (ival->stack_spill_pos == -1) { + _pick_spill(ctx, ival); + } + } + } + + if (data.allocated != IR_REGSET_EMPTY) { + IR_REGSET_FOREACH(data.allocated, reg) { + _evict_reg(ctx, &data, reg); + } IR_REGSET_FOREACH_END(); + } + } + +#ifdef IR_TARGET_X86 + if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); + } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) + && data.ra_data.cc->fp_ret_reg == IR_REG_NONE) { + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); + } else { + ctx->ret_slot = -1; + } +#endif + + ctx->flags |= IR_NO_STACK_COMBINE; + ir_fix_stack_frame(ctx); + ctx->data = NULL; + + IR_LOG("Finish Fast Local RA"); + + return 1; +} diff --git a/ext/opcache/jit/ir/ir_ra_simple.c b/ext/opcache/jit/ir/ir_ra_simple.c index 4ecdd79986b0..7a78801f4d71 100644 --- a/ext/opcache/jit/ir/ir_ra_simple.c +++ b/ext/opcache/jit/ir/ir_ra_simple.c @@ -1,6 +1,6 @@ /* * IR - Lightweight JIT Compilation Framework - * Simple Local Register Allocation (used for -O0) + * Simple Local Register Allocation (not used anymore) * This file is part of the IR Project distributed under the MIT-style LICENSE. * Authors: Dmitry Stogov * @@ -131,6 +131,15 @@ static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data return reg; } +/* The following checks for the safety of VAR slot reuse are incomplete. + * Some of the unsupported edge cases are demonstrated at https://github.com/dstogov/ir/pull/232 + * and https://github.com/dstogov/ir/pull/230 (there are may be other problems). + * Unfortunatly, the attempts to fix the problems break the simplicity of this allocator and make it ridiculous. + * The complete removing of VAR slot reuse makes the already bad generated code significanly worse. + * + * A better appoach is implemented in ir_ra_fast.c + */ + static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load) { ir_use_list *use_list = &ctx->use_lists[load]; diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 3ad6997b071e..2ca202add5e8 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -2673,16 +2673,16 @@ static bool ir_is_zero(const ir_ctx *ctx, ir_ref ref) && ctx->ir_base[ref].val.u32 == 0; } -static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) +static bool ir_is_min_max_const(const ir_ctx *ctx, const ir_insn *cond, ir_ref ref) { if (cond->op == IR_ULE) { /* (x <= 3 ? 4 : x) => (x < 4 ? 4 : x) => max(x, 4) */ /* (x <= 3 ? x : 4) => (x < 4 ? x : 4) => min(x, 4) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.u64 != 0 && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 - 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_UGE) { @@ -2690,9 +2690,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x >= 3 ? x : 2) => (x > 2 ? x : 2) => max(x, 2) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.u64 != UINT64_MAX && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 + 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_LE) { @@ -2700,9 +2700,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x <= 3 ? x : 4) => (x < 4 ? x : 4) => min(x, 4) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) - && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 - 1 + && ctx->ir_base[ref].val.i64 != INT64_MIN + && ctx->ir_base[cond->op2].val.i64 == ctx->ir_base[ref].val.i64 - 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } else if (cond->op == IR_GE) { @@ -2710,9 +2710,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref) /* (x >= 3 ? x : 2) => (x > 2 ? x : 2) => max(x, 2) */ if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.i64 != INT64_MAX && ctx->ir_base[cond->op2].val.i64 == ctx->ir_base[ref].val.i64 + 1 && ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) { - cond->op2 = ref; return 1; } } @@ -2758,18 +2758,19 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re } if (is_cmp + && !IR_IS_TYPE_FP(type) && ((insn->op2 == cond->op1 && (insn->op3 == cond->op2 || (IR_IS_CONST_REF(cond->op2) && (IR_IS_CONST_REF(insn->op3) && IR_IS_TYPE_INT(insn->type) - && ir_fix_min_max_const(ctx, cond, insn->op3))))) + && ir_is_min_max_const(ctx, cond, insn->op3))))) || (insn->op3 == cond->op1 && (insn->op2 == cond->op2 || (IR_IS_CONST_REF(cond->op2) && (IR_IS_CONST_REF(insn->op2) && IR_IS_TYPE_INT(insn->type) - && ir_fix_min_max_const(ctx, cond, insn->op2))))))) { + && ir_is_min_max_const(ctx, cond, insn->op2))))))) { /* MAX/MIN * * prev prev @@ -2804,11 +2805,16 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re ir_remove_unused_vars(ctx, start2_ref, end2_ref); } - insn->op = ( - (is_less ? cond->op1 : cond->op2) - == - ((start1->op == IR_IF_TRUE) ? insn->op2 : insn->op3) - ) ? IR_MIN : IR_MAX; + if (start1->op != IR_IF_TRUE) { + IR_ASSERT(start1->op == IR_IF_FALSE); + is_less = !is_less; + } + if (cond->op1 != insn->op2) { + IR_ASSERT(cond->op1 == insn->op3); + is_less = !is_less; + } + + insn->op = is_less ? IR_MIN : IR_MAX; insn->inputs_count = 2; if (insn->op2 > insn->op3) { insn->op1 = insn->op2; @@ -2939,7 +2945,12 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re } return 1; - } else if (insn->op2 <= root_ref && insn->op3 <= root_ref + } else if (((insn->op2 <= root_ref && insn->op3 <= root_ref) + || ((cond->op >= IR_EQ && cond->op <= IR_UNORDERED) + && ((insn->op2 == cond->op1 && (insn->op3 == cond->op2 || insn->op3 <= root_ref)) + || (insn->op2 == cond->op2 && (insn->op3 == cond->op1 || insn->op3 <= root_ref)) + || (insn->op3 == cond->op1 && insn->op2 <= root_ref) + || (insn->op3 == cond->op2 && insn->op2 <= root_ref)))) && cond->op != IR_OVERFLOW // TODO: temporary disable IF-conversion for RLOAD. // We don't track anti-dependencies in GCM and Local Scheduling. diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index f33c89f574f5..463be9e750d3 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -1622,6 +1622,7 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = { _(BINOP_INT) \ _(BINOP_SSE2) \ _(BINOP_AVX) \ + _(MIN_MAX_FP) \ _(SHIFT) \ _(SHIFT_CONST) \ _(COPY_INT) \ @@ -1636,6 +1637,7 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = { _(COND_TEST_BIT) \ _(COND_CMP_INT) \ _(COND_CMP_FP) \ + _(COND_MIN_MAX_FP) \ _(GUARD_CMP_INT) \ _(GUARD_CMP_FP) \ _(GUARD_TEST_INT) \ @@ -1998,6 +2000,28 @@ op2_const: case IR_COND_CMP_FP: flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG; break; + case IR_COND_MIN_MAX_FP: + flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG; + if (!(ctx->mflags & IR_X86_AVX)) { + insn = &ctx->ir_base[ref]; + if (!IR_IS_CONST_REF(insn->op3) || ctx->vregs[insn->op3] > 0) { + /* def_reg can't be the same as op3_reg */ + flags |= IR_DEF_CONFLICTS_WITH_INPUT_REGS; + } + } + break; + case IR_MIN_MAX_FP: + flags = IR_USE_MUST_BE_IN_REG | IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_DEF_CONFLICTS_WITH_INPUT_REGS; + insn = &ctx->ir_base[ref]; + constraints->tmp_regs[0] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n = 1; + if (IR_IS_CONST_REF(insn->op1)) { + n = ir_add_const_tmp_reg(ctx, insn->op1, 1, n, constraints); + } + if (IR_IS_CONST_REF(insn->op2)) { + n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); + } + break; case IR_VSTORE_INT: flags = IR_OP3_MUST_BE_IN_REG; insn = &ctx->ir_base[ref]; @@ -4496,7 +4520,7 @@ binop_fp: goto binop_vector; #endif } else { - goto binop_fp; + return IR_MIN_MAX_FP; } break; case IR_COPY: @@ -5047,6 +5071,11 @@ store_int: return IR_TWO_REGS | IR_COND_I64_CMP_FP; } #endif + if ((op1_insn->op >= IR_LT && op1_insn->op <= IR_GT) && + ((op1_insn->op1 == insn->op2 && op1_insn->op2 == insn->op3) || + (op1_insn->op1 == insn->op3 && op1_insn->op2 == insn->op2))) { + return IR_COND_MIN_MAX_FP; + } return IR_COND_CMP_FP; } #if IR_X86_I64 @@ -7543,6 +7572,30 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) | add Rd(def_reg), Rd(op1_reg) |.if X64 || } +|.endif + | ASM_REG_IMM_OP sar, type, def_reg, shift + } else if (shift == (uint32_t)ir_type_size[type] * 8 - 1) { +|.if X64 +|| if (ir_type_size[type] == 8) { + | mov64 Rq(def_reg), 0x8000000000000000 + | cmp Rq(def_reg), Rq(op1_reg) + | sete Rb(def_reg) + | movzx Rd(def_reg), Rb(def_reg) +|| } else { +|.endif + | xor Rd(def_reg), Rd(def_reg) + | cmp Rd(op1_reg), 0x80000000 +|.if X64 + | sete Rb(def_reg) +|| } +|.else + if (def_reg < IR_REG_R3) { + | sete Rb(def_reg) + } else { + | jne >1 + | mov Rd(def_reg), 1 + |1: + } |.endif } else { |.if X64 @@ -7564,8 +7617,8 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) |.endif | ASM_REG_REG_OP test, type, op1_reg, op1_reg | ASM_REG_REG_OP2 cmovns, type, def_reg, op1_reg + | ASM_REG_IMM_OP sar, type, def_reg, shift } - | ASM_REG_IMM_OP sar, type, def_reg, shift if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); @@ -8614,12 +8667,6 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_REG_OP divs, type, def_reg, op2_reg break; - case IR_MIN: - | ASM_SSE2_REG_REG_OP mins, type, def_reg, op2_reg - break; - case IR_MAX: - | ASM_SSE2_REG_REG_OP maxs, type, def_reg, op2_reg - break; } } else if (IR_IS_CONST_REF(op2)) { int label = ir_get_const_label(ctx, op2); @@ -8639,12 +8686,6 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_TXT_OP divs, type, def_reg, [=>label] break; - case IR_MIN: - | ASM_SSE2_REG_TXT_OP mins, type, def_reg, [=>label] - break; - case IR_MAX: - | ASM_SSE2_REG_TXT_OP maxs, type, def_reg, [=>label] - break; } } else { ir_mem mem; @@ -8669,14 +8710,88 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) case IR_DIV: | ASM_SSE2_REG_MEM_OP divs, type, def_reg, mem break; - case IR_MIN: - | ASM_SSE2_REG_MEM_OP mins, type, def_reg, mem - break; - case IR_MAX: - | ASM_SSE2_REG_MEM_OP maxs, type, def_reg, mem - break; } } + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, insn->type, def, def_reg); + } +} + +static void ir_emit_min_max_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = insn->type; + ir_ref op1 = insn->op1; + ir_ref op2 = insn->op2; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op1_reg = ctx->regs[def][1]; + ir_reg op2_reg = ctx->regs[def][2]; + ir_reg tmp_reg = ctx->regs[def][3]; + + IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); + + if (IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + ir_emit_load(ctx, type, op1_reg, op1); + } + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + if (op1 != op2) { + ir_emit_load(ctx, type, op2_reg, op2); + } + } + + if (!(ctx->mflags & IR_X86_AVX)) { + ir_emit_fp_mov(ctx, type, def_reg, op1_reg); + } + + if (type == IR_DOUBLE) { + if (ctx->mflags & IR_X86_AVX) { + if (insn->op == IR_MAX) { + | vmaxsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | vminsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | vcmpsd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), 7 + | vblendvpd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } else { + if (insn->op == IR_MAX) { + | maxsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | minsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | movapd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + | cmpsd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST), 7 + | andpd xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + | andnpd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST) + | orpd xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } + } else { + IR_ASSERT(type == IR_FLOAT); + if (ctx->mflags & IR_X86_AVX) { + if (insn->op == IR_MAX) { + | vmaxss xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | vminss xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | vcmpss xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), 7 + | vblendvps xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } else { + if (insn->op == IR_MAX) { + | maxss xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } else { + | minss xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + } + | movaps xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST) + | cmpss xmm(tmp_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST), 7 + | andps xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + | andnps xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST) + | orps xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST) + } + } + if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } @@ -9469,6 +9584,20 @@ static void ir_emit_jcc(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint } } +static bool ir_insns_may_change_cmp_flags(ir_ctx *ctx, ir_ref from, ir_ref to) +{ + ir_ref ref; + + for (ref = ctx->prev_ref[to]; ref != from; ref = ctx->prev_ref[ref]) { + if (!(ctx->rules[ref] & (IR_FUSED|IR_SKIPPED)) + && ctx->ir_base[ref].op != IR_VAR + && ctx->ir_base[ref].op != IR_PHI) { + return 1; + } + } + return 0; +} + static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next_block) { ir_insn *cmp_insn = &ctx->ir_base[insn->op2]; @@ -9518,7 +9647,9 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i if (ir_rule(ctx, prev_insn->op1) == IR_CMP_AND_BRANCH_INT) { prev_insn = &ctx->ir_base[prev_insn->op1]; prev_insn = &ctx->ir_base[prev_insn->op2]; - if (prev_insn->op1 == cmp_insn->op1 && prev_insn->op2 == cmp_insn->op2) { + if (prev_insn->op1 == cmp_insn->op1 + && prev_insn->op2 == cmp_insn->op2 + && !ir_insns_may_change_cmp_flags(ctx, insn->op1, def)) { same_comparison = true; } } @@ -9666,7 +9797,7 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, op3); if (op1 == op3) { - op1_reg = op2_reg; + op1_reg = op3_reg; } } if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { @@ -10295,6 +10426,137 @@ static void ir_emit_cond_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } +static void ir_emit_cond_min_max_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = insn->type; + ir_insn *cmp_insn = &ctx->ir_base[insn->op1]; + ir_ref op2 = insn->op2; + ir_ref op3 = insn->op3; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op2_reg = ctx->regs[def][2]; + ir_reg op3_reg = ctx->regs[def][3]; + ir_reg cmp_op1_reg, cmp_op2_reg; + ir_op op; + + IR_ASSERT(def_reg != IR_REG_NONE); + + if (UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { + cmp_op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); + cmp_op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); + } else { + cmp_op1_reg = ctx->regs[insn->op1][1]; + cmp_op2_reg = ctx->regs[insn->op1][2]; + } + if (cmp_insn->op1 == op2) { + IR_ASSERT(cmp_insn->op2 == op3); + op = (cmp_insn->op == IR_LT || cmp_insn->op == IR_LE) ? IR_MIN : IR_MAX; + if (op2_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op1_reg) == IR_REG_NUM(op2_reg)); + op2_reg |= cmp_op1_reg; /* merge spill load flag */ + } + if (op3_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op2_reg) == IR_REG_NUM(op3_reg)); + op3_reg |= cmp_op2_reg; /* merge spill load flag */ + } + } else { + IR_ASSERT(cmp_insn->op1 == op3 && cmp_insn->op2 == op2); + op = (cmp_insn->op == IR_LT || cmp_insn->op == IR_LE) ? IR_MAX : IR_MIN; + if (op2_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op2_reg) == IR_REG_NUM(op2_reg)); + op2_reg |= cmp_op2_reg; /* merge spill load flag */ + } + if (op3_reg != IR_REG_NONE) { + IR_ASSERT(IR_REG_NUM(cmp_op1_reg) == IR_REG_NUM(op3_reg)); + op3_reg |= cmp_op1_reg; /* merge spill load flag */ + } + } + + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, type, op2_reg, op2); + } + } + if (op3_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op3_reg)) { + op3_reg = IR_REG_NUM(op3_reg); + if (op2 != op3) { + ir_emit_load(ctx, type, op3_reg, op3); + } + } + } + + IR_ASSERT((ctx->mflags & IR_X86_AVX) || def_reg != op3_reg); + + if (op2_reg != IR_REG_NONE) { + if (!(ctx->mflags & IR_X86_AVX) && def_reg != op2_reg) { + ir_emit_fp_mov(ctx, type, def_reg, op2_reg); + } + } else { + ir_emit_load(ctx, type, def_reg, op2); + op2_reg = def_reg; + } + + if (op3_reg != IR_REG_NONE) { + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_REG_OP vmins, type, def_reg, op2_reg, op3_reg + } else { + | ASM_AVX_REG_REG_REG_OP vmaxs, type, def_reg, op2_reg, op3_reg + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_REG_OP mins, type, def_reg, op3_reg + } else { + | ASM_SSE2_REG_REG_OP maxs, type, def_reg, op3_reg + } + } + } else if (IR_IS_CONST_REF(op3)) { + int label = ir_get_const_label(ctx, op3); + + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_TXT_OP vmins, type, def_reg, op2_reg, [=>label] + } else { + | ASM_AVX_REG_REG_TXT_OP vmaxs, type, def_reg, op2_reg, [=>label] + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_TXT_OP mins, type, def_reg, [=>label] + } else { + | ASM_SSE2_REG_TXT_OP maxs, type, def_reg, [=>label] + } + } + } else { + ir_mem mem; + + if (ir_rule(ctx, op3) & IR_FUSED) { + mem = ir_fuse_load(ctx, def, op3); + } else { + mem = ir_ref_spill_slot(ctx, op3); + } + if (ctx->mflags & IR_X86_AVX) { + if (op == IR_MIN) { + | ASM_AVX_REG_REG_MEM_OP vmins, type, def_reg, op2_reg, mem + } else { + | ASM_AVX_REG_REG_MEM_OP vmaxs, type, def_reg, op2_reg, mem + } + } else { + if (op == IR_MIN) { + | ASM_SSE2_REG_MEM_OP mins, type, def_reg, mem + } else { + | ASM_SSE2_REG_MEM_OP maxs, type, def_reg, mem + } + } + } + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, type, def, def_reg); + } +} + static void ir_emit_return_void(ir_ctx *ctx) { ir_backend_data *data = ctx->data; @@ -15009,7 +15271,9 @@ static void ir_emit_cmp_and_branch_i64(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i if (ir_rule(ctx, prev_insn->op1) == IR_CMP_AND_BRANCH_INT) { prev_insn = &ctx->ir_base[prev_insn->op1]; prev_insn = &ctx->ir_base[prev_insn->op2]; - if (prev_insn->op1 == cmp_insn->op1 && prev_insn->op2 == cmp_insn->op2) { + if (prev_insn->op1 == cmp_insn->op1 + && prev_insn->op2 == cmp_insn->op2 + && !ir_insns_may_change_cmp_flags(ctx, insn->op1, def)) { same_comparison = true; } } @@ -17070,7 +17334,7 @@ static void ir_emit_cond_cmp_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(tmp_reg != IR_REG_NONE); - if (!IR_IS_CONST_REF(insn->op2) && UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { + if (!IR_IS_CONST_REF(insn->op1) && UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); } else { @@ -17232,7 +17496,7 @@ static void ir_emit_cond_i64_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(tmp_reg != IR_REG_NONE); - if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { + if (UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) { op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); } else { @@ -26183,6 +26447,12 @@ static void ir_emit_load_params(ir_ctx *ctx) ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_start + stack_offset); } if (dst_reg != IR_REG_NONE && IR_REG_SPILLED(ctx->regs[use][0])) { +#if IR_X86_I64 + if (insn->type == IR_I64 || insn->type == IR_U64) { + ir_emit_store_i64_lo(ctx, use, IR_REG_I64_LO(dst_reg)); + ir_emit_store_i64_hi(ctx, use, IR_REG_I64_HI(dst_reg)); + } else +#endif ir_emit_store(ctx, insn->type, use, dst_reg); } } @@ -26283,10 +26553,10 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { IR_ASSERT(ival->stack_spill_pos == -1); ival->stack_spill_pos = stack_start + stack_offset; - if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { - /* Remap VAR to PARAM stack slot */ - ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); - } + } + if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { + /* Remap VAR to PARAM stack slot */ + ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); } } if (sizeof(void*) == 8) { @@ -26636,6 +26906,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_BINOP_AVX: ir_emit_binop_avx(ctx, i, insn); break; + case IR_MIN_MAX_FP: + ir_emit_min_max_fp(ctx, i, insn); + break; case IR_MUL_INT: case IR_DIV_INT: case IR_MOD_INT: @@ -26759,6 +27032,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_COND_CMP_FP: ir_emit_cond_cmp_fp(ctx, i, insn); break; + case IR_COND_MIN_MAX_FP: + ir_emit_cond_min_max_fp(ctx, i, insn); + break; case IR_SWITCH: ir_emit_switch(ctx, b, i, insn); break; From e5ea3ae35f0c45f367ba7938340ddc94bbe519f4 Mon Sep 17 00:00:00 2001 From: Jorg Sowa Date: Thu, 5 Nov 2026 11:52:29 +0100 Subject: [PATCH 2/2] Update IR and drop unused register allocators IR commit: 4a1208ccbfc8ca32a5103baf9820ad3cbb6ff06e Stop bundling ir_ra_simple.c and ir_ra_fast.c, which are not used. --- .github/scripts/download-bundled/jit-ir.sh | 4 +- ext/opcache/config.m4 | 2 - ext/opcache/config.w32 | 2 +- ext/opcache/jit/Makefile.frag.w32 | 10 - ext/opcache/jit/ir/ir_fold.h | 15 +- ext/opcache/jit/ir/ir_ra_fast.c | 1481 -------------------- ext/opcache/jit/ir/ir_ra_simple.c | 671 --------- 7 files changed, 14 insertions(+), 2171 deletions(-) delete mode 100644 ext/opcache/jit/ir/ir_ra_fast.c delete mode 100644 ext/opcache/jit/ir/ir_ra_simple.c diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index ee9d31b41b7d..9a1f8faa1be1 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=43d1d1023a6ea811b11b39ed738f5e18696a1254 +revision=4a1208ccbfc8ca32a5103baf9820ad3cbb6ff06e git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" @@ -32,6 +32,8 @@ rm ir_load.c rm ir_load_llvm.c rm ir_main.c rm ir_mem2ssa.c +rm ir_ra_fast.c +rm ir_ra_simple.c # add extra files git restore README diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 89c1fa488e97..2a1ac4e332fc 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -48,8 +48,6 @@ AS_VAR_IF([PHP_OPCACHE_JIT], [yes], [ jit/ir/ir_patch.c jit/ir/ir_perf.c jit/ir/ir_ra.c - jit/ir/ir_ra_fast.c - jit/ir/ir_ra_simple.c jit/ir/ir_save.c jit/ir/ir_sccp.c jit/ir/ir_strtab.c diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index d3d3415fd01d..cd22b363db0b 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -25,7 +25,7 @@ if (PHP_OPCACHE_JIT == "yes") { } else if (CHECK_HEADER("ir/ir.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) { var dasm_flags = (X64 ? "-D X64=1" : "") + (X64 ? " -D X64WIN=1" : "") + " -D WIN=1"; var ir_target = (X64 ? "IR_TARGET_X64" : "IR_TARGET_X86"); - var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_ra_simple.c ir_ra_fast.c ir_save.c \ + var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_save.c \ ir_dump.c ir_check.c ir_patch.c"; DEFINE("IR_TARGET", ir_target); diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32 index e38ed60aedda..7caf81458870 100644 --- a/ext/opcache/jit/Makefile.frag.w32 +++ b/ext/opcache/jit/Makefile.frag.w32 @@ -19,16 +19,6 @@ $(BUILD_DIR)\ext\opcache\jit\ir\ir_ra.obj: \ ext\opcache\jit\ir\ir_private.h \ ext\opcache\jit\ir\ir_x86.h -$(BUILD_DIR)\ext\opcache\jit\ir\ir_ra_simple.obj: \ - ext\opcache\jit\ir\ir.h \ - ext\opcache\jit\ir\ir_private.h \ - ext\opcache\jit\ir\ir_x86.h - -$(BUILD_DIR)\ext\opcache\jit\ir\ir_ra_fast.obj: \ - ext\opcache\jit\ir\ir.h \ - ext\opcache\jit\ir\ir_private.h \ - ext\opcache\jit\ir\ir_x86.h - $(BUILD_DIR)\ext\opcache\jit\ir\ir_emit.obj: \ ext\opcache\jit\ir\ir.h \ ext\opcache\jit\ir\ir_private.h \ diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 1debe2fa25a0..c8292191b5e8 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -2670,10 +2670,6 @@ IR_FOLD(MUL(NEG, C_I8)) IR_FOLD(MUL(NEG, C_I16)) IR_FOLD(MUL(NEG, C_I32)) IR_FOLD(MUL(NEG, C_I64)) -IR_FOLD(DIV(NEG, C_I8)) -IR_FOLD(DIV(NEG, C_I16)) -IR_FOLD(DIV(NEG, C_I32)) -IR_FOLD(DIV(NEG, C_I64)) { op1 = op1_insn->op1; val.u64 = (0 - op2_insn->val.u64); @@ -2869,13 +2865,22 @@ IR_FOLD(DIV(_, C_FLOAT)) } IR_FOLD(MUL(NEG, NEG)) -IR_FOLD(DIV(NEG, NEG)) { op1 = op1_insn->op1; op2 = op2_insn->op1; IR_FOLD_RESTART; } +IR_FOLD(DIV(NEG, NEG)) +{ + if (IR_IS_TYPE_FP(IR_OPT_TYPE(opt))) { + op1 = op1_insn->op1; + op2 = op2_insn->op1; + IR_FOLD_RESTART; + } + IR_FOLD_NEXT; +} + IR_FOLD(AND(_, C_BOOL)) { IR_FOLD_COPY(op2_insn->val.b ? op1 : op2); diff --git a/ext/opcache/jit/ir/ir_ra_fast.c b/ext/opcache/jit/ir/ir_ra_fast.c deleted file mode 100644 index a3c893167561..000000000000 --- a/ext/opcache/jit/ir/ir_ra_fast.c +++ /dev/null @@ -1,1481 +0,0 @@ -/* - * IR - Lightweight JIT Compilation Framework - * Fast Local Register Allocation (used for -O0) - * This file is part of the IR Project distributed under the MIT-style LICENSE. - * Authors: Dmitry Stogov - */ - -#ifndef _GNU_SOURCE -# define _GNU_SOURCE -#endif - -#include "ir.h" - -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) -# include "ir_x86.h" -#elif defined(IR_TARGET_AARCH64) -# include "ir_aarch64.h" -#else -# error "Unknown IR target" -#endif - -#include "ir_private.h" - -typedef struct { - ir_reg_alloc_data ra_data; - ir_block *bb; - int32_t num; - int32_t tmp_num; - ir_regset scratch; - ir_regset preserved; - ir_regset fixed; - ir_regset allocated; - ir_regset clobbered_args; - ir_regset clobbered[IR_SUB_REFS_COUNT]; - struct { - ir_ref def; - ir_ref root; - ir_ref use; - ir_ref op; - } state[IR_REG_NUM]; - struct { - uint8_t type; - int8_t start; - int8_t end; - int8_t hint; - int8_t flags; - ir_ref root; - ir_ref ref; - ir_ref op; - ir_ref def; - } regs[32], tmp_regs[32]; -} ir_reg_alloc_fast_data; - -#ifdef IR_DEBUG -# define IR_DEBUG_SPILLS 1 -# if IR_X86_I64 -# define IR_REG_NAME_FMT "%s%s%s" -# define IR_REG_NAME_VAL(_reg, _type) (((_type) == IR_I64 || (_type) == IR_U64) ? \ - ir_reg_name(IR_REG_I64_LO(IR_REG_NUM(_reg)), IR_U32) : \ - ir_reg_name(IR_REG_NUM(_reg), _type)), \ - (((_type) == IR_I64 || (_type) == IR_U64) ? " and " : ""), \ - (((_type) == IR_I64 || (_type) == IR_U64) ? \ - ir_reg_name(IR_REG_I64_HI(IR_REG_NUM(_reg)), IR_U32) : "") -# else -# define IR_REG_NAME_FMT "%s" -# define IR_REG_NAME_VAL(_reg, _type) ir_reg_name(IR_REG_NUM(_reg), _type) -# endif -# define IR_LOG(comment) do { \ - if (ctx->flags & IR_DEBUG_RA) { \ - fprintf(stderr, "---- %s\n", comment); \ - } \ - } while (0) -# define IR_LOG_ASSIGN(_ref, _pos, _x, _y, _reg) do { \ - if (ctx->flags & IR_DEBUG_RA) { \ - fprintf(stderr, " ---- Assign R%d (d_%d) to " IR_REG_NAME_FMT "%s at %d (%d.%d)\n", \ - ctx->vregs[_ref], _ref, \ - IR_REG_NAME_VAL(_reg, ctx->ir_base[_ref].type), \ - (((_reg) & IR_REG_SPILL_LOAD) ? " (spill)" : ""), \ - _pos, _x, _y); \ - } \ - } while (0) -# define IR_LOG_ASSIGN_TMP(_type, _pos, _x, _y, _reg) do { \ - if (ctx->flags & IR_DEBUG_RA) { \ - fprintf(stderr, " ---- Assign TMP to " IR_REG_NAME_FMT "%s at %d (%d.%d)\n", \ - IR_REG_NAME_VAL(_reg, _type), \ - (((_reg) & IR_REG_SPILL_LOAD) ? " (spill)" : ""), \ - _pos, _x, _y); \ - } \ - } while (0) -# define IR_LOG_ASSIGN_VAR(_var, _spill) do { \ - if (IR_DEBUG_SPILLS && (ctx->flags & IR_DEBUG_RA)) { \ - fprintf(stderr, " ---- Spill VAR d_%d to 0x%x\n", \ - _var, _spill); \ - } \ - } while (0) -# define IR_LOG_ASSIGN_SPILL(_vreg, _spill) do { \ - if (IR_DEBUG_SPILLS && (ctx->flags & IR_DEBUG_RA)) { \ - fprintf(stderr, " ---- Spill R%d to 0x%x\n", \ - _vreg, _spill); \ - } \ - } while (0) -# define IR_LOG_FREE(_ref, _reg) do { \ - if (ctx->flags & IR_DEBUG_RA) { \ - fprintf(stderr, " ---- Free R%d(d_%d) assigned to " IR_REG_NAME_FMT "\n", \ - ctx->vregs[_ref], _ref, \ - IR_REG_NAME_VAL(_reg, ctx->ir_base[_ref].type)); \ - } \ - } while (0) -# define IR_LOG_EVICT(_ref, _reg) do { \ - if (ctx->flags & IR_DEBUG_RA) { \ - fprintf(stderr, " ---- Evict R%d(d_%d) assigned to " IR_REG_NAME_FMT "\n", \ - ctx->vregs[_ref], _ref, \ - IR_REG_NAME_VAL(_reg, ctx->ir_base[_ref].type)); \ - } \ - } while (0) -# define IR_LOG_SPILL_LOAD(_ref, _pos, _x, _y, _reg) do { \ - if (ctx->flags & IR_DEBUG_RA) { \ - fprintf(stderr, " ---- Add Spill Load R%d (d_%d) to " IR_REG_NAME_FMT " at %d (%d.%d)\n", \ - ctx->vregs[_ref], _ref, \ - IR_REG_NAME_VAL(_reg, ctx->ir_base[_ref].type), \ - _pos, _x, _y); \ - } \ - } while (0) -#else -# define IR_LOG(comment) -# define IR_LOG_ASSIGN(_ref, _pos, _x, _y, _reg) -# define IR_LOG_ASSIGN_TMP(_type, _pos, _x, _y, _reg) -# define IR_LOG_ASSIGN_VAR(_vreg, _spill) -# define IR_LOG_ASSIGN_SPILL(_vreg, _spill) -# define IR_LOG_FREE(_ref, _reg) -# define IR_LOG_EVICT(_ref, _reg) -# define IR_LOG_SPILL_LOAD(_ref, _pos, _x, _y, _reg) -#endif - -static void _add_scratch(ir_reg_alloc_fast_data *data, ir_reg reg, int8_t start, int8_t end) -{ - int8_t j; - - if (start < 0) { - /* ARGVAL support */ - if (reg >= IR_REG_NUM) { - data->clobbered_args = IR_REGSET_UNION(data->clobbered_args, ir_scratch_regset[reg - IR_REG_NUM]); - } else { - IR_REGSET_INCL(data->clobbered_args, reg); - } - start = 0; - } - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT); - if (reg >= IR_REG_NUM) { - for (j = start; j < end; j++) { - data->clobbered[j] = IR_REGSET_UNION(data->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]); - } - } else { - for (j = start; j < end; j++) { - IR_REGSET_INCL(data->clobbered[j], reg); - } - } -} - -static void _add_tmp_reg(ir_reg_alloc_fast_data *data, ir_type type, - int8_t start, int8_t end, uint8_t flags, - ir_ref root, ir_ref ref, ir_ref op) -{ - uintptr_t n; - - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32); - n = data->tmp_num++; - data->tmp_regs[n].type = type; - data->tmp_regs[n].start = start; - data->tmp_regs[n].end = end; - data->tmp_regs[n].flags = flags; - data->tmp_regs[n].root = root; - data->tmp_regs[n].ref = ref; - data->tmp_regs[n].op = op; -} - -static void _add_reg(ir_reg_alloc_fast_data *data, ir_type type, - int8_t start, int8_t end, ir_reg hint, int8_t flags, - ir_ref root, ir_ref ref, ir_ref op, ir_ref def) -{ - uintptr_t n; - - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32); - n = data->num++; - data->regs[n].type = type; - data->regs[n].start = start; - data->regs[n].end = end; - data->regs[n].hint = hint; - data->regs[n].flags = flags; - data->regs[n].root = root; - data->regs[n].ref = ref; - data->regs[n].op = op; - data->regs[n].def = def; -} - -static void _pick_spill(ir_ctx *ctx, ir_live_interval *ival) -{ - size_t size = ir_get_type_size(ival->type); - - IR_ASSERT(ival->stack_spill_pos == -1); -#if IR_SIMD - if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { - size = 4; - } -#endif - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); - IR_LOG_ASSIGN_SPILL(ival->vreg, ival->stack_spill_pos); -} - -static void _evict_reg(ir_ctx *ctx, ir_reg_alloc_fast_data *data, ir_reg reg) -{ - ir_live_interval *ival; - ir_ref ref = data->state[reg].def; - - IR_ASSERT(ctx->vregs[ref] > 0); - ival = ctx->live_intervals[ctx->vregs[ref]]; - - /* allocate spill slot */ - if (ival->stack_spill_pos == -1) { - _pick_spill(ctx, ival); - } - - /* free reg */ -#if IR_X86_I64 - if (ival->type == IR_I64 || ival->type == IR_U64) { - if (ival->reg == IR_REG_NONE && ival->reg_hi == IR_REG_NONE) { - /* already evicted */ - return; - } - - IR_ASSERT(ival->reg != IR_REG_NONE && ival->reg_hi != IR_REG_NONE && (ival->reg == reg || ival->reg_hi == reg)); - - ir_reg pair = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); - IR_LOG_EVICT(ref, pair); - IR_REGSET_EXCL(data->allocated, ival->reg); - IR_REGSET_EXCL(data->allocated, ival->reg_hi); - ival->reg = ival->reg_hi = IR_REG_NONE; - - if (data->state[reg].use) { - /* add spill load for the previous use */ - if (!data->state[reg].root - || ctx->ir_base[data->state[reg].use].op == IR_ARGVAL) { - ir_set_alocated_reg(ctx, data->state[reg].use, data->state[reg].op, pair | IR_REG_SPILL_LOAD); - } else { - ctx->rules[data->state[reg].use] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, data->state[reg].root, data->state[reg].use, data->state[reg].op, - pair | IR_REG_SPILL_LOAD); - } - IR_LOG_SPILL_LOAD(ref, data->state[reg].root, data->state[reg].use, data->state[reg].op, pair); - } - return; - } -#endif - IR_ASSERT(ival->reg == reg); - IR_LOG_EVICT(ref, reg); - IR_REGSET_EXCL(data->allocated, reg); - ival->reg = IR_REG_NONE; - - if (data->state[reg].use) { - /* add spill load for the previous use */ - if (!data->state[reg].root - || ctx->ir_base[data->state[reg].use].op == IR_ARGVAL) { - ir_set_alocated_reg(ctx, data->state[reg].use, data->state[reg].op, reg | IR_REG_SPILL_LOAD); - } else { - ctx->rules[data->state[reg].use] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, data->state[reg].root, data->state[reg].use, data->state[reg].op, - reg | IR_REG_SPILL_LOAD); - } - IR_LOG_SPILL_LOAD(ref, data->state[reg].root, data->state[reg].use, data->state[reg].op, reg); - } -} - -static ir_regset _preferred_regset(ir_ctx *ctx, ir_reg_alloc_fast_data *data, ir_ref use, ir_ref def) -{ - ir_ref ref; - - if (!use || def < data->bb->start || def >= use) { - return data->scratch; - } - for (ref = ctx->prev_ref[use]; ref != def; ref = ctx->prev_ref[ref]) { - if (ctx->ir_base[ref].op == IR_CALL) { - return data->preserved; - } - } - return data->scratch; -} - -static ir_reg _pick_free_reg(ir_ctx *ctx, ir_reg_alloc_fast_data *data, ir_type type, ir_ref use, ir_ref def) -{ - ir_regset available; - ir_ref reg; - - if (IR_IS_TYPE_INT(type)) { - available = IR_REGSET_GP; -#if defined(IR_TARGET_X86) - if (ir_type_size[type] == 1) { - IR_REGSET_EXCL(available, IR_REG_RBP); - IR_REGSET_EXCL(available, IR_REG_RSI); - IR_REGSET_EXCL(available, IR_REG_RDI); - } -#endif - } else { - IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); - available = IR_REGSET_FP; - } - available = IR_REGSET_DIFFERENCE(available, data->allocated); - if (!IR_REGSET_IS_EMPTY(available)) { - ir_regset preferred = IR_REGSET_INTERSECTION(_preferred_regset(ctx, data, use, def), available); - - if (!IR_REGSET_IS_EMPTY(preferred)) { - reg = IR_REGSET_FIRST(preferred); - if (IR_REGSET_IN(data->preserved, reg)) { - ir_regset tmp = (ir_regset)ctx->used_preserved_regs; - IR_REGSET_INCL(tmp, reg); - ctx->used_preserved_regs = tmp; - } - return reg; - } - reg = IR_REGSET_FIRST(available); - if (IR_REGSET_IN(data->preserved, reg)) { - ir_regset tmp = (ir_regset)ctx->used_preserved_regs; - IR_REGSET_INCL(tmp, reg); - ctx->used_preserved_regs = tmp; - } - return reg; - } - return IR_REG_NONE; -} - -static ir_reg _pick_reg(ir_ctx *ctx, ir_reg_alloc_fast_data *data, ir_type type, ir_regset used, uint8_t flags, ir_ref use, ir_ref def) -{ - ir_regset available; - ir_reg reg, best_reg; - ir_ref best_cost; - - if (IR_IS_TYPE_INT(type)) { - available = IR_REGSET_GP; -#if defined(IR_TARGET_X86) - if (ir_type_size[type] == 1) { - IR_REGSET_EXCL(available, IR_REG_RBP); - IR_REGSET_EXCL(available, IR_REG_RSI); - IR_REGSET_EXCL(available, IR_REG_RDI); - } -#endif - } else { - IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); - available = IR_REGSET_FP; - } - available = IR_REGSET_DIFFERENCE(available, used); - if (!IR_REGSET_IS_EMPTY(available)) { - ir_regset available_unused = IR_REGSET_DIFFERENCE(available, data->allocated); - ir_regset preferred = IR_REGSET_INTERSECTION(_preferred_regset(ctx, data, use, def), available); - - if (!IR_REGSET_IS_EMPTY(available_unused)) { - ir_regset available_preferred = IR_REGSET_INTERSECTION(available_unused, preferred); - - if (!IR_REGSET_IS_EMPTY(available_preferred)) { - reg = IR_REGSET_FIRST(available_preferred); - if (IR_REGSET_IN(data->preserved, reg)) { - ir_regset tmp = (ir_regset)ctx->used_preserved_regs; - IR_REGSET_INCL(tmp, reg); - ctx->used_preserved_regs = tmp; - } - return reg; - } - reg = IR_REGSET_FIRST(available_unused); - if (IR_REGSET_IN(data->preserved, reg)) { - ir_regset tmp = (ir_regset)ctx->used_preserved_regs; - IR_REGSET_INCL(tmp, reg); - ctx->used_preserved_regs = tmp; - } - return reg; - } - - if (!(flags & IR_USE_MUST_BE_IN_REG)) { - return IR_REG_NONE; - } - - /* select best reg to evict */ - available = IR_REGSET_INTERSECTION(available, data->allocated); - best_cost = 0x7fffffff; - best_reg = IR_REG_NONE; - IR_REGSET_FOREACH(available, reg) { - if (use) { - /* skip registers allocated for the same instruction */ - if (data->state[reg].root) { - if (data->state[reg].root == use) continue; - } else { - if (data->state[reg].use == use) continue; - } - } - if (data->state[reg].def < best_cost) { - best_cost = data->state[reg].def; - best_reg = reg; - } - } IR_REGSET_FOREACH_END(); - - _evict_reg(ctx, data, best_reg); - return best_reg; - } - - if (!(flags & IR_USE_MUST_BE_IN_REG)) { - return IR_REG_NONE; - } - - fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n"); - IR_ASSERT(0); - exit(-1); -} - -static ir_live_interval *_create_ival(ir_ctx *ctx, uint32_t vreg, ir_type type) -{ - ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); - - memset(ival, 0, sizeof(ir_live_interval)); - ctx->live_intervals[vreg] = ival; - ival->type = type; - ival->reg = IR_REG_NONE; - ival->vreg = vreg; - ival->stack_spill_pos = -1; - return ival; -} - -IR_ALWAYS_INLINE ir_live_interval *_get_ival(ir_ctx *ctx, ir_ref ref) -{ - int32_t vreg = ctx->vregs[ref]; - ir_live_interval *ival; - - IR_ASSERT(vreg > 0); - ival = ctx->live_intervals[vreg]; - if (!ival) { - return _create_ival(ctx, vreg, ctx->ir_base[ref].type); - } - return ival; -} - -static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn) -{ - size_t size = ir_get_type_size(var_insn->type); - size_t align = var_insn->op3; - int32_t offset; - -#if IR_SIMD - if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) { - size = 4; - align = IR_MAX(align, 4); - } -#endif - - offset = ir_allocate_spill_slot(ctx, size, align); - ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); - IR_LOG_ASSIGN_VAR(var, offset); -} - -static bool _may_coalesce(ir_ctx *ctx, ir_ref input, ir_ref def) -{ - /* Coalescing of values connected trough PHI requires global live range analyzes */ - return 0; -} - -static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_fast_data *data, ir_ref start) -{ - ir_ref stack[4]; - int stack_pos = 0; - ir_target_constraints constraints; - ir_insn *insn; - uint32_t j, n, def_flags; - ir_ref *p, child; - - while (1) { - IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED); - - if (!(ctx->rules[input] & IR_SIMPLE)) { - def_flags = ir_get_target_constraints(ctx, input, &constraints); - n = constraints.tmps_count; - while (n > 0) { - n--; - if (constraints.tmp_regs[n].type) { - uint8_t flags = 0; - ir_ref op = constraints.tmp_regs[n].num; - - if (op > 0 && op <= ctx->ir_base[input].inputs_count) { - ir_ref *ops = ctx->ir_base[input].ops; - - if (IR_IS_CONST_REF(ops[op])) { - /* rematerialization */ - flags = IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { - /* local address rematerialization */ - flags = IR_REG_SPILL_LOAD; - } - } - _add_tmp_reg(data, constraints.tmp_regs[n].type, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, flags, - ref, input, op); - } else { - _add_scratch(data, constraints.tmp_regs[n].reg, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); - } - } - } else { - def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; - constraints.hints_count = 0; - } - - insn = &ctx->ir_base[input]; - n = insn->inputs_count; - j = 1; - p = insn->ops + j; - if (ir_op_flags[insn->op] & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) { - j++; - p++; - } - for (; j <= n; j++, p++) { - IR_ASSERT(IR_OPND_KIND(ir_op_flags[insn->op], j) == IR_OPND_DATA); - child = *p; - if (child > 0) { - if (ctx->vregs[child] > 0) { - uint32_t use_flags = IR_USE_FLAGS(def_flags, j); - ir_live_interval *ival = ctx->live_intervals[ctx->vregs[child]]; - - if (!ival) { - ival = _create_ival(ctx, ctx->vregs[child], ctx->ir_base[child].type); - } - if (child < start || child >= ref) { - /* live before BB start - spill it */ - if (ival->stack_spill_pos == -1) { - _pick_spill(ctx, ival); - } - if (!(use_flags & IR_USE_MUST_BE_IN_REG)) { - ctx->rules[input] |= IR_FUSED_REG; - continue; - } - } - - ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; - int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF; - - _add_reg(data, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, use_flags, - ref, input, j, child); - } else if (ctx->rules[child] & IR_FUSED) { - IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos))); - stack[stack_pos++] = child; - } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) { - ir_set_alocated_reg(ctx, input, j, ctx->ir_base[child].op2); - } - } - } - if (!stack_pos) { - break; - } - input = stack[--stack_pos]; - } -} - -int ir_reg_alloc_fast(ir_ctx *ctx) -{ - uint32_t b; - ir_block *bb; - ir_insn *insn; - ir_ref i, n, j, *p; - ir_target_constraints constraints; - uint32_t def_flags; - ir_reg reg; - ir_regset param_regs; - ir_reg_alloc_fast_data data; - int32_t offset; - - ir_compute_dessa_moves(ctx); - - IR_LOG("Start Fast Local RA"); - - memset(&data.ra_data, 0, sizeof(data.ra_data)); - data.ra_data.cc = ir_get_call_conv_dsc(ctx->flags); - ctx->data = &data.ra_data; - - ctx->stack_frame_size = 0; - ctx->call_stack_size = 0; - ctx->used_preserved_regs = ctx->fixed_save_regset; - - data.fixed = (ir_regset)ctx->fixed_regset; - if (ctx->flags & IR_USE_FRAME_POINTER) { - IR_REGSET_INCL(data.fixed, IR_REG_FRAME_POINTER); - } - - data.scratch = IR_REGSET_DIFFERENCE(ir_scratch_regset[data.ra_data.cc->scratch_reg - IR_REG_NUM], data.fixed); - data.preserved = IR_REGSET_DIFFERENCE(data.ra_data.cc->preserved_regs, data.fixed); - param_regs = IR_REGSET_EMPTY; - - ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count); - memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count); - - /* vregs + tmp + fixed + SRATCH + ALL */ - ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*)); - - if (!ctx->arena) { - ctx->arena = ir_arena_create(16 * 1024); - } - - /* For all basic blocks in reverse order */ - for (b = ctx->cfg_blocks_count, bb = ctx->cfg_blocks + b; b > 0; bb--, b--) { - IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE)); - - if (bb->flags & IR_BB_EMPTY) { - continue; - } - - data.allocated = IR_REGSET_EMPTY; - data.bb = bb; - - if (bb->flags & IR_BB_DESSA_MOVES) { - ir_block *succ_bb; - ir_use_list *use_list; - ir_ref k, *p, n; - bool needs_tmp_gp = 0; - bool needs_tmp_fp = 0; - - /* For all PHI functions in the successor of this block */ - IR_ASSERT(bb->successors_count == 1); - succ_bb = &ctx->cfg_blocks[ctx->cfg_edges[bb->successors]]; - k = ir_phi_input_number(ctx, succ_bb, b); - use_list = &ctx->use_lists[succ_bb->start]; - n = use_list->count; - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - ir_ref use = *p; - ir_insn *use_insn = &ctx->ir_base[use]; - - if (use_insn->op == IR_PHI && ctx->vregs[use] > 0) { - ir_ref input = ir_insn_op(use_insn, k); - - /* Require tmp register for potential MEM->MEM DESSA move */ - if (use < bb->start || ctx->regs[use][0] == IR_REG_NONE) { - if (IR_IS_TYPE_INT(use_insn->type)) { - needs_tmp_gp = 1; - } else { - needs_tmp_fp = 1; - } - } - - if (input > 0 && ctx->vregs[input] > 0) { - ir_live_interval *ival = ctx->live_intervals[ctx->vregs[input]]; - - if (!ival) { - ival = _create_ival(ctx, ctx->vregs[input], ctx->ir_base[input].type); - } - - /* If input of PHI, coming from this block, is defined in this block - * (otherwise it's treated as "global" and spilled). - */ - if (input > bb->start && input < bb->end) { -#if IR_X86_I64 - if (use_insn->type == IR_I64 || use_insn->type == IR_U64) { - ir_reg reg_hi, reg_lo; - - if (ival->reg != IR_REG_NONE) { - /* reuse previously allocated registers */ - reg = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); - ir_set_alocated_reg(ctx, use, k, reg); - IR_LOG_ASSIGN(input, bb->end, use, k, reg); - continue; - } else if (ctx->regs[use][0] != IR_REG_NONE - && !IR_REG_SPILLED(ctx->regs[use][0]) - && !IR_REGSET_IN(data.allocated, IR_REG_I64_LO(ctx->regs[use][0])) - && !IR_REGSET_IN(data.allocated, IR_REG_I64_HI(ctx->regs[use][0]))) { - /* lightweight register coalescing */ - reg = ctx->regs[use][0]; - reg_lo = IR_REG_I64_LO(reg); - reg_hi = IR_REG_I64_HI(reg); - } else { - reg_lo = _pick_free_reg(ctx, &data, ival->type, bb->end, input); - if (reg_lo == IR_REG_NONE) { - reg = IR_REG_NONE; - } else { - IR_REGSET_INCL(data.allocated, reg_lo); - reg_hi = _pick_free_reg(ctx, &data, ival->type, bb->end, input); - if (reg_hi == IR_REG_NONE) { - IR_REGSET_EXCL(data.allocated, reg_lo); - reg = IR_REG_NONE; - } else { - IR_ASSERT(reg_lo != reg_hi); - if (reg_lo > reg_hi) { - SWAP_REGS(reg_lo, reg_hi); - } - reg = IR_REG_I64_PAIR(reg_lo, reg_hi); - } - } - } - if (reg != IR_REG_NONE) { - data.state[reg_hi].def = input; - data.state[reg_hi].root = IR_UNUSED; - data.state[reg_hi].use = use; - data.state[reg_hi].op = k; - IR_REGSET_INCL(data.allocated, reg_hi); - ival->reg_hi = reg_hi; - data.state[reg_lo].def = input; - data.state[reg_lo].root = IR_UNUSED; - data.state[reg_lo].use = use; - data.state[reg_lo].op = k; - IR_REGSET_INCL(data.allocated, reg_lo); - ival->reg = reg_lo; - ir_set_alocated_reg(ctx, use, k, reg); - IR_LOG_ASSIGN(input, bb->end, use, k, reg); - continue; - } - } else -#endif - { - if (ival->reg != IR_REG_NONE) { - /* reuse previously allocated register */ - reg = ival->reg; - ir_set_alocated_reg(ctx, use, k, reg); - IR_LOG_ASSIGN(input, bb->end, use, k, reg); - continue; - } else if (ctx->regs[use][0] != IR_REG_NONE - && !IR_REG_SPILLED(ctx->regs[use][0]) - && !IR_REGSET_IN(data.allocated, ctx->regs[use][0])) { - /* lightweight register coalescing */ - reg = ctx->regs[use][0]; - } else { - reg = _pick_free_reg(ctx, &data, ival->type, bb->end, input); - } - if (reg != IR_REG_NONE) { - data.state[reg].def = input; - data.state[reg].root = IR_UNUSED; - data.state[reg].use = use; - data.state[reg].op = k; - IR_REGSET_INCL(data.allocated, reg); - ival->reg = reg; - ir_set_alocated_reg(ctx, use, k, reg); - IR_LOG_ASSIGN(input, bb->end, use, k, reg); - continue; - } - } - } - - if (ival->stack_spill_pos == -1) { - if (ctx->vregs[use] > 0 - && ctx->live_intervals[ctx->vregs[use]] - && ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos != -1 - && _may_coalesce(ctx, input, use)) { - /* lightweight spill slot coalescing */ - ival->stack_spill_pos = ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos; - IR_LOG_ASSIGN_SPILL(ival->vreg, ival->stack_spill_pos); - } else { - _pick_spill(ctx, ival); - } - } - } - } - } - - /* Add tmp registers for MEM->MEM DESSA moves */ - if (needs_tmp_gp) { - ctx->regs[bb->end][0] = _pick_reg(ctx, &data, IR_I32, IR_REGSET_EMPTY, IR_USE_MUST_BE_IN_REG, 0, 0); - } - if (needs_tmp_fp) { - ctx->regs[bb->end][1] = _pick_reg(ctx, &data, IR_DOUBLE, IR_REGSET_EMPTY, IR_USE_MUST_BE_IN_REG, 0, 0); - } - } - - /* For all basic block's instructions in reverse order */ - for (i = bb->end; i != bb->start; i = ctx->prev_ref[i]) { - uint32_t rule = ctx->rules[i]; - - if (rule & (IR_FUSED|IR_SKIPPED)) { - if ((rule & IR_RULE_MASK) == IR_ALLOCA) { - insn = ctx->ir_base + i; - - /* assign "static" spill slots for VARs and ALLOCAs */ - if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) { - /* spill slot already allocated */ - } else if (insn->op == IR_VAR) { - if (ctx->use_lists[i].count > 0) { - ir_allocate_var_spill_slot(ctx, i, insn); - } - } else if (insn->op == IR_ALLOCA) { - if (ctx->use_lists[i].count > 0) { - ir_insn *val = &ctx->ir_base[insn->op2]; - - IR_ASSERT(IR_IS_CONST_REF(insn->op2)); - IR_ASSERT(IR_IS_TYPE_INT(val->type)); - IR_ASSERT(!IR_IS_SYM_CONST(val->op)); - IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); - IR_ASSERT(val->val.i64 < 0x7fffffff); - offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); - ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset); - IR_LOG_ASSIGN_VAR(i, offset); - } - } else if (insn->op == IR_VADDR) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) { - ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]); - } - ctx->vregs[i] = ctx->vregs[insn->op1]; - } - } - /* registers for fused instructions are allocated together with their "root"s */ - continue; - } - - insn = ctx->ir_base + i; - if (ctx->vregs[i] > 0) { - ir_live_interval *ival = ctx->live_intervals[ctx->vregs[i]]; - - if (!ival || (ival->reg == IR_REG_NONE && ival->stack_spill_pos == -1)) { - if (IR_IS_FOLDABLE_OP(insn->op) || (ir_op_flags[insn->op] & IR_OP_FLAG_NO_SIDE_EFFECT)) { - /* lightweigh dead code elimination */ - ctx->rules[i] = IR_SKIPPED | IR_NOP; - ctx->vregs[i] = 0; - continue; - } else if (insn->op == IR_PARAM) { - /* unused PARAM */ - ctx->rules[i] = IR_SKIPPED | IR_PARAM; - ctx->vregs[i] = 0; - continue; - } else if (insn->op == IR_CALL) { - /* unused CALL result */ - ctx->vregs[i] = 0; - } - - /* Handle PHI specially (inputs come from other blocks) */ - } else if (insn->op == IR_PHI) { - IR_ASSERT(ival); - if (ival->reg != IR_REG_NONE) { - /* use the previously allocated register */ -#if IR_X86_I64 - if (ival->type == IR_I64 || ival->type == IR_U64) { - IR_REGSET_EXCL(data.allocated, ival->reg); - IR_REGSET_EXCL(data.allocated, ival->reg_hi); - reg = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); - if (ival->stack_spill_pos != -1) { -#if 1 - /* Spilled PHI var is passed through memory */ - _evict_reg(ctx, &data, ival->reg); - reg = IR_REG_NONE; -#else - - reg |= IR_REG_SPILL_STORE; -#endif - } - ival->reg = ival->reg_hi = IR_REG_NONE; - ir_set_alocated_reg(ctx, i, 0, reg); - if (reg != IR_REG_NONE) { - IR_LOG_ASSIGN(i, i, i, 0, reg); - } - continue; - } -#endif - reg = ival->reg; - IR_REGSET_EXCL(data.allocated, reg); - if (ival->stack_spill_pos != -1) { -#if 1 - /* Spilled PHI var is passed through memory */ - _evict_reg(ctx, &data, reg); - reg = IR_REG_NONE; -#else - reg |= IR_REG_SPILL_STORE; -#endif - } - ival->reg = IR_REG_NONE; - ir_set_alocated_reg(ctx, i, 0, reg); - if (reg != IR_REG_NONE) { - IR_LOG_ASSIGN(i, i, i, 0, reg); - } - } else { - IR_ASSERT(ival->stack_spill_pos != -1); - } - continue; - - /* An attempt to swap instruction operands for better op1 register reuse */ - } else if ((rule & IR_MAY_SWAP) - && insn->op2 > 0 - && ctx->vregs[insn->op2] > 0 - && insn->op2 > bb->start - && (!ctx->live_intervals[ctx->vregs[insn->op2]] - || ctx->live_intervals[ctx->vregs[insn->op2]]->reg == IR_REG_NONE) - && !(insn->op1 > 0 - && ctx->vregs[insn->op1] > 0 - && insn->op1 > bb->start - && (!ctx->live_intervals[ctx->vregs[insn->op1]] - || ctx->live_intervals[ctx->vregs[insn->op1]]->reg == IR_REG_NONE))) { - SWAP_REFS(insn->op1, insn->op2); - } - } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { - ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); - } - if (b == 1 - && insn->op == IR_VSTORE - && !IR_IS_CONST_REF(insn->op3) - && ctx->ir_base[insn->op3].op == IR_PARAM - /* first use of VAR and PARAM is VSTORE(_, VAR, PARAM) */ - && ctx->use_edges[ctx->use_lists[insn->op2].refs] == i - && ctx->use_edges[ctx->use_lists[insn->op3].refs] == i) { - ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op3]]; - - if (!ival) { - ival = _create_ival(ctx, ctx->vregs[insn->op3], ctx->ir_base[insn->op3].type); - } - if (ival->stack_spill_pos == -1) { - ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]); - IR_LOG_ASSIGN_SPILL(ival->vreg, ival->stack_spill_pos); - continue; - } - } - } - - /* Reset information for the current instruction */ - data.num = data.tmp_num = 0; - data.clobbered_args = data.fixed; - for (j = 0; j < IR_SUB_REFS_COUNT; j++) { - data.clobbered[j] = IR_REGSET_EMPTY; - } - - /* Get infirmation about target constraints */ - def_flags = ir_get_target_constraints(ctx, i, &constraints); - - /* Collect information about scratch and temporary registers */ - n = constraints.tmps_count; - while (n) { - n--; - IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT); - if (constraints.tmp_regs[n].type) { - ir_reg flags = 0; - ir_ref op = constraints.tmp_regs[n].num; - - if (op > 0 && op <= insn->inputs_count) { - ir_ref *ops = insn->ops; - - if (IR_IS_CONST_REF(ops[op])) { - /* rematerialization */ - flags = IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { - /* local address rematerialization */ - flags = IR_REG_SPILL_LOAD; - } - } - _add_tmp_reg(&data, constraints.tmp_regs[n].type, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, flags, - IR_UNUSED, i, op); - } else { - _add_scratch(&data, constraints.tmp_regs[n].reg, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); - } - } - - /* Collect information about necessary input registers and related constaints */ - n = insn->inputs_count; - j = 1; - p = insn->ops + 1; - if (ir_op_flags[insn->op] & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) { - j++; - p++; - } - for (; j <= n; j++, p++) { - ir_ref input = *p; - ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; - ir_live_pos use_pos; - uint32_t use_flags = IR_USE_FLAGS(def_flags, j); - - if (reg != IR_REG_NONE) { -#if IR_X86_I64 - if (use_flags & IR_HINT_TWO_REGS) { - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); - } else -#endif - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg); - } - - if (input > 0) { - if (ctx->vregs[input] > 0) { - ir_live_interval *ival = ctx->live_intervals[ctx->vregs[input]]; - - if (!ival) { - ival = _create_ival(ctx, ctx->vregs[input], ctx->ir_base[input].type); - } - if (input < bb->start || input >= i) { - /* live before BB start - spill it */ - if (ival->stack_spill_pos == -1) { - _pick_spill(ctx, ival); - } - if (!(use_flags & IR_USE_MUST_BE_IN_REG)) { - continue; - } - } - - use_pos = IR_USE_SUB_REF; - if (reg != IR_REG_NONE) { - use_pos = IR_LOAD_SUB_REF; -#if IR_X86_I64 - if (use_flags & IR_HINT_TWO_REGS) { - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); - } else -#endif - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg); - } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (j == 1) { - if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - use_pos = IR_USE_SUB_REF; - } else { - use_pos = IR_LOAD_SUB_REF; - } - } else if (input == insn->op1) { - /* Input is the same as "op1" */ - use_pos = IR_LOAD_SUB_REF; - } - } - _add_reg(&data, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, use_flags, - IR_UNUSED, i, j, input); - } else { - if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) { - ir_add_fusion_data(ctx, i, input, &data, bb->start); - } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { - ir_set_alocated_reg(ctx, i, j, ctx->ir_base[input].op2); - } - } - } - } - - /* allocate output register */ - if (ctx->vregs[i] > 0) { - ir_live_interval *ival; - ir_live_pos def_pos; - ir_regset used; - - if (constraints.def_reg != IR_REG_NONE) { - def_pos = IR_SAVE_SUB_REF; - if (insn->op == IR_PARAM) { - /* parameter register must not be used before its loaded */ - IR_REGSET_INCL(param_regs, constraints.def_reg); - } - } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - def_pos = IR_USE_SUB_REF; - } else { - def_pos = IR_LOAD_SUB_REF; - } - } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - def_pos = IR_LOAD_SUB_REF; - } else { - if (insn->op == IR_PARAM) { - /* We may reuse parameter stack slot for spilling */ - ir_live_interval *ival = ctx->live_intervals[ctx->vregs[i]]; - - do { - if (ctx->use_lists[i].count > 0) { - ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; - if (use < bb->end) { - ir_insn *use_insn = &ctx->ir_base[use]; - if (use_insn->op == IR_VSTORE - && !(ctx->rules[use] & IR_SKIPPED) - /* first use of VAR is VSTORE(_, VAR, PARAM) */ - && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) { - IR_ASSERT(use_insn->op3 == i); - /* skip VSTORE (VAR is going to be remapped to PARAM) */ - ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; - ival->use_pos = NULL; - ctx->rules[use] = IR_SKIPPED | IR_NOP; - if (ctx->use_lists[i].count != 1) { - break; - } - } - } - } - ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - ival->stack_spill_pos = -1; // ??? - IR_LOG_ASSIGN_SPILL(ival->vreg, ival->stack_spill_pos); - } while (0); - } - def_pos = IR_DEF_SUB_REF; - } - - used = data.fixed; - for (n = def_pos; n <= IR_SAVE_SUB_REF; n++) { - used = IR_REGSET_UNION(used, data.clobbered[n]); - } - - ival = ctx->live_intervals[ctx->vregs[i]]; - if (!ival) { - ival = _create_ival(ctx, ctx->vregs[i], insn->type); - } - - -#if IR_X86_I64 - if (insn->type == IR_I64 || insn->type == IR_U64) { - /* allocate output register pair */ - ir_reg reg_lo, reg_hi; - - do { - if (ival->reg != IR_REG_NONE) { - if (!IR_REGSET_IN(used, ival->reg) && !IR_REGSET_IN(used, ival->reg_hi)) { - /* use the previously allocated register */ - reg_lo = ival->reg; - reg_hi = ival->reg_hi; - reg = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); - IR_REGSET_EXCL(data.allocated, ival->reg); - IR_REGSET_EXCL(data.allocated, ival->reg_hi); - ival->reg = ival->reg_hi = IR_REG_NONE; - IR_LOG_FREE(i, reg); - break; - } else { - _evict_reg(ctx, &data, ival->reg); - } - } - - if (def_flags & IR_USE_MUST_BE_IN_REG) { - reg = constraints.def_reg; - - if (reg != IR_REG_NONE - && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), IR_REG_I64_LO(reg)) - && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), IR_REG_I64_HI(reg))) { - /* use fixed register hint */ - reg_hi = IR_REG_I64_HI(reg); - reg_lo = IR_REG_I64_LO(reg); - } else { - reg_lo = _pick_reg(ctx, &data, insn->type, used, IR_USE_MUST_BE_IN_REG, 0, 0); - IR_REGSET_INCL(used, reg_lo); - reg_hi = _pick_reg(ctx, &data, insn->type, used, IR_USE_MUST_BE_IN_REG, 0, 0); - reg = IR_REG_I64_PAIR(ival->reg, ival->reg_hi); - IR_ASSERT(reg_lo != reg_hi); - if (reg_lo > reg_hi) { - SWAP_REGS(reg_lo, reg_hi); - } - reg = IR_REG_I64_PAIR(reg_lo, reg_hi); - } - } else { - reg = IR_REG_NONE; - } - } while (0); - - if (reg != IR_REG_NONE) { - for (n = def_pos; n <= IR_SAVE_SUB_REF; n++) { - IR_REGSET_INCL(data.clobbered[n], reg_lo); - IR_REGSET_INCL(data.clobbered[n], reg_hi); - } - if (ival && ival->stack_spill_pos != -1) { - reg |= IR_REG_SPILL_STORE; - } - ir_set_alocated_reg(ctx, i, 0, reg); - IR_LOG_ASSIGN(i, i, i, 0, reg); - } - } else -#endif - do { - /* allocate output register */ - do { - if (ival->reg != IR_REG_NONE) { - if (!IR_REGSET_IN(used, ival->reg)) { - /* use the previously allocated register */ - reg = ival->reg; - IR_REGSET_EXCL(data.allocated, reg); - ival->reg = IR_REG_NONE; - IR_LOG_FREE(i, reg); - break; - } else { - _evict_reg(ctx, &data, ival->reg); - } - } - if (def_flags & IR_USE_MUST_BE_IN_REG) { - reg = constraints.def_reg; - if (reg != IR_REG_NONE && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg)) { - /* use fixed register hint */ - } else { - reg = _pick_reg(ctx, &data, insn->type, used, IR_USE_MUST_BE_IN_REG, 0, 0); - } - } else { - reg = IR_REG_NONE; - } - } while (0); - - if (reg != IR_REG_NONE) { - for (n = def_pos; n <= IR_SAVE_SUB_REF; n++) { - IR_REGSET_INCL(data.clobbered[n], reg); - } - if (ival && ival->stack_spill_pos != -1) { - reg |= IR_REG_SPILL_STORE; - } - ir_set_alocated_reg(ctx, i, 0, reg); - IR_LOG_ASSIGN(i, i, i, 0, reg); - } - } while (0); - } - - /* evict clobbered registers */ - if (data.allocated != IR_REGSET_EMPTY) { - ir_regset used = param_regs; - - for (n = 0; n < IR_SUB_REFS_COUNT; n++) { - used = IR_REGSET_UNION(used, data.clobbered[n]); - } - used = IR_REGSET_INTERSECTION(used, data.allocated); - if (used != IR_REGSET_EMPTY) { - IR_REGSET_FOREACH(used, reg) { - _evict_reg(ctx, &data, reg); - } IR_REGSET_FOREACH_END(); - } - } - - /* allocate temporary registers */ - for (j = 0; j < data.tmp_num; j++) { - ir_regset used = data.fixed; - - for (n = data.tmp_regs[j].start; n < data.tmp_regs[j].end; n++) { - used = IR_REGSET_UNION(used, data.clobbered[n]); - } - - reg = _pick_reg(ctx, &data, data.tmp_regs[j].type, used, IR_USE_MUST_BE_IN_REG, 0, 0); - for (n = data.tmp_regs[j].start; n < data.tmp_regs[j].end; n++) { - IR_REGSET_INCL(data.clobbered[n], reg); - } -#if IR_X86_I64 - if (data.tmp_regs[j].type == IR_I64 || data.tmp_regs[j].type == IR_U64) { - ir_reg reg_hi; - - IR_REGSET_INCL(used, reg); - reg_hi = _pick_reg(ctx, &data, data.tmp_regs[j].type, used, IR_USE_MUST_BE_IN_REG, 0, 0); - for (n = data.tmp_regs[j].start; n < data.tmp_regs[j].end; n++) { - IR_REGSET_INCL(data.clobbered[n], reg_hi); - } - IR_ASSERT(reg != reg_hi); - if (reg > reg_hi) { - SWAP_REGS(reg, reg_hi); - } - reg = IR_REG_I64_PAIR(reg, reg_hi); - } -#endif - reg = reg | data.tmp_regs[j].flags; - if (data.tmp_regs[j].op == 4 && insn->inputs_count < 4) { - if (!ctx->tmp_regs) { - ctx->tmp_regs = ir_mem_malloc(ctx->insns_count); - memset(ctx->tmp_regs, -1, ctx->insns_count); - } - ctx->tmp_regs[data.tmp_regs[j].ref] = reg; - } else if (!data.tmp_regs[j].root - || ir_get_alocated_reg(ctx, data.tmp_regs[j].ref, data.tmp_regs[j].op) == IR_REG_NONE) { - ir_set_alocated_reg(ctx, data.tmp_regs[j].ref, data.tmp_regs[j].op, reg); - } else if (ir_get_alocated_reg(ctx, data.tmp_regs[j].ref, data.tmp_regs[j].op) != reg) { - ctx->rules[data.tmp_regs[j].ref] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, data.tmp_regs[j].root, data.tmp_regs[j].ref, data.tmp_regs[j].op, reg); - } - IR_LOG_ASSIGN_TMP(data.tmp_regs[j].type, i, data.tmp_regs[j].ref, data.tmp_regs[j].op, reg); - } - - /* allocate input registers */ - for (j = 0; j < data.num; j++) { - ir_regset used = data.clobbered_args; - ir_live_interval *ival; - - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - used = IR_REGSET_UNION(used, data.clobbered[n]); - } - -#if IR_X86_I64 - if (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64) { - ir_reg reg_lo, reg_hi; - - /* allocate input register pair */ - do { - IR_ASSERT(data.regs[j].def > 0 && ctx->vregs[data.regs[j].def] > 0); - ival = ctx->live_intervals[ctx->vregs[data.regs[j].def]]; - reg_lo = ival->reg; - reg_hi = ival->reg_hi; - - /* reuse previously allocated registers if possible */ - if (reg_lo != IR_REG_NONE && reg_hi != IR_REG_NONE) { - IR_ASSERT(data.state[reg_lo].def == data.regs[j].def); - IR_ASSERT(data.state[reg_hi].def == data.regs[j].def); - if ((data.state[reg_lo].root == 0 && data.state[reg_lo].use == i) - || data.state[reg_lo].root == i) { - /* keep previous operand of the same instraction as "last use" */ - break; - } else { - /* update "last use" */ - IR_ASSERT(!IR_REGSET_IN(used, reg_lo) && !IR_REGSET_IN(used, ival->reg_hi)); - data.state[reg_lo].root = data.regs[j].root; - data.state[reg_lo].use = data.regs[j].ref; - data.state[reg_lo].op = data.regs[j].op; - data.state[reg_hi].root = data.regs[j].root; - data.state[reg_hi].use = data.regs[j].ref; - data.state[reg_hi].op = data.regs[j].op; - reg = IR_REG_I64_PAIR(reg_lo, reg_hi); - break; - } - } - - /* try fixed register hint */ - reg = data.regs[j].hint; - reg_hi = IR_REG_NONE; - reg_lo = IR_REG_NONE; - if (reg != IR_REG_NONE) { - if (data.regs[j].flags & IR_HINT_TWO_REGS) { - reg_hi = IR_REG_I64_HI(reg); - reg_lo = IR_REG_I64_LO(reg); - } else { - reg_lo = reg; - } - } - - if (reg_lo != IR_REG_NONE - && reg_hi != IR_REG_NONE - && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg_lo) - && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg_hi)) { - /* use fixed register hint */ - } else if ((def_flags & IR_DEF_REUSES_OP1_REG) - && data.regs[j].op == 1 - && data.regs[j].root == IR_UNUSED - && ctx->regs[i][0] != IR_REG_NONE - && (insn->type == IR_I64 || insn->type == IR_U64) - && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), IR_REG_I64_LO(IR_REG_NUM(ctx->regs[i][0]))) - && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), IR_REG_I64_HI(IR_REG_NUM(ctx->regs[i][0]))) - && (ival->reg == IR_REG_NONE && ival->stack_spill_pos == -1)) { - /* reuse for op1 the register allocated for instruction result (lightweigh coalescing) */ - reg = IR_REG_NUM(ctx->regs[i][0]); - reg_hi = IR_REG_I64_HI(reg); - reg_lo = IR_REG_I64_LO(reg); - } else { - reg_lo = _pick_reg(ctx, &data, data.regs[j].type, used, data.regs[j].flags, i, data.regs[j].def); - if (reg_lo == IR_REG_NONE) { - reg = IR_REG_NONE; - break; - } - IR_REGSET_INCL(used, reg_lo); - reg_hi = _pick_reg(ctx, &data, data.regs[j].type, used, data.regs[j].flags, i, data.regs[j].def); - IR_REGSET_EXCL(used, reg_lo); - if (reg_hi == IR_REG_NONE) { - reg = IR_REG_NONE; - break; - } - IR_ASSERT(reg_lo != reg_hi); - if (reg_lo > reg_hi) { - SWAP_REGS(reg_lo, reg_hi); - } - reg = IR_REG_I64_PAIR(reg_lo, reg_hi); - } - - data.state[reg_lo].def = data.regs[j].def; - data.state[reg_lo].root = data.regs[j].root; - data.state[reg_lo].use = data.regs[j].ref; - data.state[reg_lo].op = data.regs[j].op; - data.state[reg_hi].def = data.regs[j].def; - data.state[reg_hi].root = data.regs[j].root; - data.state[reg_hi].use = data.regs[j].ref; - data.state[reg_hi].op = data.regs[j].op; - IR_REGSET_INCL(data.allocated, reg_lo); - IR_REGSET_INCL(data.allocated, reg_hi); - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - IR_REGSET_INCL(data.clobbered[n], reg_lo); - IR_REGSET_INCL(data.clobbered[n], reg_hi); - } - } while (0); - - if (reg != IR_REG_NONE) { - ival->reg = reg_lo; - ival->reg_hi = reg_hi; - if (!data.regs[j].root - || (!(ctx->rules[data.regs[j].ref] & IR_FUSED_REG) - && ir_get_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op) == IR_REG_NONE)) { - ir_set_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op, reg); - } else if (ir_get_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op) != reg) { - ctx->rules[data.regs[j].ref] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg); - } - IR_LOG_ASSIGN(data.regs[j].def, i, data.regs[j].ref, data.regs[j].op, reg); - } else if (ival->stack_spill_pos == -1) { - _pick_spill(ctx, ival); - } - - continue; - } -#endif - - /* allocate input register */ - do { - IR_ASSERT(data.regs[j].def > 0 && ctx->vregs[data.regs[j].def] > 0); - ival = ctx->live_intervals[ctx->vregs[data.regs[j].def]]; - reg = ival->reg; - - /* reuse previously allocated registers if possible */ - if (reg != IR_REG_NONE) { - IR_ASSERT(data.state[reg].def == data.regs[j].def); - if ((data.state[reg].root == 0 && data.state[reg].use == i) - || data.state[reg].root == i) { - /* keep previous operand of the same instraction as "last use" */ - break; - } else { - /* update "last use" */ - IR_ASSERT(!IR_REGSET_IN(used, reg)); - data.state[reg].root = data.regs[j].root; - data.state[reg].use = data.regs[j].ref; - data.state[reg].op = data.regs[j].op; - break; - } - } - - reg = data.regs[j].hint; - if (reg != IR_REG_NONE && !IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg)) { - /* use fixed register hint */ - } else if ((def_flags & IR_DEF_REUSES_OP1_REG) - && data.regs[j].op == 1 - && data.regs[j].root == IR_UNUSED - && ctx->regs[i][0] != IR_REG_NONE - && IR_IS_TYPE_INT(insn->type) == IR_IS_TYPE_INT(data.regs[j].type) - && (ival->reg == IR_REG_NONE && ival->stack_spill_pos == -1)) { - /* reuse for op1 the register allocated for instruction result (lightweigh coalescing) */ - reg = IR_REG_NUM(ctx->regs[i][0]); -#if IR_X86_I64 - if (insn->type == IR_I64 || insn->type == IR_U64) { - reg = IR_REG_I64_LO(reg); - } -#endif - if (IR_REGSET_IN(IR_REGSET_UNION(used, data.allocated), reg)) { - goto pick; - } -#if defined(IR_TARGET_X86) - if (IR_IS_TYPE_INT(data.regs[j].type) - && ir_type_size[data.regs[j].type] == 1 - && (reg == IR_REG_RBP || reg == IR_REG_RSI || reg == IR_REG_RDI)) { - goto pick; - } -#endif - } else { -pick: - reg = _pick_reg(ctx, &data, data.regs[j].type, used, data.regs[j].flags, i, data.regs[j].def); - if (reg == IR_REG_NONE) { - break; - } - } - data.state[reg].def = data.regs[j].def; - data.state[reg].root = data.regs[j].root; - data.state[reg].use = data.regs[j].ref; - data.state[reg].op = data.regs[j].op; - IR_REGSET_INCL(data.allocated, reg); - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - IR_REGSET_INCL(data.clobbered[n], reg); - } - } while (0); - - if (reg != IR_REG_NONE) { - ival->reg = reg; - if (!data.regs[j].root - || (!(ctx->rules[data.regs[j].ref] & IR_FUSED_REG) - && ir_get_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op) == IR_REG_NONE)) { - ir_set_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op, reg); - } else if (ir_get_alocated_reg(ctx, data.regs[j].ref, data.regs[j].op) != reg) { - ctx->rules[data.regs[j].ref] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg); - } - IR_LOG_ASSIGN(data.regs[j].def, i, data.regs[j].ref, data.regs[j].op, reg); - } else if (ival->stack_spill_pos == -1) { - _pick_spill(ctx, ival); - } - } - } - - if (data.allocated != IR_REGSET_EMPTY) { - IR_REGSET_FOREACH(data.allocated, reg) { - _evict_reg(ctx, &data, reg); - } IR_REGSET_FOREACH_END(); - } - } - -#ifdef IR_TARGET_X86 - if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); - } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) - && data.ra_data.cc->fp_ret_reg == IR_REG_NONE) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); - } else { - ctx->ret_slot = -1; - } -#endif - - ctx->flags |= IR_NO_STACK_COMBINE; - ir_fix_stack_frame(ctx); - ctx->data = NULL; - - IR_LOG("Finish Fast Local RA"); - - return 1; -} diff --git a/ext/opcache/jit/ir/ir_ra_simple.c b/ext/opcache/jit/ir/ir_ra_simple.c deleted file mode 100644 index 7a78801f4d71..000000000000 --- a/ext/opcache/jit/ir/ir_ra_simple.c +++ /dev/null @@ -1,671 +0,0 @@ -/* - * IR - Lightweight JIT Compilation Framework - * Simple Local Register Allocation (not used anymore) - * This file is part of the IR Project distributed under the MIT-style LICENSE. - * Authors: Dmitry Stogov - * - * Simple RA assign registers for each IR instruction separately and always pass - * values between instrutios in stack spill slots. - */ - -#ifndef _GNU_SOURCE -# define _GNU_SOURCE -#endif - -#include "ir.h" - -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) -# include "ir_x86.h" -#elif defined(IR_TARGET_AARCH64) -# include "ir_aarch64.h" -#else -# error "Unknown IR target" -#endif - -#include "ir_private.h" - -typedef struct { - ir_reg_alloc_data ra_data; - int32_t num; - ir_regset preserved_regs; - ir_regset clobbered[IR_SUB_REFS_COUNT]; - struct { - uint8_t type; - int8_t start; - int8_t end; - int8_t hint; - int8_t flags; - ir_ref root; - ir_ref ref; - ir_ref op; - } regs[32]; -} ir_reg_alloc_simple_data; - -static void _add_scratch(ir_reg_alloc_simple_data *data, ir_reg reg, int8_t start, int8_t end) -{ - int8_t j; - - if (start < 0) start = 0; // TODO: ARGVAL support ??? - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT); - if (reg >= IR_REG_NUM) { - for (j = start; j < end; j++) { - data->clobbered[j] = IR_REGSET_UNION(data->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]); - } - } else { - for (j = start; j < end; j++) { - IR_REGSET_INCL(data->clobbered[j], reg); - } - } -} - -static void _add_reg(ir_reg_alloc_simple_data *data, ir_type type, - int8_t start, int8_t end, ir_reg hint, int8_t flags, - ir_ref root, ir_ref ref, ir_ref op) -{ - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32); - data->regs[data->num].type = type; - data->regs[data->num].start = start; - data->regs[data->num].end = end; - data->regs[data->num].hint = hint; - data->regs[data->num].flags = flags; - data->regs[data->num].root = root; - data->regs[data->num].ref = ref; - data->regs[data->num].op = op; - data->num++; -} - -static ir_reg _get_free_reg(ir_type type, ir_regset available) -{ - if (IR_IS_TYPE_INT(type)) { - available = IR_REGSET_INTERSECTION(available, IR_REGSET_GP); - } else { - IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); - available = IR_REGSET_INTERSECTION(available, IR_REGSET_FP); - } - if (!IR_REGSET_IS_EMPTY(available)) { - return IR_REGSET_FIRST(available); - } else { - return IR_REG_NONE; - } -} - -static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data *data, int j) -{ - int n; - ir_regset available; - ir_reg reg; - - if (IR_IS_TYPE_INT(type)) { - available = IR_REGSET_GP; - if (ctx->flags & IR_USE_FRAME_POINTER) { - IR_REGSET_EXCL(available, IR_REG_FRAME_POINTER); - } - -#if defined(IR_TARGET_X86) - if (ir_type_size[type] == 1) { - /* TODO: if no registers avialivle, we may use of one this register for already allocated interval ??? */ - IR_REGSET_EXCL(available, IR_REG_RBP); - IR_REGSET_EXCL(available, IR_REG_RSI); - IR_REGSET_EXCL(available, IR_REG_RDI); - } -#endif - } else { - IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); - available = IR_REGSET_FP; - } - for (n = data->regs[j].start; n < data->regs[j].end; n++) { - available = IR_REGSET_DIFFERENCE(available, data->clobbered[n]); - } - if (IR_REGSET_IS_EMPTY(available)) { - fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n"); - IR_ASSERT(0); - exit(-1); - } - - reg = IR_REGSET_FIRST(available); - if (IR_REGSET_IN(data->preserved_regs, reg)) { - ir_regset tmp = (ir_regset)ctx->used_preserved_regs; - IR_REGSET_INCL(tmp, reg); - ctx->used_preserved_regs = tmp; - } - return reg; -} - -/* The following checks for the safety of VAR slot reuse are incomplete. - * Some of the unsupported edge cases are demonstrated at https://github.com/dstogov/ir/pull/232 - * and https://github.com/dstogov/ir/pull/230 (there are may be other problems). - * Unfortunatly, the attempts to fix the problems break the simplicity of this allocator and make it ridiculous. - * The complete removing of VAR slot reuse makes the already bad generated code significanly worse. - * - * A better appoach is implemented in ir_ra_fast.c - */ - -static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load) -{ - ir_use_list *use_list = &ctx->use_lists[load]; - ir_ref *p, use, i, n = use_list->count; - ir_ref last_use = IR_UNUSED; - ir_insn *insn; - int address_taken = -1; /* unknown */ - - if (n) { - /* Check if all uses of the result of VLOAD are in the same basic block (after VLOAD) */ - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - use = *p; - if (use < load || use > bb->end) return 0; - if (use > last_use) last_use = use; - } - /* Check if the result of VLOAD can not be clobbered between VLOAD and the last use */ - for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) { - if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) { - return 0; - } else if (address_taken - && (insn->op == IR_STORE - || insn->op == IR_STORE_v - || insn->op == IR_CALL)) { - if (address_taken == -1) { - ir_use_list *use_list = &ctx->use_lists[var]; - ir_ref *p, n = use_list->count; - - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - if (ctx->ir_base[*p].op == IR_VADDR) { - return 0; - } - } - address_taken = 0; - } - } - n = ir_insn_len(insn); - i += n; - insn += n; - } - } - return 1; -} - -static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref store, ir_ref val) -{ - ir_ref i, n; - ir_insn *insn; - int address_taken = -1; /* unknown */ - - /* Check if the value stored by VSTORE is produced in the same basic block (before VSTORE) */ - if (val < bb->start || val > store || store > bb->end) return 0; - - /* Check if the instructions between the "producer" and VSTORE can not access the same VAR */ - for (i = val, insn = &ctx->ir_base[i]; i < store;) { - if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) - && insn->op2 == var) { - return 0; - } else if (address_taken - && (insn->op == IR_LOAD - || insn->op == IR_LOAD_v - || insn->op == IR_STORE - || insn->op == IR_STORE_v - || insn->op == IR_CALL)) { - if (address_taken == -1) { - ir_use_list *use_list = &ctx->use_lists[var]; - ir_ref *p, n = use_list->count; - - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - if (ctx->ir_base[*p].op == IR_VADDR) { - return 0; - } - } - address_taken = 0; - } - } - n = ir_insn_len(insn); - i += n; - insn += n; - } - return 1; -} - -static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_simple_data *data) -{ - ir_ref stack[4]; - int stack_pos = 0; - ir_target_constraints constraints; - ir_insn *insn; - uint32_t j, n, flags, def_flags; - ir_ref *p, child; - - while (1) { - IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED); - - if (!(ctx->rules[input] & IR_SIMPLE)) { - def_flags = ir_get_target_constraints(ctx, input, &constraints); - n = constraints.tmps_count; - while (n > 0) { - n--; - if (constraints.tmp_regs[n].type) { - ir_reg flags = 0; - ir_ref op = constraints.tmp_regs[n].num; - - if (op > 0 && op <= ctx->ir_base[input].inputs_count) { - ir_ref *ops = ctx->ir_base[input].ops; - - if (IR_IS_CONST_REF(ops[op])) { - /* rematerialization */ - flags = IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { - /* local address rematerialization */ - flags = IR_REG_SPILL_LOAD; - } - } - _add_reg(data, constraints.tmp_regs[n].type, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags, - IR_UNUSED, input, op); - } else { - _add_scratch(data, constraints.tmp_regs[n].reg, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); - } - } - } else { - def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; - constraints.hints_count = 0; - } - - insn = &ctx->ir_base[input]; - flags = ir_op_flags[insn->op]; - n = IR_INPUT_EDGES_COUNT(flags); - j = 1; - p = insn->ops + j; - if (flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) { - j++; - p++; - } - for (; j <= n; j++, p++) { - IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); - child = *p; - if (child > 0) { - if (ctx->vregs[child] > 0) { - if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) { - ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; - int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF; - - _add_reg(data, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD, - ref, input, j); - } - } else if (ctx->rules[child] & IR_FUSED) { - IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos))); - stack[stack_pos++] = child; - } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) { - ctx->regs[input][j] = ctx->ir_base[child].op2; - } - } - } - if (!stack_pos) { - break; - } - input = stack[--stack_pos]; - } -} - -static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn) -{ - size_t size = ir_get_type_size(var_insn->type); - size_t align = var_insn->op3; - int32_t offset; - -#if IR_SIMD - if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) { - size = 4; - align = IR_MAX(align, 4); - } -#endif - - offset = ir_allocate_spill_slot(ctx, size, align); - ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); -} - -int ir_reg_alloc_simple(ir_ctx *ctx) -{ - uint32_t b; - ir_block *bb; - ir_insn *insn; - ir_ref i, n, j, *p; - uint32_t *rule, insn_flags; - ir_target_constraints constraints; - uint32_t def_flags; - ir_reg reg; - ir_regset scratch; - ir_reg_alloc_simple_data data; - int32_t offset; - - ir_compute_dessa_moves(ctx); - - memset(&data.ra_data, 0, sizeof(data.ra_data)); - data.ra_data.cc = ir_get_call_conv_dsc(ctx->flags); - ctx->data = &data.ra_data; - - ctx->stack_frame_size = 0; - ctx->call_stack_size = 0; - ctx->used_preserved_regs = 0; - ctx->used_preserved_regs = ctx->fixed_save_regset; - - scratch = ir_scratch_regset[data.ra_data.cc->scratch_reg - IR_REG_NUM]; - data.preserved_regs = IR_REGSET_DIFFERENCE(data.ra_data.cc->preserved_regs, ctx->fixed_save_regset); - - ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count); - memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count); - - /* vregs + tmp + fixed + SRATCH + ALL */ - ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*)); - - if (!ctx->arena) { - ctx->arena = ir_arena_create(16 * 1024); - } - - for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) { - IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE)); - for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) { - if (*rule & (IR_FUSED|IR_SKIPPED)) { - if ((*rule & IR_RULE_MASK) == IR_ALLOCA) { - if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) { - /* spill slot already allocated */ - } else if (insn->op == IR_VAR) { - if (ctx->use_lists[i].count > 0) { - ir_allocate_var_spill_slot(ctx, i, insn); - } - } else if (insn->op == IR_ALLOCA) { - if (ctx->use_lists[i].count > 0) { - ir_insn *val = &ctx->ir_base[insn->op2]; - - IR_ASSERT(IR_IS_CONST_REF(insn->op2)); - IR_ASSERT(IR_IS_TYPE_INT(val->type)); - IR_ASSERT(!IR_IS_SYM_CONST(val->op)); - IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); - IR_ASSERT(val->val.i64 < 0x7fffffff); - offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); - ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset); - } - } else if (insn->op == IR_VADDR) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) { - ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]); - } - ctx->vregs[i] = ctx->vregs[insn->op1]; - } - } - } else { - data.num = 0; - for (j = 0; j < IR_SUB_REFS_COUNT; j++) { - data.clobbered[j] = IR_REGSET_EMPTY; - } - - def_flags = ir_get_target_constraints(ctx, i, &constraints); - n = constraints.tmps_count; - while (n) { - n--; - - IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT); - if (constraints.tmp_regs[n].type) { - ir_reg flags = 0; - ir_ref op = constraints.tmp_regs[n].num; - - if (op > 0 && op <= insn->inputs_count) { - ir_ref *ops = insn->ops; - - if (IR_IS_CONST_REF(ops[op])) { - /* rematerialization */ - flags = IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { - /* local address rematerialization */ - flags = IR_REG_SPILL_LOAD; - } - } - _add_reg(&data, constraints.tmp_regs[n].type, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags, - IR_UNUSED, i, op); - } else { - _add_scratch(&data, constraints.tmp_regs[n].reg, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); - } - } - - if (ctx->vregs[i] > 0) { - reg = constraints.def_reg; - if (!ctx->live_intervals[ctx->vregs[i]]) { - do { - ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); - memset(ival, 0, sizeof(ir_live_interval)); - ctx->live_intervals[ctx->vregs[i]] = ival; - ival->type = insn->type; - ival->reg = IR_REG_NONE; - ival->vreg = ctx->vregs[i]; - ival->stack_spill_pos = -1; - - if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { - ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); - } - if (ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) { - ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]); - reg = IR_REG_NONE; - def_flags &= ~IR_USE_MUST_BE_IN_REG; - break; - } - } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) { - /* We may reuse parameter stack slot for spilling */ - ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - if (ctx->use_lists[i].count > 0) { - ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; - if (use < bb->end) { - ir_insn *use_insn = &ctx->ir_base[use]; - if (use_insn->op == IR_VSTORE - && !(ctx->rules[use] & IR_SKIPPED) - /* first use of VAR is VSTORE(_, VAR, PARAM) */ - && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) { - IR_ASSERT(use_insn->op3 == i); - /* skip VSTORE (VAR is going to be remapped to PARAM) */ - ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; - ival->use_pos = NULL; - ctx->rules[use] = IR_SKIPPED | IR_NOP; - } - } - } - break; - } - - if (ctx->use_lists[i].count == 1) { - ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; - ir_insn *use_insn = &ctx->ir_base[use]; - - if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) - && use_insn->op3 == i - && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[use_insn->op2])) { - ir_allocate_var_spill_slot(ctx, use_insn->op2, &ctx->ir_base[use_insn->op2]); - } - ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[use_insn->op2]); - break; - } - } - - size_t size = ir_get_type_size(ival->type); -#if IR_SIMD - if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { - size = 4; - } -#endif - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); - } while (0); - } - - if (def_flags & IR_USE_MUST_BE_IN_REG) { - ir_live_pos def_pos; - - if (reg != IR_REG_NONE) { - def_pos = IR_SAVE_SUB_REF; - } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - def_pos = IR_USE_SUB_REF; - } else { - def_pos = IR_LOAD_SUB_REF; - } - } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - def_pos = IR_LOAD_SUB_REF; - } else { - def_pos = IR_DEF_SUB_REF; - } - - _add_reg(&data, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE, - IR_UNUSED, i, 0); - } - } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { - ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); - } - } - - n = insn->inputs_count; - insn_flags = ir_op_flags[insn->op]; - j = 1; - p = insn->ops + 1; - if (insn_flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) { - j++; - p++; - } - for (; j <= n; j++, p++) { - ir_ref input = *p; - ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; - ir_live_pos use_pos; - uint32_t use_flags = IR_USE_FLAGS(def_flags, j); - - if (reg != IR_REG_NONE) { -#if IR_X86_I64 - if (use_flags & IR_HINT_TWO_REGS) { - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); - } else -#endif - IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg); - } - - if (input > 0) { - if (ctx->vregs[input] > 0) { - use_pos = IR_USE_SUB_REF; - if (reg != IR_REG_NONE) { - use_pos = IR_LOAD_SUB_REF; - } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (j == 1) { - if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - use_pos = IR_USE_SUB_REF; - } else { - use_pos = IR_LOAD_SUB_REF; - } - } else if (input == insn->op1) { - /* Input is the same as "op1" */ - use_pos = IR_LOAD_SUB_REF; - } - } - if (use_flags & IR_USE_MUST_BE_IN_REG) { - _add_reg(&data, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD, - IR_UNUSED, i, j); - } - } else { - if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) { - ir_add_fusion_data(ctx, i, input, &data); - } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { - ctx->regs[i][j] = ctx->ir_base[input].op2; - } - } - } - } - - for (j = 0; j < data.num; j++) { - ir_regset available = scratch; -#if IR_X86_I64 - ir_reg reg2; -#endif - - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - available = IR_REGSET_DIFFERENCE(available, data.clobbered[n]); - } - reg = data.regs[j].hint; -#if IR_X86_I64 - reg2 = IR_REG_NONE; - if (reg != IR_REG_NONE && (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64)) { - reg2 = IR_REG_I64_HI(reg); - reg = IR_REG_I64_LO(reg); - } -#endif - if (reg == IR_REG_NONE || !IR_REGSET_IN(available, reg)) { - reg = _get_free_reg(data.regs[j].type, available); - if (UNEXPECTED(reg == IR_REG_NONE)) { - reg = _get_free_reg2(ctx, data.regs[j].type, &data, j); - } - } - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - IR_REGSET_INCL(data.clobbered[n], reg); - } -#if IR_X86_I64 - if (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64) { - IR_REGSET_EXCL(available, reg); - if (reg2 == IR_REG_NONE || !IR_REGSET_IN(available, reg2)) { - reg2 = _get_free_reg(data.regs[j].type, available); - if (UNEXPECTED(reg2 == IR_REG_NONE)) { - reg2 = _get_free_reg2(ctx, data.regs[j].type, &data, j); - } - } - for (n = data.regs[j].start; n < data.regs[j].end; n++) { - IR_REGSET_INCL(data.clobbered[n], reg2); - } - if (reg > reg2) { - SWAP_REGS(reg, reg2); - } - reg = IR_REG_I64_PAIR(reg, reg2); - } -#endif - reg = reg | data.regs[j].flags; - if (data.regs[j].op == 4 && insn->inputs_count < 4) { - if (!ctx->tmp_regs) { - ctx->tmp_regs = ir_mem_malloc(ctx->insns_count); - memset(ctx->tmp_regs, -1, ctx->insns_count); - } - ctx->tmp_regs[data.regs[j].ref] = reg; - } else if (!data.regs[j].root || ctx->regs[data.regs[j].ref][data.regs[j].op] == IR_REG_NONE) { - ctx->regs[data.regs[j].ref][data.regs[j].op] = reg; - } else if (ctx->regs[data.regs[j].ref][data.regs[j].op] != reg) { - ctx->rules[data.regs[j].ref] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg); - } - } - } - - n = ir_insn_len(insn); - i += n; - insn += n; - rule += n; - } - if (bb->flags & IR_BB_DESSA_MOVES) { -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) - ctx->regs[bb->end][0] = IR_REG_RAX; - ctx->regs[bb->end][1] = IR_REG_XMM0; -#elif defined(IR_TARGET_AARCH64) - ctx->regs[bb->end][0] = IR_REG_X0; - ctx->regs[bb->end][1] = IR_REG_V0; -#else -# error "Unknown IR target" -#endif - } - } - -#ifdef IR_TARGET_X86 - if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); - } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) - && data.ra_data.cc->fp_ret_reg == IR_REG_NONE) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); - } else { - ctx->ret_slot = -1; - } -#endif - - ctx->flags |= IR_NO_STACK_COMBINE; - ir_fix_stack_frame(ctx); - ctx->data = NULL; - - return 1; -}