diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index bc9c0a428b7b..f3c7242e9d9b 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=93f92f5b5a49a02295685e340ebb0b83562cbe79 +revision=ed32c881a0d7c14ce7ace6ac00dead75647b69c6 git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm.h b/ext/opcache/jit/ir/dynasm/dasm_arm.h index eaa94d9c2f7f..97f541891d59 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm.h +++ b/ext/opcache/jit/ir/dynasm/dasm_arm.h @@ -1,6 +1,6 @@ /* ** DynASM ARM encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm.lua b/ext/opcache/jit/ir/dynasm/dasm_arm.lua index edb575366acd..7d95e4530640 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_arm.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM ARM module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm64.h b/ext/opcache/jit/ir/dynasm/dasm_arm64.h index 8a22974ad3ff..df8d5434d3a3 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm64.h +++ b/ext/opcache/jit/ir/dynasm/dasm_arm64.h @@ -1,6 +1,6 @@ /* ** DynASM ARM64 encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ @@ -178,44 +178,46 @@ static int dasm_imm12(unsigned int n) return -1; } -static int dasm_ffs(unsigned long long x) +#if defined(__GNUC__) || defined(__clang__) +#define dasm_ctz(x) __builtin_ctzll(x) +#define dasm_clz(x) __builtin_clzll(x) +#else +static int dasm_ctz(unsigned long long x) +{ + int n = 0; + x = ~x; + while (x & 1) { x >>= 1; n++; } + return n; +} + +static int dasm_clz(unsigned long long x) { - int n = -1; - while (x) { x >>= 1; n++; } + int n = 0; + x = ~x; + while ((long long)x < 0) { x <<= 1; n++; } return n; } +#endif + +static unsigned long long dasm_ror(unsigned long long x, int n) +{ + return (x << (-n & 63)) | (x >> n); +} static int dasm_imm13(int lo, int hi) { - int inv = 0, w = 64, s = 0xfff, xa, xb; + /* Thanks to: https://dougallj.wordpress.com/2021/10/30/ */ + int rot, ones, size, immr, imms; unsigned long long n = (((unsigned long long)hi) << 32) | (unsigned int)lo; - unsigned long long m = 1ULL, a, b, c; - if (n & 1) { n = ~n; inv = 1; } - a = n & (unsigned long long)-(long long)n; - b = (n+a)&(unsigned long long)-(long long)(n+a); - c = (n+a-b)&(unsigned long long)-(long long)(n+a-b); - xa = dasm_ffs(a); xb = dasm_ffs(b); - if (c) { - w = dasm_ffs(c) - xa; - if (w == 32) m = 0x0000000100000001UL; - else if (w == 16) m = 0x0001000100010001UL; - else if (w == 8) m = 0x0101010101010101UL; - else if (w == 4) m = 0x1111111111111111UL; - else if (w == 2) m = 0x5555555555555555UL; - else return -1; - s = (-2*w & 0x3f) - 1; - } else if (!a) { - return -1; - } else if (xb == -1) { - xb = 64; - } - if ((b-a) * m != n) return -1; - if (inv) { - return ((w - xb) << 6) | (s+w+xa-xb); - } else { - return ((w - xa) << 6) | (s+xb-xa); - } - return -1; + if ((n+1u) <= 1u) return -1; /* Neither all-zero nor all-ones are allowed. */ + rot = (n & (n+1u)) ? dasm_ctz(n & (n+1u)) : 64; + n = dasm_ror(n, rot & 63); + ones = dasm_ctz(~n); + size = dasm_clz(n) + ones; + if (dasm_ror(n, size & 63) != n) return -1; /* Non-repeating? */ + immr = -rot & (size - 1); + imms = (-(size << 1) | (ones - 1)) & 63; + return ((immr | (size & 64)) << 6) | imms; } /* Pass 1: Store actions and args, link branches/labels, estimate offsets. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm64.lua b/ext/opcache/jit/ir/dynasm/dasm_arm64.lua index 501c4bd2abf6..2d6fe202bcca 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_arm64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM ARM64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ @@ -244,6 +244,10 @@ local map_cond = { hs = 2, lo = 3, } +local map_bti = { + c = 0x40, j = 0x80, jc = 0xc0, +} + ------------------------------------------------------------------------------ local parse_reg_type @@ -475,6 +479,12 @@ local function parse_cond(expr, inv) return shl(bit.bxor(c, inv), 12) end +local function parse_map(expr, map) + local x = map[expr] + if not x then werror("bad operand") end + return x +end + local function parse_load(params, nparams, n, op) if params[n+2] then werror("too many operands") end local scale = shr(op, 30) @@ -823,11 +833,21 @@ map_op = { tbz_3 = "36000000DTBw|36000000DTBx", tbnz_3 = "37000000DTBw|37000000DTBx", + -- Branch Target Identification. + bti_1 = "d503241ft", + -- ARM64e: Pointer authentication codes (PAC). blraaz_1 = "d63f081fNx", + blrabz_1 = "d63f0c1fNx", braa_2 = "d71f0800NDx", + brab_2 = "d71f0c00NDx", braaz_1 = "d61f081fNx", + brabz_1 = "d61f0c1fNx", + paciasp_0 = "d503233f", pacibsp_0 = "d503237f", + autiasp_0 = "d50323bf", + autibsp_0 = "d50323ff", + retaa_0 = "d65f0bff", retab_0 = "d65f0fff", -- Miscellaneous instructions. @@ -996,6 +1016,8 @@ local function parse_template(params, template, nparams, pos) op = op + parse_cond(q, 0); n = n + 1 elseif p == "c" then op = op + parse_cond(q, 1); n = n + 1 + elseif p == "t" then + op = op + parse_map(q, map_bti); n = n + 1 else assert(false) diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips.h b/ext/opcache/jit/ir/dynasm/dasm_mips.h index 7800e9338773..e80cf8e758d8 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips.h +++ b/ext/opcache/jit/ir/dynasm/dasm_mips.h @@ -1,6 +1,6 @@ /* ** DynASM MIPS encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips.lua b/ext/opcache/jit/ir/dynasm/dasm_mips.lua index 1c605b681f15..4d7f5069e445 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_mips.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM MIPS32/MIPS64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips64.lua b/ext/opcache/jit/ir/dynasm/dasm_mips64.lua index c97d666b319a..c0db89650cef 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_mips64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM MIPS64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ -- This module just sets 64 bit mode for the combined MIPS/MIPS64 module. diff --git a/ext/opcache/jit/ir/dynasm/dasm_ppc.h b/ext/opcache/jit/ir/dynasm/dasm_ppc.h index 4c7d7289f9d7..85324e3c3166 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_ppc.h +++ b/ext/opcache/jit/ir/dynasm/dasm_ppc.h @@ -1,6 +1,6 @@ /* ** DynASM PPC/PPC64 encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_ppc.lua b/ext/opcache/jit/ir/dynasm/dasm_ppc.lua index d66ae4a0d616..88aa193395a0 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_ppc.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_ppc.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM PPC/PPC64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. -- -- Support for various extensions contributed by Caio Souza Oliveira. diff --git a/ext/opcache/jit/ir/dynasm/dasm_proto.h b/ext/opcache/jit/ir/dynasm/dasm_proto.h index 3f50f5028f3d..f7b379dfc335 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_proto.h +++ b/ext/opcache/jit/ir/dynasm/dasm_proto.h @@ -1,6 +1,6 @@ /* ** DynASM encoding engine prototypes. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_x64.lua b/ext/opcache/jit/ir/dynasm/dasm_x64.lua index 72d9bb87bff4..688e1a33180f 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_x64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM x64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ -- This module just sets 64 bit mode for the combined x86/x64 module. diff --git a/ext/opcache/jit/ir/dynasm/dasm_x86.h b/ext/opcache/jit/ir/dynasm/dasm_x86.h index d50299b13c1b..55d6f8386e63 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x86.h +++ b/ext/opcache/jit/ir/dynasm/dasm_x86.h @@ -1,6 +1,6 @@ /* ** DynASM x86 encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_x86.lua b/ext/opcache/jit/ir/dynasm/dasm_x86.lua index 0467bf913298..e339b3161639 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x86.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_x86.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM x86/x64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ diff --git a/ext/opcache/jit/ir/dynasm/dynasm.lua b/ext/opcache/jit/ir/dynasm/dynasm.lua index 0d15a87282f2..dc5d99de0665 100644 --- a/ext/opcache/jit/ir/dynasm/dynasm.lua +++ b/ext/opcache/jit/ir/dynasm/dynasm.lua @@ -2,7 +2,7 @@ -- DynASM. A dynamic assembler for code generation engines. -- Originally designed and implemented for LuaJIT. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See below for full copyright notice. ------------------------------------------------------------------------------ @@ -17,7 +17,7 @@ local _info = { url = "https://luajit.org/dynasm.html", license = "MIT", copyright = [[ -Copyright (C) 2005-2023 Mike Pall. All rights reserved. +Copyright (C) 2005-2026 Mike Pall. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 120722b2dcad..5f9b30980518 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -379,6 +379,7 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted #define ir_op_flag_xN (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_CALL | IR_OP_FLAG_VAR_INPUTS) #define ir_op_flag_a1 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT)) #define ir_op_flag_a2 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT)) +#define ir_op_flag_a2X1 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 2 | (3 << IR_OP_FLAG_OPERANDS_SHIFT)) #define ir_op_kind____ IR_OPND_UNUSED #define ir_op_kind_def IR_OPND_DATA @@ -391,8 +392,9 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted #define ir_op_kind_num IR_OPND_NUM #define ir_op_kind_fld IR_OPND_STR #define ir_op_kind_var IR_OPND_DATA -#define ir_op_kind_prb IR_OPND_PROB -#define ir_op_kind_opt IR_OPND_PROB +#define ir_op_kind_opt IR_OPND_OPT +#define ir_op_kind_prb IR_OPND_OPT +#define ir_op_kind_alg IR_OPND_OPT #define ir_op_kind_pro IR_OPND_PROTO #define ir_op_kind_lbl IR_OPND_LABEL_REF @@ -620,7 +622,7 @@ static IR_NEVER_INLINE void ir_const_hash_rehash(ir_ctx *ctx) ctx->const_hash = ir_mem_calloc(ctx->const_hash_mask + 1, sizeof(ir_ref)); for (ref = 1 - ctx->consts_count, insn = ctx->ir_base + ref; ref < IR_TRUE; ref++, insn++) { if (insn->op == IR_LONG_CONST) { - hash = insn->val.u64; + hash = insn->val.addr; insn->prev_const = ctx->const_hash[hash & ctx->const_hash_mask]; ctx->const_hash[hash & ctx->const_hash_mask] = ref; ref += IR_ALIGNED_SIZE(insn->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn); diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 1470606a1012..ee0dc259dee4 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -65,6 +65,8 @@ extern "C" { # define IR_TARGET_TRIPLET "x86-freebsd-sysv" # elif defined(__NetBSD__) # define IR_TARGET_TRIPLET "x86-netbsd-sysv" +# elif defined(__OpenBSD__) +# define IR_TARGET_TRIPLET "x86-openbsd-sysv" # else # define IR_TARGET_TRIPLET "x86-unknown-sysv" # endif @@ -79,6 +81,8 @@ extern "C" { # define IR_TARGET_TRIPLET "x86_64-freebsd-sysv" # elif defined(__NetBSD__) # define IR_TARGET_TRIPLET "x86_64-netbsd-sysv" +# elif defined(__OpenBSD__) +# define IR_TARGET_TRIPLET "x86_64-openbsd-sysv" # else # define IR_TARGET_TRIPLET "x86_64-unknown-sysv" # endif @@ -93,6 +97,8 @@ extern "C" { # define IR_TARGET_TRIPLET "aarch64-freebsd-sysv" # elif defined(__NetBSD__) # define IR_TARGET_TRIPLET "aarch64-netbsd-sysv" +# elif defined(__OpenBSD__) +# define IR_TARGET_TRIPLET "aarch64-openbsd-sysv" # else # define IR_TARGET_TRIPLET "aarch64-unknown-sysv" # endif @@ -256,9 +262,9 @@ typedef enum _ir_type { * E - control IR_OP_FLAG_CONTROL + IR_OP_FLAG_BB_END * T - control IR_OP_FLAG_CONTROL + IR_OP_FLAG_BB_END + IR_OP_FLAG_TERMINATOR * l - load IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_LOAD - * s - store IR_OP_FLAG_MEM + IR_OP_FLAG_STORE - * x - call IR_OP_FLAG_MEM + IR_OP_FLAG_CALL - * a - alloc IR_OP_FLAG_MEM + IR_OP_FLAG_ALLOC + * s - store IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_STORE + * x - call IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_CALL + * a - alloc IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_ALLOC * 0-3 - number of input edges * N - number of arguments is defined in the insn->inputs_count (MERGE, PHI, CALL) * X1-X3 - number of extra data ops @@ -279,6 +285,7 @@ typedef enum _ir_type { * num - number: argument number (PARAM) * prb - branch probability 1-99 (0 - unspecified): (IF_TRUE, IF_FALSE, CASE_VAL, CASE_DEFAULT) * opt - optional number + * alg - optional alignment * pro - function prototype * lbl - label used as value (a reference to constant): (BEGIN) * @@ -389,7 +396,7 @@ typedef enum _ir_type { \ /* data ops */ \ _(PARAM, p1X2, reg, str, num) /* incoming parameter proj. */ \ - _(VAR, p1X1, reg, str, ___) /* local variable */ \ + _(VAR, p1X2, reg, str, alg) /* local variable */ \ _(FUNC_ADDR, r0, ___, ___, ___) /* constant func ref */ \ _(FUNC, r0, ___, ___, ___) /* constant func ref */ \ _(SYM, r0, ___, ___, ___) /* constant symbol ref */ \ @@ -402,10 +409,10 @@ typedef enum _ir_type { _(TAILCALL, xN, src, def, def) /* CALL+RETURN */ \ \ /* memory reference and load/store ops */ \ - _(ALLOCA, a2, src, def, ___) /* alloca(def) */ \ - _(AFREE, a2, src, def, ___) /* revert alloca(def) */ \ + _(ALLOCA, a2X1, src, def, alg) /* alloca(def) */ \ + _(AFREE, s2, src, def, ___) /* revert alloca(def) */ \ _(BLOCK_BEGIN, a1, src, ___, ___) /* stacksave */ \ - _(BLOCK_END, a2, src, def, ___) /* stackrestore */ \ + _(BLOCK_END, s2, src, def, ___) /* stackrestore */ \ _(VLOAD, l2, src, var, ___) /* load value of local var */ \ _(VLOAD_v, l2, src, var, ___) /* volatile variant of VLOAD */ \ _(VSTORE, s3, src, var, def) /* store value to local var */ \ @@ -712,7 +719,6 @@ typedef struct _ir_code_buffer { typedef struct { int size; int align; - int offset; } ir_value_param; typedef struct _ir_bitqueue ir_bitqueue; @@ -747,8 +753,8 @@ struct _ir_ctx { uint32_t *cfg_map; /* map of instructions to basic block number */ uint32_t *cfg_schedule; /* BB order for code generation */ uint32_t *rules; /* array of target specific code-generation rules (for each instruction) */ - uint32_t *vregs; - ir_ref vregs_count; + int32_t *vregs; /* map of insns to virtual regs (positive) or stack slots (negative) */ + int32_t vregs_count; ir_str func_name; /* Function name (should be set through ir_string()/ir_stringl()) */ int32_t spill_base; /* base register for special spill area (e.g. PHP VM frame pointer) */ uint64_t fixed_regset; /* fixed registers, excluded for regular register allocation */ diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 539108dec9ac..43d918179118 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -390,6 +390,7 @@ const ir_call_conv_dsc ir_call_conv_aarch64_preserve_none = { _(OP_INT) \ _(OP_FP) \ _(BINOP_INT) \ + _(BIC) \ _(BINOP_FP) \ _(SHIFT) \ _(SHIFT_CONST) \ @@ -476,6 +477,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co constraints->hints_count = 0; switch (rule & IR_RULE_MASK) { case IR_BINOP_INT: + case IR_BIC: insn = &ctx->ir_base[ref]; n = 0; if (IR_IS_CONST_REF(insn->op1)) { @@ -1016,7 +1018,7 @@ static void ir_match_fuse_addr(ir_ctx *ctx, ir_ref addr_ref, ir_type type) && !IR_IS_CONST_REF(addr_insn->op1) && IR_IS_CONST_REF(addr_insn->op2) // TODO: temporary workaround && !IR_IS_SYM_CONST(ctx->ir_base[addr_insn->op2].op) - && aarch64_may_encode_addr_offset(ctx->ir_base[addr_insn->op2].val.i64, ir_type_size[type])) { + && aarch64_may_encode_addr_offset(ctx->ir_base[addr_insn->op2].val.i64, ir_get_type_size(type))) { ir_use_list *use_list = &ctx->use_lists[addr_ref]; ir_ref j = use_list->count; @@ -1043,6 +1045,18 @@ static uint32_t ir_match_builtin_call(ir_ctx *ctx, const ir_insn *func) return 0; } +static bool ir_have_calls_between(ir_ctx *ctx, ir_ref from, ir_ref to) +{ + ir_ref ref; + + for (ref = ctx->prev_ref[to]; ref > from; ref = ctx->prev_ref[ref]) { + if (ctx->ir_base[ref].op == IR_CALL) { + return 1; + } + } + return 0; +} + static bool all_usages_are_fusable(ir_ctx *ctx, ir_ref ref) { ir_insn *insn = &ctx->ir_base[ref]; @@ -1453,6 +1467,30 @@ binop_fp: return IR_COPY_INT | IR_MAY_REUSE; } } + if (IR_IS_TYPE_INT(insn->type) && insn->type != IR_BOOL) { + if (ctx->ir_base[insn->op1].op == IR_NOT + && !IR_IS_CONST_REF(ctx->ir_base[insn->op1].op1) + && ir_in_same_block(ctx, insn->op1) + && ctx->use_lists[insn->op1].count == 1 + && !ir_have_calls_between(ctx, insn->op1, ref) + && (!IR_IS_CONST_REF(insn->op2) || + !aarch64_may_encode_logical_imm(ctx->ir_base[insn->op2].val.u64, ir_type_size[insn->type]))) { + /* AND(NOT(X), Y) => BIC */ + ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_NOT; + return IR_BIC; + } else if (ctx->ir_base[insn->op2].op == IR_NOT + && !IR_IS_CONST_REF(ctx->ir_base[insn->op2].op1) + && ir_in_same_block(ctx, insn->op2) + && ctx->use_lists[insn->op2].count == 1 + && !ir_have_calls_between(ctx, insn->op2, ref) + && (!IR_IS_CONST_REF(insn->op1) || + !aarch64_may_encode_logical_imm(ctx->ir_base[insn->op1].val.u64, ir_type_size[insn->type]))) { + /* AND(X, NOT(Y)) => AND(NOT(Y), X) => BIC */ + SWAP_REFS(insn->op1, insn->op2); + ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_NOT; + return IR_BIC; + } + } goto binop_int; case IR_XOR: #if IR_SIMD @@ -1835,32 +1873,34 @@ static bool ir_is_same_spill_slot(ir_ctx *ctx, ir_ref ref, ir_mem mem) return IR_MEM_VAL(ir_ref_spill_slot(ctx, ref)) == IR_MEM_VAL(mem); } -static int32_t ir_var_spill_slot(ir_ctx *ctx, ir_ref ref, ir_reg *reg) +static int32_t ir_local_slot(ir_ctx *ctx, ir_ref ref) { - ir_insn *var_insn = &ctx->ir_base[ref]; - - IR_ASSERT(var_insn->op == IR_VAR); - *reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - return IR_SPILL_POS_TO_OFFSET(var_insn->op3); + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR || + ctx->ir_base[ref].op == IR_ALLOCA || + ctx->ir_base[ref].op == IR_VADDR || + ctx->ir_base[ref].op == IR_PARAM); + IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref])); + return IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]); } -static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use) +static int32_t ir_local_offset(ir_ctx *ctx, ir_ref ref) { - ir_live_interval *ival; + int32_t offset; - IR_ASSERT(ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); - ival = ctx->live_intervals[ctx->vregs[ref]]; - while (ival) { - ir_use_pos *use_pos = ival->use_pos; - while (use_pos) { - if (IR_LIVE_POS_TO_REF(use_pos->pos) == use) { - return !use_pos->next || use_pos->next->op_num == 0; - } - use_pos = use_pos->next; - } - ival = ival->next; - } - return 0; + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR || + ctx->ir_base[ref].op == IR_ALLOCA || + ctx->ir_base[ref].op == IR_VADDR || + ctx->ir_base[ref].op == IR_PARAM); + IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref])); + offset = IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]); + return IR_SPILL_POS_TO_OFFSET(offset); +} + +static int32_t ir_var_spill_slot(ir_ctx *ctx, ir_ref ref, ir_reg *reg) +{ + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR); + *reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + return ir_local_offset(ctx, ref); } static void ir_emit_load_imm_int(ir_ctx *ctx, ir_type type, ir_reg reg, int64_t val) @@ -1925,6 +1965,42 @@ static void ir_emit_load_imm_int(ir_ctx *ctx, ir_type type, ir_reg reg, int64_t } } +static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + int32_t offset; + + IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); + offset = ir_local_offset(ctx, src); + if (aarch64_may_encode_imm12(offset)) { + | add Rx(reg), Rx(base), #offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | add Rx(reg), Rx(base), Rx(IR_REG_INT_TMP) + } +} + +static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use) +{ + ir_live_interval *ival; + + IR_ASSERT(ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); + ival = ctx->live_intervals[ctx->vregs[ref]]; + while (ival) { + ir_use_pos *use_pos = ival->use_pos; + while (use_pos) { + if (IR_LIVE_POS_TO_REF(use_pos->pos) == use) { + return !use_pos->next || use_pos->next->op_num == 0; + } + use_pos = use_pos->next; + } + ival = ival->next; + } + return 0; +} + static void ir_emit_load_mem_int(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) { ir_backend_data *data = ctx->data; @@ -2162,34 +2238,6 @@ static void ir_emit_load_mem(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) } } -static int32_t ir_local_offset(ir_ctx *ctx, ir_insn *insn) -{ - IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA || insn->op == IR_VADDR); - return IR_SPILL_POS_TO_OFFSET(insn->op3); -} - -static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) -{ - ir_backend_data *data = ctx->data; - dasm_State **Dst = &data->dasm_state; - ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - ir_insn *var_insn; - int32_t offset; - - IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); - var_insn = &ctx->ir_base[src]; - if (var_insn->op == IR_VADDR) { - var_insn = &ctx->ir_base[var_insn->op1]; - } - offset = ir_local_offset(ctx, var_insn); - if (aarch64_may_encode_imm12(offset)) { - | add Rx(reg), Rx(base), #offset - } else { - ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); - | add Rx(reg), Rx(base), Rx(IR_REG_INT_TMP) - } -} - static void ir_resolve_label_syms(ir_ctx *ctx) { uint32_t b; @@ -2246,7 +2294,7 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) } else { ir_emit_load_imm_fp(ctx, type, reg, src); } - } else if (ctx->vregs[src]) { + } else if (ctx->vregs[src] > 0) { ir_mem mem = ir_ref_spill_slot(ctx, src); ir_emit_load_mem(ctx, type, reg, mem); } else { @@ -2959,6 +3007,52 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } +static void ir_emit_bic(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = insn->type; + ir_ref not_ref = insn->op1; + ir_ref op1 = ctx->ir_base[not_ref].op1; + ir_ref op2 = insn->op2; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op1_reg; + ir_reg op2_reg = ctx->regs[def][2]; + + IR_ASSERT(ctx->ir_base[not_ref].op == IR_NOT); + if (UNEXPECTED(ctx->rules[not_ref] & IR_FUSED_REG)) { + op1_reg = ir_get_fused_reg(ctx, def, not_ref * sizeof(ir_ref) + 1); + } else { + op1_reg = ctx->regs[not_ref][1]; + } + + IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); + + if (IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + ir_emit_load(ctx, type, op1_reg, op1); + } + + if (op1 == op2) { + op2_reg = IR_REG_NUM(op1_reg); + } else if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, type, op2_reg, op2); + } + } else { + IR_ASSERT(IR_IS_CONST_REF(insn->op2)); + op2_reg = IR_REG_INT_TMP; + ir_emit_load(ctx, type, op2_reg, op2); + } + + | ASM_REG_REG_REG_OP bic, type, def_reg, op2_reg, op1_reg + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, type, def, def_reg); + } +} + static void ir_emit_min_max_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) { ir_backend_data *data = ctx->data; @@ -4600,10 +4694,10 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) | fmov Rw(def_reg), Rs(op1_reg-IR_REG_FP_FIRST) } } else if (IR_IS_CONST_REF(insn->op1)) { - if (src_type == IR_DOUBLE) { + if (size == 8) { ir_emit_load_imm_int(ctx, dst_type, def_reg, ctx->ir_base[insn->op1].val.i64); } else { - IR_ASSERT(src_type == IR_FLOAT); + IR_ASSERT(size <= 4); ir_emit_load_imm_int(ctx, dst_type, def_reg, ctx->ir_base[insn->op1].val.i32); } } else { @@ -4642,10 +4736,10 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } else if (IR_IS_CONST_REF(insn->op1)) { ir_emit_load_imm_int(ctx, src_type, IR_REG_INT_TMP, ctx->ir_base[insn->op1].val.i64); - if (dst_type == IR_DOUBLE) { + if (size == 8) { | fmov Rd(def_reg-IR_REG_FP_FIRST), Rx(IR_REG_INT_TMP) } else { - IR_ASSERT(dst_type == IR_FLOAT); + IR_ASSERT(size <= 4); | fmov Rs(def_reg-IR_REG_FP_FIRST), Rw(IR_REG_INT_TMP) } } else { @@ -4903,22 +4997,22 @@ static void ir_emit_vaddr(ir_ctx *ctx, ir_ref def, ir_insn *insn) static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_ref type = insn->type; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); ir_reg fp; - int32_t offset; + int32_t slot, offset; ir_mem mem; if (ctx->use_lists[def].count == 1) { /* dead load */ return; } - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = IR_SPILL_POS_TO_OFFSET(var_insn->op3); + slot = ir_local_slot(ctx, insn->op2); + offset = IR_SPILL_POS_TO_OFFSET(slot); mem = IR_MEM_BO(fp, offset); - if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, var_insn->op3)) { + if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, slot)) { return; // fake load } IR_ASSERT(def_reg != IR_REG_NONE); @@ -4930,22 +5024,22 @@ static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn) static void ir_emit_vstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_insn *val_insn = &ctx->ir_base[insn->op3]; ir_ref type = val_insn->type; ir_reg op3_reg = ctx->regs[ref][3]; ir_reg fp; - int32_t offset; + int32_t slot, offset; ir_mem mem; - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = IR_SPILL_POS_TO_OFFSET(var_insn->op3); + slot = ir_local_slot(ctx, insn->op2); + offset = IR_SPILL_POS_TO_OFFSET(slot); IR_ASSERT(op3_reg != IR_REG_NONE); if (IR_REG_SPILLED(op3_reg) && !IR_IS_CONST_REF(insn->op3) && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA - && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) { + && ir_is_same_mem_var(ctx, insn->op3, slot)) { return; // fake store } if (IR_REG_SPILLED(op3_reg)) { @@ -4967,7 +5061,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[addr_insn->op2].op)); if (ir_rule(ctx, addr_insn->op1) == IR_STATIC_ALLOCA) { reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[addr_insn->op1]); + offset = ir_local_offset(ctx, addr_insn->op1); offset += ctx->ir_base[addr_insn->op2].val.i32; return IR_MEM_BO(reg, offset); } else { @@ -4985,7 +5079,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) } else { IR_ASSERT(addr_insn->op == IR_ALLOCA || addr_insn->op == IR_VADDR); reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[ref]); + offset = ir_local_offset(ctx, ref); return IR_MEM_BO(reg, offset); } } @@ -5157,7 +5251,7 @@ static void ir_emit_rload(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_type type = insn->type; if (IR_REGSET_IN(IR_REGSET_UNION((ir_regset)ctx->fixed_regset, IR_REGSET_FIXED), src_reg)) { - if (ctx->vregs[def] + if (ctx->vregs[def] > 0 && ctx->live_intervals[ctx->vregs[def]] && ctx->live_intervals[ctx->vregs[def]]->stack_spill_pos != -1) { ir_emit_store(ctx, type, def, src_reg); @@ -5384,7 +5478,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -5394,8 +5488,18 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) fp = IR_REG_STACK_POINTER; arg_area_offset = ctx->call_stack_size + ctx->stack_frame_size + ctx->param_stack_size; } - | add Rx(tmp_reg), Rx(fp), #arg_area_offset - | str Rx(tmp_reg), [Rx(op2_reg), #offset] + if (aarch64_may_encode_imm12(arg_area_offset)) { + | add Rx(tmp_reg), Rx(fp), #arg_area_offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, arg_area_offset); + | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } + if (aarch64_may_encode_addr_offset(offset, sizeof(void*))) { + | str Rx(tmp_reg), [Rx(op2_reg), #offset] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } else { ir_reg fp; int reg_save_area_offset; @@ -5414,7 +5518,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -5428,35 +5532,85 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } /* Set va_list.stack */ - | add Rx(tmp_reg), Rx(fp), #overflow_arg_area_offset - | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, stack))] + if (aarch64_may_encode_imm12(overflow_arg_area_offset)) { + | add Rx(tmp_reg), Rx(fp), #overflow_arg_area_offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, overflow_arg_area_offset); + | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, stack), sizeof(void*))) { + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, stack))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, stack)); + | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } if ((ctx->flags2 & (IR_HAS_VA_ARG_GP|IR_HAS_VA_COPY)) && ctx->gp_reg_params < cc->int_param_regs_count) { reg_save_area_offset += sizeof(void*) * cc->int_param_regs_count; /* Set va_list.gr_top */ if (overflow_arg_area_offset != reg_save_area_offset) { - | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset + if (aarch64_may_encode_imm12(reg_save_area_offset)) { + | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, reg_save_area_offset); + | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } + } + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_top), sizeof(void*))) { + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_top))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_top)); + | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] } - | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_top))] /* Set va_list.gr_offset */ | movn Rw(tmp_reg), #~(0 - (sizeof(void*) * (cc->int_param_regs_count - ctx->gp_reg_params))) - | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset), sizeof(void*))) { + | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_offset)); + | str Rw(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } else { /* Set va_list.gr_offset */ - | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset), sizeof(void*))) { + | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_offset)); + | str wzr, [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } if ((ctx->flags2 & (IR_HAS_VA_ARG_FP|IR_HAS_VA_COPY)) && ctx->fp_reg_params < cc->fp_param_regs_count) { reg_save_area_offset += 16 * cc->fp_param_regs_count; /* Set va_list.vr_top */ if (overflow_arg_area_offset != reg_save_area_offset || ctx->gp_reg_params < cc->int_param_regs_count) { - | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset + if (aarch64_may_encode_imm12(reg_save_area_offset)) { + | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, reg_save_area_offset); + | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } + } + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_top), sizeof(void*))) { + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_top))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_top)); + | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] } - | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_top))] /* Set va_list.vr_offset */ | movn Rw(tmp_reg), #~(0 - (16 * (cc->fp_param_regs_count - ctx->fp_reg_params))) - | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))] + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset), sizeof(void*))) { + | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_offset)); + | str Rw(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } else { /* Set va_list.vr_offset */ - | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))] + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset), sizeof(void*))) { + | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_offset)); + | str wzr, [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } } } @@ -5483,7 +5637,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + op2_offset = ir_local_offset(ctx, insn->op2); } if (op3_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op3_reg)) { @@ -5494,7 +5648,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]); + op3_offset = ir_local_offset(ctx, insn->op3); } | ldr Rx(tmp_reg), [Rx(op3_reg), #op3_offset] | str Rx(tmp_reg), [Rx(op2_reg), #op2_offset] @@ -5514,7 +5668,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + op2_offset = ir_local_offset(ctx, insn->op2); } if (op3_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op3_reg)) { @@ -5525,7 +5679,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]); + op3_offset = ir_local_offset(ctx, insn->op3); } | ldr Rx(tmp_reg), [Rx(op3_reg), #op3_offset] | str Rx(tmp_reg), [Rx(op2_reg), #op2_offset] @@ -5561,7 +5715,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } | ldr Rx(tmp_reg), [Rx(op2_reg), #offset] if (def_reg != IR_REG_NONE) { @@ -5589,7 +5743,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (IR_IS_TYPE_INT(type)) { | ldr Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] @@ -5690,10 +5844,12 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(val2->op)); if (IR_IS_TYPE_SIGNED(type)) { IR_ASSERT(IR_IS_TYPE_SIGNED(val->type)); + IR_ASSERT(IR_IS_TYPE_SIGNED(val2->type)); min.i64 = IR_MIN(min.i64, val->val.i64); max.i64 = IR_MAX(max.i64, val2->val.i64); } else { IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type)); + IR_ASSERT(!IR_IS_TYPE_SIGNED(val2->type)); min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64); max.u64 = (int64_t)IR_MAX(max.u64, val2->val.u64); } @@ -6219,7 +6375,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_cal if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } - } else if (ctx->use_lists[def].count > 1) { + } else if (ctx->vregs[def] > 0) { ir_emit_store(ctx, insn->type, def, cc->int_ret_reg); } } else { @@ -6232,7 +6388,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_cal if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } - } else if (ctx->use_lists[def].count > 1) { + } else if (ctx->vregs[def] > 0) { ir_emit_store(ctx, insn->type, def, cc->fp_ret_reg); } } @@ -8835,7 +8991,7 @@ static void ir_emit_load_params(ir_ctx *ctx) } fp_param_num++; } - if (ctx->vregs[use]) { + if (ctx->vregs[use] > 0) { dst_reg = IR_REG_NUM(ctx->regs[use][0]); IR_ASSERT(src_reg != IR_REG_NONE || dst_reg != IR_REG_NONE || stack_start + stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos); @@ -8921,7 +9077,7 @@ static void ir_fix_param_spills(ir_ctx *ctx) fp_param_num++; } if (src_reg == IR_REG_NONE) { - if (ctx->vregs[use]) { + if (ctx->vregs[use] > 0) { ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) && ival->stack_spill_pos == -1 @@ -9187,6 +9343,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_BINOP_INT: ir_emit_binop_int(ctx, i, insn); break; + case IR_BIC: + ir_emit_bic(ctx, i, insn); + break; case IR_BINOP_FP: ir_emit_binop_fp(ctx, i, insn); break; diff --git a/ext/opcache/jit/ir/ir_disasm.c b/ext/opcache/jit/ir/ir_disasm.c index 0fe8aaea6e93..30cfb694cff6 100644 --- a/ext/opcache/jit/ir/ir_disasm.c +++ b/ext/opcache/jit/ir/ir_disasm.c @@ -599,7 +599,7 @@ int ir_disasm(const char *name, } if (p != q && *(q-1) == '-') { q--; - addr = (uint32_t)(-(int64_t)addr); + addr = (uint32_t)(0 - addr); } if (addr >= (uint64_t)(uintptr_t)start && addr < (uint64_t)(uintptr_t)orig_end) { entry = ir_hashtab_find(&labels, (uint32_t)((uintptr_t)addr - (uintptr_t)start)); diff --git a/ext/opcache/jit/ir/ir_dump.c b/ext/opcache/jit/ir/ir_dump.c index 8d15e12f842d..ac751328411f 100644 --- a/ext/opcache/jit/ir/ir_dump.c +++ b/ext/opcache/jit/ir/ir_dump.c @@ -400,13 +400,13 @@ void ir_dump_live_ranges(const ir_ctx *ctx, FILE *f) fprintf(f, "TMP"); } else { for (j = 1; j < ctx->insns_count; j++) { - if (ctx->vregs[j] == (uint32_t)i) { + if (ctx->vregs[j] == i) { break; } } fprintf(f, "R%d (d_%d", i, j); for (j++; j < ctx->insns_count; j++) { - if (ctx->vregs[j] == (uint32_t)i) { + if (ctx->vregs[j] == i) { fprintf(f, ", d_%d", j); } } @@ -667,7 +667,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) fprintf(f, "\t"); ir_print_type_cname(insn->type, f); fprintf(f, " d_%d", i); - if (ctx->vregs && ctx->vregs[i]) { + if (ctx->vregs && ctx->vregs[i] > 0) { fprintf(f, " {R%d}", ctx->vregs[i]); } if (ctx->regs) { @@ -683,7 +683,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) if (flags & IR_OP_FLAG_DATA) { ir_print_type_cname(insn->type, f); fprintf(f, " d_%d", i); - if (ctx->vregs && ctx->vregs[i]) { + if (ctx->vregs && ctx->vregs[i] > 0) { fprintf(f, " {R%d}", ctx->vregs[i]); } if (ctx->regs) { @@ -719,7 +719,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) } else { fprintf(f, "%sd_%d", first ? "(" : ", ", ref); } - if (ctx->vregs && ref > 0 && ctx->vregs[ref]) { + if (ctx->vregs && ref > 0 && ctx->vregs[ref] > 0) { fprintf(f, " {R%d}", ctx->vregs[ref]); } if (ctx->regs) { @@ -746,7 +746,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) fprintf(f, "%sfunc ", first ? "(" : ", "); ir_print_proto(ctx, ref, f); break; - case IR_OPND_PROB: + case IR_OPND_OPT: if (ref == 0) { break; } diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index 8c01f56c9e30..71555ff5da5a 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -250,7 +250,7 @@ static bool ir_is_same_mem_var(const ir_ctx *ctx, ir_ref r1, int32_t offset) return 0; } - IR_ASSERT(ctx->vregs[r1]); + IR_ASSERT(ctx->vregs[r1] > 0); ival1 = ctx->live_intervals[ctx->vregs[r1]]; IR_ASSERT(ival1); o1 = ival1->stack_spill_pos; @@ -439,7 +439,7 @@ static IR_NEVER_INLINE void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block for (i = 0; i < count; i++, pos++) { ref = ir_list_at(list, pos); - IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); + IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); if (!(ctx->live_intervals[ctx->vregs[ref]]->flags & IR_LIVE_INTERVAL_SPILLED)) { /* not spilled */ ir_reg reg = ctx->live_intervals[ctx->vregs[ref]]->reg; @@ -1393,7 +1393,7 @@ int32_t ir_get_spill_slot_offset(const ir_ctx *ctx, ir_ref ref) { int32_t offset; - IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); + IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); offset = ctx->live_intervals[ctx->vregs[ref]]->stack_spill_pos; IR_ASSERT(offset != -1); return IR_SPILL_POS_TO_OFFSET(offset); @@ -1528,7 +1528,7 @@ static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data reg = IR_REGSET_FIRST(available); if (IR_REGSET_IN(x->preserved_regs, reg)) { - ir_regset tmp = ctx->used_preserved_regs; + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; IR_REGSET_INCL(tmp, reg); ctx->used_preserved_regs = tmp; } @@ -1554,16 +1554,34 @@ static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir ir_ref *p, use, i, n = use_list->count; ir_ref last_use = IR_UNUSED; ir_insn *insn; + int address_taken = -1; /* unknown */ if (n) { + /* Check if all uses of the result of VLOAD are in the same basic block (after VLOAD) */ for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { use = *p; if (use < load || use > bb->end) return 0; if (use > last_use) last_use = use; } + /* Check if the result of VLOAD can not be clobbered between VLOAD and the last use */ for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) { if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) { return 0; + } else if (address_taken + && (insn->op == IR_STORE + || insn->op == IR_STORE_v + || insn->op == IR_CALL)) { + if (address_taken == -1) { + ir_use_list *use_list = &ctx->use_lists[var]; + ir_ref *p, n = use_list->count; + + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + if (ctx->ir_base[*p].op == IR_VADDR) { + return 0; + } + } + address_taken = 0; + } } n = ir_insn_len(insn); i += n; @@ -1577,13 +1595,33 @@ static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, i { ir_ref i, n; ir_insn *insn; + int address_taken = -1; /* unknown */ - if (val < bb->start && val > store) return 0; + /* Check if the value stored by VSTORE is produced in the same basic block (before VSTORE) */ + if (val < bb->start || val > store || store > bb->end) return 0; + /* Check if the instructions between the "producer" and VSTORE can not access the same VAR */ for (i = val, insn = &ctx->ir_base[i]; i < store;) { if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) { return 0; + } else if (address_taken + && (insn->op == IR_LOAD + || insn->op == IR_LOAD_v + || insn->op == IR_STORE + || insn->op == IR_STORE_v + || insn->op == IR_CALL)) { + if (address_taken == -1) { + ir_use_list *use_list = &ctx->use_lists[var]; + ir_ref *p, n = use_list->count; + + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + if (ctx->ir_base[*p].op == IR_VADDR) { + return 0; + } + } + address_taken = 0; + } } n = ir_insn_len(insn); i += n; @@ -1650,7 +1688,7 @@ static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_all IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); child = *p; if (child > 0) { - if (ctx->vregs[child]) { + if (ctx->vregs[child] > 0) { if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) { ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF; @@ -1673,6 +1711,23 @@ static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_all } } +static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn) +{ + size_t size = ir_get_type_size(var_insn->type); + size_t align = var_insn->op3; + int32_t offset; + +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) { + size = 4; + align = IR_MAX(align, 4); + } +#endif + + offset = ir_allocate_spill_slot(ctx, size, align); + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); +} + int ir_reg_alloc_simple(ir_ctx *ctx) { ir_reg_alloc_data data; @@ -1686,6 +1741,7 @@ int ir_reg_alloc_simple(ir_ctx *ctx) ir_reg reg; ir_regset scratch; ir_reg_alloc_simple_data x; + int32_t offset; memset(&data, 0, sizeof(data)); data.cc = ir_get_call_conv_dsc(ctx->flags); @@ -1714,9 +1770,11 @@ int ir_reg_alloc_simple(ir_ctx *ctx) for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) { if (*rule & (IR_FUSED|IR_SKIPPED)) { if ((*rule & IR_RULE_MASK) == IR_ALLOCA) { - if (insn->op == IR_VAR) { + if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) { + /* spill slot already allocated */ + } else if (insn->op == IR_VAR) { if (ctx->use_lists[i].count > 0) { - insn->op3 = ir_allocate_spill_slot(ctx, insn->type); + ir_allocate_var_spill_slot(ctx, i, insn); } } else if (insn->op == IR_ALLOCA) { if (ctx->use_lists[i].count > 0) { @@ -1727,10 +1785,14 @@ int ir_reg_alloc_simple(ir_ctx *ctx) IR_ASSERT(!IR_IS_SYM_CONST(val->op)); IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); IR_ASSERT(val->val.i64 < 0x7fffffff); - insn->op3 = ir_allocate_big_spill_slot(ctx, val->val.i32); + offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); + ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset); } } else if (insn->op == IR_VADDR) { - insn->op3 = ctx->ir_base[insn->op1].op3; + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) { + ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]); + } + ctx->vregs[i] = ctx->vregs[insn->op1]; } } } else { @@ -1769,45 +1831,56 @@ int ir_reg_alloc_simple(ir_ctx *ctx) } } - if (ctx->vregs[i]) { + if (ctx->vregs[i] > 0) { reg = constraints.def_reg; if (!ctx->live_intervals[ctx->vregs[i]]) { - ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); - memset(ival, 0, sizeof(ir_live_interval)); - ctx->live_intervals[ctx->vregs[i]] = ival; - ival->type = insn->type; - ival->reg = IR_REG_NONE; - ival->vreg = ctx->vregs[i]; - ival->stack_spill_pos = -1; - if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) - && ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) { - ival->stack_spill_pos = ctx->ir_base[insn->op2].op3; - reg = IR_REG_NONE; - def_flags &= ~IR_USE_MUST_BE_IN_REG; - } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) { - ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - } else if (ctx->use_lists[i].count == 1) { - ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; - ir_insn *use_insn = &ctx->ir_base[use]; - - if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) - && use_insn->op3 == i - && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) { - if (use_insn->op2 < i) { - ival->stack_spill_pos = ctx->ir_base[use_insn->op2].op3; - } else { - ival->stack_spill_pos = ctx->ir_base[use_insn->op2].op3 = - ir_allocate_spill_slot(ctx, ival->type); + do { + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); + ctx->live_intervals[ctx->vregs[i]] = ival; + ival->type = insn->type; + ival->reg = IR_REG_NONE; + ival->vreg = ctx->vregs[i]; + ival->stack_spill_pos = -1; + + if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { + ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); } - } else { - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type); + if (ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) { + ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]); + reg = IR_REG_NONE; + def_flags &= ~IR_USE_MUST_BE_IN_REG; + break; + } + } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) { + ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; + break; } - } else { - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type); - } - } else if (insn->op == IR_PARAM) { - IR_ASSERT(0 && "unexpected PARAM"); - return 0; + + if (ctx->use_lists[i].count == 1) { + ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; + ir_insn *use_insn = &ctx->ir_base[use]; + + if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) + && use_insn->op3 == i + && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[use_insn->op2])) { + ir_allocate_var_spill_slot(ctx, use_insn->op2, &ctx->ir_base[use_insn->op2]); + } + ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[use_insn->op2]); + break; + } + } + + size_t size = ir_get_type_size(ival->type); +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { + size = 4; + } +#endif + ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); + } while (0); } if (def_flags & IR_USE_MUST_BE_IN_REG) { @@ -1834,6 +1907,10 @@ int ir_reg_alloc_simple(ir_ctx *ctx) _add_reg(&x, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE, IR_UNUSED, i, 0); } + } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { + ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); + } } n = insn->inputs_count; @@ -1850,18 +1927,21 @@ int ir_reg_alloc_simple(ir_ctx *ctx) ir_live_pos use_pos; uint32_t use_flags = IR_USE_FLAGS(def_flags, j); + if (reg != IR_REG_NONE) { +#if IR_X86_I64 + if (use_flags & IR_HINT_TWO_REGS) { + IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); + IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); + } else +#endif + IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], reg); + } + if (input > 0) { - if (ctx->vregs[input]) { + if (ctx->vregs[input] > 0) { use_pos = IR_USE_SUB_REF; if (reg != IR_REG_NONE) { use_pos = IR_LOAD_SUB_REF; -#if IR_X86_I64 - if (use_flags & IR_HINT_TWO_REGS) { - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); - } else -#endif - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], reg); } else if (def_flags & IR_DEF_REUSES_OP1_REG) { if (j == 1) { if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { @@ -1960,10 +2040,10 @@ int ir_reg_alloc_simple(ir_ctx *ctx) #ifdef IR_TARGET_X86 if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, IR_DOUBLE); + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) && data.cc->fp_ret_reg == IR_REG_NONE) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ctx->ret_type); + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); } else { ctx->ret_slot = -1; } diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index abab986e0a30..85ed6c74b7e5 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -890,7 +890,7 @@ IR_FOLD(ABS(C_I32)) if (op1_insn->val.i64 >= 0) { IR_FOLD_COPY(op1); } else { - IR_FOLD_CONST_I((int32_t)-op1_insn->val.u32); + IR_FOLD_CONST_I((int32_t)(0 - op1_insn->val.u32)); } } @@ -900,7 +900,7 @@ IR_FOLD(ABS(C_I64)) if (op1_insn->val.i64 >= 0) { IR_FOLD_COPY(op1); } else { - IR_FOLD_CONST_I(-op1_insn->val.u64); + IR_FOLD_CONST_I(0 - op1_insn->val.u64); } } @@ -2675,7 +2675,7 @@ IR_FOLD(DIV(NEG, C_I32)) IR_FOLD(DIV(NEG, C_I64)) { op1 = op1_insn->op1; - val.i64 = -(uint64_t)op2_insn->val.i64; + val.u64 = (0 - op2_insn->val.u64); op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; } @@ -3462,6 +3462,66 @@ IR_FOLD(AND(SHR, C_U64)) IR_FOLD_NEXT; } +IR_FOLD(AND(SAR, C_I8)) +IR_FOLD(AND(SAR, C_U8)) +{ + if (IR_IS_CONST_REF(op1_insn->op2)) { + if (((uint8_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u8 & 0x7) == op2_insn->val.u8) { + /* (x >>s N) & (~0 >> N) => x >>u N */ + opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt)); + op1 = op1_insn->op1; + op2 = op1_insn->op2; + IR_FOLD_RESTART; + } + } + IR_FOLD_NEXT; +} + +IR_FOLD(AND(SAR, C_I16)) +IR_FOLD(AND(SAR, C_U16)) +{ + if (IR_IS_CONST_REF(op1_insn->op2)) { + if (((uint16_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u16 & 0xf) == op2_insn->val.u16) { + /* (x >>s N) & (~0 >> N) => x >>u N */ + opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt)); + op1 = op1_insn->op1; + op2 = op1_insn->op2; + IR_FOLD_RESTART; + } + } + IR_FOLD_NEXT; +} + +IR_FOLD(AND(SAR, C_I32)) +IR_FOLD(AND(SAR, C_U32)) +{ + if (IR_IS_CONST_REF(op1_insn->op2)) { + if (((uint32_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u32 & 0x1f) == op2_insn->val.u32) { + /* (x >>s N) & (~0 >> N) => x >>u N */ + opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt)); + op1 = op1_insn->op1; + op2 = op1_insn->op2; + IR_FOLD_RESTART; + } + } + IR_FOLD_NEXT; +} + +IR_FOLD(AND(SAR, C_I64)) +IR_FOLD(AND(SAR, C_U64)) +{ + if (IR_IS_CONST_REF(op1_insn->op2)) { + if (((uint64_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u64 & 0x3f) == op2_insn->val.u64) { + /* (x >>s N) & (~0 >> N) => x >>u N */ + opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt)); + op1 = op1_insn->op1; + op2 = op1_insn->op2; + IR_FOLD_RESTART; + } + } + IR_FOLD_NEXT; +} + IR_FOLD(AND(SHL, C_I8)) IR_FOLD(AND(SHL, C_U8)) { @@ -3974,7 +4034,7 @@ IR_FOLD(MIN(MIN, _)) IR_FOLD(MAX(MAX, _)) { if (op1_insn->op1 == op2 || op1_insn->op2 == op2) { - IR_FOLD_COPY(op2); + IR_FOLD_COPY(op1); } IR_FOLD_NEXT; } @@ -4029,6 +4089,51 @@ IR_FOLD(ADD(SHR, SHL)) } +/* ROL/ROR by 1: SHL(x, 1) is folded to ADD(x, x) and then to MUL(x, 2), + * so the SHL/SHR rule above doesn't see it. */ +IR_FOLD(OR(MUL, SHR)) +IR_FOLD(ADD(MUL, SHR)) +{ + const ir_insn *mul_insn = op1_insn; + const ir_insn *shr_insn = op2_insn; + + if (IR_IS_TYPE_INT(IR_OPT_TYPE(opt)) + && mul_insn->op1 == shr_insn->op1 + && IR_IS_CONST_REF(mul_insn->op2) + && IR_IS_CONST_REF(shr_insn->op2) + && ctx->ir_base[mul_insn->op2].val.i64 == 2 + && ctx->ir_base[shr_insn->op2].val.i64 == ir_type_size[IR_OPT_TYPE(opt)] * 8 - 1) { + /* (x * 2) | (x >> (N - 1)) -> ROR(x, N - 1) == ROL(x, 1) */ + op1 = shr_insn->op1; + op2 = shr_insn->op2; + opt = shr_insn->opt + 3; /* SHR -> ROR */ + IR_FOLD_RESTART; + } + IR_FOLD_NEXT; +} + +IR_FOLD(OR(SHR, MUL)) +IR_FOLD(ADD(SHR, MUL)) +{ + const ir_insn *mul_insn = op2_insn; + const ir_insn *shr_insn = op1_insn; + + if (IR_IS_TYPE_INT(IR_OPT_TYPE(opt)) + && mul_insn->op1 == shr_insn->op1 + && IR_IS_CONST_REF(mul_insn->op2) + && IR_IS_CONST_REF(shr_insn->op2) + && ctx->ir_base[mul_insn->op2].val.i64 == 2 + && ctx->ir_base[shr_insn->op2].val.i64 == ir_type_size[IR_OPT_TYPE(opt)] * 8 - 1) { + /* (x * 2) | (x >> (N - 1)) -> ROR(x, N - 1) == ROL(x, 1) */ + op1 = shr_insn->op1; + op2 = shr_insn->op2; + opt = shr_insn->opt + 3; /* SHR -> ROR */ + IR_FOLD_RESTART; + } + IR_FOLD_NEXT; +} + + /* Swap operands (move lower ref to op2) for better CSE */ IR_FOLD(MUL(_, _)) IR_FOLD(ORDERED(_, _)) diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c index 60422f12f6be..7a0755b74774 100644 --- a/ext/opcache/jit/ir/ir_gcm.c +++ b/ext/opcache/jit/ir/ir_gcm.c @@ -322,9 +322,13 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) ir_hashtab_init(&hash, use_list->count); n = use_list->count; for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { + ir_ref src; + use = *p; + src = ctx->cfg_map[use]; insn = &ctx->ir_base[use]; - if (insn->op == IR_PHI) { + /* dead PHIs (src == NULL) are not scheduled, so they go together with other unscheduled uses */ + if (src && insn->op == IR_PHI) { ir_ref *p = insn->ops + 2; /* PHI data inputs */ ir_ref *q = ctx->ir_base[insn->op1].ops + 1; /* MERGE inputs */ ir_ref n = insn->inputs_count - 1; @@ -359,9 +363,9 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) } } } else { - j = i = ctx->cfg_map[use]; - if (i) { - IR_ASSERT(i > 0); + j = src; + if (j) { + IR_ASSERT(j > 0); while (ir_sparse_set_in(&data->totally_useful, ctx->cfg_blocks[j].idom)) { j = ctx->cfg_blocks[j].idom; } @@ -371,14 +375,14 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) clone = clones_count++; ir_hashtab_add(&hash, j, clone); clones[clone].block = j; - clones[clone].lca = i; + clones[clone].lca = src; clones[clone].use_count = 0; clones[clone].use = -1; } else { - clones[clone].lca = ir_gcm_find_lca(ctx, clones[clone].lca, i); + clones[clone].lca = ir_gcm_find_lca(ctx, clones[clone].lca, src); } uses[uses_count].ref = use; - uses[uses_count].block = i; + uses[uses_count].block = src; uses[uses_count].next = clones[clone].use; clones[clone].use_count++; clones[clone].use = uses_count++; @@ -478,7 +482,7 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) ir_insn *insn = &ctx->ir_base[use]; ir_ref k, l = insn->inputs_count; - if (insn->op == IR_PHI) { + if (src && insn->op == IR_PHI) { ir_insn *merge = &ctx->ir_base[insn->op1]; for (k = 2; k <= l; k++) { j = ctx->cfg_map[ir_insn_op(merge, k - 1)]; @@ -1359,11 +1363,11 @@ int ir_schedule(ir_ctx *ctx) *dst = *src; _xlat[i] = j; if (dst->op == IR_LONG_CONST) { - uintptr_t n; + ir_ref n; memset(dst + 1, 0, dst->long_const_size); memcpy(dst + 1, src + 1, dst->long_const_size); - n = IR_ALIGNED_SIZE(dst->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn); + n = (ir_ref)IR_ALIGNED_SIZE(dst->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn); dst += n + 1; src += n + 1; i += n + 1; diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index b4409f97df7c..950b6535f577 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -975,12 +975,15 @@ IR_ALWAYS_INLINE bool ir_ref_is_true(const ir_ctx *ctx, ir_ref ref) #define IR_OP_FLAG_TERMINATOR (1<<14) #define IR_OP_FLAG_PINNED (1<<15) -#define IR_OP_FLAG_MEM_LOAD ((0<<6)|(0<<7)) -#define IR_OP_FLAG_MEM_STORE ((0<<6)|(1<<7)) -#define IR_OP_FLAG_MEM_CALL ((1<<6)|(0<<7)) -#define IR_OP_FLAG_MEM_ALLOC ((1<<6)|(1<<7)) +#define IR_OP_FLAG_MEM_CALL ((0<<6)|(0<<7)) +#define IR_OP_FLAG_MEM_LOAD ((0<<6)|(1<<7)) /* no side effect (may be removed if not used) */ +#define IR_OP_FLAG_MEM_STORE ((1<<6)|(0<<7)) +#define IR_OP_FLAG_MEM_ALLOC ((1<<6)|(1<<7)) /* no side effect (may be removed if not used) */ #define IR_OP_FLAG_MEM_MASK ((1<<6)|(1<<7)) +#define IR_OP_FLAG_NO_SIDE_EFFECT ((0<<6)|(1<<7)) + + #define IR_OPND_UNUSED 0x0U #define IR_OPND_DATA 0x1U #define IR_OPND_CONTROL 0x2U @@ -990,7 +993,7 @@ IR_ALWAYS_INLINE bool ir_ref_is_true(const ir_ctx *ctx, ir_ref ref) #define IR_OPND_CONTROL_GUARD 0x6U #define IR_OPND_STR 0x7U #define IR_OPND_NUM 0x8U -#define IR_OPND_PROB 0x9U +#define IR_OPND_OPT 0x9U /* optional number (flags, branch propability, align, size+align) */ #define IR_OPND_PROTO 0xaU #define IR_OP_FLAGS(op_flags, op1_flags, op2_flags, op3_flags) \ @@ -1324,6 +1327,11 @@ struct _ir_live_range { #define IR_LIVE_INTERVAL_SPLIT_CHILD (1<<8) #define IR_LIVE_INTERVAL_TWO_REGS (1<<9) +/* macros to encode and decode stack offset stored in ctx->vregs[] */ +#define IR_VREG_IS_STACK_SLOT(vreg) ((vreg) < 0) +#define IR_VREG_TO_STACK_SLOT(vreg) ((vreg) + 0x40000000) +#define IR_STACK_SLOT_TO_VREG(offset) ((int32_t)0xc0000000 + (offset)) + struct _ir_live_interval { uint8_t type; int8_t reg; @@ -1478,8 +1486,7 @@ typedef struct _ir_reg_alloc_data { ir_live_interval **handled; } ir_reg_alloc_data; -int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type); -int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size); +int32_t ir_allocate_spill_slot(ir_ctx *ctx, size_t size, size_t align); void ir_dump_reg(const ir_ctx *ctx, int8_t reg, ir_ref ref, bool store, FILE *f); IR_ALWAYS_INLINE void ir_set_alocated_reg(ir_ctx *ctx, ir_ref ref, int op_num, int8_t reg) diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index fe052b2a097b..6cfc779e6c3c 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -40,8 +40,8 @@ bool ir_reg_is_int(int32_t reg) static int ir_assign_virtual_registers_slow(ir_ctx *ctx) { - uint32_t *vregs; - uint32_t vregs_count = 0; + int32_t *vregs; + int32_t vregs_count = 0; uint32_t b; ir_ref i, n; ir_block *bb; @@ -81,8 +81,8 @@ static int ir_assign_virtual_registers_slow(ir_ctx *ctx) int ir_assign_virtual_registers(ir_ctx *ctx) { - uint32_t *vregs; - uint32_t vregs_count = 0; + int32_t *vregs; + int32_t vregs_count = 0; ir_ref i; ir_insn *insn; @@ -94,7 +94,7 @@ int ir_assign_virtual_registers(ir_ctx *ctx) vregs = ir_mem_malloc(ctx->insns_count * sizeof(ir_ref)); for (i = 1, insn = &ctx->ir_base[1]; i < ctx->insns_count; i++, insn++) { - uint32_t v = 0; + int32_t v = 0; if (ctx->rules[i] && !(ctx->rules[i] & (IR_FUSED|IR_SKIPPED|IR_NO_REG))) { uint32_t flags = ir_op_flags[insn->op]; @@ -548,9 +548,9 @@ static void ir_add_fusion_ranges(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_block IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); child = *p; if (child > 0) { - uint32_t v = ctx->vregs[child]; + int32_t v = ctx->vregs[child]; - if (v) { + if (v > 0) { use_flags = IR_FUSED_USE | IR_USE_FLAGS(def_flags, j); reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); @@ -694,9 +694,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (insn->op == IR_PHI) { ir_ref input = ir_insn_op(insn, k); if (input > 0) { - uint32_t v = ctx->vregs[input]; + int32_t v = ctx->vregs[input]; - if (v) { + if (v > 0) { /* live.add(phi.inputOf(b)) */ ir_bitset_incl(live, v); /* intervals[phi.inputOf(b)].addRange(b.from, b.to) */ @@ -722,7 +722,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) uint32_t flags; ir_ref *p; ir_target_constraints constraints; - uint32_t v; + int32_t v; if (ctx->rules) { int n; @@ -730,7 +730,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) #if IR_X86_I64 if (ctx->rules[ref] & IR_TWO_REGS) { v = ctx->vregs[ref]; - if (v) { + if (v > 0) { IR_ASSERT(ctx->live_intervals[v]); ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_TWO_REGS; } @@ -742,7 +742,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) && ctx->use_lists[ref].count > 0) { insn = &ctx->ir_base[ref]; if (insn->op == IR_VAR || insn->op == IR_ALLOCA) { - insn->op3 = ctx->vars; + ctx->vregs[ref] = IR_STACK_SLOT_TO_VREG(ctx->vars); ctx->vars = ref; } } @@ -770,7 +770,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) insn = &ctx->ir_base[ref]; v = ctx->vregs[ref]; - if (v) { + if (v > 0) { if (insn->op != IR_PHI) { ir_live_pos def_pos; ir_ref hint_ref = 0; @@ -789,7 +789,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_add_fixed_live_range(ctx, reg, IR_START_LIVE_POS_FROM_REF(bb->start), def_pos); } } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1]) { + if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1] > 0) { hint_ref = insn->op1; } def_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); @@ -799,6 +799,17 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (insn->op == IR_PARAM) { /* We may reuse parameter stack slot for spilling */ ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; +#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) + if (param_ival->use_pos) { + ir_use_pos *use_pos = param_ival->use_pos; + ir_ref use = IR_LIVE_POS_TO_REF(use_pos->pos); + ir_insn *use_insn = &ctx->ir_base[use]; + if (use_insn->op == IR_VSTORE) { + /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */ + param_ival->use_pos = use_pos->next; + } + } +#endif } def_pos = IR_DEF_LIVE_POS_FROM_REF(ref); } @@ -831,11 +842,11 @@ int ir_compute_live_ranges(ir_ctx *ctx) p = insn->ops + 2; for (; j <= insn->inputs_count; j++, p++) { ir_ref input = *p; - uint32_t v; + int32_t v; if (input > 0) { v = ctx->vregs[input]; - IR_ASSERT(v); + IR_ASSERT(v > 0); use_pos = IR_USE_LIVE_POS_FROM_REF(next); if (!ir_bitset_in(live, v)) { /* live.add(opd) */ @@ -865,12 +876,12 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; ir_live_pos use_pos; ir_ref hint_ref = 0; - uint32_t v; + int32_t v; uint32_t use_flags = IR_USE_FLAGS(def_flags, j); if (input > 0) { v = ctx->vregs[input]; - if (v) { + if (v > 0) { use_pos = IR_USE_LIVE_POS_FROM_REF(ref); if (reg != IR_REG_NONE) { use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); @@ -895,7 +906,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) } else if (def_flags & IR_DEF_REUSES_OP1_REG) { if (j == 1) { use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); - IR_ASSERT(ctx->vregs[ref]); + IR_ASSERT(ctx->vregs[ref] > 0); hint_ref = ref; } else if (input == insn->op1) { /* Input is the same as "op1" */ @@ -1022,7 +1033,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) } while (0) /* Returns the last virtual register alive at the end of the block (it is used as an already-visited marker) */ -IR_ALWAYS_INLINE uint32_t ir_live_out_top(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists, uint32_t b) +IR_ALWAYS_INLINE int32_t ir_live_out_top(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists, uint32_t b) { #if 0 return live_outs[b]; @@ -1035,7 +1046,7 @@ IR_ALWAYS_INLINE uint32_t ir_live_out_top(ir_ctx *ctx, uint32_t *live_outs, ir_l } /* Remember a virtual register alive at the end of the block */ -IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists, uint32_t b, uint32_t v) +IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists, uint32_t b, int32_t v) { #if 0 ir_block *bb = &ctx->cfg_blocks[b]; @@ -1061,7 +1072,7 @@ IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, uint32_t *live_outs, ir_list * "Computing Liveness Sets for SSA-Form Programs", Florian Brandner, Benoit Boissinot. * Alain Darte, Benoit Dupont de Dinechin, Fabrice Rastello. TR Inria RR-7503, 2011 */ -static void ir_compute_live_sets(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists) +static void ir_compute_live_sets(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists) { ir_list block_queue, fuse_queue; ir_ref i; @@ -1071,9 +1082,9 @@ static void ir_compute_live_sets(ir_ctx *ctx, uint32_t *live_outs, ir_list *live /* For each virtual register explore paths from all uses to definition */ for (i = ctx->insns_count - 1; i > 0; i--) { - uint32_t v = ctx->vregs[i]; + int32_t v = ctx->vregs[i]; - if (v) { + if (v > 0) { uint32_t def_block = ctx->cfg_map[i]; ir_use_list *use_list = &ctx->use_lists[i]; ir_ref *p, n = use_list->count; @@ -1285,9 +1296,9 @@ static void ir_add_fusion_ranges(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_block IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); child = *p; if (child > 0) { - uint32_t v = ctx->vregs[child]; + int32_t v = ctx->vregs[child]; - if (v) { + if (v > 0) { reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; use_pos = pos; if (EXPECTED(reg == IR_REG_NONE)) { @@ -1330,7 +1341,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_ref ref; ir_insn *insn; ir_block *bb, *succ_bb; - uint32_t *live_outs; + int32_t *live_outs; uint32_t *live_in_block; ir_list live_lists; ir_live_interval *ival; @@ -1395,9 +1406,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (insn->op == IR_PHI) { ir_ref input = ir_insn_op(insn, k); if (input > 0) { - uint32_t v = ctx->vregs[input]; + int32_t v = ctx->vregs[input]; - if (v) { + if (v > 0) { ival = ctx->live_intervals[v]; ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use); } @@ -1418,7 +1429,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) uint32_t flags; ir_ref *p; ir_target_constraints constraints; - uint32_t v; + int32_t v; if (ctx->rules) { int n; @@ -1426,7 +1437,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) #if IR_X86_I64 if (ctx->rules[ref] & IR_TWO_REGS) { v = ctx->vregs[ref]; - if (v) { + if (v > 0) { IR_ASSERT(ctx->live_intervals[v]); ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_TWO_REGS; } @@ -1438,7 +1449,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) && ctx->use_lists[ref].count > 0) { insn = &ctx->ir_base[ref]; if (insn->op == IR_VAR || insn->op == IR_ALLOCA) { - insn->op3 = ctx->vars; + ctx->vregs[ref] = IR_STACK_SLOT_TO_VREG(ctx->vars); ctx->vars = ref; } } @@ -1466,7 +1477,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) insn = &ctx->ir_base[ref]; v = ctx->vregs[ref]; - if (v) { + if (v > 0) { if (insn->op != IR_PHI) { ir_live_pos def_pos; ir_ref hint_ref = 0; @@ -1485,7 +1496,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_add_fixed_live_range(ctx, reg, IR_START_LIVE_POS_FROM_REF(bb->start), def_pos); } } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1]) { + if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1] > 0) { hint_ref = insn->op1; } if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { @@ -1498,7 +1509,20 @@ int ir_compute_live_ranges(ir_ctx *ctx) } else { if (insn->op == IR_PARAM) { /* We may reuse parameter stack slot for spilling */ - ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; + ir_live_interval *param_ival = ctx->live_intervals[v]; + + param_ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; +#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) + if (param_ival->use_pos) { + ir_use_pos *use_pos = param_ival->use_pos; + ir_ref use = IR_LIVE_POS_TO_REF(use_pos->pos); + ir_insn *use_insn = &ctx->ir_base[use]; + if (use_insn->op == IR_VSTORE) { + /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */ + param_ival->use_pos = use_pos->next; + } + } +#endif } def_pos = IR_DEF_LIVE_POS_FROM_REF(ref); } @@ -1527,11 +1551,11 @@ int ir_compute_live_ranges(ir_ctx *ctx) p = insn->ops + 2; for (; j <= insn->inputs_count; j++, p++) { ir_ref input = *p; - uint32_t v; + int32_t v; if (input > 0) { v = ctx->vregs[input]; - IR_ASSERT(v); + IR_ASSERT(v > 0); use_pos = IR_USE_LIVE_POS_FROM_REF(next); if (!IS_LIVE_IN_BLOCK(v, b)) { /* live.add(opd) */ @@ -1561,12 +1585,12 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; ir_live_pos use_pos; ir_ref hint_ref = 0; - uint32_t v; + int32_t v; uint32_t use_flags = IR_USE_FLAGS(def_flags, j); if (input > 0) { v = ctx->vregs[input]; - if (v) { + if (v > 0) { use_pos = IR_USE_LIVE_POS_FROM_REF(ref); if (reg != IR_REG_NONE) { use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); @@ -1595,7 +1619,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) } else { use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); } - IR_ASSERT(ctx->vregs[ref]); + IR_ASSERT(ctx->vregs[ref] > 0); hint_ref = ref; } else if (input == insn->op1) { /* Input is the same as "op1" */ @@ -1688,7 +1712,7 @@ static ir_live_pos ir_ivals_overlap(ir_live_range *lrg1, ir_live_range *lrg2) } } -static ir_live_pos ir_vregs_overlap(ir_ctx *ctx, uint32_t r1, uint32_t r2) +static ir_live_pos ir_vregs_overlap(ir_ctx *ctx, int32_t r1, int32_t r2) { ir_live_interval *ival1 = ctx->live_intervals[r1]; ir_live_interval *ival2 = ctx->live_intervals[r2]; @@ -1716,7 +1740,7 @@ static bool ir_ivals_inside(ir_live_range *parent, ir_live_range *child) return 1; } -static bool ir_vregs_inside(ir_ctx *ctx, uint32_t parent, uint32_t child) +static bool ir_vregs_inside(ir_ctx *ctx, int32_t parent, int32_t child) { ir_live_interval *child_ival = ctx->live_intervals[child]; ir_live_interval *parent_ival = ctx->live_intervals[parent]; @@ -1733,7 +1757,7 @@ static bool ir_vregs_inside(ir_ctx *ctx, uint32_t parent, uint32_t child) return ir_ivals_inside(&parent_ival->range, &child_ival->range); } -static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2) +static void ir_vregs_join(ir_ctx *ctx, int32_t r1, int32_t r2) { ir_live_interval *ival = ctx->live_intervals[r2]; ir_live_range *live_range = &ival->range; @@ -1795,7 +1819,7 @@ static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2) //ir_mem_free(ival); } -static void ir_vregs_coalesce(ir_ctx *ctx, uint32_t v1, uint32_t v2, ir_ref from, ir_ref to) +static void ir_vregs_coalesce(ir_ctx *ctx, int32_t v1, int32_t v2, ir_ref from, ir_ref to) { ir_ref i; uint16_t f1 = ctx->live_intervals[v1]->flags; @@ -1914,7 +1938,7 @@ static void ir_swap_operands(ir_ctx *ctx, ir_ref i, ir_insn *insn) p = p->next; } - if (insn->op2 > 0 && ctx->vregs[insn->op2]) { + if (insn->op2 > 0 && ctx->vregs[insn->op2] > 0) { ival = ctx->live_intervals[ctx->vregs[insn->op2]]; r = &ival->range; while (r) { @@ -1977,13 +2001,13 @@ static int ir_hint_conflict(ir_ctx *ctx, ir_ref ref, int use, int def) static int ir_try_swap_operands(ir_ctx *ctx, ir_ref i, ir_insn *insn) { - if (ctx->vregs[insn->op1] + if (ctx->vregs[insn->op1] > 0 && ctx->vregs[insn->op1] != ctx->vregs[i] && !ir_vregs_overlap(ctx, ctx->vregs[insn->op1], ctx->vregs[i]) && !ir_hint_conflict(ctx, i, ctx->vregs[insn->op1], ctx->vregs[i])) { /* pass */ } else { - if (ctx->vregs[insn->op2] && ctx->vregs[insn->op2] != ctx->vregs[i]) { + if (ctx->vregs[insn->op2] > 0 && ctx->vregs[insn->op2] != ctx->vregs[i]) { ir_live_pos pos = IR_USE_LIVE_POS_FROM_REF(i); ir_live_pos load_pos = IR_LOAD_LIVE_POS_FROM_REF(i); ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op2]]; @@ -2081,7 +2105,7 @@ int ir_coalesce(ir_ctx *ctx) insn = &ctx->ir_base[use]; if (insn->op == IR_PHI) { input = ir_insn_op(insn, k); - if (input > 0 && ctx->vregs[input]) { + if (input > 0 && ctx->vregs[input] > 0) { uint32_t v1 = ctx->vregs[input]; uint32_t v2 = ctx->vregs[use]; @@ -2151,7 +2175,7 @@ int ir_coalesce(ir_ctx *ctx) for (i = 1; i < ctx->insns_count; rule++, i++) { if ((*rule) & (IR_MAY_SWAP|IR_MAY_REUSE)) { insn = &ctx->ir_base[i]; - IR_ASSERT(ctx->vregs[i]); + IR_ASSERT(ctx->vregs[i] > 0); if ((*rule) & IR_MAY_SWAP) { IR_ASSERT(ir_op_flags[insn->op] & IR_OP_FLAG_COMMUTATIVE); if (ctx->live_intervals[ctx->vregs[i]]->use_pos @@ -2164,7 +2188,7 @@ int ir_coalesce(ir_ctx *ctx) } else { IR_ASSERT((*rule) & IR_MAY_REUSE); if (insn->op1 > 0 - && ctx->vregs[insn->op1] + && ctx->vregs[insn->op1] > 0 && ctx->vregs[i] != ctx->vregs[insn->op1]) { if (ir_vregs_inside(ctx, ctx->vregs[insn->op1], ctx->vregs[i])) { if (ctx->binding) { @@ -2208,7 +2232,7 @@ int ir_coalesce(ir_ctx *ctx) } } for (j = 1; j < ctx->insns_count; j++) { - if (ctx->vregs[j]) { + if (ctx->vregs[j] > 0) { ctx->vregs[j] = xlat[ctx->vregs[j]]; } } @@ -2325,7 +2349,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy, void *dat insn = &ctx->ir_base[ref]; if (insn->op == IR_PHI) { input = ir_insn_op(insn, k); - if (IR_IS_CONST_REF(input) || !ctx->vregs[input]) { + if (IR_IS_CONST_REF(input) || !(ctx->vregs[input] > 0)) { have_constants_or_addresses = 1; } else if (ctx->vregs[input] != ctx->vregs[ref]) { s = ctx->vregs[input]; @@ -2394,7 +2418,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy, void *dat insn = &ctx->ir_base[ref]; if (insn->op == IR_PHI) { input = ir_insn_op(insn, k); - if (IR_IS_CONST_REF(input) || !ctx->vregs[input]) { + if (IR_IS_CONST_REF(input) || !(ctx->vregs[input] > 0)) { emit_copy(ctx, insn->type, input, ref, data); } } @@ -2712,7 +2736,7 @@ static int32_t ir_allocate_small_spill_slot(ir_ctx *ctx, size_t size) } IR_ASSERT(size == 1 || size == 2 || size == 4 || size == 8); - n = ir_ntz(size); + n = IR_LOG2(size); if (data->handled && data->handled[n]) { ret = data->handled[n]->stack_spill_pos; data->handled[n] = data->handled[n]->list_next; @@ -2808,21 +2832,13 @@ static int32_t ir_allocate_small_spill_slot(ir_ctx *ctx, size_t size) return ret; } -int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size) +static int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, size_t size, size_t align) { int32_t ret; - if (size <= 8) { - if (size == 3) { - size = 4; - } else if (size > 4 && size < 8) { - size = 8; - } - return ir_allocate_small_spill_slot(ctx, size); - } - + IR_ASSERT(size < 0xfffffff); if (size <= 64 && (size & (size - 1)) == 0) { - uint32_t n = ir_ntz(size); + uint32_t n = IR_LOG2(size); ir_reg_alloc_data *data = ctx->data; if (data->handled && data->handled[n]) { @@ -2833,25 +2849,27 @@ int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size) } /* Align stack allocated data to 16 byte */ + // TODO: support for alignment larger than 16 ??? ctx->flags2 |= IR_16B_FRAME_ALIGNMENT; ret = IR_ALIGNED_SIZE(ctx->stack_frame_size, 16); size = IR_ALIGNED_SIZE(size, 8); - ctx->stack_frame_size = ret + size; + ctx->stack_frame_size = ret + (int32_t)size; return ret; } -int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type) +int32_t ir_allocate_spill_slot(ir_ctx *ctx, size_t size, size_t align) { - if (IR_IS_TYPE_SCALAR(type)) { - return ir_allocate_small_spill_slot(ctx, ir_type_size[type]); + IR_ASSERT(align == 0 || (align & (align -1)) == 0); /* align must be a power of 2 */ + if (size <= 8 && align < 16) { + if (align > size) { + size = align; + } else if ((size & (size - 1)) != 0) { + size = (size == 3) ? 4 : 8; + } + return ir_allocate_small_spill_slot(ctx, size); } else { - int32_t size; - - IR_ASSERT(IR_IS_TYPE_VECTOR(type)); - size = IR_VECTOR_SIZE(type); - size = IR_MAX(size, 4); - return ir_allocate_big_spill_slot(ctx, size); + return ir_allocate_big_spill_slot(ctx, size, align); } } @@ -3917,12 +3935,12 @@ static void ir_assign_bound_spill_slots(ir_ctx *ctx) { ir_hashtab_bucket *b = ctx->binding->data; uint32_t n = ctx->binding->count; - uint32_t v; + int32_t v; ir_live_interval *ival; while (n > 0) { v = ctx->vregs[b->key]; - if (v) { + if (v > 0) { ival = ctx->live_intervals[v]; if (ival && ival->stack_spill_pos == -1 @@ -3984,25 +4002,35 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars) ir_insn *insn = &ctx->ir_base[var]; IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA); - vars = insn->op3; /* list next */ + vars = IR_VREG_TO_STACK_SLOT(ctx->vregs[var]); /* list next */ if (insn->op == IR_VAR) { - ir_ref slot = ir_allocate_spill_slot(ctx, insn->type); + ir_ref slot; ir_use_list *use_list; ir_ref n, *p; + size_t size = ir_get_type_size(insn->type); + size_t align = insn->op3; - insn->op3 = slot; +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(insn->type) && size < 4) { + size = 4; + align = IR_MAX(align, 4); + } +#endif + slot = ir_allocate_spill_slot(ctx, size, align); + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(slot); use_list = &ctx->use_lists[var]; n = use_list->count; p = &ctx->use_edges[use_list->refs]; for (; n > 0; p++, n--) { insn = &ctx->ir_base[*p]; if (insn->op == IR_VADDR) { - insn->op3 = slot; + ctx->vregs[*p] = IR_STACK_SLOT_TO_VREG(slot); } } } else { ir_insn *val = &ctx->ir_base[insn->op2]; + int32_t offset; IR_ASSERT(IR_IS_CONST_REF(insn->op2)); IR_ASSERT(IR_IS_TYPE_INT(val->type)); @@ -4010,7 +4038,8 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars) IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); IR_ASSERT(val->val.i64 < 0x7fffffff); - insn->op3 = ir_allocate_big_spill_slot(ctx, val->val.i32); + offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); } } @@ -4228,7 +4257,13 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars) other = prev ? prev->list_next : active; } - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type); + size_t size = ir_get_type_size(ival->type); +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { + size = 4; + } +#endif + ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); if (unhandled && ival->end > unhandled->range.start) { ival->list_next = active; active = ival; @@ -4256,10 +4291,10 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars) #ifdef IR_TARGET_X86 if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, IR_DOUBLE); + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) && ((ir_reg_alloc_data*)(ctx->data))->cc->fp_ret_reg == IR_REG_NONE) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ctx->ret_type); + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); } else { ctx->ret_slot = -1; } @@ -4430,7 +4465,7 @@ static void assign_regs(ir_ctx *ctx) || ctx->ir_base[ref].op == IR_BITCAST || ctx->ir_base[ref].op == IR_TRUNC) && !IR_IS_CONST_REF(ctx->ir_base[ref].op1) - && ctx->vregs[ctx->ir_base[ref].op1] == (uint32_t)i) { + && ctx->vregs[ctx->ir_base[ref].op1] == i) { /* register reuse */ ir_set_alocated_reg(ctx, ref, use_pos->op_num, reg); prev_use_ref = ref; @@ -4446,13 +4481,6 @@ static void assign_regs(ir_ctx *ctx) && (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM)) { /* Stack PARAM var is passed through memory */ reg = IR_REG_NONE; -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) - if (use_pos->next - && ctx->ir_base[IR_LIVE_POS_TO_REF(use_pos->next->pos)].op == IR_VSTORE) { - /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */ - use_pos = use_pos->next; - } -#endif } else { uint32_t use_b = ctx->cfg_map[ref]; @@ -4509,7 +4537,7 @@ static void assign_regs(ir_ctx *ctx) if (use_pos->hint_ref < 0) { if (use_pos->flags & IR_PHI_USE) { IR_ASSERT(use_pos->hint_ref < 0); - IR_ASSERT(ctx->vregs[-use_pos->hint_ref]); + IR_ASSERT(ctx->vregs[-use_pos->hint_ref] > 0); IR_ASSERT(ctx->live_intervals[ctx->vregs[-use_pos->hint_ref]]); if (ctx->live_intervals[ctx->vregs[-use_pos->hint_ref]]->flags & IR_LIVE_INTERVAL_SPILLED) { /* Spilled PHI var is passed through memory */ @@ -4571,10 +4599,12 @@ static void assign_regs(ir_ctx *ctx) if (ival->tmp_op_num <= insn->inputs_count) { ir_ref *ops = insn->ops; + if (IR_IS_CONST_REF(ops[ival->tmp_op_num])) { /* constant rematerialization */ reg |= IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[ival->tmp_op_num]] == (IR_SKIPPED|IR_FUSED|IR_SIMPLE|IR_ALLOCA)) { + } else if (ctx->rules[ops[ival->tmp_op_num]] == (IR_SKIPPED|IR_FUSED|IR_SIMPLE|IR_ALLOCA) + && IR_OPND_KIND(ir_op_flags[insn->op], ival->tmp_op_num) == IR_OPND_DATA) { /* local address rematerialization */ reg |= IR_REG_SPILL_LOAD; } diff --git a/ext/opcache/jit/ir/ir_save.c b/ext/opcache/jit/ir/ir_save.c index 1286f5503856..f3b885d3bff4 100644 --- a/ext/opcache/jit/ir/ir_save.c +++ b/ext/opcache/jit/ir/ir_save.c @@ -299,7 +299,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) ir_print_type_cname(insn->type, f); fprintf(f, " d_%d", i); if (save_flags & IR_SAVE_REGS) { - if (ctx->vregs && ctx->vregs[i]) { + if (ctx->vregs && ctx->vregs[i] > 0) { fprintf(f, " {R%d}", ctx->vregs[i]); } if (ctx->regs) { @@ -317,7 +317,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) ir_print_type_cname(insn->type, f); fprintf(f, " d_%d", i); if (save_flags & IR_SAVE_REGS) { - if (ctx->vregs && ctx->vregs[i]) { + if (ctx->vregs && ctx->vregs[i] > 0) { fprintf(f, " {R%d}", ctx->vregs[i]); } if (ctx->regs) { @@ -355,7 +355,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) fprintf(f, "%sd_%d", first ? "(" : ", ", ref); } if (save_flags & IR_SAVE_REGS) { - if (ctx->vregs && ref > 0 && ctx->vregs[ref]) { + if (ctx->vregs && ref > 0 && ctx->vregs[ref] > 0) { fprintf(f, " {R%d}", ctx->vregs[ref]); } if (ctx->regs) { @@ -383,7 +383,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) fprintf(f, "%sfunc ", first ? "(" : ", "); ir_print_proto(ctx, ref, f); break; - case IR_OPND_PROB: + case IR_OPND_OPT: if (ref == 0) { break; } diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 4745c7000d72..c14d34b6c4d4 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -520,25 +520,13 @@ static bool ir_sccp_analyze_phi(const ir_ctx *ctx, ir_sccp_val *_values, ir_bitq static bool ir_is_dead_load_ex(const ir_ctx *ctx, ir_ref ref, uint32_t flags, const ir_insn *insn) { - if ((flags & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD)) { - return ctx->use_lists[ref].count == 1; - } else if (insn->op == IR_ALLOCA || insn->op == IR_BLOCK_BEGIN) { - return ctx->use_lists[ref].count == 1; - } - return 0; + return (flags & IR_OP_FLAG_NO_SIDE_EFFECT) && ctx->use_lists[ref].count == 1; } static bool ir_is_dead_load(const ir_ctx *ctx, ir_ref ref) { if (ctx->use_lists[ref].count == 1) { - const ir_insn *insn = &ctx->ir_base[ref]; - uint32_t flags = ir_op_flags[insn->op]; - - if ((flags & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD)) { - return 1; - } else if (insn->op == IR_ALLOCA || insn->op == IR_BLOCK_BEGIN) { - return 1; - } + return (ir_op_flags[ctx->ir_base[ref].op] & IR_OP_FLAG_NO_SIDE_EFFECT) != 0; } return 0; } @@ -3752,7 +3740,7 @@ static ir_ref ir_iter_optimize_condition(ir_ctx *ctx, ir_ref control, ir_ref con && ctx->use_lists[condition].count == 1 && ctx->use_lists[condition_insn->op1].count == 1 && (op1_insn->op == IR_ZEXT || op1_insn->op == IR_SEXT) - && val_insn->val.u64 <= (((uint64_t)-1ULL) >> (64 - ir_type_size[ctx->ir_base[op1_insn->op1].type] * 8))) { + && val_insn->val.u64 <= (((uint64_t)(0-1ULL)) >> (64 - ir_type_size[ctx->ir_base[op1_insn->op1].type] * 8))) { /* IF(AND(ZEXT(X), C)) => IF(AND(X, C)) */ if (op1_insn->op1 > 0) { ir_use_list_replace_one(ctx, op1_insn->op1, condition_insn->op1, condition); diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 5ce20343bd00..3f1a3be180f8 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -2031,7 +2031,6 @@ op2_const: n = 1; } break; - case IR_LOAD_FP: case IR_LOAD_INT: case IR_MEM_OP_INT: case IR_MEM_INC: @@ -2049,6 +2048,19 @@ op2_const: n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); } break; + case IR_LOAD_FP: + flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; + insn = &ctx->ir_base[ref]; + if (IR_IS_CONST_REF(insn->op2)) { + n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); + } +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(insn->type) && IR_VECTOR_SIZE(insn->type) < 4) { + constraints->tmp_regs[n] = IR_TMP_REG(3, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; + } +#endif + break; case IR_STORE_FP: flags = IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; insn = &ctx->ir_base[ref]; @@ -2060,6 +2072,13 @@ op2_const: constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; } +#if IR_SIMD + else if (IR_IS_TYPE_VECTOR(ctx->ir_base[insn->op3].type) + && IR_VECTOR_SIZE(ctx->ir_base[insn->op3].type) < 4) { + constraints->tmp_regs[n] = IR_TMP_REG(1, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; + } +#endif break; case IR_SWITCH: flags = IR_OP2_MUST_BE_IN_REG; @@ -2757,7 +2776,7 @@ get_arg_hints: && IR_VECTOR_SIZE(insn->type) == 32) { constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n++; - if (ctx->vregs[insn->op1]) { + if (ctx->vregs[insn->op1] > 0) { flags &= ~IR_DEF_CONFLICTS_WITH_INPUT_REGS; flags |= IR_DEF_REUSES_OP1_REG; constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); @@ -2907,7 +2926,7 @@ get_arg_hints: if (!(ctx->mflags & IR_X86_AVX2) && IR_VECTOR_SIZE(insn->type) == 32) { constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n++; - if (ctx->vregs[insn->op2] || insn->op == IR_NE) { + if (ctx->vregs[insn->op2] > 0 || insn->op == IR_NE) { constraints->tmp_regs[n] = IR_TMP_REG(4, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n++; } @@ -3288,6 +3307,11 @@ static void ir_match_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root) if (ir_in_same_block(ctx, ref) && (ctx->ir_base[ref].op == IR_LOAD || ctx->ir_base[ref].op == IR_LOAD_v || ctx->ir_base[ref].op == IR_VLOAD || ctx->ir_base[ref].op == IR_VLOAD_v)) { +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ctx->ir_base[ref].type) && IR_VECTOR_SIZE(ctx->ir_base[ref].type) < 2) { + return; + } +#endif if (ctx->use_lists[ref].count == 2 && !ir_match_has_mem_deps(ctx, ref, root)) { ir_ref addr_ref = ctx->ir_base[ref].op2; @@ -3314,6 +3338,11 @@ static bool ir_match_try_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root) if (ir_in_same_block(ctx, ref) && (insn->op == IR_LOAD || insn->op == IR_LOAD_v || insn->op == IR_VLOAD || insn->op == IR_VLOAD_v)) { +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ctx->ir_base[ref].type) && IR_VECTOR_SIZE(ctx->ir_base[ref].type) < 2) { + return 0; + } +#endif if (ctx->use_lists[ref].count == 2 && !ir_match_has_mem_deps(ctx, ref, root)) { ir_ref addr_ref = ctx->ir_base[ref].op2; @@ -5373,7 +5402,7 @@ static int32_t ir_ref_spill_slot_offset(ir_ctx *ctx, ir_ref ref, ir_reg *reg) { int32_t offset; - IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); + IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); offset = ctx->live_intervals[ctx->vregs[ref]]->stack_spill_pos; IR_ASSERT(offset != -1); if (ctx->live_intervals[ctx->vregs[ref]]->flags & IR_LIVE_INTERVAL_SPILL_SPECIAL) { @@ -5414,21 +5443,59 @@ static bool ir_is_same_spill_slot(ir_ctx *ctx, ir_ref ref, ir_mem mem) return IR_MEM_VAL(m) == IR_MEM_VAL(mem); } +static int32_t ir_local_slot(ir_ctx *ctx, ir_ref ref) +{ + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR || + ctx->ir_base[ref].op == IR_ALLOCA || + ctx->ir_base[ref].op == IR_VADDR || + ctx->ir_base[ref].op == IR_PARAM); + IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref])); + return IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]); +} + +static int32_t ir_local_offset(ir_ctx *ctx, ir_ref ref) +{ + int32_t offset; + + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR || + ctx->ir_base[ref].op == IR_ALLOCA || + ctx->ir_base[ref].op == IR_VADDR || + ctx->ir_base[ref].op == IR_PARAM); + IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref])); + offset = IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]); + return IR_SPILL_POS_TO_OFFSET(offset); +} + static ir_mem ir_var_spill_slot(ir_ctx *ctx, ir_ref ref) { - ir_insn *var_insn = &ctx->ir_base[ref]; ir_reg reg; - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR); reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - return IR_MEM_BO(reg, IR_SPILL_POS_TO_OFFSET(var_insn->op3)); + return IR_MEM_BO(reg, ir_local_offset(ctx, ref)); +} + +static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + int32_t offset; + + IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); + offset = ir_local_offset(ctx, src); + if (offset == 0) { + | mov Ra(reg), Ra(base) + } else { + | lea Ra(reg), [Ra(base)+offset] + } } static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use) { ir_live_interval *ival; - IR_ASSERT(ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); + IR_ASSERT(ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); ival = ctx->live_intervals[ctx->vregs[ref]]; while (ival) { ir_use_pos *use_pos = ival->use_pos; @@ -5633,6 +5700,49 @@ static void ir_emit_load_imm_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref sr } } +#if IR_SIMD +/* Vectos with size less than 4 have to be loaded throug general purpose register */ +static void ir_emit_load_short_vector(ir_ctx *ctx, uint32_t width, ir_reg reg, ir_mem mem, ir_reg tmp_reg) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + bool do_pop = 0; + + IR_ASSERT(width == 1 || width == 2); + if (tmp_reg == IR_REG_NONE) { + IR_ASSERT(0 && "this is not expected"); + if (IR_MEM_BASE(mem) != IR_REG_RAX && IR_MEM_INDEX(mem) != IR_REG_RAX) { + tmp_reg = IR_REG_RAX; + } else if (IR_MEM_BASE(mem) != IR_REG_RCX && IR_MEM_INDEX(mem) != IR_REG_RCX) { + tmp_reg = IR_REG_RCX; + } else if (IR_MEM_BASE(mem) != IR_REG_RDX && IR_MEM_INDEX(mem) != IR_REG_RDX) { + tmp_reg = IR_REG_RDX; + } + if (IR_MEM_BASE(mem) == IR_REG_RSP) { + mem = IR_MEM_ADD(mem, sizeof(void*)); + } + + | push Ra(tmp_reg) + do_pop = 1; + } + + if (width == 1) { + | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), byte, mem + } else { + | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), word, mem + } + if (ctx->mflags & IR_X86_AVX) { + | vmovd xmm(reg-IR_REG_FP_FIRST), Rd(tmp_reg) + } else { + | movd xmm(reg-IR_REG_FP_FIRST), Rd(tmp_reg) + } + + if (do_pop) { + | pop Ra(tmp_reg) + } +} +#endif + static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) { ir_backend_data *data = ctx->data; @@ -5650,13 +5760,13 @@ static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem me | ASM_TXT_TMEM_OP vmovdqa, xmm(reg-IR_REG_FP_FIRST), oword, mem } else if (width == 8) { | ASM_TXT_TMEM_OP vmovq, xmm(reg-IR_REG_FP_FIRST), qword, mem - } else if (width == 4) { + } else if (width == 4 || + (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) || + ((ctx->flags & IR_USE_FRAME_POINTER) && + IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) { | ASM_TXT_TMEM_OP vmovd, xmm(reg-IR_REG_FP_FIRST), dword, mem } else { - // TODO: 4-byte read may be unsafe ??? - // IR_ASSERT(width >= 4 && width <= 16); - IR_ASSERT(width <= 4); - | ASM_TXT_TMEM_OP vmovd, xmm(reg-IR_REG_FP_FIRST), dword, mem + ir_emit_load_short_vector(ctx, width, reg, mem, IR_REG_NONE); } } else if (element_type == IR_DOUBLE) { if (width == 16) { @@ -5699,13 +5809,13 @@ static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem me |.else | ASM_TXT_TMEM_OP movlpd, xmm(reg-IR_REG_FP_FIRST), qword, mem |.endif - } else if (width == 4) { + } else if (width == 4 || + (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) || + ((ctx->flags & IR_USE_FRAME_POINTER) && + IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) { | ASM_TXT_TMEM_OP movd, xmm(reg-IR_REG_FP_FIRST), dword, mem } else { - // TODO: 4-byte read may be unsafe ??? - // IR_ASSERT(width >= 4 && width <= 16); - IR_ASSERT(width <= 4); - | ASM_TXT_TMEM_OP movd, xmm(reg-IR_REG_FP_FIRST), dword, mem + ir_emit_load_short_vector(ctx, width, reg, mem, IR_REG_NONE); } } else if (element_type == IR_DOUBLE) { if (width == 16) { @@ -5741,38 +5851,6 @@ static void ir_emit_load_mem(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) } } -static int32_t ir_local_offset(ir_ctx *ctx, ir_insn *insn) -{ - if (insn->op != IR_PARAM) { - IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA || insn->op == IR_VADDR); - return IR_SPILL_POS_TO_OFFSET(insn->op3); - } else { - IR_ASSERT(ctx->value_params && ctx->value_params[insn->op3 - 1].align); - return IR_SPILL_POS_TO_OFFSET(ctx->value_params[insn->op3 - 1].offset); - } -} - -static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) -{ - ir_backend_data *data = ctx->data; - dasm_State **Dst = &data->dasm_state; - ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - ir_insn *var_insn; - int32_t offset; - - IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); - var_insn = &ctx->ir_base[src]; - if (var_insn->op == IR_VADDR) { - var_insn = &ctx->ir_base[var_insn->op1]; - } - offset = ir_local_offset(ctx, var_insn); - if (offset == 0) { - | mov Ra(reg), Ra(base) - } else { - | lea Ra(reg), [Ra(base)+offset] - } -} - static void ir_resolve_label_syms(ir_ctx *ctx) { uint32_t b; @@ -5828,7 +5906,7 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) } else { ir_emit_load_imm_fp(ctx, type, reg, src); } - } else if (ctx->vregs[src]) { + } else if (ctx->vregs[src] > 0) { ir_emit_load_mem(ctx, type, reg, ir_ref_spill_slot(ctx, src)); } else { ir_load_local_addr(ctx, reg, src); @@ -5843,6 +5921,49 @@ static void ir_emit_store_mem_int(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg | ASM_MEM_REG_OP mov, type, mem, reg } +#if IR_SIMD +/* Vectos with size less than 4 have to be stored throug general purpose register */ +static void ir_emit_store_short_vector(ir_ctx *ctx, uint32_t width, ir_mem mem, ir_reg reg, ir_reg tmp_reg) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + bool do_pop = 0; + + IR_ASSERT(width == 1 || width == 2); + if (tmp_reg == IR_REG_NONE) { + IR_ASSERT(0 && "this is not expected"); + if (IR_MEM_BASE(mem) != IR_REG_RAX && IR_MEM_INDEX(mem) != IR_REG_RAX) { + tmp_reg = IR_REG_RAX; + } else if (IR_MEM_BASE(mem) != IR_REG_RCX && IR_MEM_INDEX(mem) != IR_REG_RCX) { + tmp_reg = IR_REG_RCX; + } else if (IR_MEM_BASE(mem) != IR_REG_RDX && IR_MEM_INDEX(mem) != IR_REG_RDX) { + tmp_reg = IR_REG_RDX; + } + if (IR_MEM_BASE(mem) == IR_REG_RSP) { + mem = IR_MEM_ADD(mem, sizeof(void*)); + } + + | push Ra(tmp_reg) + do_pop = 1; + } + + if (ctx->mflags & IR_X86_AVX) { + | vmovd Rd(tmp_reg), xmm(reg-IR_REG_FP_FIRST) + } else { + | movd Rd(tmp_reg), xmm(reg-IR_REG_FP_FIRST) + } + if (width == 1) { + | ASM_TMEM_TXT_OP mov, byte, mem, Rb(tmp_reg) + } else { + | ASM_TMEM_TXT_OP mov, word, mem, Rw(tmp_reg) + } + + if (do_pop) { + | pop Ra(tmp_reg) + } +} +#endif + static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg reg) { ir_backend_data *data = ctx->data; @@ -5860,11 +5981,13 @@ static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg r | ASM_TMEM_TXT_OP vmovdqa, oword, mem, xmm(reg-IR_REG_FP_FIRST) } else if (width == 8) { | ASM_TMEM_TXT_OP vmovq, qword, mem, xmm(reg-IR_REG_FP_FIRST) - } else { - // TODO: 4-byte write may be unsafe ??? - // IR_ASSERT(width >= 4 && width <= 16); - IR_ASSERT(width <= 4); + } else if (width == 4 || + (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) || + ((ctx->flags & IR_USE_FRAME_POINTER) && + IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) { | ASM_TMEM_TXT_OP vmovd, dword, mem, xmm(reg-IR_REG_FP_FIRST) + } else { + ir_emit_store_short_vector(ctx, width, mem, reg, IR_REG_NONE); } } else if (element_type == IR_DOUBLE) { if (width == 16) { @@ -5907,11 +6030,13 @@ static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg r |.else | ASM_TMEM_TXT_OP movlpd, qword, mem, xmm(reg-IR_REG_FP_FIRST) |.endif - } else { - // TODO: 4-byte write may be unsafe ??? - // IR_ASSERT(width >= 4 && width <= 16); - IR_ASSERT(width <= 4); + } else if (width == 4 || + (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) || + ((ctx->flags & IR_USE_FRAME_POINTER) && + IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) { | ASM_TMEM_TXT_OP movd, dword, mem, xmm(reg-IR_REG_FP_FIRST) + } else { + ir_emit_store_short_vector(ctx, width, mem, reg, IR_REG_NONE); } } else if (element_type == IR_DOUBLE) { if (width == 16) { @@ -5955,7 +6080,7 @@ static void ir_emit_load_i64_lo(ir_ctx *ctx, ir_reg reg, ir_ref src) IR_ASSERT(IR_IS_TYPE_INT(insn->type) && !IR_IS_SYM_CONST(insn->op)); ir_emit_load_imm_int(ctx, IR_U32, reg, insn->val.u32); - } else if (ctx->vregs[src]) { + } else if (ctx->vregs[src] > 0) { ir_emit_load_mem_int(ctx, IR_U32, reg, ir_ref_spill_slot(ctx, src)); } else { IR_ASSERT(0); @@ -5969,7 +6094,7 @@ static void ir_emit_load_i64_hi(ir_ctx *ctx, ir_reg reg, ir_ref src) IR_ASSERT(IR_IS_TYPE_INT(insn->type) && !IR_IS_SYM_CONST(insn->op)); ir_emit_load_imm_int(ctx, IR_U32, reg, insn->val.u32_hi); - } else if (ctx->vregs[src]) { + } else if (ctx->vregs[src] > 0) { ir_mem mem = IR_MEM_I64_HI(ir_ref_spill_slot(ctx, src)); ir_emit_load_mem_int(ctx, IR_U32, reg, mem); } else { @@ -5979,7 +6104,7 @@ static void ir_emit_load_i64_hi(ir_ctx *ctx, ir_reg reg, ir_ref src) static void ir_emit_store_i64_lo(ir_ctx *ctx, ir_ref dst, ir_reg reg) { - if (ctx->vregs[dst]) { + if (ctx->vregs[dst] > 0) { ir_emit_store_mem_int(ctx, IR_U32, ir_ref_spill_slot(ctx, dst), reg); } else { IR_ASSERT(0); @@ -5988,7 +6113,7 @@ static void ir_emit_store_i64_lo(ir_ctx *ctx, ir_ref dst, ir_reg reg) static void ir_emit_store_i64_hi(ir_ctx *ctx, ir_ref dst, ir_reg reg) { - if (ctx->vregs[dst]) { + if (ctx->vregs[dst] > 0) { ir_mem mem = IR_MEM_I64_HI(ir_ref_spill_slot(ctx, dst)); ir_emit_store_mem_int(ctx, IR_U32, mem, reg); } else { @@ -6207,6 +6332,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) ir_ref base_reg_ref, index_reg_ref; ir_reg base_reg = IR_REG_NONE, index_reg; int32_t offset = 0, scale; + intptr_t long_offset; IR_ASSERT(((rule & IR_RULE_MASK) >= IR_LEA_FIRST && (rule & IR_RULE_MASK) <= IR_LEA_LAST) || @@ -6217,7 +6343,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) case IR_LEA_OB: offset_insn = insn; if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]); + offset = ir_local_offset(ctx, insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6239,12 +6365,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) break; case IR_LEA_IB: if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]); + offset = ir_local_offset(ctx, insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = ref * sizeof(ir_ref) + 2; } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = ref * sizeof(ir_ref) + 1; @@ -6260,12 +6386,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = op1_insn; scale = 1; if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; } else if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]); + offset = ir_local_offset(ctx, op1_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = ref * sizeof(ir_ref) + 2; @@ -6279,12 +6405,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = op2_insn; scale = 1; if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]); + offset = ir_local_offset(ctx, insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = insn->op2 * sizeof(ir_ref) + 1; } else if (ir_rule(ctx, op2_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op2_insn->op1]); + offset = ir_local_offset(ctx, op2_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = ref * sizeof(ir_ref) + 1; @@ -6311,12 +6437,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = insn; scale = 1; if (ir_rule(ctx, op1_insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op2]); + offset = ir_local_offset(ctx, op1_insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; } else if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]); + offset = ir_local_offset(ctx, op1_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = insn->op1 * sizeof(ir_ref) + 2; @@ -6332,7 +6458,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) op2_insn = &ctx->ir_base[insn->op2]; scale = ctx->ir_base[op2_insn->op2].val.i32; if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]); + offset = ir_local_offset(ctx, op1_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6346,7 +6472,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) op2_insn = &ctx->ir_base[insn->op2]; offset_insn = op2_insn; if (ir_rule(ctx, op2_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op2_insn->op1]); + offset = ir_local_offset(ctx, op2_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6355,7 +6481,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) break; case IR_LEA_B_SI: if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]); + offset = ir_local_offset(ctx, insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6369,7 +6495,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) case IR_LEA_SI_B: index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6383,7 +6509,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = insn; op1_insn = &ctx->ir_base[insn->op1]; if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]); + offset = ir_local_offset(ctx, op1_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6398,7 +6524,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) op1_insn = &ctx->ir_base[insn->op1]; index_reg_ref = op1_insn->op1 * sizeof(ir_ref) + 1; if (ir_rule(ctx, op1_insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op2]); + offset = ir_local_offset(ctx, op1_insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6410,7 +6536,9 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) case IR_LEA_SYM_O: op1_insn = &ctx->ir_base[insn->op1]; op2_insn = &ctx->ir_base[insn->op2]; - offset = (intptr_t)ir_sym_val(ctx, op1_insn) + (intptr_t)op2_insn->val.i64; + long_offset = (intptr_t)ir_sym_val(ctx, op1_insn) + (intptr_t)op2_insn->val.i64; + IR_ASSERT(IR_IS_SIGNED_32BIT(long_offset)); + offset = (int32_t)long_offset; base_reg_ref = index_reg_ref = IR_UNUSED; scale = 1; offset_insn = NULL; @@ -6418,13 +6546,15 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) case IR_LEA_O_SYM: op1_insn = &ctx->ir_base[insn->op1]; op2_insn = &ctx->ir_base[insn->op2]; - offset = (intptr_t)ir_sym_val(ctx, op2_insn) + (intptr_t)op1_insn->val.i64; + long_offset = (intptr_t)ir_sym_val(ctx, op2_insn) + (intptr_t)op1_insn->val.i64; + IR_ASSERT(IR_IS_SIGNED_32BIT(long_offset)); + offset = (int32_t)long_offset; base_reg_ref = index_reg_ref = IR_UNUSED; scale = 1; offset_insn = NULL; break; case IR_ALLOCA: - offset = ir_local_offset(ctx, insn); + offset = ir_local_offset(ctx, ref); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = index_reg_ref = IR_UNUSED; scale = 1; @@ -11377,20 +11507,21 @@ static void ir_emit_vaddr(ir_ctx *ctx, ir_ref def, ir_insn *insn) static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_ref type = insn->type; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); ir_reg fp; ir_mem mem; + int32_t slot; if (ctx->use_lists[def].count == 1) { /* dead load */ return; } - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3)); - if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, var_insn->op3)) { + slot = ir_local_slot(ctx, insn->op2); + mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot)); + if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, slot)) { return; // fake load } IR_ASSERT(def_reg != IR_REG_NONE); @@ -11420,20 +11551,21 @@ static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn) static void ir_emit_vstore_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_insn *val_insn = &ctx->ir_base[insn->op3]; ir_ref type = val_insn->type; ir_reg op3_reg = ctx->regs[ref][3]; ir_reg fp; ir_mem mem; + int32_t slot; - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3)); + slot = ir_local_slot(ctx, insn->op2); + mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot)); if ((op3_reg == IR_REG_NONE || IR_REG_SPILLED(op3_reg)) && !IR_IS_CONST_REF(insn->op3) && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA - && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) { + && ir_is_same_mem_var(ctx, insn->op3, slot)) { return; // fake store } if (IR_IS_CONST_REF(insn->op3)) { @@ -11470,19 +11602,20 @@ static void ir_emit_vstore_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) static void ir_emit_vstore_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_ref type = ctx->ir_base[insn->op3].type; ir_reg op3_reg = ctx->regs[ref][3]; ir_reg fp; ir_mem mem; + int32_t slot; - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3)); + slot = ir_local_slot(ctx, insn->op2); + mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot)); if ((op3_reg == IR_REG_NONE || IR_REG_SPILLED(op3_reg)) && !IR_IS_CONST_REF(insn->op3) && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA - && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) { + && ir_is_same_mem_var(ctx, insn->op3, slot)) { return; // fake store } if (IR_IS_CONST_REF(insn->op3)) { @@ -11565,6 +11698,7 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op2_reg = ctx->regs[def][2]; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); ir_mem mem; + bool avoid_spill_store = 0; if (ctx->use_lists[def].count == 1) { /* dead load */ @@ -11584,16 +11718,38 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); mem = ir_fuse_addr(ctx, def, insn->op2); if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, mem)) { - if (!ir_may_avoid_spill_load(ctx, def, def)) { - ir_emit_load_mem_fp(ctx, type, def_reg, mem); + + if (ir_may_avoid_spill_load(ctx, def, def)) { + avoid_spill_store = 1; } - /* avoid load to the same location (valid only when register is not reused) */ - return; } } +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) { + uint32_t width = IR_VECTOR_SIZE(type); + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg tmp_reg = ctx->regs[def][3]; + + IR_ASSERT(tmp_reg != IR_REG_NONE); + IR_ASSERT(width == 1 || width == 2); + if (width == 1) { + | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), byte, mem + } else { + | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), word, mem + } + if (ctx->mflags & IR_X86_AVX) { + | vmovd xmm(def_reg-IR_REG_FP_FIRST), Rd(tmp_reg) + } else { + | movd xmm(def_reg-IR_REG_FP_FIRST), Rd(tmp_reg) + } + } else +#endif + ir_emit_load_mem_fp(ctx, type, def_reg, mem); - if (IR_REG_SPILLED(ctx->regs[def][0])) { + + if (IR_REG_SPILLED(ctx->regs[def][0]) && !avoid_spill_store) { ir_emit_store(ctx, type, def, def_reg); } } @@ -11738,6 +11894,26 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) } if (IR_IS_CONST_REF(insn->op3)) { +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) { + uint32_t width = IR_VECTOR_SIZE(type); + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + + IR_ASSERT(width == 1 || width == 2); + if (width == 1) { + uint8_t *p = ir_long_const_ptr(ctx, insn->op3); + int32_t val = *p; + + | ASM_TMEM_TXT_OP mov, byte, mem, val + } else { + uint16_t *p = ir_long_const_ptr(ctx, insn->op3); + int32_t val = *p; + + | ASM_TMEM_TXT_OP mov, word, mem, val + } + } else +#endif ir_emit_store_mem_fp_const(ctx, type, mem, insn->op3, IR_REG_NONE, op3_reg); } else { IR_ASSERT(op3_reg != IR_REG_NONE); @@ -11745,6 +11921,28 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); } + +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) { + uint32_t width = IR_VECTOR_SIZE(type); + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg tmp_reg = ctx->regs[ref][1]; + + IR_ASSERT(width == 1 || width == 2); + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (ctx->mflags & IR_X86_AVX) { + | vmovd Rd(tmp_reg), xmm(op3_reg-IR_REG_FP_FIRST) + } else { + | movd Rd(tmp_reg), xmm(op3_reg-IR_REG_FP_FIRST) + } + if (width == 1) { + | ASM_TMEM_TXT_OP mov, byte, mem, Rb(tmp_reg) + } else { + | ASM_TMEM_TXT_OP mov, word, mem, Rw(tmp_reg) + } + } else +#endif ir_emit_store_mem_fp(ctx, type, mem, op3_reg); } } @@ -11755,7 +11953,7 @@ static void ir_emit_rload(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_type type = insn->type; if (IR_REGSET_IN(IR_REGSET_UNION((ir_regset)ctx->fixed_regset, IR_REGSET_FIXED), src_reg)) { - if (ctx->vregs[def] + if (ctx->vregs[def] > 0 && ctx->live_intervals[ctx->vregs[def]] && ctx->live_intervals[ctx->vregs[def]]->stack_spill_pos != -1) { ir_emit_store(ctx, type, def, src_reg); @@ -11978,7 +12176,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -12012,7 +12210,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -12080,7 +12278,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + op2_offset = ir_local_offset(ctx, insn->op2); } if (op3_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op3_reg)) { @@ -12091,7 +12289,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]); + op3_offset = ir_local_offset(ctx, insn->op3); } | mov Ra(tmp_reg), aword [Ra(op3_reg)+op3_offset] | mov aword [Ra(op2_reg)+op2_offset], Ra(tmp_reg) @@ -12114,7 +12312,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + op2_offset = ir_local_offset(ctx, insn->op2); } if (op3_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op3_reg)) { @@ -12125,7 +12323,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]); + op3_offset = ir_local_offset(ctx, insn->op3); } | mov Rd(tmp_reg), dword [Ra(op3_reg)+(op3_offset+offsetof(ir_x86_64_sysv_va_list, gp_offset))] | mov dword [Ra(op2_reg)+(op2_offset+offsetof(ir_x86_64_sysv_va_list, gp_offset))], Rd(tmp_reg) @@ -12163,7 +12361,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } | mov Ra(tmp_reg), aword [Ra(op2_reg)+offset] #if IR_X86_I64 @@ -12233,7 +12431,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (insn->op3) { /* long struct arguemnt */ @@ -12266,9 +12464,9 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) |2: if (def_reg != IR_REG_NONE) { if (ir_type_size[type] == 8) { - | mov Rq(def_reg), qword [Ra(tmp_reg)-sizeof(void*)] + | mov Rq(def_reg), qword [Ra(tmp_reg)+(0-sizeof(void*))] } else { - | mov Rd(def_reg), dword [Ra(tmp_reg)-sizeof(void*)] + | mov Rd(def_reg), dword [Ra(tmp_reg)+(0-sizeof(void*))] } } } else { @@ -12309,7 +12507,7 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) uint32_t n, *p, use_block; int i; int label, default_label = 0; - int count = 0; + uint32_t count = 0; ir_val min, max; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; @@ -12342,8 +12540,8 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) max.i64 = IR_MAX(max.i64, val->val.i64); } else { IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type)); - min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64); - max.u64 = (int64_t)IR_MAX(max.u64, val->val.u64); + min.u64 = IR_MIN(min.u64, val->val.u64); + max.u64 = IR_MAX(max.u64, val->val.u64); } count++; } else if (use_insn->op == IR_CASE_RANGE) { @@ -12354,12 +12552,14 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(val2->op)); if (IR_IS_TYPE_SIGNED(type)) { IR_ASSERT(IR_IS_TYPE_SIGNED(val->type)); + IR_ASSERT(IR_IS_TYPE_SIGNED(val2->type)); min.i64 = IR_MIN(min.i64, val->val.i64); max.i64 = IR_MAX(max.i64, val2->val.i64); } else { IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type)); - min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64); - max.u64 = (int64_t)IR_MAX(max.u64, val2->val.u64); + IR_ASSERT(!IR_IS_TYPE_SIGNED(val2->type)); + min.u64 = IR_MIN(min.u64, val->val.u64); + max.u64 = IR_MAX(max.u64, val2->val.u64); } } else { IR_ASSERT(use_insn->op == IR_CASE_DEFAULT); @@ -12388,8 +12588,10 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) } /* Generate a table jmp or a seqence of calls */ - if (!has_case_range && count > 2 && (max.i64-min.i64) < count * 8) { - int *labels = ir_mem_malloc(sizeof(int) * (size_t)(max.i64 - min.i64 + 1)); + if ((sizeof(void*) == 8 || ir_type_size[type] <= 4) + && !has_case_range && count > 2 && (max.u64 - min.u64) < count * 8) { + int *labels = ir_mem_malloc(sizeof(int) * (size_t)(max.u64 - min.u64 + 1)); + ir_reg idx_reg; for (i = 0; i <= (max.i64 - min.i64); i++) { labels[i] = default_label; @@ -12442,62 +12644,69 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) } if (min.i64 != 0) { - int64_t offset = -min.i64; + int64_t offset = 0 - min.u64; if (IR_IS_SIGNED_32BIT(offset)) { | lea Ra(tmp_reg), [Ra(op2_reg)+(int32_t)offset] } else { - IR_ASSERT(sizeof(void*) == 8); |.if X64 | mov64 Rq(tmp_reg), offset | add Ra(tmp_reg), Ra(op2_reg) |.endif } - if (default_label) { - offset = max.i64 - min.i64; + idx_reg = tmp_reg; + } else { + idx_reg = op2_reg; + } - IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); - | cmp Ra(tmp_reg), (int32_t)offset - | ja =>default_label - } -|.if X64 - if (ctx->code_buffer - && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start) - && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end)) { - | jmp aword [Ra(tmp_reg)*8+>1] - } else { - int64_t offset = -min.i64; + if (default_label) { + int64_t offset = max.u64 - min.u64; - IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); - offset *= 8; - IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); - | lea Ra(tmp_reg), aword [>1] - | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8+offset] - } -|.else - | jmp aword [Ra(tmp_reg)*4+>1] + IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); + | cmp Ra(idx_reg), (int32_t)offset + | ja =>default_label + } + +#ifdef IR_TARGET_X86 +|.if not X64 + | jmp aword [Ra(idx_reg)*4+>1] |.endif +#else +|.if X64 + bool use_table_address = 0; + + if ((ctx->code_buffer + && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start) + && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end))) { + /* Use SIB with absolute jmp_table displacement */ + | jmp aword [Ra(idx_reg)*8+>1] + } else { - if (default_label) { - int64_t offset = max.i64; + /* Use IP-relative addressing mode to get the jmp_table address */ - IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); - | cmp Ra(op2_reg), (int32_t)offset - | ja =>default_label - } -|.if X64 - if (ctx->code_buffer - && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start) - && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end)) { - | jmp aword [Ra(op2_reg)*8+>1] - } else { + // TODO: it's possible to use an extra temporary register (not conflicting with "killed" op2) ??? +// ir_reg op1_reg = ctx->regs[def][1]; +// if (op1_reg != IR_REG_NONE) { +// | lea Ra(op1_reg), aword [>1] +// | jmp aword [Ra(op1_reg)+Ra(idx_reg)*8] +// } else + if ((min.i64 >= (INT32_MIN+1)/8) && (min.i64 <= INT32_MAX/8)) { + tmp_reg = ctx->regs[def][3]; | lea Ra(tmp_reg), aword [>1] - | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8] + | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8+(-min.i32 * 8)] + } else { + use_table_address = 1; + if (idx_reg == tmp_reg) { + | shl Ra(tmp_reg), 3 + } else { + | lea Ra(tmp_reg), aword [Ra(idx_reg)*8] + } + | add Ra(tmp_reg), aword [>2] + | jmp aword [Ra(tmp_reg)] } -|.else - | jmp aword [Ra(op2_reg)*4+>1] -|.endif } +|.endif +#endif |.jmp_table if (!data->jmp_table_label) { @@ -12505,6 +12714,12 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) |=>data->jmp_table_label: } |.align aword +#ifndef IR_TARGET_X86 + if (use_table_address) { + |2: + | .aword >1 + } +#endif |1: for (i = 0; i <= (max.i64 - min.i64); i++) { int b = labels[i]; @@ -13073,7 +13288,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, const i type = IR_U32; } ir_emit_load(ctx, type, dst_reg, arg); - } else if (ctx->vregs[arg]) { + } else if (ctx->vregs[arg] > 0) { ir_mem mem = ir_ref_spill_slot(ctx, arg); uint32_t size = ir_type_size[type]; @@ -13294,7 +13509,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_pro ir_emit_store(ctx, insn->type, def, def_reg); } } - } else if (ctx->use_lists[def].count > 1) { + } else if (ctx->vregs[def] > 0) { #if IR_X86_I64 if (insn->type == IR_I64 || insn->type == IR_U64) { IR_ASSERT(cc->int_ret2_reg != IR_REG_NONE); @@ -13322,14 +13537,14 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_pro if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } - } else if (ctx->use_lists[def].count > 1) { + } else if (ctx->vregs[def] > 0) { ir_emit_store(ctx, insn->type, def, ret_reg); } } #ifdef IR_TARGET_X86 if (insn->op == IR_TAILCALL) { /* pass */ - } else if (ctx->use_lists[def].count > 1 && ret_reg == IR_REG_NONE) { + } else if (ctx->vregs[def] > 0 && ret_reg == IR_REG_NONE) { int32_t offset; ir_reg fp; @@ -13407,7 +13622,7 @@ static void ir_emit_tailcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) bool is_spill_slot = op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg) - && ctx->vregs[func]; + && ctx->vregs[func] > 0; if (op2_reg != IR_REG_NONE && !is_spill_slot) { if (IR_REGSET_IN(preserved_regs, IR_REG_NUM(op2_reg))) { @@ -17660,7 +17875,7 @@ static void ir_emit_vector_extract(ir_ctx *ctx, ir_ref def, ir_insn *insn) } /* extract through stack memory */ - int offset = -width; + int offset = 0 - width; ir_mem mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1); ir_mem mem2 = IR_MEM(IR_REG_RSP, offset, op2_reg, ir_type_size[element_type]); @@ -17789,7 +18004,7 @@ static void ir_emit_vector_replace(ir_ctx *ctx, ir_ref def, ir_insn *insn) int offset = width; | movdqu [Ra(IR_REG_RSP)-offset], xmm(tmp_reg-IR_REG_FP_FIRST) - | mov [Ra(IR_REG_RSP)-(offset-(lane%16))], Rb(op3_reg) + | mov [Ra(IR_REG_RSP)+((lane%16)-offset)], Rb(op3_reg) | movdqu xmm(tmp_reg-IR_REG_FP_FIRST), [Ra(IR_REG_RSP)-offset] } } else if (element_type == IR_I16 || element_type == IR_U16) { @@ -17936,7 +18151,7 @@ static void ir_emit_vector_replace(ir_ctx *ctx, ir_ref def, ir_insn *insn) } /* modify through stack memory */ - int offset = -width; + int offset = 0 - width; ir_mem mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1); ir_mem mem2 = IR_MEM(IR_REG_RSP, offset, op2_reg, ir_type_size[element_type]); @@ -18790,7 +19005,7 @@ static void ir_emit_vector_shuffle(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_type mask_element_type = IR_VECTOR_BASE_TYPE(mask_type); uint32_t mask_element_size = ir_type_size[mask_element_type]; uint32_t mask_len = IR_VECTOR_LENGTH(mask_type); - int dst_offset = -width; + int dst_offset = 0 - width; int src_offset = dst_offset; uint32_t mod = 0; ir_reg tmp_reg = ctx->tmp_regs[def]; @@ -18818,22 +19033,22 @@ static void ir_emit_vector_shuffle(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_mem src, dst; if (mask_element_size == 1) { - j = *(uint8_t*)ptr; + j = *(uint8_t*)ptr % mod; ptr = ((uint8_t*)ptr) + 1; } else if (mask_element_size == 2) { - j = *(uint16_t*)ptr; + j = *(uint16_t*)ptr % mod; ptr = ((uint16_t*)ptr) + 1; } else if (mask_element_size == 4) { - j = *(uint32_t*)ptr; + j = *(uint32_t*)ptr % mod; ptr = ((uint32_t*)ptr) + 1; } else if (mask_element_size == 8) { - j = *(uint64_t*)ptr; + j = *(uint64_t*)ptr % mod; ptr = ((uint64_t*)ptr) + 1; } else { IR_ASSERT(0); j = 0; } - src = IR_MEM(IR_REG_RSP, src_offset + (j % mod) * element_size, IR_REG_NONE, 1); + src = IR_MEM(IR_REG_RSP, src_offset + j * element_size, IR_REG_NONE, 1); dst = IR_MEM(IR_REG_RSP, dst_offset + i * element_size, IR_REG_NONE, 1); #if IR_X86_I64 if (element_type == IR_I64 || element_type == IR_U64) { @@ -21978,7 +22193,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn) dst_mem = ir_ref_spill_slot(ctx, def); def_reg = IR_REG_NUM(def_reg); } else { - offset = -width; + offset = 0 - width; dst_mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1); } @@ -21991,7 +22206,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (offset != 0) { src1_mem = dst_mem; } else { - offset = -width; + offset = 0 - width; src1_mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1); } ir_emit_store_mem_fp(ctx, type, src1_mem, op1_reg); @@ -22144,7 +22359,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (!scalar_shift) { if (element_size == 8) { if (IR_IS_SIGNED_32BIT(((int64_t*)ptr2)[i])) { - val = ((int64_t*)ptr2)[i]; + val = ((int64_t*)ptr2)[i] & 0xffffffff; } else { #if IR_X86_I64 val_hi = ((uint64_t*)ptr2)[i] >> 32; @@ -25739,7 +25954,7 @@ static void ir_emit_load_params(ir_ctx *ctx) int_param_num++; } } - if (ctx->vregs[use]) { + if (ctx->vregs[use] > 0) { dst_reg = IR_REG_NUM(ctx->regs[use][0]); IR_ASSERT(src_reg != IR_REG_NONE || dst_reg != IR_REG_NONE || stack_start + stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos); @@ -25805,7 +26020,6 @@ static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_s if (param_store_insn->op == IR_VSTORE) { ir_ref var = param_store_insn->op2; - ir_insn *var_insn = &ctx->ir_base[var]; IR_ASSERT(param_store_insn->op3 == param); IR_ASSERT(ctx->ir_base[var].op == IR_VAR); @@ -25814,19 +26028,28 @@ static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_s ir_use_list *var_use_list = &ctx->use_lists[var]; ir_ref n = var_use_list->count; ir_ref *p = &ctx->use_edges[var_use_list->refs]; + int32_t old_slot = IR_VREG_TO_STACK_SLOT(ctx->vregs[var]); + for (; n > 0; p++, n--) { ir_ref use = *p; ir_insn *insn = &ctx->ir_base[use]; if (insn->op == IR_VADDR) { if (ctx->rules[use] == IR_STATIC_ALLOCA) { - IR_ASSERT(insn->op3 == var_insn->op3); - insn->op3 = spill_slot; + IR_ASSERT(ctx->vregs[use] == ctx->vregs[var]); + ctx->vregs[use] = IR_STACK_SLOT_TO_VREG(spill_slot); } - } else if (insn->op == IR_VLOAD) { - if (ctx->vregs[use]) { + } else if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { + if (ctx->vregs[use] > 0) { ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; - if (ival->stack_spill_pos == var_insn->op3) { + if (ival && ival->stack_spill_pos == old_slot) { + ival->stack_spill_pos = spill_slot; + } + } + } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { + if (insn->op3 > 0 && ctx->vregs[insn->op3] > 0) { + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op3]]; + if (ival && ival->stack_spill_pos == old_slot) { ival->stack_spill_pos = spill_slot; } } @@ -25834,7 +26057,7 @@ static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_s } /* Remap VAR itself */ - var_insn->op3 = spill_slot; + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(spill_slot); /* Avoid copying to itself */ ctx->rules[param_store] = IR_SKIPPED | IR_NOP; @@ -25877,7 +26100,7 @@ static void ir_fix_param_spills(ir_ctx *ctx) align = IR_MAX(sizeof(void*), align); stack_offset = IR_ALIGNED_SIZE(stack_offset, align); - ctx->value_params[insn->op3 - 1].offset = stack_start + stack_offset; + ctx->vregs[use] = IR_STACK_SLOT_TO_VREG(stack_start + stack_offset); stack_offset += ctx->value_params[insn->op3 - 1].size; stack_offset = IR_ALIGNED_SIZE(stack_offset, sizeof(void*)); continue; @@ -25923,7 +26146,7 @@ static void ir_fix_param_spills(ir_ctx *ctx) } } if (src_reg == IR_REG_NONE) { - if (ctx->vregs[use]) { + if (ctx->vregs[use] > 0) { ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) && ival->stack_spill_pos == -1) { @@ -26003,7 +26226,7 @@ void ir_fix_stack_frame(ir_ctx *ctx) #ifdef IR_TARGET_X86 if (ctx->flags2 & IR_HAS_MEMCPY) { - ir_regset tmp = ctx->used_preserved_regs; + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; IR_REGSET_INCL(tmp, IR_REG_RSI); IR_REGSET_INCL(tmp, IR_REG_RDI); ctx->used_preserved_regs = tmp;