From 503cd8278d382af9850b18be42f917914ce0e65e Mon Sep 17 00:00:00 2001 From: Jorg Sowa Date: Tue, 13 Oct 2026 17:27:26 +0200 Subject: [PATCH 1/3] Update IR IR commit: ed32c881a0d7c14ce7ace6ac00dead75647b69c6 --- .github/scripts/download-bundled/jit-ir.sh | 2 +- ext/opcache/jit/ir/dynasm/dasm_arm.h | 2 +- ext/opcache/jit/ir/dynasm/dasm_arm.lua | 2 +- ext/opcache/jit/ir/dynasm/dasm_arm64.h | 66 +-- ext/opcache/jit/ir/dynasm/dasm_arm64.lua | 24 +- ext/opcache/jit/ir/dynasm/dasm_mips.h | 2 +- ext/opcache/jit/ir/dynasm/dasm_mips.lua | 2 +- ext/opcache/jit/ir/dynasm/dasm_mips64.lua | 2 +- ext/opcache/jit/ir/dynasm/dasm_ppc.h | 2 +- ext/opcache/jit/ir/dynasm/dasm_ppc.lua | 2 +- ext/opcache/jit/ir/dynasm/dasm_proto.h | 2 +- ext/opcache/jit/ir/dynasm/dasm_x64.lua | 2 +- ext/opcache/jit/ir/dynasm/dasm_x86.h | 2 +- ext/opcache/jit/ir/dynasm/dasm_x86.lua | 2 +- ext/opcache/jit/ir/dynasm/dynasm.lua | 4 +- ext/opcache/jit/ir/ir.c | 8 +- ext/opcache/jit/ir/ir.h | 26 +- ext/opcache/jit/ir/ir_aarch64.dasc | 343 ++++++++---- ext/opcache/jit/ir/ir_disasm.c | 2 +- ext/opcache/jit/ir/ir_dump.c | 12 +- ext/opcache/jit/ir/ir_emit.c | 190 +++++-- ext/opcache/jit/ir/ir_fold.h | 113 +++- ext/opcache/jit/ir/ir_gcm.c | 24 +- ext/opcache/jit/ir/ir_private.h | 21 +- ext/opcache/jit/ir/ir_ra.c | 220 ++++---- ext/opcache/jit/ir/ir_save.c | 8 +- ext/opcache/jit/ir/ir_sccp.c | 18 +- ext/opcache/jit/ir/ir_x86.dasc | 589 ++++++++++++++------- 28 files changed, 1160 insertions(+), 532 deletions(-) diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index bc9c0a428b7b..f3c7242e9d9b 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=93f92f5b5a49a02295685e340ebb0b83562cbe79 +revision=ed32c881a0d7c14ce7ace6ac00dead75647b69c6 git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm.h b/ext/opcache/jit/ir/dynasm/dasm_arm.h index eaa94d9c2f7f..97f541891d59 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm.h +++ b/ext/opcache/jit/ir/dynasm/dasm_arm.h @@ -1,6 +1,6 @@ /* ** DynASM ARM encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm.lua b/ext/opcache/jit/ir/dynasm/dasm_arm.lua index edb575366acd..7d95e4530640 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_arm.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM ARM module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm64.h b/ext/opcache/jit/ir/dynasm/dasm_arm64.h index 8a22974ad3ff..df8d5434d3a3 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm64.h +++ b/ext/opcache/jit/ir/dynasm/dasm_arm64.h @@ -1,6 +1,6 @@ /* ** DynASM ARM64 encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ @@ -178,44 +178,46 @@ static int dasm_imm12(unsigned int n) return -1; } -static int dasm_ffs(unsigned long long x) +#if defined(__GNUC__) || defined(__clang__) +#define dasm_ctz(x) __builtin_ctzll(x) +#define dasm_clz(x) __builtin_clzll(x) +#else +static int dasm_ctz(unsigned long long x) +{ + int n = 0; + x = ~x; + while (x & 1) { x >>= 1; n++; } + return n; +} + +static int dasm_clz(unsigned long long x) { - int n = -1; - while (x) { x >>= 1; n++; } + int n = 0; + x = ~x; + while ((long long)x < 0) { x <<= 1; n++; } return n; } +#endif + +static unsigned long long dasm_ror(unsigned long long x, int n) +{ + return (x << (-n & 63)) | (x >> n); +} static int dasm_imm13(int lo, int hi) { - int inv = 0, w = 64, s = 0xfff, xa, xb; + /* Thanks to: https://dougallj.wordpress.com/2021/10/30/ */ + int rot, ones, size, immr, imms; unsigned long long n = (((unsigned long long)hi) << 32) | (unsigned int)lo; - unsigned long long m = 1ULL, a, b, c; - if (n & 1) { n = ~n; inv = 1; } - a = n & (unsigned long long)-(long long)n; - b = (n+a)&(unsigned long long)-(long long)(n+a); - c = (n+a-b)&(unsigned long long)-(long long)(n+a-b); - xa = dasm_ffs(a); xb = dasm_ffs(b); - if (c) { - w = dasm_ffs(c) - xa; - if (w == 32) m = 0x0000000100000001UL; - else if (w == 16) m = 0x0001000100010001UL; - else if (w == 8) m = 0x0101010101010101UL; - else if (w == 4) m = 0x1111111111111111UL; - else if (w == 2) m = 0x5555555555555555UL; - else return -1; - s = (-2*w & 0x3f) - 1; - } else if (!a) { - return -1; - } else if (xb == -1) { - xb = 64; - } - if ((b-a) * m != n) return -1; - if (inv) { - return ((w - xb) << 6) | (s+w+xa-xb); - } else { - return ((w - xa) << 6) | (s+xb-xa); - } - return -1; + if ((n+1u) <= 1u) return -1; /* Neither all-zero nor all-ones are allowed. */ + rot = (n & (n+1u)) ? dasm_ctz(n & (n+1u)) : 64; + n = dasm_ror(n, rot & 63); + ones = dasm_ctz(~n); + size = dasm_clz(n) + ones; + if (dasm_ror(n, size & 63) != n) return -1; /* Non-repeating? */ + immr = -rot & (size - 1); + imms = (-(size << 1) | (ones - 1)) & 63; + return ((immr | (size & 64)) << 6) | imms; } /* Pass 1: Store actions and args, link branches/labels, estimate offsets. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm64.lua b/ext/opcache/jit/ir/dynasm/dasm_arm64.lua index 501c4bd2abf6..2d6fe202bcca 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_arm64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM ARM64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ @@ -244,6 +244,10 @@ local map_cond = { hs = 2, lo = 3, } +local map_bti = { + c = 0x40, j = 0x80, jc = 0xc0, +} + ------------------------------------------------------------------------------ local parse_reg_type @@ -475,6 +479,12 @@ local function parse_cond(expr, inv) return shl(bit.bxor(c, inv), 12) end +local function parse_map(expr, map) + local x = map[expr] + if not x then werror("bad operand") end + return x +end + local function parse_load(params, nparams, n, op) if params[n+2] then werror("too many operands") end local scale = shr(op, 30) @@ -823,11 +833,21 @@ map_op = { tbz_3 = "36000000DTBw|36000000DTBx", tbnz_3 = "37000000DTBw|37000000DTBx", + -- Branch Target Identification. + bti_1 = "d503241ft", + -- ARM64e: Pointer authentication codes (PAC). blraaz_1 = "d63f081fNx", + blrabz_1 = "d63f0c1fNx", braa_2 = "d71f0800NDx", + brab_2 = "d71f0c00NDx", braaz_1 = "d61f081fNx", + brabz_1 = "d61f0c1fNx", + paciasp_0 = "d503233f", pacibsp_0 = "d503237f", + autiasp_0 = "d50323bf", + autibsp_0 = "d50323ff", + retaa_0 = "d65f0bff", retab_0 = "d65f0fff", -- Miscellaneous instructions. @@ -996,6 +1016,8 @@ local function parse_template(params, template, nparams, pos) op = op + parse_cond(q, 0); n = n + 1 elseif p == "c" then op = op + parse_cond(q, 1); n = n + 1 + elseif p == "t" then + op = op + parse_map(q, map_bti); n = n + 1 else assert(false) diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips.h b/ext/opcache/jit/ir/dynasm/dasm_mips.h index 7800e9338773..e80cf8e758d8 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips.h +++ b/ext/opcache/jit/ir/dynasm/dasm_mips.h @@ -1,6 +1,6 @@ /* ** DynASM MIPS encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips.lua b/ext/opcache/jit/ir/dynasm/dasm_mips.lua index 1c605b681f15..4d7f5069e445 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_mips.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM MIPS32/MIPS64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips64.lua b/ext/opcache/jit/ir/dynasm/dasm_mips64.lua index c97d666b319a..c0db89650cef 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_mips64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM MIPS64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ -- This module just sets 64 bit mode for the combined MIPS/MIPS64 module. diff --git a/ext/opcache/jit/ir/dynasm/dasm_ppc.h b/ext/opcache/jit/ir/dynasm/dasm_ppc.h index 4c7d7289f9d7..85324e3c3166 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_ppc.h +++ b/ext/opcache/jit/ir/dynasm/dasm_ppc.h @@ -1,6 +1,6 @@ /* ** DynASM PPC/PPC64 encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_ppc.lua b/ext/opcache/jit/ir/dynasm/dasm_ppc.lua index d66ae4a0d616..88aa193395a0 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_ppc.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_ppc.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM PPC/PPC64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. -- -- Support for various extensions contributed by Caio Souza Oliveira. diff --git a/ext/opcache/jit/ir/dynasm/dasm_proto.h b/ext/opcache/jit/ir/dynasm/dasm_proto.h index 3f50f5028f3d..f7b379dfc335 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_proto.h +++ b/ext/opcache/jit/ir/dynasm/dasm_proto.h @@ -1,6 +1,6 @@ /* ** DynASM encoding engine prototypes. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_x64.lua b/ext/opcache/jit/ir/dynasm/dasm_x64.lua index 72d9bb87bff4..688e1a33180f 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_x64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM x64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ -- This module just sets 64 bit mode for the combined x86/x64 module. diff --git a/ext/opcache/jit/ir/dynasm/dasm_x86.h b/ext/opcache/jit/ir/dynasm/dasm_x86.h index d50299b13c1b..55d6f8386e63 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x86.h +++ b/ext/opcache/jit/ir/dynasm/dasm_x86.h @@ -1,6 +1,6 @@ /* ** DynASM x86 encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. +** Copyright (C) 2005-2026 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_x86.lua b/ext/opcache/jit/ir/dynasm/dasm_x86.lua index 0467bf913298..e339b3161639 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x86.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_x86.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM x86/x64 module. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ diff --git a/ext/opcache/jit/ir/dynasm/dynasm.lua b/ext/opcache/jit/ir/dynasm/dynasm.lua index 0d15a87282f2..dc5d99de0665 100644 --- a/ext/opcache/jit/ir/dynasm/dynasm.lua +++ b/ext/opcache/jit/ir/dynasm/dynasm.lua @@ -2,7 +2,7 @@ -- DynASM. A dynamic assembler for code generation engines. -- Originally designed and implemented for LuaJIT. -- --- Copyright (C) 2005-2023 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2026 Mike Pall. All rights reserved. -- See below for full copyright notice. ------------------------------------------------------------------------------ @@ -17,7 +17,7 @@ local _info = { url = "https://luajit.org/dynasm.html", license = "MIT", copyright = [[ -Copyright (C) 2005-2023 Mike Pall. All rights reserved. +Copyright (C) 2005-2026 Mike Pall. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 120722b2dcad..5f9b30980518 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -379,6 +379,7 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted #define ir_op_flag_xN (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_CALL | IR_OP_FLAG_VAR_INPUTS) #define ir_op_flag_a1 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT)) #define ir_op_flag_a2 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT)) +#define ir_op_flag_a2X1 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 2 | (3 << IR_OP_FLAG_OPERANDS_SHIFT)) #define ir_op_kind____ IR_OPND_UNUSED #define ir_op_kind_def IR_OPND_DATA @@ -391,8 +392,9 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted #define ir_op_kind_num IR_OPND_NUM #define ir_op_kind_fld IR_OPND_STR #define ir_op_kind_var IR_OPND_DATA -#define ir_op_kind_prb IR_OPND_PROB -#define ir_op_kind_opt IR_OPND_PROB +#define ir_op_kind_opt IR_OPND_OPT +#define ir_op_kind_prb IR_OPND_OPT +#define ir_op_kind_alg IR_OPND_OPT #define ir_op_kind_pro IR_OPND_PROTO #define ir_op_kind_lbl IR_OPND_LABEL_REF @@ -620,7 +622,7 @@ static IR_NEVER_INLINE void ir_const_hash_rehash(ir_ctx *ctx) ctx->const_hash = ir_mem_calloc(ctx->const_hash_mask + 1, sizeof(ir_ref)); for (ref = 1 - ctx->consts_count, insn = ctx->ir_base + ref; ref < IR_TRUE; ref++, insn++) { if (insn->op == IR_LONG_CONST) { - hash = insn->val.u64; + hash = insn->val.addr; insn->prev_const = ctx->const_hash[hash & ctx->const_hash_mask]; ctx->const_hash[hash & ctx->const_hash_mask] = ref; ref += IR_ALIGNED_SIZE(insn->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn); diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 1470606a1012..ee0dc259dee4 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -65,6 +65,8 @@ extern "C" { # define IR_TARGET_TRIPLET "x86-freebsd-sysv" # elif defined(__NetBSD__) # define IR_TARGET_TRIPLET "x86-netbsd-sysv" +# elif defined(__OpenBSD__) +# define IR_TARGET_TRIPLET "x86-openbsd-sysv" # else # define IR_TARGET_TRIPLET "x86-unknown-sysv" # endif @@ -79,6 +81,8 @@ extern "C" { # define IR_TARGET_TRIPLET "x86_64-freebsd-sysv" # elif defined(__NetBSD__) # define IR_TARGET_TRIPLET "x86_64-netbsd-sysv" +# elif defined(__OpenBSD__) +# define IR_TARGET_TRIPLET "x86_64-openbsd-sysv" # else # define IR_TARGET_TRIPLET "x86_64-unknown-sysv" # endif @@ -93,6 +97,8 @@ extern "C" { # define IR_TARGET_TRIPLET "aarch64-freebsd-sysv" # elif defined(__NetBSD__) # define IR_TARGET_TRIPLET "aarch64-netbsd-sysv" +# elif defined(__OpenBSD__) +# define IR_TARGET_TRIPLET "aarch64-openbsd-sysv" # else # define IR_TARGET_TRIPLET "aarch64-unknown-sysv" # endif @@ -256,9 +262,9 @@ typedef enum _ir_type { * E - control IR_OP_FLAG_CONTROL + IR_OP_FLAG_BB_END * T - control IR_OP_FLAG_CONTROL + IR_OP_FLAG_BB_END + IR_OP_FLAG_TERMINATOR * l - load IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_LOAD - * s - store IR_OP_FLAG_MEM + IR_OP_FLAG_STORE - * x - call IR_OP_FLAG_MEM + IR_OP_FLAG_CALL - * a - alloc IR_OP_FLAG_MEM + IR_OP_FLAG_ALLOC + * s - store IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_STORE + * x - call IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_CALL + * a - alloc IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_ALLOC * 0-3 - number of input edges * N - number of arguments is defined in the insn->inputs_count (MERGE, PHI, CALL) * X1-X3 - number of extra data ops @@ -279,6 +285,7 @@ typedef enum _ir_type { * num - number: argument number (PARAM) * prb - branch probability 1-99 (0 - unspecified): (IF_TRUE, IF_FALSE, CASE_VAL, CASE_DEFAULT) * opt - optional number + * alg - optional alignment * pro - function prototype * lbl - label used as value (a reference to constant): (BEGIN) * @@ -389,7 +396,7 @@ typedef enum _ir_type { \ /* data ops */ \ _(PARAM, p1X2, reg, str, num) /* incoming parameter proj. */ \ - _(VAR, p1X1, reg, str, ___) /* local variable */ \ + _(VAR, p1X2, reg, str, alg) /* local variable */ \ _(FUNC_ADDR, r0, ___, ___, ___) /* constant func ref */ \ _(FUNC, r0, ___, ___, ___) /* constant func ref */ \ _(SYM, r0, ___, ___, ___) /* constant symbol ref */ \ @@ -402,10 +409,10 @@ typedef enum _ir_type { _(TAILCALL, xN, src, def, def) /* CALL+RETURN */ \ \ /* memory reference and load/store ops */ \ - _(ALLOCA, a2, src, def, ___) /* alloca(def) */ \ - _(AFREE, a2, src, def, ___) /* revert alloca(def) */ \ + _(ALLOCA, a2X1, src, def, alg) /* alloca(def) */ \ + _(AFREE, s2, src, def, ___) /* revert alloca(def) */ \ _(BLOCK_BEGIN, a1, src, ___, ___) /* stacksave */ \ - _(BLOCK_END, a2, src, def, ___) /* stackrestore */ \ + _(BLOCK_END, s2, src, def, ___) /* stackrestore */ \ _(VLOAD, l2, src, var, ___) /* load value of local var */ \ _(VLOAD_v, l2, src, var, ___) /* volatile variant of VLOAD */ \ _(VSTORE, s3, src, var, def) /* store value to local var */ \ @@ -712,7 +719,6 @@ typedef struct _ir_code_buffer { typedef struct { int size; int align; - int offset; } ir_value_param; typedef struct _ir_bitqueue ir_bitqueue; @@ -747,8 +753,8 @@ struct _ir_ctx { uint32_t *cfg_map; /* map of instructions to basic block number */ uint32_t *cfg_schedule; /* BB order for code generation */ uint32_t *rules; /* array of target specific code-generation rules (for each instruction) */ - uint32_t *vregs; - ir_ref vregs_count; + int32_t *vregs; /* map of insns to virtual regs (positive) or stack slots (negative) */ + int32_t vregs_count; ir_str func_name; /* Function name (should be set through ir_string()/ir_stringl()) */ int32_t spill_base; /* base register for special spill area (e.g. PHP VM frame pointer) */ uint64_t fixed_regset; /* fixed registers, excluded for regular register allocation */ diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 539108dec9ac..43d918179118 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -390,6 +390,7 @@ const ir_call_conv_dsc ir_call_conv_aarch64_preserve_none = { _(OP_INT) \ _(OP_FP) \ _(BINOP_INT) \ + _(BIC) \ _(BINOP_FP) \ _(SHIFT) \ _(SHIFT_CONST) \ @@ -476,6 +477,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co constraints->hints_count = 0; switch (rule & IR_RULE_MASK) { case IR_BINOP_INT: + case IR_BIC: insn = &ctx->ir_base[ref]; n = 0; if (IR_IS_CONST_REF(insn->op1)) { @@ -1016,7 +1018,7 @@ static void ir_match_fuse_addr(ir_ctx *ctx, ir_ref addr_ref, ir_type type) && !IR_IS_CONST_REF(addr_insn->op1) && IR_IS_CONST_REF(addr_insn->op2) // TODO: temporary workaround && !IR_IS_SYM_CONST(ctx->ir_base[addr_insn->op2].op) - && aarch64_may_encode_addr_offset(ctx->ir_base[addr_insn->op2].val.i64, ir_type_size[type])) { + && aarch64_may_encode_addr_offset(ctx->ir_base[addr_insn->op2].val.i64, ir_get_type_size(type))) { ir_use_list *use_list = &ctx->use_lists[addr_ref]; ir_ref j = use_list->count; @@ -1043,6 +1045,18 @@ static uint32_t ir_match_builtin_call(ir_ctx *ctx, const ir_insn *func) return 0; } +static bool ir_have_calls_between(ir_ctx *ctx, ir_ref from, ir_ref to) +{ + ir_ref ref; + + for (ref = ctx->prev_ref[to]; ref > from; ref = ctx->prev_ref[ref]) { + if (ctx->ir_base[ref].op == IR_CALL) { + return 1; + } + } + return 0; +} + static bool all_usages_are_fusable(ir_ctx *ctx, ir_ref ref) { ir_insn *insn = &ctx->ir_base[ref]; @@ -1453,6 +1467,30 @@ binop_fp: return IR_COPY_INT | IR_MAY_REUSE; } } + if (IR_IS_TYPE_INT(insn->type) && insn->type != IR_BOOL) { + if (ctx->ir_base[insn->op1].op == IR_NOT + && !IR_IS_CONST_REF(ctx->ir_base[insn->op1].op1) + && ir_in_same_block(ctx, insn->op1) + && ctx->use_lists[insn->op1].count == 1 + && !ir_have_calls_between(ctx, insn->op1, ref) + && (!IR_IS_CONST_REF(insn->op2) || + !aarch64_may_encode_logical_imm(ctx->ir_base[insn->op2].val.u64, ir_type_size[insn->type]))) { + /* AND(NOT(X), Y) => BIC */ + ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_NOT; + return IR_BIC; + } else if (ctx->ir_base[insn->op2].op == IR_NOT + && !IR_IS_CONST_REF(ctx->ir_base[insn->op2].op1) + && ir_in_same_block(ctx, insn->op2) + && ctx->use_lists[insn->op2].count == 1 + && !ir_have_calls_between(ctx, insn->op2, ref) + && (!IR_IS_CONST_REF(insn->op1) || + !aarch64_may_encode_logical_imm(ctx->ir_base[insn->op1].val.u64, ir_type_size[insn->type]))) { + /* AND(X, NOT(Y)) => AND(NOT(Y), X) => BIC */ + SWAP_REFS(insn->op1, insn->op2); + ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_NOT; + return IR_BIC; + } + } goto binop_int; case IR_XOR: #if IR_SIMD @@ -1835,32 +1873,34 @@ static bool ir_is_same_spill_slot(ir_ctx *ctx, ir_ref ref, ir_mem mem) return IR_MEM_VAL(ir_ref_spill_slot(ctx, ref)) == IR_MEM_VAL(mem); } -static int32_t ir_var_spill_slot(ir_ctx *ctx, ir_ref ref, ir_reg *reg) +static int32_t ir_local_slot(ir_ctx *ctx, ir_ref ref) { - ir_insn *var_insn = &ctx->ir_base[ref]; - - IR_ASSERT(var_insn->op == IR_VAR); - *reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - return IR_SPILL_POS_TO_OFFSET(var_insn->op3); + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR || + ctx->ir_base[ref].op == IR_ALLOCA || + ctx->ir_base[ref].op == IR_VADDR || + ctx->ir_base[ref].op == IR_PARAM); + IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref])); + return IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]); } -static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use) +static int32_t ir_local_offset(ir_ctx *ctx, ir_ref ref) { - ir_live_interval *ival; + int32_t offset; - IR_ASSERT(ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); - ival = ctx->live_intervals[ctx->vregs[ref]]; - while (ival) { - ir_use_pos *use_pos = ival->use_pos; - while (use_pos) { - if (IR_LIVE_POS_TO_REF(use_pos->pos) == use) { - return !use_pos->next || use_pos->next->op_num == 0; - } - use_pos = use_pos->next; - } - ival = ival->next; - } - return 0; + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR || + ctx->ir_base[ref].op == IR_ALLOCA || + ctx->ir_base[ref].op == IR_VADDR || + ctx->ir_base[ref].op == IR_PARAM); + IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref])); + offset = IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]); + return IR_SPILL_POS_TO_OFFSET(offset); +} + +static int32_t ir_var_spill_slot(ir_ctx *ctx, ir_ref ref, ir_reg *reg) +{ + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR); + *reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + return ir_local_offset(ctx, ref); } static void ir_emit_load_imm_int(ir_ctx *ctx, ir_type type, ir_reg reg, int64_t val) @@ -1925,6 +1965,42 @@ static void ir_emit_load_imm_int(ir_ctx *ctx, ir_type type, ir_reg reg, int64_t } } +static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + int32_t offset; + + IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); + offset = ir_local_offset(ctx, src); + if (aarch64_may_encode_imm12(offset)) { + | add Rx(reg), Rx(base), #offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | add Rx(reg), Rx(base), Rx(IR_REG_INT_TMP) + } +} + +static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use) +{ + ir_live_interval *ival; + + IR_ASSERT(ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); + ival = ctx->live_intervals[ctx->vregs[ref]]; + while (ival) { + ir_use_pos *use_pos = ival->use_pos; + while (use_pos) { + if (IR_LIVE_POS_TO_REF(use_pos->pos) == use) { + return !use_pos->next || use_pos->next->op_num == 0; + } + use_pos = use_pos->next; + } + ival = ival->next; + } + return 0; +} + static void ir_emit_load_mem_int(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) { ir_backend_data *data = ctx->data; @@ -2162,34 +2238,6 @@ static void ir_emit_load_mem(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) } } -static int32_t ir_local_offset(ir_ctx *ctx, ir_insn *insn) -{ - IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA || insn->op == IR_VADDR); - return IR_SPILL_POS_TO_OFFSET(insn->op3); -} - -static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) -{ - ir_backend_data *data = ctx->data; - dasm_State **Dst = &data->dasm_state; - ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - ir_insn *var_insn; - int32_t offset; - - IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); - var_insn = &ctx->ir_base[src]; - if (var_insn->op == IR_VADDR) { - var_insn = &ctx->ir_base[var_insn->op1]; - } - offset = ir_local_offset(ctx, var_insn); - if (aarch64_may_encode_imm12(offset)) { - | add Rx(reg), Rx(base), #offset - } else { - ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); - | add Rx(reg), Rx(base), Rx(IR_REG_INT_TMP) - } -} - static void ir_resolve_label_syms(ir_ctx *ctx) { uint32_t b; @@ -2246,7 +2294,7 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) } else { ir_emit_load_imm_fp(ctx, type, reg, src); } - } else if (ctx->vregs[src]) { + } else if (ctx->vregs[src] > 0) { ir_mem mem = ir_ref_spill_slot(ctx, src); ir_emit_load_mem(ctx, type, reg, mem); } else { @@ -2959,6 +3007,52 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } +static void ir_emit_bic(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = insn->type; + ir_ref not_ref = insn->op1; + ir_ref op1 = ctx->ir_base[not_ref].op1; + ir_ref op2 = insn->op2; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op1_reg; + ir_reg op2_reg = ctx->regs[def][2]; + + IR_ASSERT(ctx->ir_base[not_ref].op == IR_NOT); + if (UNEXPECTED(ctx->rules[not_ref] & IR_FUSED_REG)) { + op1_reg = ir_get_fused_reg(ctx, def, not_ref * sizeof(ir_ref) + 1); + } else { + op1_reg = ctx->regs[not_ref][1]; + } + + IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); + + if (IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + ir_emit_load(ctx, type, op1_reg, op1); + } + + if (op1 == op2) { + op2_reg = IR_REG_NUM(op1_reg); + } else if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, type, op2_reg, op2); + } + } else { + IR_ASSERT(IR_IS_CONST_REF(insn->op2)); + op2_reg = IR_REG_INT_TMP; + ir_emit_load(ctx, type, op2_reg, op2); + } + + | ASM_REG_REG_REG_OP bic, type, def_reg, op2_reg, op1_reg + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, type, def, def_reg); + } +} + static void ir_emit_min_max_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) { ir_backend_data *data = ctx->data; @@ -4600,10 +4694,10 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) | fmov Rw(def_reg), Rs(op1_reg-IR_REG_FP_FIRST) } } else if (IR_IS_CONST_REF(insn->op1)) { - if (src_type == IR_DOUBLE) { + if (size == 8) { ir_emit_load_imm_int(ctx, dst_type, def_reg, ctx->ir_base[insn->op1].val.i64); } else { - IR_ASSERT(src_type == IR_FLOAT); + IR_ASSERT(size <= 4); ir_emit_load_imm_int(ctx, dst_type, def_reg, ctx->ir_base[insn->op1].val.i32); } } else { @@ -4642,10 +4736,10 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } else if (IR_IS_CONST_REF(insn->op1)) { ir_emit_load_imm_int(ctx, src_type, IR_REG_INT_TMP, ctx->ir_base[insn->op1].val.i64); - if (dst_type == IR_DOUBLE) { + if (size == 8) { | fmov Rd(def_reg-IR_REG_FP_FIRST), Rx(IR_REG_INT_TMP) } else { - IR_ASSERT(dst_type == IR_FLOAT); + IR_ASSERT(size <= 4); | fmov Rs(def_reg-IR_REG_FP_FIRST), Rw(IR_REG_INT_TMP) } } else { @@ -4903,22 +4997,22 @@ static void ir_emit_vaddr(ir_ctx *ctx, ir_ref def, ir_insn *insn) static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_ref type = insn->type; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); ir_reg fp; - int32_t offset; + int32_t slot, offset; ir_mem mem; if (ctx->use_lists[def].count == 1) { /* dead load */ return; } - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = IR_SPILL_POS_TO_OFFSET(var_insn->op3); + slot = ir_local_slot(ctx, insn->op2); + offset = IR_SPILL_POS_TO_OFFSET(slot); mem = IR_MEM_BO(fp, offset); - if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, var_insn->op3)) { + if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, slot)) { return; // fake load } IR_ASSERT(def_reg != IR_REG_NONE); @@ -4930,22 +5024,22 @@ static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn) static void ir_emit_vstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_insn *val_insn = &ctx->ir_base[insn->op3]; ir_ref type = val_insn->type; ir_reg op3_reg = ctx->regs[ref][3]; ir_reg fp; - int32_t offset; + int32_t slot, offset; ir_mem mem; - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = IR_SPILL_POS_TO_OFFSET(var_insn->op3); + slot = ir_local_slot(ctx, insn->op2); + offset = IR_SPILL_POS_TO_OFFSET(slot); IR_ASSERT(op3_reg != IR_REG_NONE); if (IR_REG_SPILLED(op3_reg) && !IR_IS_CONST_REF(insn->op3) && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA - && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) { + && ir_is_same_mem_var(ctx, insn->op3, slot)) { return; // fake store } if (IR_REG_SPILLED(op3_reg)) { @@ -4967,7 +5061,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[addr_insn->op2].op)); if (ir_rule(ctx, addr_insn->op1) == IR_STATIC_ALLOCA) { reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[addr_insn->op1]); + offset = ir_local_offset(ctx, addr_insn->op1); offset += ctx->ir_base[addr_insn->op2].val.i32; return IR_MEM_BO(reg, offset); } else { @@ -4985,7 +5079,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) } else { IR_ASSERT(addr_insn->op == IR_ALLOCA || addr_insn->op == IR_VADDR); reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[ref]); + offset = ir_local_offset(ctx, ref); return IR_MEM_BO(reg, offset); } } @@ -5157,7 +5251,7 @@ static void ir_emit_rload(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_type type = insn->type; if (IR_REGSET_IN(IR_REGSET_UNION((ir_regset)ctx->fixed_regset, IR_REGSET_FIXED), src_reg)) { - if (ctx->vregs[def] + if (ctx->vregs[def] > 0 && ctx->live_intervals[ctx->vregs[def]] && ctx->live_intervals[ctx->vregs[def]]->stack_spill_pos != -1) { ir_emit_store(ctx, type, def, src_reg); @@ -5384,7 +5478,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -5394,8 +5488,18 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) fp = IR_REG_STACK_POINTER; arg_area_offset = ctx->call_stack_size + ctx->stack_frame_size + ctx->param_stack_size; } - | add Rx(tmp_reg), Rx(fp), #arg_area_offset - | str Rx(tmp_reg), [Rx(op2_reg), #offset] + if (aarch64_may_encode_imm12(arg_area_offset)) { + | add Rx(tmp_reg), Rx(fp), #arg_area_offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, arg_area_offset); + | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } + if (aarch64_may_encode_addr_offset(offset, sizeof(void*))) { + | str Rx(tmp_reg), [Rx(op2_reg), #offset] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } else { ir_reg fp; int reg_save_area_offset; @@ -5414,7 +5518,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -5428,35 +5532,85 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } /* Set va_list.stack */ - | add Rx(tmp_reg), Rx(fp), #overflow_arg_area_offset - | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, stack))] + if (aarch64_may_encode_imm12(overflow_arg_area_offset)) { + | add Rx(tmp_reg), Rx(fp), #overflow_arg_area_offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, overflow_arg_area_offset); + | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, stack), sizeof(void*))) { + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, stack))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, stack)); + | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } if ((ctx->flags2 & (IR_HAS_VA_ARG_GP|IR_HAS_VA_COPY)) && ctx->gp_reg_params < cc->int_param_regs_count) { reg_save_area_offset += sizeof(void*) * cc->int_param_regs_count; /* Set va_list.gr_top */ if (overflow_arg_area_offset != reg_save_area_offset) { - | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset + if (aarch64_may_encode_imm12(reg_save_area_offset)) { + | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, reg_save_area_offset); + | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } + } + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_top), sizeof(void*))) { + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_top))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_top)); + | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] } - | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_top))] /* Set va_list.gr_offset */ | movn Rw(tmp_reg), #~(0 - (sizeof(void*) * (cc->int_param_regs_count - ctx->gp_reg_params))) - | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset), sizeof(void*))) { + | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_offset)); + | str Rw(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } else { /* Set va_list.gr_offset */ - | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset), sizeof(void*))) { + | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_offset)); + | str wzr, [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } if ((ctx->flags2 & (IR_HAS_VA_ARG_FP|IR_HAS_VA_COPY)) && ctx->fp_reg_params < cc->fp_param_regs_count) { reg_save_area_offset += 16 * cc->fp_param_regs_count; /* Set va_list.vr_top */ if (overflow_arg_area_offset != reg_save_area_offset || ctx->gp_reg_params < cc->int_param_regs_count) { - | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset + if (aarch64_may_encode_imm12(reg_save_area_offset)) { + | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, reg_save_area_offset); + | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } + } + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_top), sizeof(void*))) { + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_top))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_top)); + | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] } - | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_top))] /* Set va_list.vr_offset */ | movn Rw(tmp_reg), #~(0 - (16 * (cc->fp_param_regs_count - ctx->fp_reg_params))) - | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))] + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset), sizeof(void*))) { + | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_offset)); + | str Rw(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } else { /* Set va_list.vr_offset */ - | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))] + if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset), sizeof(void*))) { + | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_offset)); + | str wzr, [Rx(op2_reg), Rx(IR_REG_INT_TMP)] + } } } } @@ -5483,7 +5637,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + op2_offset = ir_local_offset(ctx, insn->op2); } if (op3_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op3_reg)) { @@ -5494,7 +5648,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]); + op3_offset = ir_local_offset(ctx, insn->op3); } | ldr Rx(tmp_reg), [Rx(op3_reg), #op3_offset] | str Rx(tmp_reg), [Rx(op2_reg), #op2_offset] @@ -5514,7 +5668,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + op2_offset = ir_local_offset(ctx, insn->op2); } if (op3_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op3_reg)) { @@ -5525,7 +5679,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]); + op3_offset = ir_local_offset(ctx, insn->op3); } | ldr Rx(tmp_reg), [Rx(op3_reg), #op3_offset] | str Rx(tmp_reg), [Rx(op2_reg), #op2_offset] @@ -5561,7 +5715,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } | ldr Rx(tmp_reg), [Rx(op2_reg), #offset] if (def_reg != IR_REG_NONE) { @@ -5589,7 +5743,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (IR_IS_TYPE_INT(type)) { | ldr Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))] @@ -5690,10 +5844,12 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(val2->op)); if (IR_IS_TYPE_SIGNED(type)) { IR_ASSERT(IR_IS_TYPE_SIGNED(val->type)); + IR_ASSERT(IR_IS_TYPE_SIGNED(val2->type)); min.i64 = IR_MIN(min.i64, val->val.i64); max.i64 = IR_MAX(max.i64, val2->val.i64); } else { IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type)); + IR_ASSERT(!IR_IS_TYPE_SIGNED(val2->type)); min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64); max.u64 = (int64_t)IR_MAX(max.u64, val2->val.u64); } @@ -6219,7 +6375,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_cal if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } - } else if (ctx->use_lists[def].count > 1) { + } else if (ctx->vregs[def] > 0) { ir_emit_store(ctx, insn->type, def, cc->int_ret_reg); } } else { @@ -6232,7 +6388,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_cal if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } - } else if (ctx->use_lists[def].count > 1) { + } else if (ctx->vregs[def] > 0) { ir_emit_store(ctx, insn->type, def, cc->fp_ret_reg); } } @@ -8835,7 +8991,7 @@ static void ir_emit_load_params(ir_ctx *ctx) } fp_param_num++; } - if (ctx->vregs[use]) { + if (ctx->vregs[use] > 0) { dst_reg = IR_REG_NUM(ctx->regs[use][0]); IR_ASSERT(src_reg != IR_REG_NONE || dst_reg != IR_REG_NONE || stack_start + stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos); @@ -8921,7 +9077,7 @@ static void ir_fix_param_spills(ir_ctx *ctx) fp_param_num++; } if (src_reg == IR_REG_NONE) { - if (ctx->vregs[use]) { + if (ctx->vregs[use] > 0) { ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) && ival->stack_spill_pos == -1 @@ -9187,6 +9343,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_BINOP_INT: ir_emit_binop_int(ctx, i, insn); break; + case IR_BIC: + ir_emit_bic(ctx, i, insn); + break; case IR_BINOP_FP: ir_emit_binop_fp(ctx, i, insn); break; diff --git a/ext/opcache/jit/ir/ir_disasm.c b/ext/opcache/jit/ir/ir_disasm.c index 0fe8aaea6e93..30cfb694cff6 100644 --- a/ext/opcache/jit/ir/ir_disasm.c +++ b/ext/opcache/jit/ir/ir_disasm.c @@ -599,7 +599,7 @@ int ir_disasm(const char *name, } if (p != q && *(q-1) == '-') { q--; - addr = (uint32_t)(-(int64_t)addr); + addr = (uint32_t)(0 - addr); } if (addr >= (uint64_t)(uintptr_t)start && addr < (uint64_t)(uintptr_t)orig_end) { entry = ir_hashtab_find(&labels, (uint32_t)((uintptr_t)addr - (uintptr_t)start)); diff --git a/ext/opcache/jit/ir/ir_dump.c b/ext/opcache/jit/ir/ir_dump.c index 8d15e12f842d..ac751328411f 100644 --- a/ext/opcache/jit/ir/ir_dump.c +++ b/ext/opcache/jit/ir/ir_dump.c @@ -400,13 +400,13 @@ void ir_dump_live_ranges(const ir_ctx *ctx, FILE *f) fprintf(f, "TMP"); } else { for (j = 1; j < ctx->insns_count; j++) { - if (ctx->vregs[j] == (uint32_t)i) { + if (ctx->vregs[j] == i) { break; } } fprintf(f, "R%d (d_%d", i, j); for (j++; j < ctx->insns_count; j++) { - if (ctx->vregs[j] == (uint32_t)i) { + if (ctx->vregs[j] == i) { fprintf(f, ", d_%d", j); } } @@ -667,7 +667,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) fprintf(f, "\t"); ir_print_type_cname(insn->type, f); fprintf(f, " d_%d", i); - if (ctx->vregs && ctx->vregs[i]) { + if (ctx->vregs && ctx->vregs[i] > 0) { fprintf(f, " {R%d}", ctx->vregs[i]); } if (ctx->regs) { @@ -683,7 +683,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) if (flags & IR_OP_FLAG_DATA) { ir_print_type_cname(insn->type, f); fprintf(f, " d_%d", i); - if (ctx->vregs && ctx->vregs[i]) { + if (ctx->vregs && ctx->vregs[i] > 0) { fprintf(f, " {R%d}", ctx->vregs[i]); } if (ctx->regs) { @@ -719,7 +719,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) } else { fprintf(f, "%sd_%d", first ? "(" : ", ", ref); } - if (ctx->vregs && ref > 0 && ctx->vregs[ref]) { + if (ctx->vregs && ref > 0 && ctx->vregs[ref] > 0) { fprintf(f, " {R%d}", ctx->vregs[ref]); } if (ctx->regs) { @@ -746,7 +746,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) fprintf(f, "%sfunc ", first ? "(" : ", "); ir_print_proto(ctx, ref, f); break; - case IR_OPND_PROB: + case IR_OPND_OPT: if (ref == 0) { break; } diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index 8c01f56c9e30..71555ff5da5a 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -250,7 +250,7 @@ static bool ir_is_same_mem_var(const ir_ctx *ctx, ir_ref r1, int32_t offset) return 0; } - IR_ASSERT(ctx->vregs[r1]); + IR_ASSERT(ctx->vregs[r1] > 0); ival1 = ctx->live_intervals[ctx->vregs[r1]]; IR_ASSERT(ival1); o1 = ival1->stack_spill_pos; @@ -439,7 +439,7 @@ static IR_NEVER_INLINE void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block for (i = 0; i < count; i++, pos++) { ref = ir_list_at(list, pos); - IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); + IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); if (!(ctx->live_intervals[ctx->vregs[ref]]->flags & IR_LIVE_INTERVAL_SPILLED)) { /* not spilled */ ir_reg reg = ctx->live_intervals[ctx->vregs[ref]]->reg; @@ -1393,7 +1393,7 @@ int32_t ir_get_spill_slot_offset(const ir_ctx *ctx, ir_ref ref) { int32_t offset; - IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); + IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); offset = ctx->live_intervals[ctx->vregs[ref]]->stack_spill_pos; IR_ASSERT(offset != -1); return IR_SPILL_POS_TO_OFFSET(offset); @@ -1528,7 +1528,7 @@ static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data reg = IR_REGSET_FIRST(available); if (IR_REGSET_IN(x->preserved_regs, reg)) { - ir_regset tmp = ctx->used_preserved_regs; + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; IR_REGSET_INCL(tmp, reg); ctx->used_preserved_regs = tmp; } @@ -1554,16 +1554,34 @@ static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir ir_ref *p, use, i, n = use_list->count; ir_ref last_use = IR_UNUSED; ir_insn *insn; + int address_taken = -1; /* unknown */ if (n) { + /* Check if all uses of the result of VLOAD are in the same basic block (after VLOAD) */ for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { use = *p; if (use < load || use > bb->end) return 0; if (use > last_use) last_use = use; } + /* Check if the result of VLOAD can not be clobbered between VLOAD and the last use */ for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) { if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) { return 0; + } else if (address_taken + && (insn->op == IR_STORE + || insn->op == IR_STORE_v + || insn->op == IR_CALL)) { + if (address_taken == -1) { + ir_use_list *use_list = &ctx->use_lists[var]; + ir_ref *p, n = use_list->count; + + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + if (ctx->ir_base[*p].op == IR_VADDR) { + return 0; + } + } + address_taken = 0; + } } n = ir_insn_len(insn); i += n; @@ -1577,13 +1595,33 @@ static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, i { ir_ref i, n; ir_insn *insn; + int address_taken = -1; /* unknown */ - if (val < bb->start && val > store) return 0; + /* Check if the value stored by VSTORE is produced in the same basic block (before VSTORE) */ + if (val < bb->start || val > store || store > bb->end) return 0; + /* Check if the instructions between the "producer" and VSTORE can not access the same VAR */ for (i = val, insn = &ctx->ir_base[i]; i < store;) { if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) { return 0; + } else if (address_taken + && (insn->op == IR_LOAD + || insn->op == IR_LOAD_v + || insn->op == IR_STORE + || insn->op == IR_STORE_v + || insn->op == IR_CALL)) { + if (address_taken == -1) { + ir_use_list *use_list = &ctx->use_lists[var]; + ir_ref *p, n = use_list->count; + + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + if (ctx->ir_base[*p].op == IR_VADDR) { + return 0; + } + } + address_taken = 0; + } } n = ir_insn_len(insn); i += n; @@ -1650,7 +1688,7 @@ static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_all IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); child = *p; if (child > 0) { - if (ctx->vregs[child]) { + if (ctx->vregs[child] > 0) { if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) { ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF; @@ -1673,6 +1711,23 @@ static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_all } } +static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn) +{ + size_t size = ir_get_type_size(var_insn->type); + size_t align = var_insn->op3; + int32_t offset; + +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) { + size = 4; + align = IR_MAX(align, 4); + } +#endif + + offset = ir_allocate_spill_slot(ctx, size, align); + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); +} + int ir_reg_alloc_simple(ir_ctx *ctx) { ir_reg_alloc_data data; @@ -1686,6 +1741,7 @@ int ir_reg_alloc_simple(ir_ctx *ctx) ir_reg reg; ir_regset scratch; ir_reg_alloc_simple_data x; + int32_t offset; memset(&data, 0, sizeof(data)); data.cc = ir_get_call_conv_dsc(ctx->flags); @@ -1714,9 +1770,11 @@ int ir_reg_alloc_simple(ir_ctx *ctx) for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) { if (*rule & (IR_FUSED|IR_SKIPPED)) { if ((*rule & IR_RULE_MASK) == IR_ALLOCA) { - if (insn->op == IR_VAR) { + if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) { + /* spill slot already allocated */ + } else if (insn->op == IR_VAR) { if (ctx->use_lists[i].count > 0) { - insn->op3 = ir_allocate_spill_slot(ctx, insn->type); + ir_allocate_var_spill_slot(ctx, i, insn); } } else if (insn->op == IR_ALLOCA) { if (ctx->use_lists[i].count > 0) { @@ -1727,10 +1785,14 @@ int ir_reg_alloc_simple(ir_ctx *ctx) IR_ASSERT(!IR_IS_SYM_CONST(val->op)); IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); IR_ASSERT(val->val.i64 < 0x7fffffff); - insn->op3 = ir_allocate_big_spill_slot(ctx, val->val.i32); + offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); + ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset); } } else if (insn->op == IR_VADDR) { - insn->op3 = ctx->ir_base[insn->op1].op3; + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) { + ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]); + } + ctx->vregs[i] = ctx->vregs[insn->op1]; } } } else { @@ -1769,45 +1831,56 @@ int ir_reg_alloc_simple(ir_ctx *ctx) } } - if (ctx->vregs[i]) { + if (ctx->vregs[i] > 0) { reg = constraints.def_reg; if (!ctx->live_intervals[ctx->vregs[i]]) { - ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); - memset(ival, 0, sizeof(ir_live_interval)); - ctx->live_intervals[ctx->vregs[i]] = ival; - ival->type = insn->type; - ival->reg = IR_REG_NONE; - ival->vreg = ctx->vregs[i]; - ival->stack_spill_pos = -1; - if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) - && ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) { - ival->stack_spill_pos = ctx->ir_base[insn->op2].op3; - reg = IR_REG_NONE; - def_flags &= ~IR_USE_MUST_BE_IN_REG; - } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) { - ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - } else if (ctx->use_lists[i].count == 1) { - ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; - ir_insn *use_insn = &ctx->ir_base[use]; - - if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) - && use_insn->op3 == i - && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) { - if (use_insn->op2 < i) { - ival->stack_spill_pos = ctx->ir_base[use_insn->op2].op3; - } else { - ival->stack_spill_pos = ctx->ir_base[use_insn->op2].op3 = - ir_allocate_spill_slot(ctx, ival->type); + do { + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); + ctx->live_intervals[ctx->vregs[i]] = ival; + ival->type = insn->type; + ival->reg = IR_REG_NONE; + ival->vreg = ctx->vregs[i]; + ival->stack_spill_pos = -1; + + if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { + ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); } - } else { - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type); + if (ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) { + ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]); + reg = IR_REG_NONE; + def_flags &= ~IR_USE_MUST_BE_IN_REG; + break; + } + } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) { + ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; + break; } - } else { - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type); - } - } else if (insn->op == IR_PARAM) { - IR_ASSERT(0 && "unexpected PARAM"); - return 0; + + if (ctx->use_lists[i].count == 1) { + ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; + ir_insn *use_insn = &ctx->ir_base[use]; + + if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) + && use_insn->op3 == i + && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[use_insn->op2])) { + ir_allocate_var_spill_slot(ctx, use_insn->op2, &ctx->ir_base[use_insn->op2]); + } + ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[use_insn->op2]); + break; + } + } + + size_t size = ir_get_type_size(ival->type); +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { + size = 4; + } +#endif + ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); + } while (0); } if (def_flags & IR_USE_MUST_BE_IN_REG) { @@ -1834,6 +1907,10 @@ int ir_reg_alloc_simple(ir_ctx *ctx) _add_reg(&x, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE, IR_UNUSED, i, 0); } + } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { + ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); + } } n = insn->inputs_count; @@ -1850,18 +1927,21 @@ int ir_reg_alloc_simple(ir_ctx *ctx) ir_live_pos use_pos; uint32_t use_flags = IR_USE_FLAGS(def_flags, j); + if (reg != IR_REG_NONE) { +#if IR_X86_I64 + if (use_flags & IR_HINT_TWO_REGS) { + IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); + IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); + } else +#endif + IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], reg); + } + if (input > 0) { - if (ctx->vregs[input]) { + if (ctx->vregs[input] > 0) { use_pos = IR_USE_SUB_REF; if (reg != IR_REG_NONE) { use_pos = IR_LOAD_SUB_REF; -#if IR_X86_I64 - if (use_flags & IR_HINT_TWO_REGS) { - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); - } else -#endif - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], reg); } else if (def_flags & IR_DEF_REUSES_OP1_REG) { if (j == 1) { if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { @@ -1960,10 +2040,10 @@ int ir_reg_alloc_simple(ir_ctx *ctx) #ifdef IR_TARGET_X86 if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, IR_DOUBLE); + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) && data.cc->fp_ret_reg == IR_REG_NONE) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ctx->ret_type); + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); } else { ctx->ret_slot = -1; } diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index abab986e0a30..85ed6c74b7e5 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -890,7 +890,7 @@ IR_FOLD(ABS(C_I32)) if (op1_insn->val.i64 >= 0) { IR_FOLD_COPY(op1); } else { - IR_FOLD_CONST_I((int32_t)-op1_insn->val.u32); + IR_FOLD_CONST_I((int32_t)(0 - op1_insn->val.u32)); } } @@ -900,7 +900,7 @@ IR_FOLD(ABS(C_I64)) if (op1_insn->val.i64 >= 0) { IR_FOLD_COPY(op1); } else { - IR_FOLD_CONST_I(-op1_insn->val.u64); + IR_FOLD_CONST_I(0 - op1_insn->val.u64); } } @@ -2675,7 +2675,7 @@ IR_FOLD(DIV(NEG, C_I32)) IR_FOLD(DIV(NEG, C_I64)) { op1 = op1_insn->op1; - val.i64 = -(uint64_t)op2_insn->val.i64; + val.u64 = (0 - op2_insn->val.u64); op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; } @@ -3462,6 +3462,66 @@ IR_FOLD(AND(SHR, C_U64)) IR_FOLD_NEXT; } +IR_FOLD(AND(SAR, C_I8)) +IR_FOLD(AND(SAR, C_U8)) +{ + if (IR_IS_CONST_REF(op1_insn->op2)) { + if (((uint8_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u8 & 0x7) == op2_insn->val.u8) { + /* (x >>s N) & (~0 >> N) => x >>u N */ + opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt)); + op1 = op1_insn->op1; + op2 = op1_insn->op2; + IR_FOLD_RESTART; + } + } + IR_FOLD_NEXT; +} + +IR_FOLD(AND(SAR, C_I16)) +IR_FOLD(AND(SAR, C_U16)) +{ + if (IR_IS_CONST_REF(op1_insn->op2)) { + if (((uint16_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u16 & 0xf) == op2_insn->val.u16) { + /* (x >>s N) & (~0 >> N) => x >>u N */ + opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt)); + op1 = op1_insn->op1; + op2 = op1_insn->op2; + IR_FOLD_RESTART; + } + } + IR_FOLD_NEXT; +} + +IR_FOLD(AND(SAR, C_I32)) +IR_FOLD(AND(SAR, C_U32)) +{ + if (IR_IS_CONST_REF(op1_insn->op2)) { + if (((uint32_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u32 & 0x1f) == op2_insn->val.u32) { + /* (x >>s N) & (~0 >> N) => x >>u N */ + opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt)); + op1 = op1_insn->op1; + op2 = op1_insn->op2; + IR_FOLD_RESTART; + } + } + IR_FOLD_NEXT; +} + +IR_FOLD(AND(SAR, C_I64)) +IR_FOLD(AND(SAR, C_U64)) +{ + if (IR_IS_CONST_REF(op1_insn->op2)) { + if (((uint64_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u64 & 0x3f) == op2_insn->val.u64) { + /* (x >>s N) & (~0 >> N) => x >>u N */ + opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt)); + op1 = op1_insn->op1; + op2 = op1_insn->op2; + IR_FOLD_RESTART; + } + } + IR_FOLD_NEXT; +} + IR_FOLD(AND(SHL, C_I8)) IR_FOLD(AND(SHL, C_U8)) { @@ -3974,7 +4034,7 @@ IR_FOLD(MIN(MIN, _)) IR_FOLD(MAX(MAX, _)) { if (op1_insn->op1 == op2 || op1_insn->op2 == op2) { - IR_FOLD_COPY(op2); + IR_FOLD_COPY(op1); } IR_FOLD_NEXT; } @@ -4029,6 +4089,51 @@ IR_FOLD(ADD(SHR, SHL)) } +/* ROL/ROR by 1: SHL(x, 1) is folded to ADD(x, x) and then to MUL(x, 2), + * so the SHL/SHR rule above doesn't see it. */ +IR_FOLD(OR(MUL, SHR)) +IR_FOLD(ADD(MUL, SHR)) +{ + const ir_insn *mul_insn = op1_insn; + const ir_insn *shr_insn = op2_insn; + + if (IR_IS_TYPE_INT(IR_OPT_TYPE(opt)) + && mul_insn->op1 == shr_insn->op1 + && IR_IS_CONST_REF(mul_insn->op2) + && IR_IS_CONST_REF(shr_insn->op2) + && ctx->ir_base[mul_insn->op2].val.i64 == 2 + && ctx->ir_base[shr_insn->op2].val.i64 == ir_type_size[IR_OPT_TYPE(opt)] * 8 - 1) { + /* (x * 2) | (x >> (N - 1)) -> ROR(x, N - 1) == ROL(x, 1) */ + op1 = shr_insn->op1; + op2 = shr_insn->op2; + opt = shr_insn->opt + 3; /* SHR -> ROR */ + IR_FOLD_RESTART; + } + IR_FOLD_NEXT; +} + +IR_FOLD(OR(SHR, MUL)) +IR_FOLD(ADD(SHR, MUL)) +{ + const ir_insn *mul_insn = op2_insn; + const ir_insn *shr_insn = op1_insn; + + if (IR_IS_TYPE_INT(IR_OPT_TYPE(opt)) + && mul_insn->op1 == shr_insn->op1 + && IR_IS_CONST_REF(mul_insn->op2) + && IR_IS_CONST_REF(shr_insn->op2) + && ctx->ir_base[mul_insn->op2].val.i64 == 2 + && ctx->ir_base[shr_insn->op2].val.i64 == ir_type_size[IR_OPT_TYPE(opt)] * 8 - 1) { + /* (x * 2) | (x >> (N - 1)) -> ROR(x, N - 1) == ROL(x, 1) */ + op1 = shr_insn->op1; + op2 = shr_insn->op2; + opt = shr_insn->opt + 3; /* SHR -> ROR */ + IR_FOLD_RESTART; + } + IR_FOLD_NEXT; +} + + /* Swap operands (move lower ref to op2) for better CSE */ IR_FOLD(MUL(_, _)) IR_FOLD(ORDERED(_, _)) diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c index 60422f12f6be..7a0755b74774 100644 --- a/ext/opcache/jit/ir/ir_gcm.c +++ b/ext/opcache/jit/ir/ir_gcm.c @@ -322,9 +322,13 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) ir_hashtab_init(&hash, use_list->count); n = use_list->count; for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { + ir_ref src; + use = *p; + src = ctx->cfg_map[use]; insn = &ctx->ir_base[use]; - if (insn->op == IR_PHI) { + /* dead PHIs (src == NULL) are not scheduled, so they go together with other unscheduled uses */ + if (src && insn->op == IR_PHI) { ir_ref *p = insn->ops + 2; /* PHI data inputs */ ir_ref *q = ctx->ir_base[insn->op1].ops + 1; /* MERGE inputs */ ir_ref n = insn->inputs_count - 1; @@ -359,9 +363,9 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) } } } else { - j = i = ctx->cfg_map[use]; - if (i) { - IR_ASSERT(i > 0); + j = src; + if (j) { + IR_ASSERT(j > 0); while (ir_sparse_set_in(&data->totally_useful, ctx->cfg_blocks[j].idom)) { j = ctx->cfg_blocks[j].idom; } @@ -371,14 +375,14 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) clone = clones_count++; ir_hashtab_add(&hash, j, clone); clones[clone].block = j; - clones[clone].lca = i; + clones[clone].lca = src; clones[clone].use_count = 0; clones[clone].use = -1; } else { - clones[clone].lca = ir_gcm_find_lca(ctx, clones[clone].lca, i); + clones[clone].lca = ir_gcm_find_lca(ctx, clones[clone].lca, src); } uses[uses_count].ref = use; - uses[uses_count].block = i; + uses[uses_count].block = src; uses[uses_count].next = clones[clone].use; clones[clone].use_count++; clones[clone].use = uses_count++; @@ -478,7 +482,7 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) ir_insn *insn = &ctx->ir_base[use]; ir_ref k, l = insn->inputs_count; - if (insn->op == IR_PHI) { + if (src && insn->op == IR_PHI) { ir_insn *merge = &ctx->ir_base[insn->op1]; for (k = 2; k <= l; k++) { j = ctx->cfg_map[ir_insn_op(merge, k - 1)]; @@ -1359,11 +1363,11 @@ int ir_schedule(ir_ctx *ctx) *dst = *src; _xlat[i] = j; if (dst->op == IR_LONG_CONST) { - uintptr_t n; + ir_ref n; memset(dst + 1, 0, dst->long_const_size); memcpy(dst + 1, src + 1, dst->long_const_size); - n = IR_ALIGNED_SIZE(dst->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn); + n = (ir_ref)IR_ALIGNED_SIZE(dst->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn); dst += n + 1; src += n + 1; i += n + 1; diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index b4409f97df7c..950b6535f577 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -975,12 +975,15 @@ IR_ALWAYS_INLINE bool ir_ref_is_true(const ir_ctx *ctx, ir_ref ref) #define IR_OP_FLAG_TERMINATOR (1<<14) #define IR_OP_FLAG_PINNED (1<<15) -#define IR_OP_FLAG_MEM_LOAD ((0<<6)|(0<<7)) -#define IR_OP_FLAG_MEM_STORE ((0<<6)|(1<<7)) -#define IR_OP_FLAG_MEM_CALL ((1<<6)|(0<<7)) -#define IR_OP_FLAG_MEM_ALLOC ((1<<6)|(1<<7)) +#define IR_OP_FLAG_MEM_CALL ((0<<6)|(0<<7)) +#define IR_OP_FLAG_MEM_LOAD ((0<<6)|(1<<7)) /* no side effect (may be removed if not used) */ +#define IR_OP_FLAG_MEM_STORE ((1<<6)|(0<<7)) +#define IR_OP_FLAG_MEM_ALLOC ((1<<6)|(1<<7)) /* no side effect (may be removed if not used) */ #define IR_OP_FLAG_MEM_MASK ((1<<6)|(1<<7)) +#define IR_OP_FLAG_NO_SIDE_EFFECT ((0<<6)|(1<<7)) + + #define IR_OPND_UNUSED 0x0U #define IR_OPND_DATA 0x1U #define IR_OPND_CONTROL 0x2U @@ -990,7 +993,7 @@ IR_ALWAYS_INLINE bool ir_ref_is_true(const ir_ctx *ctx, ir_ref ref) #define IR_OPND_CONTROL_GUARD 0x6U #define IR_OPND_STR 0x7U #define IR_OPND_NUM 0x8U -#define IR_OPND_PROB 0x9U +#define IR_OPND_OPT 0x9U /* optional number (flags, branch propability, align, size+align) */ #define IR_OPND_PROTO 0xaU #define IR_OP_FLAGS(op_flags, op1_flags, op2_flags, op3_flags) \ @@ -1324,6 +1327,11 @@ struct _ir_live_range { #define IR_LIVE_INTERVAL_SPLIT_CHILD (1<<8) #define IR_LIVE_INTERVAL_TWO_REGS (1<<9) +/* macros to encode and decode stack offset stored in ctx->vregs[] */ +#define IR_VREG_IS_STACK_SLOT(vreg) ((vreg) < 0) +#define IR_VREG_TO_STACK_SLOT(vreg) ((vreg) + 0x40000000) +#define IR_STACK_SLOT_TO_VREG(offset) ((int32_t)0xc0000000 + (offset)) + struct _ir_live_interval { uint8_t type; int8_t reg; @@ -1478,8 +1486,7 @@ typedef struct _ir_reg_alloc_data { ir_live_interval **handled; } ir_reg_alloc_data; -int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type); -int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size); +int32_t ir_allocate_spill_slot(ir_ctx *ctx, size_t size, size_t align); void ir_dump_reg(const ir_ctx *ctx, int8_t reg, ir_ref ref, bool store, FILE *f); IR_ALWAYS_INLINE void ir_set_alocated_reg(ir_ctx *ctx, ir_ref ref, int op_num, int8_t reg) diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index fe052b2a097b..6cfc779e6c3c 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -40,8 +40,8 @@ bool ir_reg_is_int(int32_t reg) static int ir_assign_virtual_registers_slow(ir_ctx *ctx) { - uint32_t *vregs; - uint32_t vregs_count = 0; + int32_t *vregs; + int32_t vregs_count = 0; uint32_t b; ir_ref i, n; ir_block *bb; @@ -81,8 +81,8 @@ static int ir_assign_virtual_registers_slow(ir_ctx *ctx) int ir_assign_virtual_registers(ir_ctx *ctx) { - uint32_t *vregs; - uint32_t vregs_count = 0; + int32_t *vregs; + int32_t vregs_count = 0; ir_ref i; ir_insn *insn; @@ -94,7 +94,7 @@ int ir_assign_virtual_registers(ir_ctx *ctx) vregs = ir_mem_malloc(ctx->insns_count * sizeof(ir_ref)); for (i = 1, insn = &ctx->ir_base[1]; i < ctx->insns_count; i++, insn++) { - uint32_t v = 0; + int32_t v = 0; if (ctx->rules[i] && !(ctx->rules[i] & (IR_FUSED|IR_SKIPPED|IR_NO_REG))) { uint32_t flags = ir_op_flags[insn->op]; @@ -548,9 +548,9 @@ static void ir_add_fusion_ranges(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_block IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); child = *p; if (child > 0) { - uint32_t v = ctx->vregs[child]; + int32_t v = ctx->vregs[child]; - if (v) { + if (v > 0) { use_flags = IR_FUSED_USE | IR_USE_FLAGS(def_flags, j); reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); @@ -694,9 +694,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (insn->op == IR_PHI) { ir_ref input = ir_insn_op(insn, k); if (input > 0) { - uint32_t v = ctx->vregs[input]; + int32_t v = ctx->vregs[input]; - if (v) { + if (v > 0) { /* live.add(phi.inputOf(b)) */ ir_bitset_incl(live, v); /* intervals[phi.inputOf(b)].addRange(b.from, b.to) */ @@ -722,7 +722,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) uint32_t flags; ir_ref *p; ir_target_constraints constraints; - uint32_t v; + int32_t v; if (ctx->rules) { int n; @@ -730,7 +730,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) #if IR_X86_I64 if (ctx->rules[ref] & IR_TWO_REGS) { v = ctx->vregs[ref]; - if (v) { + if (v > 0) { IR_ASSERT(ctx->live_intervals[v]); ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_TWO_REGS; } @@ -742,7 +742,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) && ctx->use_lists[ref].count > 0) { insn = &ctx->ir_base[ref]; if (insn->op == IR_VAR || insn->op == IR_ALLOCA) { - insn->op3 = ctx->vars; + ctx->vregs[ref] = IR_STACK_SLOT_TO_VREG(ctx->vars); ctx->vars = ref; } } @@ -770,7 +770,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) insn = &ctx->ir_base[ref]; v = ctx->vregs[ref]; - if (v) { + if (v > 0) { if (insn->op != IR_PHI) { ir_live_pos def_pos; ir_ref hint_ref = 0; @@ -789,7 +789,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_add_fixed_live_range(ctx, reg, IR_START_LIVE_POS_FROM_REF(bb->start), def_pos); } } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1]) { + if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1] > 0) { hint_ref = insn->op1; } def_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); @@ -799,6 +799,17 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (insn->op == IR_PARAM) { /* We may reuse parameter stack slot for spilling */ ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; +#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) + if (param_ival->use_pos) { + ir_use_pos *use_pos = param_ival->use_pos; + ir_ref use = IR_LIVE_POS_TO_REF(use_pos->pos); + ir_insn *use_insn = &ctx->ir_base[use]; + if (use_insn->op == IR_VSTORE) { + /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */ + param_ival->use_pos = use_pos->next; + } + } +#endif } def_pos = IR_DEF_LIVE_POS_FROM_REF(ref); } @@ -831,11 +842,11 @@ int ir_compute_live_ranges(ir_ctx *ctx) p = insn->ops + 2; for (; j <= insn->inputs_count; j++, p++) { ir_ref input = *p; - uint32_t v; + int32_t v; if (input > 0) { v = ctx->vregs[input]; - IR_ASSERT(v); + IR_ASSERT(v > 0); use_pos = IR_USE_LIVE_POS_FROM_REF(next); if (!ir_bitset_in(live, v)) { /* live.add(opd) */ @@ -865,12 +876,12 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; ir_live_pos use_pos; ir_ref hint_ref = 0; - uint32_t v; + int32_t v; uint32_t use_flags = IR_USE_FLAGS(def_flags, j); if (input > 0) { v = ctx->vregs[input]; - if (v) { + if (v > 0) { use_pos = IR_USE_LIVE_POS_FROM_REF(ref); if (reg != IR_REG_NONE) { use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); @@ -895,7 +906,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) } else if (def_flags & IR_DEF_REUSES_OP1_REG) { if (j == 1) { use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); - IR_ASSERT(ctx->vregs[ref]); + IR_ASSERT(ctx->vregs[ref] > 0); hint_ref = ref; } else if (input == insn->op1) { /* Input is the same as "op1" */ @@ -1022,7 +1033,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) } while (0) /* Returns the last virtual register alive at the end of the block (it is used as an already-visited marker) */ -IR_ALWAYS_INLINE uint32_t ir_live_out_top(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists, uint32_t b) +IR_ALWAYS_INLINE int32_t ir_live_out_top(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists, uint32_t b) { #if 0 return live_outs[b]; @@ -1035,7 +1046,7 @@ IR_ALWAYS_INLINE uint32_t ir_live_out_top(ir_ctx *ctx, uint32_t *live_outs, ir_l } /* Remember a virtual register alive at the end of the block */ -IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists, uint32_t b, uint32_t v) +IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists, uint32_t b, int32_t v) { #if 0 ir_block *bb = &ctx->cfg_blocks[b]; @@ -1061,7 +1072,7 @@ IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, uint32_t *live_outs, ir_list * "Computing Liveness Sets for SSA-Form Programs", Florian Brandner, Benoit Boissinot. * Alain Darte, Benoit Dupont de Dinechin, Fabrice Rastello. TR Inria RR-7503, 2011 */ -static void ir_compute_live_sets(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists) +static void ir_compute_live_sets(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists) { ir_list block_queue, fuse_queue; ir_ref i; @@ -1071,9 +1082,9 @@ static void ir_compute_live_sets(ir_ctx *ctx, uint32_t *live_outs, ir_list *live /* For each virtual register explore paths from all uses to definition */ for (i = ctx->insns_count - 1; i > 0; i--) { - uint32_t v = ctx->vregs[i]; + int32_t v = ctx->vregs[i]; - if (v) { + if (v > 0) { uint32_t def_block = ctx->cfg_map[i]; ir_use_list *use_list = &ctx->use_lists[i]; ir_ref *p, n = use_list->count; @@ -1285,9 +1296,9 @@ static void ir_add_fusion_ranges(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_block IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); child = *p; if (child > 0) { - uint32_t v = ctx->vregs[child]; + int32_t v = ctx->vregs[child]; - if (v) { + if (v > 0) { reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; use_pos = pos; if (EXPECTED(reg == IR_REG_NONE)) { @@ -1330,7 +1341,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_ref ref; ir_insn *insn; ir_block *bb, *succ_bb; - uint32_t *live_outs; + int32_t *live_outs; uint32_t *live_in_block; ir_list live_lists; ir_live_interval *ival; @@ -1395,9 +1406,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (insn->op == IR_PHI) { ir_ref input = ir_insn_op(insn, k); if (input > 0) { - uint32_t v = ctx->vregs[input]; + int32_t v = ctx->vregs[input]; - if (v) { + if (v > 0) { ival = ctx->live_intervals[v]; ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use); } @@ -1418,7 +1429,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) uint32_t flags; ir_ref *p; ir_target_constraints constraints; - uint32_t v; + int32_t v; if (ctx->rules) { int n; @@ -1426,7 +1437,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) #if IR_X86_I64 if (ctx->rules[ref] & IR_TWO_REGS) { v = ctx->vregs[ref]; - if (v) { + if (v > 0) { IR_ASSERT(ctx->live_intervals[v]); ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_TWO_REGS; } @@ -1438,7 +1449,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) && ctx->use_lists[ref].count > 0) { insn = &ctx->ir_base[ref]; if (insn->op == IR_VAR || insn->op == IR_ALLOCA) { - insn->op3 = ctx->vars; + ctx->vregs[ref] = IR_STACK_SLOT_TO_VREG(ctx->vars); ctx->vars = ref; } } @@ -1466,7 +1477,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) insn = &ctx->ir_base[ref]; v = ctx->vregs[ref]; - if (v) { + if (v > 0) { if (insn->op != IR_PHI) { ir_live_pos def_pos; ir_ref hint_ref = 0; @@ -1485,7 +1496,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_add_fixed_live_range(ctx, reg, IR_START_LIVE_POS_FROM_REF(bb->start), def_pos); } } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1]) { + if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1] > 0) { hint_ref = insn->op1; } if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { @@ -1498,7 +1509,20 @@ int ir_compute_live_ranges(ir_ctx *ctx) } else { if (insn->op == IR_PARAM) { /* We may reuse parameter stack slot for spilling */ - ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; + ir_live_interval *param_ival = ctx->live_intervals[v]; + + param_ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; +#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) + if (param_ival->use_pos) { + ir_use_pos *use_pos = param_ival->use_pos; + ir_ref use = IR_LIVE_POS_TO_REF(use_pos->pos); + ir_insn *use_insn = &ctx->ir_base[use]; + if (use_insn->op == IR_VSTORE) { + /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */ + param_ival->use_pos = use_pos->next; + } + } +#endif } def_pos = IR_DEF_LIVE_POS_FROM_REF(ref); } @@ -1527,11 +1551,11 @@ int ir_compute_live_ranges(ir_ctx *ctx) p = insn->ops + 2; for (; j <= insn->inputs_count; j++, p++) { ir_ref input = *p; - uint32_t v; + int32_t v; if (input > 0) { v = ctx->vregs[input]; - IR_ASSERT(v); + IR_ASSERT(v > 0); use_pos = IR_USE_LIVE_POS_FROM_REF(next); if (!IS_LIVE_IN_BLOCK(v, b)) { /* live.add(opd) */ @@ -1561,12 +1585,12 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; ir_live_pos use_pos; ir_ref hint_ref = 0; - uint32_t v; + int32_t v; uint32_t use_flags = IR_USE_FLAGS(def_flags, j); if (input > 0) { v = ctx->vregs[input]; - if (v) { + if (v > 0) { use_pos = IR_USE_LIVE_POS_FROM_REF(ref); if (reg != IR_REG_NONE) { use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); @@ -1595,7 +1619,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) } else { use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); } - IR_ASSERT(ctx->vregs[ref]); + IR_ASSERT(ctx->vregs[ref] > 0); hint_ref = ref; } else if (input == insn->op1) { /* Input is the same as "op1" */ @@ -1688,7 +1712,7 @@ static ir_live_pos ir_ivals_overlap(ir_live_range *lrg1, ir_live_range *lrg2) } } -static ir_live_pos ir_vregs_overlap(ir_ctx *ctx, uint32_t r1, uint32_t r2) +static ir_live_pos ir_vregs_overlap(ir_ctx *ctx, int32_t r1, int32_t r2) { ir_live_interval *ival1 = ctx->live_intervals[r1]; ir_live_interval *ival2 = ctx->live_intervals[r2]; @@ -1716,7 +1740,7 @@ static bool ir_ivals_inside(ir_live_range *parent, ir_live_range *child) return 1; } -static bool ir_vregs_inside(ir_ctx *ctx, uint32_t parent, uint32_t child) +static bool ir_vregs_inside(ir_ctx *ctx, int32_t parent, int32_t child) { ir_live_interval *child_ival = ctx->live_intervals[child]; ir_live_interval *parent_ival = ctx->live_intervals[parent]; @@ -1733,7 +1757,7 @@ static bool ir_vregs_inside(ir_ctx *ctx, uint32_t parent, uint32_t child) return ir_ivals_inside(&parent_ival->range, &child_ival->range); } -static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2) +static void ir_vregs_join(ir_ctx *ctx, int32_t r1, int32_t r2) { ir_live_interval *ival = ctx->live_intervals[r2]; ir_live_range *live_range = &ival->range; @@ -1795,7 +1819,7 @@ static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2) //ir_mem_free(ival); } -static void ir_vregs_coalesce(ir_ctx *ctx, uint32_t v1, uint32_t v2, ir_ref from, ir_ref to) +static void ir_vregs_coalesce(ir_ctx *ctx, int32_t v1, int32_t v2, ir_ref from, ir_ref to) { ir_ref i; uint16_t f1 = ctx->live_intervals[v1]->flags; @@ -1914,7 +1938,7 @@ static void ir_swap_operands(ir_ctx *ctx, ir_ref i, ir_insn *insn) p = p->next; } - if (insn->op2 > 0 && ctx->vregs[insn->op2]) { + if (insn->op2 > 0 && ctx->vregs[insn->op2] > 0) { ival = ctx->live_intervals[ctx->vregs[insn->op2]]; r = &ival->range; while (r) { @@ -1977,13 +2001,13 @@ static int ir_hint_conflict(ir_ctx *ctx, ir_ref ref, int use, int def) static int ir_try_swap_operands(ir_ctx *ctx, ir_ref i, ir_insn *insn) { - if (ctx->vregs[insn->op1] + if (ctx->vregs[insn->op1] > 0 && ctx->vregs[insn->op1] != ctx->vregs[i] && !ir_vregs_overlap(ctx, ctx->vregs[insn->op1], ctx->vregs[i]) && !ir_hint_conflict(ctx, i, ctx->vregs[insn->op1], ctx->vregs[i])) { /* pass */ } else { - if (ctx->vregs[insn->op2] && ctx->vregs[insn->op2] != ctx->vregs[i]) { + if (ctx->vregs[insn->op2] > 0 && ctx->vregs[insn->op2] != ctx->vregs[i]) { ir_live_pos pos = IR_USE_LIVE_POS_FROM_REF(i); ir_live_pos load_pos = IR_LOAD_LIVE_POS_FROM_REF(i); ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op2]]; @@ -2081,7 +2105,7 @@ int ir_coalesce(ir_ctx *ctx) insn = &ctx->ir_base[use]; if (insn->op == IR_PHI) { input = ir_insn_op(insn, k); - if (input > 0 && ctx->vregs[input]) { + if (input > 0 && ctx->vregs[input] > 0) { uint32_t v1 = ctx->vregs[input]; uint32_t v2 = ctx->vregs[use]; @@ -2151,7 +2175,7 @@ int ir_coalesce(ir_ctx *ctx) for (i = 1; i < ctx->insns_count; rule++, i++) { if ((*rule) & (IR_MAY_SWAP|IR_MAY_REUSE)) { insn = &ctx->ir_base[i]; - IR_ASSERT(ctx->vregs[i]); + IR_ASSERT(ctx->vregs[i] > 0); if ((*rule) & IR_MAY_SWAP) { IR_ASSERT(ir_op_flags[insn->op] & IR_OP_FLAG_COMMUTATIVE); if (ctx->live_intervals[ctx->vregs[i]]->use_pos @@ -2164,7 +2188,7 @@ int ir_coalesce(ir_ctx *ctx) } else { IR_ASSERT((*rule) & IR_MAY_REUSE); if (insn->op1 > 0 - && ctx->vregs[insn->op1] + && ctx->vregs[insn->op1] > 0 && ctx->vregs[i] != ctx->vregs[insn->op1]) { if (ir_vregs_inside(ctx, ctx->vregs[insn->op1], ctx->vregs[i])) { if (ctx->binding) { @@ -2208,7 +2232,7 @@ int ir_coalesce(ir_ctx *ctx) } } for (j = 1; j < ctx->insns_count; j++) { - if (ctx->vregs[j]) { + if (ctx->vregs[j] > 0) { ctx->vregs[j] = xlat[ctx->vregs[j]]; } } @@ -2325,7 +2349,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy, void *dat insn = &ctx->ir_base[ref]; if (insn->op == IR_PHI) { input = ir_insn_op(insn, k); - if (IR_IS_CONST_REF(input) || !ctx->vregs[input]) { + if (IR_IS_CONST_REF(input) || !(ctx->vregs[input] > 0)) { have_constants_or_addresses = 1; } else if (ctx->vregs[input] != ctx->vregs[ref]) { s = ctx->vregs[input]; @@ -2394,7 +2418,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy, void *dat insn = &ctx->ir_base[ref]; if (insn->op == IR_PHI) { input = ir_insn_op(insn, k); - if (IR_IS_CONST_REF(input) || !ctx->vregs[input]) { + if (IR_IS_CONST_REF(input) || !(ctx->vregs[input] > 0)) { emit_copy(ctx, insn->type, input, ref, data); } } @@ -2712,7 +2736,7 @@ static int32_t ir_allocate_small_spill_slot(ir_ctx *ctx, size_t size) } IR_ASSERT(size == 1 || size == 2 || size == 4 || size == 8); - n = ir_ntz(size); + n = IR_LOG2(size); if (data->handled && data->handled[n]) { ret = data->handled[n]->stack_spill_pos; data->handled[n] = data->handled[n]->list_next; @@ -2808,21 +2832,13 @@ static int32_t ir_allocate_small_spill_slot(ir_ctx *ctx, size_t size) return ret; } -int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size) +static int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, size_t size, size_t align) { int32_t ret; - if (size <= 8) { - if (size == 3) { - size = 4; - } else if (size > 4 && size < 8) { - size = 8; - } - return ir_allocate_small_spill_slot(ctx, size); - } - + IR_ASSERT(size < 0xfffffff); if (size <= 64 && (size & (size - 1)) == 0) { - uint32_t n = ir_ntz(size); + uint32_t n = IR_LOG2(size); ir_reg_alloc_data *data = ctx->data; if (data->handled && data->handled[n]) { @@ -2833,25 +2849,27 @@ int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size) } /* Align stack allocated data to 16 byte */ + // TODO: support for alignment larger than 16 ??? ctx->flags2 |= IR_16B_FRAME_ALIGNMENT; ret = IR_ALIGNED_SIZE(ctx->stack_frame_size, 16); size = IR_ALIGNED_SIZE(size, 8); - ctx->stack_frame_size = ret + size; + ctx->stack_frame_size = ret + (int32_t)size; return ret; } -int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type) +int32_t ir_allocate_spill_slot(ir_ctx *ctx, size_t size, size_t align) { - if (IR_IS_TYPE_SCALAR(type)) { - return ir_allocate_small_spill_slot(ctx, ir_type_size[type]); + IR_ASSERT(align == 0 || (align & (align -1)) == 0); /* align must be a power of 2 */ + if (size <= 8 && align < 16) { + if (align > size) { + size = align; + } else if ((size & (size - 1)) != 0) { + size = (size == 3) ? 4 : 8; + } + return ir_allocate_small_spill_slot(ctx, size); } else { - int32_t size; - - IR_ASSERT(IR_IS_TYPE_VECTOR(type)); - size = IR_VECTOR_SIZE(type); - size = IR_MAX(size, 4); - return ir_allocate_big_spill_slot(ctx, size); + return ir_allocate_big_spill_slot(ctx, size, align); } } @@ -3917,12 +3935,12 @@ static void ir_assign_bound_spill_slots(ir_ctx *ctx) { ir_hashtab_bucket *b = ctx->binding->data; uint32_t n = ctx->binding->count; - uint32_t v; + int32_t v; ir_live_interval *ival; while (n > 0) { v = ctx->vregs[b->key]; - if (v) { + if (v > 0) { ival = ctx->live_intervals[v]; if (ival && ival->stack_spill_pos == -1 @@ -3984,25 +4002,35 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars) ir_insn *insn = &ctx->ir_base[var]; IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA); - vars = insn->op3; /* list next */ + vars = IR_VREG_TO_STACK_SLOT(ctx->vregs[var]); /* list next */ if (insn->op == IR_VAR) { - ir_ref slot = ir_allocate_spill_slot(ctx, insn->type); + ir_ref slot; ir_use_list *use_list; ir_ref n, *p; + size_t size = ir_get_type_size(insn->type); + size_t align = insn->op3; - insn->op3 = slot; +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(insn->type) && size < 4) { + size = 4; + align = IR_MAX(align, 4); + } +#endif + slot = ir_allocate_spill_slot(ctx, size, align); + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(slot); use_list = &ctx->use_lists[var]; n = use_list->count; p = &ctx->use_edges[use_list->refs]; for (; n > 0; p++, n--) { insn = &ctx->ir_base[*p]; if (insn->op == IR_VADDR) { - insn->op3 = slot; + ctx->vregs[*p] = IR_STACK_SLOT_TO_VREG(slot); } } } else { ir_insn *val = &ctx->ir_base[insn->op2]; + int32_t offset; IR_ASSERT(IR_IS_CONST_REF(insn->op2)); IR_ASSERT(IR_IS_TYPE_INT(val->type)); @@ -4010,7 +4038,8 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars) IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); IR_ASSERT(val->val.i64 < 0x7fffffff); - insn->op3 = ir_allocate_big_spill_slot(ctx, val->val.i32); + offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); } } @@ -4228,7 +4257,13 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars) other = prev ? prev->list_next : active; } - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type); + size_t size = ir_get_type_size(ival->type); +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { + size = 4; + } +#endif + ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); if (unhandled && ival->end > unhandled->range.start) { ival->list_next = active; active = ival; @@ -4256,10 +4291,10 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars) #ifdef IR_TARGET_X86 if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, IR_DOUBLE); + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) && ((ir_reg_alloc_data*)(ctx->data))->cc->fp_ret_reg == IR_REG_NONE) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ctx->ret_type); + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); } else { ctx->ret_slot = -1; } @@ -4430,7 +4465,7 @@ static void assign_regs(ir_ctx *ctx) || ctx->ir_base[ref].op == IR_BITCAST || ctx->ir_base[ref].op == IR_TRUNC) && !IR_IS_CONST_REF(ctx->ir_base[ref].op1) - && ctx->vregs[ctx->ir_base[ref].op1] == (uint32_t)i) { + && ctx->vregs[ctx->ir_base[ref].op1] == i) { /* register reuse */ ir_set_alocated_reg(ctx, ref, use_pos->op_num, reg); prev_use_ref = ref; @@ -4446,13 +4481,6 @@ static void assign_regs(ir_ctx *ctx) && (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM)) { /* Stack PARAM var is passed through memory */ reg = IR_REG_NONE; -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) - if (use_pos->next - && ctx->ir_base[IR_LIVE_POS_TO_REF(use_pos->next->pos)].op == IR_VSTORE) { - /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */ - use_pos = use_pos->next; - } -#endif } else { uint32_t use_b = ctx->cfg_map[ref]; @@ -4509,7 +4537,7 @@ static void assign_regs(ir_ctx *ctx) if (use_pos->hint_ref < 0) { if (use_pos->flags & IR_PHI_USE) { IR_ASSERT(use_pos->hint_ref < 0); - IR_ASSERT(ctx->vregs[-use_pos->hint_ref]); + IR_ASSERT(ctx->vregs[-use_pos->hint_ref] > 0); IR_ASSERT(ctx->live_intervals[ctx->vregs[-use_pos->hint_ref]]); if (ctx->live_intervals[ctx->vregs[-use_pos->hint_ref]]->flags & IR_LIVE_INTERVAL_SPILLED) { /* Spilled PHI var is passed through memory */ @@ -4571,10 +4599,12 @@ static void assign_regs(ir_ctx *ctx) if (ival->tmp_op_num <= insn->inputs_count) { ir_ref *ops = insn->ops; + if (IR_IS_CONST_REF(ops[ival->tmp_op_num])) { /* constant rematerialization */ reg |= IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[ival->tmp_op_num]] == (IR_SKIPPED|IR_FUSED|IR_SIMPLE|IR_ALLOCA)) { + } else if (ctx->rules[ops[ival->tmp_op_num]] == (IR_SKIPPED|IR_FUSED|IR_SIMPLE|IR_ALLOCA) + && IR_OPND_KIND(ir_op_flags[insn->op], ival->tmp_op_num) == IR_OPND_DATA) { /* local address rematerialization */ reg |= IR_REG_SPILL_LOAD; } diff --git a/ext/opcache/jit/ir/ir_save.c b/ext/opcache/jit/ir/ir_save.c index 1286f5503856..f3b885d3bff4 100644 --- a/ext/opcache/jit/ir/ir_save.c +++ b/ext/opcache/jit/ir/ir_save.c @@ -299,7 +299,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) ir_print_type_cname(insn->type, f); fprintf(f, " d_%d", i); if (save_flags & IR_SAVE_REGS) { - if (ctx->vregs && ctx->vregs[i]) { + if (ctx->vregs && ctx->vregs[i] > 0) { fprintf(f, " {R%d}", ctx->vregs[i]); } if (ctx->regs) { @@ -317,7 +317,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) ir_print_type_cname(insn->type, f); fprintf(f, " d_%d", i); if (save_flags & IR_SAVE_REGS) { - if (ctx->vregs && ctx->vregs[i]) { + if (ctx->vregs && ctx->vregs[i] > 0) { fprintf(f, " {R%d}", ctx->vregs[i]); } if (ctx->regs) { @@ -355,7 +355,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) fprintf(f, "%sd_%d", first ? "(" : ", ", ref); } if (save_flags & IR_SAVE_REGS) { - if (ctx->vregs && ref > 0 && ctx->vregs[ref]) { + if (ctx->vregs && ref > 0 && ctx->vregs[ref] > 0) { fprintf(f, " {R%d}", ctx->vregs[ref]); } if (ctx->regs) { @@ -383,7 +383,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) fprintf(f, "%sfunc ", first ? "(" : ", "); ir_print_proto(ctx, ref, f); break; - case IR_OPND_PROB: + case IR_OPND_OPT: if (ref == 0) { break; } diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 4745c7000d72..c14d34b6c4d4 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -520,25 +520,13 @@ static bool ir_sccp_analyze_phi(const ir_ctx *ctx, ir_sccp_val *_values, ir_bitq static bool ir_is_dead_load_ex(const ir_ctx *ctx, ir_ref ref, uint32_t flags, const ir_insn *insn) { - if ((flags & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD)) { - return ctx->use_lists[ref].count == 1; - } else if (insn->op == IR_ALLOCA || insn->op == IR_BLOCK_BEGIN) { - return ctx->use_lists[ref].count == 1; - } - return 0; + return (flags & IR_OP_FLAG_NO_SIDE_EFFECT) && ctx->use_lists[ref].count == 1; } static bool ir_is_dead_load(const ir_ctx *ctx, ir_ref ref) { if (ctx->use_lists[ref].count == 1) { - const ir_insn *insn = &ctx->ir_base[ref]; - uint32_t flags = ir_op_flags[insn->op]; - - if ((flags & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD)) { - return 1; - } else if (insn->op == IR_ALLOCA || insn->op == IR_BLOCK_BEGIN) { - return 1; - } + return (ir_op_flags[ctx->ir_base[ref].op] & IR_OP_FLAG_NO_SIDE_EFFECT) != 0; } return 0; } @@ -3752,7 +3740,7 @@ static ir_ref ir_iter_optimize_condition(ir_ctx *ctx, ir_ref control, ir_ref con && ctx->use_lists[condition].count == 1 && ctx->use_lists[condition_insn->op1].count == 1 && (op1_insn->op == IR_ZEXT || op1_insn->op == IR_SEXT) - && val_insn->val.u64 <= (((uint64_t)-1ULL) >> (64 - ir_type_size[ctx->ir_base[op1_insn->op1].type] * 8))) { + && val_insn->val.u64 <= (((uint64_t)(0-1ULL)) >> (64 - ir_type_size[ctx->ir_base[op1_insn->op1].type] * 8))) { /* IF(AND(ZEXT(X), C)) => IF(AND(X, C)) */ if (op1_insn->op1 > 0) { ir_use_list_replace_one(ctx, op1_insn->op1, condition_insn->op1, condition); diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 5ce20343bd00..3f1a3be180f8 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -2031,7 +2031,6 @@ op2_const: n = 1; } break; - case IR_LOAD_FP: case IR_LOAD_INT: case IR_MEM_OP_INT: case IR_MEM_INC: @@ -2049,6 +2048,19 @@ op2_const: n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); } break; + case IR_LOAD_FP: + flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; + insn = &ctx->ir_base[ref]; + if (IR_IS_CONST_REF(insn->op2)) { + n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); + } +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(insn->type) && IR_VECTOR_SIZE(insn->type) < 4) { + constraints->tmp_regs[n] = IR_TMP_REG(3, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; + } +#endif + break; case IR_STORE_FP: flags = IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; insn = &ctx->ir_base[ref]; @@ -2060,6 +2072,13 @@ op2_const: constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; } +#if IR_SIMD + else if (IR_IS_TYPE_VECTOR(ctx->ir_base[insn->op3].type) + && IR_VECTOR_SIZE(ctx->ir_base[insn->op3].type) < 4) { + constraints->tmp_regs[n] = IR_TMP_REG(1, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; + } +#endif break; case IR_SWITCH: flags = IR_OP2_MUST_BE_IN_REG; @@ -2757,7 +2776,7 @@ get_arg_hints: && IR_VECTOR_SIZE(insn->type) == 32) { constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n++; - if (ctx->vregs[insn->op1]) { + if (ctx->vregs[insn->op1] > 0) { flags &= ~IR_DEF_CONFLICTS_WITH_INPUT_REGS; flags |= IR_DEF_REUSES_OP1_REG; constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); @@ -2907,7 +2926,7 @@ get_arg_hints: if (!(ctx->mflags & IR_X86_AVX2) && IR_VECTOR_SIZE(insn->type) == 32) { constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n++; - if (ctx->vregs[insn->op2] || insn->op == IR_NE) { + if (ctx->vregs[insn->op2] > 0 || insn->op == IR_NE) { constraints->tmp_regs[n] = IR_TMP_REG(4, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n++; } @@ -3288,6 +3307,11 @@ static void ir_match_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root) if (ir_in_same_block(ctx, ref) && (ctx->ir_base[ref].op == IR_LOAD || ctx->ir_base[ref].op == IR_LOAD_v || ctx->ir_base[ref].op == IR_VLOAD || ctx->ir_base[ref].op == IR_VLOAD_v)) { +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ctx->ir_base[ref].type) && IR_VECTOR_SIZE(ctx->ir_base[ref].type) < 2) { + return; + } +#endif if (ctx->use_lists[ref].count == 2 && !ir_match_has_mem_deps(ctx, ref, root)) { ir_ref addr_ref = ctx->ir_base[ref].op2; @@ -3314,6 +3338,11 @@ static bool ir_match_try_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root) if (ir_in_same_block(ctx, ref) && (insn->op == IR_LOAD || insn->op == IR_LOAD_v || insn->op == IR_VLOAD || insn->op == IR_VLOAD_v)) { +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ctx->ir_base[ref].type) && IR_VECTOR_SIZE(ctx->ir_base[ref].type) < 2) { + return 0; + } +#endif if (ctx->use_lists[ref].count == 2 && !ir_match_has_mem_deps(ctx, ref, root)) { ir_ref addr_ref = ctx->ir_base[ref].op2; @@ -5373,7 +5402,7 @@ static int32_t ir_ref_spill_slot_offset(ir_ctx *ctx, ir_ref ref, ir_reg *reg) { int32_t offset; - IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); + IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); offset = ctx->live_intervals[ctx->vregs[ref]]->stack_spill_pos; IR_ASSERT(offset != -1); if (ctx->live_intervals[ctx->vregs[ref]]->flags & IR_LIVE_INTERVAL_SPILL_SPECIAL) { @@ -5414,21 +5443,59 @@ static bool ir_is_same_spill_slot(ir_ctx *ctx, ir_ref ref, ir_mem mem) return IR_MEM_VAL(m) == IR_MEM_VAL(mem); } +static int32_t ir_local_slot(ir_ctx *ctx, ir_ref ref) +{ + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR || + ctx->ir_base[ref].op == IR_ALLOCA || + ctx->ir_base[ref].op == IR_VADDR || + ctx->ir_base[ref].op == IR_PARAM); + IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref])); + return IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]); +} + +static int32_t ir_local_offset(ir_ctx *ctx, ir_ref ref) +{ + int32_t offset; + + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR || + ctx->ir_base[ref].op == IR_ALLOCA || + ctx->ir_base[ref].op == IR_VADDR || + ctx->ir_base[ref].op == IR_PARAM); + IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref])); + offset = IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]); + return IR_SPILL_POS_TO_OFFSET(offset); +} + static ir_mem ir_var_spill_slot(ir_ctx *ctx, ir_ref ref) { - ir_insn *var_insn = &ctx->ir_base[ref]; ir_reg reg; - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[ref].op == IR_VAR); reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - return IR_MEM_BO(reg, IR_SPILL_POS_TO_OFFSET(var_insn->op3)); + return IR_MEM_BO(reg, ir_local_offset(ctx, ref)); +} + +static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + int32_t offset; + + IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); + offset = ir_local_offset(ctx, src); + if (offset == 0) { + | mov Ra(reg), Ra(base) + } else { + | lea Ra(reg), [Ra(base)+offset] + } } static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use) { ir_live_interval *ival; - IR_ASSERT(ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]); + IR_ASSERT(ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]); ival = ctx->live_intervals[ctx->vregs[ref]]; while (ival) { ir_use_pos *use_pos = ival->use_pos; @@ -5633,6 +5700,49 @@ static void ir_emit_load_imm_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref sr } } +#if IR_SIMD +/* Vectos with size less than 4 have to be loaded throug general purpose register */ +static void ir_emit_load_short_vector(ir_ctx *ctx, uint32_t width, ir_reg reg, ir_mem mem, ir_reg tmp_reg) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + bool do_pop = 0; + + IR_ASSERT(width == 1 || width == 2); + if (tmp_reg == IR_REG_NONE) { + IR_ASSERT(0 && "this is not expected"); + if (IR_MEM_BASE(mem) != IR_REG_RAX && IR_MEM_INDEX(mem) != IR_REG_RAX) { + tmp_reg = IR_REG_RAX; + } else if (IR_MEM_BASE(mem) != IR_REG_RCX && IR_MEM_INDEX(mem) != IR_REG_RCX) { + tmp_reg = IR_REG_RCX; + } else if (IR_MEM_BASE(mem) != IR_REG_RDX && IR_MEM_INDEX(mem) != IR_REG_RDX) { + tmp_reg = IR_REG_RDX; + } + if (IR_MEM_BASE(mem) == IR_REG_RSP) { + mem = IR_MEM_ADD(mem, sizeof(void*)); + } + + | push Ra(tmp_reg) + do_pop = 1; + } + + if (width == 1) { + | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), byte, mem + } else { + | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), word, mem + } + if (ctx->mflags & IR_X86_AVX) { + | vmovd xmm(reg-IR_REG_FP_FIRST), Rd(tmp_reg) + } else { + | movd xmm(reg-IR_REG_FP_FIRST), Rd(tmp_reg) + } + + if (do_pop) { + | pop Ra(tmp_reg) + } +} +#endif + static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) { ir_backend_data *data = ctx->data; @@ -5650,13 +5760,13 @@ static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem me | ASM_TXT_TMEM_OP vmovdqa, xmm(reg-IR_REG_FP_FIRST), oword, mem } else if (width == 8) { | ASM_TXT_TMEM_OP vmovq, xmm(reg-IR_REG_FP_FIRST), qword, mem - } else if (width == 4) { + } else if (width == 4 || + (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) || + ((ctx->flags & IR_USE_FRAME_POINTER) && + IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) { | ASM_TXT_TMEM_OP vmovd, xmm(reg-IR_REG_FP_FIRST), dword, mem } else { - // TODO: 4-byte read may be unsafe ??? - // IR_ASSERT(width >= 4 && width <= 16); - IR_ASSERT(width <= 4); - | ASM_TXT_TMEM_OP vmovd, xmm(reg-IR_REG_FP_FIRST), dword, mem + ir_emit_load_short_vector(ctx, width, reg, mem, IR_REG_NONE); } } else if (element_type == IR_DOUBLE) { if (width == 16) { @@ -5699,13 +5809,13 @@ static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem me |.else | ASM_TXT_TMEM_OP movlpd, xmm(reg-IR_REG_FP_FIRST), qword, mem |.endif - } else if (width == 4) { + } else if (width == 4 || + (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) || + ((ctx->flags & IR_USE_FRAME_POINTER) && + IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) { | ASM_TXT_TMEM_OP movd, xmm(reg-IR_REG_FP_FIRST), dword, mem } else { - // TODO: 4-byte read may be unsafe ??? - // IR_ASSERT(width >= 4 && width <= 16); - IR_ASSERT(width <= 4); - | ASM_TXT_TMEM_OP movd, xmm(reg-IR_REG_FP_FIRST), dword, mem + ir_emit_load_short_vector(ctx, width, reg, mem, IR_REG_NONE); } } else if (element_type == IR_DOUBLE) { if (width == 16) { @@ -5741,38 +5851,6 @@ static void ir_emit_load_mem(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) } } -static int32_t ir_local_offset(ir_ctx *ctx, ir_insn *insn) -{ - if (insn->op != IR_PARAM) { - IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA || insn->op == IR_VADDR); - return IR_SPILL_POS_TO_OFFSET(insn->op3); - } else { - IR_ASSERT(ctx->value_params && ctx->value_params[insn->op3 - 1].align); - return IR_SPILL_POS_TO_OFFSET(ctx->value_params[insn->op3 - 1].offset); - } -} - -static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) -{ - ir_backend_data *data = ctx->data; - dasm_State **Dst = &data->dasm_state; - ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - ir_insn *var_insn; - int32_t offset; - - IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); - var_insn = &ctx->ir_base[src]; - if (var_insn->op == IR_VADDR) { - var_insn = &ctx->ir_base[var_insn->op1]; - } - offset = ir_local_offset(ctx, var_insn); - if (offset == 0) { - | mov Ra(reg), Ra(base) - } else { - | lea Ra(reg), [Ra(base)+offset] - } -} - static void ir_resolve_label_syms(ir_ctx *ctx) { uint32_t b; @@ -5828,7 +5906,7 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) } else { ir_emit_load_imm_fp(ctx, type, reg, src); } - } else if (ctx->vregs[src]) { + } else if (ctx->vregs[src] > 0) { ir_emit_load_mem(ctx, type, reg, ir_ref_spill_slot(ctx, src)); } else { ir_load_local_addr(ctx, reg, src); @@ -5843,6 +5921,49 @@ static void ir_emit_store_mem_int(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg | ASM_MEM_REG_OP mov, type, mem, reg } +#if IR_SIMD +/* Vectos with size less than 4 have to be stored throug general purpose register */ +static void ir_emit_store_short_vector(ir_ctx *ctx, uint32_t width, ir_mem mem, ir_reg reg, ir_reg tmp_reg) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + bool do_pop = 0; + + IR_ASSERT(width == 1 || width == 2); + if (tmp_reg == IR_REG_NONE) { + IR_ASSERT(0 && "this is not expected"); + if (IR_MEM_BASE(mem) != IR_REG_RAX && IR_MEM_INDEX(mem) != IR_REG_RAX) { + tmp_reg = IR_REG_RAX; + } else if (IR_MEM_BASE(mem) != IR_REG_RCX && IR_MEM_INDEX(mem) != IR_REG_RCX) { + tmp_reg = IR_REG_RCX; + } else if (IR_MEM_BASE(mem) != IR_REG_RDX && IR_MEM_INDEX(mem) != IR_REG_RDX) { + tmp_reg = IR_REG_RDX; + } + if (IR_MEM_BASE(mem) == IR_REG_RSP) { + mem = IR_MEM_ADD(mem, sizeof(void*)); + } + + | push Ra(tmp_reg) + do_pop = 1; + } + + if (ctx->mflags & IR_X86_AVX) { + | vmovd Rd(tmp_reg), xmm(reg-IR_REG_FP_FIRST) + } else { + | movd Rd(tmp_reg), xmm(reg-IR_REG_FP_FIRST) + } + if (width == 1) { + | ASM_TMEM_TXT_OP mov, byte, mem, Rb(tmp_reg) + } else { + | ASM_TMEM_TXT_OP mov, word, mem, Rw(tmp_reg) + } + + if (do_pop) { + | pop Ra(tmp_reg) + } +} +#endif + static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg reg) { ir_backend_data *data = ctx->data; @@ -5860,11 +5981,13 @@ static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg r | ASM_TMEM_TXT_OP vmovdqa, oword, mem, xmm(reg-IR_REG_FP_FIRST) } else if (width == 8) { | ASM_TMEM_TXT_OP vmovq, qword, mem, xmm(reg-IR_REG_FP_FIRST) - } else { - // TODO: 4-byte write may be unsafe ??? - // IR_ASSERT(width >= 4 && width <= 16); - IR_ASSERT(width <= 4); + } else if (width == 4 || + (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) || + ((ctx->flags & IR_USE_FRAME_POINTER) && + IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) { | ASM_TMEM_TXT_OP vmovd, dword, mem, xmm(reg-IR_REG_FP_FIRST) + } else { + ir_emit_store_short_vector(ctx, width, mem, reg, IR_REG_NONE); } } else if (element_type == IR_DOUBLE) { if (width == 16) { @@ -5907,11 +6030,13 @@ static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg r |.else | ASM_TMEM_TXT_OP movlpd, qword, mem, xmm(reg-IR_REG_FP_FIRST) |.endif - } else { - // TODO: 4-byte write may be unsafe ??? - // IR_ASSERT(width >= 4 && width <= 16); - IR_ASSERT(width <= 4); + } else if (width == 4 || + (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) || + ((ctx->flags & IR_USE_FRAME_POINTER) && + IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) { | ASM_TMEM_TXT_OP movd, dword, mem, xmm(reg-IR_REG_FP_FIRST) + } else { + ir_emit_store_short_vector(ctx, width, mem, reg, IR_REG_NONE); } } else if (element_type == IR_DOUBLE) { if (width == 16) { @@ -5955,7 +6080,7 @@ static void ir_emit_load_i64_lo(ir_ctx *ctx, ir_reg reg, ir_ref src) IR_ASSERT(IR_IS_TYPE_INT(insn->type) && !IR_IS_SYM_CONST(insn->op)); ir_emit_load_imm_int(ctx, IR_U32, reg, insn->val.u32); - } else if (ctx->vregs[src]) { + } else if (ctx->vregs[src] > 0) { ir_emit_load_mem_int(ctx, IR_U32, reg, ir_ref_spill_slot(ctx, src)); } else { IR_ASSERT(0); @@ -5969,7 +6094,7 @@ static void ir_emit_load_i64_hi(ir_ctx *ctx, ir_reg reg, ir_ref src) IR_ASSERT(IR_IS_TYPE_INT(insn->type) && !IR_IS_SYM_CONST(insn->op)); ir_emit_load_imm_int(ctx, IR_U32, reg, insn->val.u32_hi); - } else if (ctx->vregs[src]) { + } else if (ctx->vregs[src] > 0) { ir_mem mem = IR_MEM_I64_HI(ir_ref_spill_slot(ctx, src)); ir_emit_load_mem_int(ctx, IR_U32, reg, mem); } else { @@ -5979,7 +6104,7 @@ static void ir_emit_load_i64_hi(ir_ctx *ctx, ir_reg reg, ir_ref src) static void ir_emit_store_i64_lo(ir_ctx *ctx, ir_ref dst, ir_reg reg) { - if (ctx->vregs[dst]) { + if (ctx->vregs[dst] > 0) { ir_emit_store_mem_int(ctx, IR_U32, ir_ref_spill_slot(ctx, dst), reg); } else { IR_ASSERT(0); @@ -5988,7 +6113,7 @@ static void ir_emit_store_i64_lo(ir_ctx *ctx, ir_ref dst, ir_reg reg) static void ir_emit_store_i64_hi(ir_ctx *ctx, ir_ref dst, ir_reg reg) { - if (ctx->vregs[dst]) { + if (ctx->vregs[dst] > 0) { ir_mem mem = IR_MEM_I64_HI(ir_ref_spill_slot(ctx, dst)); ir_emit_store_mem_int(ctx, IR_U32, mem, reg); } else { @@ -6207,6 +6332,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) ir_ref base_reg_ref, index_reg_ref; ir_reg base_reg = IR_REG_NONE, index_reg; int32_t offset = 0, scale; + intptr_t long_offset; IR_ASSERT(((rule & IR_RULE_MASK) >= IR_LEA_FIRST && (rule & IR_RULE_MASK) <= IR_LEA_LAST) || @@ -6217,7 +6343,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) case IR_LEA_OB: offset_insn = insn; if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]); + offset = ir_local_offset(ctx, insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6239,12 +6365,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) break; case IR_LEA_IB: if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]); + offset = ir_local_offset(ctx, insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = ref * sizeof(ir_ref) + 2; } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = ref * sizeof(ir_ref) + 1; @@ -6260,12 +6386,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = op1_insn; scale = 1; if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; } else if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]); + offset = ir_local_offset(ctx, op1_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = ref * sizeof(ir_ref) + 2; @@ -6279,12 +6405,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = op2_insn; scale = 1; if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]); + offset = ir_local_offset(ctx, insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = insn->op2 * sizeof(ir_ref) + 1; } else if (ir_rule(ctx, op2_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op2_insn->op1]); + offset = ir_local_offset(ctx, op2_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = ref * sizeof(ir_ref) + 1; @@ -6311,12 +6437,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = insn; scale = 1; if (ir_rule(ctx, op1_insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op2]); + offset = ir_local_offset(ctx, op1_insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; } else if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]); + offset = ir_local_offset(ctx, op1_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; index_reg_ref = insn->op1 * sizeof(ir_ref) + 2; @@ -6332,7 +6458,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) op2_insn = &ctx->ir_base[insn->op2]; scale = ctx->ir_base[op2_insn->op2].val.i32; if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]); + offset = ir_local_offset(ctx, op1_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6346,7 +6472,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) op2_insn = &ctx->ir_base[insn->op2]; offset_insn = op2_insn; if (ir_rule(ctx, op2_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op2_insn->op1]); + offset = ir_local_offset(ctx, op2_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6355,7 +6481,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) break; case IR_LEA_B_SI: if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]); + offset = ir_local_offset(ctx, insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6369,7 +6495,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) case IR_LEA_SI_B: index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6383,7 +6509,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = insn; op1_insn = &ctx->ir_base[insn->op1]; if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]); + offset = ir_local_offset(ctx, op1_insn->op1); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6398,7 +6524,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) op1_insn = &ctx->ir_base[insn->op1]; index_reg_ref = op1_insn->op1 * sizeof(ir_ref) + 1; if (ir_rule(ctx, op1_insn->op2) == IR_STATIC_ALLOCA) { - offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op2]); + offset = ir_local_offset(ctx, op1_insn->op2); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = IR_UNUSED; } else { @@ -6410,7 +6536,9 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) case IR_LEA_SYM_O: op1_insn = &ctx->ir_base[insn->op1]; op2_insn = &ctx->ir_base[insn->op2]; - offset = (intptr_t)ir_sym_val(ctx, op1_insn) + (intptr_t)op2_insn->val.i64; + long_offset = (intptr_t)ir_sym_val(ctx, op1_insn) + (intptr_t)op2_insn->val.i64; + IR_ASSERT(IR_IS_SIGNED_32BIT(long_offset)); + offset = (int32_t)long_offset; base_reg_ref = index_reg_ref = IR_UNUSED; scale = 1; offset_insn = NULL; @@ -6418,13 +6546,15 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) case IR_LEA_O_SYM: op1_insn = &ctx->ir_base[insn->op1]; op2_insn = &ctx->ir_base[insn->op2]; - offset = (intptr_t)ir_sym_val(ctx, op2_insn) + (intptr_t)op1_insn->val.i64; + long_offset = (intptr_t)ir_sym_val(ctx, op2_insn) + (intptr_t)op1_insn->val.i64; + IR_ASSERT(IR_IS_SIGNED_32BIT(long_offset)); + offset = (int32_t)long_offset; base_reg_ref = index_reg_ref = IR_UNUSED; scale = 1; offset_insn = NULL; break; case IR_ALLOCA: - offset = ir_local_offset(ctx, insn); + offset = ir_local_offset(ctx, ref); base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; base_reg_ref = index_reg_ref = IR_UNUSED; scale = 1; @@ -11377,20 +11507,21 @@ static void ir_emit_vaddr(ir_ctx *ctx, ir_ref def, ir_insn *insn) static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_ref type = insn->type; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); ir_reg fp; ir_mem mem; + int32_t slot; if (ctx->use_lists[def].count == 1) { /* dead load */ return; } - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3)); - if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, var_insn->op3)) { + slot = ir_local_slot(ctx, insn->op2); + mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot)); + if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, slot)) { return; // fake load } IR_ASSERT(def_reg != IR_REG_NONE); @@ -11420,20 +11551,21 @@ static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn) static void ir_emit_vstore_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_insn *val_insn = &ctx->ir_base[insn->op3]; ir_ref type = val_insn->type; ir_reg op3_reg = ctx->regs[ref][3]; ir_reg fp; ir_mem mem; + int32_t slot; - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3)); + slot = ir_local_slot(ctx, insn->op2); + mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot)); if ((op3_reg == IR_REG_NONE || IR_REG_SPILLED(op3_reg)) && !IR_IS_CONST_REF(insn->op3) && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA - && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) { + && ir_is_same_mem_var(ctx, insn->op3, slot)) { return; // fake store } if (IR_IS_CONST_REF(insn->op3)) { @@ -11470,19 +11602,20 @@ static void ir_emit_vstore_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) static void ir_emit_vstore_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) { - ir_insn *var_insn = &ctx->ir_base[insn->op2]; ir_ref type = ctx->ir_base[insn->op3].type; ir_reg op3_reg = ctx->regs[ref][3]; ir_reg fp; ir_mem mem; + int32_t slot; - IR_ASSERT(var_insn->op == IR_VAR); + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3)); + slot = ir_local_slot(ctx, insn->op2); + mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot)); if ((op3_reg == IR_REG_NONE || IR_REG_SPILLED(op3_reg)) && !IR_IS_CONST_REF(insn->op3) && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA - && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) { + && ir_is_same_mem_var(ctx, insn->op3, slot)) { return; // fake store } if (IR_IS_CONST_REF(insn->op3)) { @@ -11565,6 +11698,7 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op2_reg = ctx->regs[def][2]; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); ir_mem mem; + bool avoid_spill_store = 0; if (ctx->use_lists[def].count == 1) { /* dead load */ @@ -11584,16 +11718,38 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); mem = ir_fuse_addr(ctx, def, insn->op2); if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, mem)) { - if (!ir_may_avoid_spill_load(ctx, def, def)) { - ir_emit_load_mem_fp(ctx, type, def_reg, mem); + + if (ir_may_avoid_spill_load(ctx, def, def)) { + avoid_spill_store = 1; } - /* avoid load to the same location (valid only when register is not reused) */ - return; } } +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) { + uint32_t width = IR_VECTOR_SIZE(type); + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg tmp_reg = ctx->regs[def][3]; + + IR_ASSERT(tmp_reg != IR_REG_NONE); + IR_ASSERT(width == 1 || width == 2); + if (width == 1) { + | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), byte, mem + } else { + | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), word, mem + } + if (ctx->mflags & IR_X86_AVX) { + | vmovd xmm(def_reg-IR_REG_FP_FIRST), Rd(tmp_reg) + } else { + | movd xmm(def_reg-IR_REG_FP_FIRST), Rd(tmp_reg) + } + } else +#endif + ir_emit_load_mem_fp(ctx, type, def_reg, mem); - if (IR_REG_SPILLED(ctx->regs[def][0])) { + + if (IR_REG_SPILLED(ctx->regs[def][0]) && !avoid_spill_store) { ir_emit_store(ctx, type, def, def_reg); } } @@ -11738,6 +11894,26 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) } if (IR_IS_CONST_REF(insn->op3)) { +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) { + uint32_t width = IR_VECTOR_SIZE(type); + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + + IR_ASSERT(width == 1 || width == 2); + if (width == 1) { + uint8_t *p = ir_long_const_ptr(ctx, insn->op3); + int32_t val = *p; + + | ASM_TMEM_TXT_OP mov, byte, mem, val + } else { + uint16_t *p = ir_long_const_ptr(ctx, insn->op3); + int32_t val = *p; + + | ASM_TMEM_TXT_OP mov, word, mem, val + } + } else +#endif ir_emit_store_mem_fp_const(ctx, type, mem, insn->op3, IR_REG_NONE, op3_reg); } else { IR_ASSERT(op3_reg != IR_REG_NONE); @@ -11745,6 +11921,28 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); } + +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) { + uint32_t width = IR_VECTOR_SIZE(type); + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg tmp_reg = ctx->regs[ref][1]; + + IR_ASSERT(width == 1 || width == 2); + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (ctx->mflags & IR_X86_AVX) { + | vmovd Rd(tmp_reg), xmm(op3_reg-IR_REG_FP_FIRST) + } else { + | movd Rd(tmp_reg), xmm(op3_reg-IR_REG_FP_FIRST) + } + if (width == 1) { + | ASM_TMEM_TXT_OP mov, byte, mem, Rb(tmp_reg) + } else { + | ASM_TMEM_TXT_OP mov, word, mem, Rw(tmp_reg) + } + } else +#endif ir_emit_store_mem_fp(ctx, type, mem, op3_reg); } } @@ -11755,7 +11953,7 @@ static void ir_emit_rload(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_type type = insn->type; if (IR_REGSET_IN(IR_REGSET_UNION((ir_regset)ctx->fixed_regset, IR_REGSET_FIXED), src_reg)) { - if (ctx->vregs[def] + if (ctx->vregs[def] > 0 && ctx->live_intervals[ctx->vregs[def]] && ctx->live_intervals[ctx->vregs[def]]->stack_spill_pos != -1) { ir_emit_store(ctx, type, def, src_reg); @@ -11978,7 +12176,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -12012,7 +12210,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -12080,7 +12278,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + op2_offset = ir_local_offset(ctx, insn->op2); } if (op3_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op3_reg)) { @@ -12091,7 +12289,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]); + op3_offset = ir_local_offset(ctx, insn->op3); } | mov Ra(tmp_reg), aword [Ra(op3_reg)+op3_offset] | mov aword [Ra(op2_reg)+op2_offset], Ra(tmp_reg) @@ -12114,7 +12312,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + op2_offset = ir_local_offset(ctx, insn->op2); } if (op3_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op3_reg)) { @@ -12125,7 +12323,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]); + op3_offset = ir_local_offset(ctx, insn->op3); } | mov Rd(tmp_reg), dword [Ra(op3_reg)+(op3_offset+offsetof(ir_x86_64_sysv_va_list, gp_offset))] | mov dword [Ra(op2_reg)+(op2_offset+offsetof(ir_x86_64_sysv_va_list, gp_offset))], Rd(tmp_reg) @@ -12163,7 +12361,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } | mov Ra(tmp_reg), aword [Ra(op2_reg)+offset] #if IR_X86_I64 @@ -12233,7 +12431,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; - offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]); + offset = ir_local_offset(ctx, insn->op2); } if (insn->op3) { /* long struct arguemnt */ @@ -12266,9 +12464,9 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) |2: if (def_reg != IR_REG_NONE) { if (ir_type_size[type] == 8) { - | mov Rq(def_reg), qword [Ra(tmp_reg)-sizeof(void*)] + | mov Rq(def_reg), qword [Ra(tmp_reg)+(0-sizeof(void*))] } else { - | mov Rd(def_reg), dword [Ra(tmp_reg)-sizeof(void*)] + | mov Rd(def_reg), dword [Ra(tmp_reg)+(0-sizeof(void*))] } } } else { @@ -12309,7 +12507,7 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) uint32_t n, *p, use_block; int i; int label, default_label = 0; - int count = 0; + uint32_t count = 0; ir_val min, max; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; @@ -12342,8 +12540,8 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) max.i64 = IR_MAX(max.i64, val->val.i64); } else { IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type)); - min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64); - max.u64 = (int64_t)IR_MAX(max.u64, val->val.u64); + min.u64 = IR_MIN(min.u64, val->val.u64); + max.u64 = IR_MAX(max.u64, val->val.u64); } count++; } else if (use_insn->op == IR_CASE_RANGE) { @@ -12354,12 +12552,14 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(val2->op)); if (IR_IS_TYPE_SIGNED(type)) { IR_ASSERT(IR_IS_TYPE_SIGNED(val->type)); + IR_ASSERT(IR_IS_TYPE_SIGNED(val2->type)); min.i64 = IR_MIN(min.i64, val->val.i64); max.i64 = IR_MAX(max.i64, val2->val.i64); } else { IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type)); - min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64); - max.u64 = (int64_t)IR_MAX(max.u64, val2->val.u64); + IR_ASSERT(!IR_IS_TYPE_SIGNED(val2->type)); + min.u64 = IR_MIN(min.u64, val->val.u64); + max.u64 = IR_MAX(max.u64, val2->val.u64); } } else { IR_ASSERT(use_insn->op == IR_CASE_DEFAULT); @@ -12388,8 +12588,10 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) } /* Generate a table jmp or a seqence of calls */ - if (!has_case_range && count > 2 && (max.i64-min.i64) < count * 8) { - int *labels = ir_mem_malloc(sizeof(int) * (size_t)(max.i64 - min.i64 + 1)); + if ((sizeof(void*) == 8 || ir_type_size[type] <= 4) + && !has_case_range && count > 2 && (max.u64 - min.u64) < count * 8) { + int *labels = ir_mem_malloc(sizeof(int) * (size_t)(max.u64 - min.u64 + 1)); + ir_reg idx_reg; for (i = 0; i <= (max.i64 - min.i64); i++) { labels[i] = default_label; @@ -12442,62 +12644,69 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) } if (min.i64 != 0) { - int64_t offset = -min.i64; + int64_t offset = 0 - min.u64; if (IR_IS_SIGNED_32BIT(offset)) { | lea Ra(tmp_reg), [Ra(op2_reg)+(int32_t)offset] } else { - IR_ASSERT(sizeof(void*) == 8); |.if X64 | mov64 Rq(tmp_reg), offset | add Ra(tmp_reg), Ra(op2_reg) |.endif } - if (default_label) { - offset = max.i64 - min.i64; + idx_reg = tmp_reg; + } else { + idx_reg = op2_reg; + } - IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); - | cmp Ra(tmp_reg), (int32_t)offset - | ja =>default_label - } -|.if X64 - if (ctx->code_buffer - && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start) - && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end)) { - | jmp aword [Ra(tmp_reg)*8+>1] - } else { - int64_t offset = -min.i64; + if (default_label) { + int64_t offset = max.u64 - min.u64; - IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); - offset *= 8; - IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); - | lea Ra(tmp_reg), aword [>1] - | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8+offset] - } -|.else - | jmp aword [Ra(tmp_reg)*4+>1] + IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); + | cmp Ra(idx_reg), (int32_t)offset + | ja =>default_label + } + +#ifdef IR_TARGET_X86 +|.if not X64 + | jmp aword [Ra(idx_reg)*4+>1] |.endif +#else +|.if X64 + bool use_table_address = 0; + + if ((ctx->code_buffer + && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start) + && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end))) { + /* Use SIB with absolute jmp_table displacement */ + | jmp aword [Ra(idx_reg)*8+>1] + } else { - if (default_label) { - int64_t offset = max.i64; + /* Use IP-relative addressing mode to get the jmp_table address */ - IR_ASSERT(IR_IS_SIGNED_32BIT(offset)); - | cmp Ra(op2_reg), (int32_t)offset - | ja =>default_label - } -|.if X64 - if (ctx->code_buffer - && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start) - && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end)) { - | jmp aword [Ra(op2_reg)*8+>1] - } else { + // TODO: it's possible to use an extra temporary register (not conflicting with "killed" op2) ??? +// ir_reg op1_reg = ctx->regs[def][1]; +// if (op1_reg != IR_REG_NONE) { +// | lea Ra(op1_reg), aword [>1] +// | jmp aword [Ra(op1_reg)+Ra(idx_reg)*8] +// } else + if ((min.i64 >= (INT32_MIN+1)/8) && (min.i64 <= INT32_MAX/8)) { + tmp_reg = ctx->regs[def][3]; | lea Ra(tmp_reg), aword [>1] - | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8] + | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8+(-min.i32 * 8)] + } else { + use_table_address = 1; + if (idx_reg == tmp_reg) { + | shl Ra(tmp_reg), 3 + } else { + | lea Ra(tmp_reg), aword [Ra(idx_reg)*8] + } + | add Ra(tmp_reg), aword [>2] + | jmp aword [Ra(tmp_reg)] } -|.else - | jmp aword [Ra(op2_reg)*4+>1] -|.endif } +|.endif +#endif |.jmp_table if (!data->jmp_table_label) { @@ -12505,6 +12714,12 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) |=>data->jmp_table_label: } |.align aword +#ifndef IR_TARGET_X86 + if (use_table_address) { + |2: + | .aword >1 + } +#endif |1: for (i = 0; i <= (max.i64 - min.i64); i++) { int b = labels[i]; @@ -13073,7 +13288,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, const i type = IR_U32; } ir_emit_load(ctx, type, dst_reg, arg); - } else if (ctx->vregs[arg]) { + } else if (ctx->vregs[arg] > 0) { ir_mem mem = ir_ref_spill_slot(ctx, arg); uint32_t size = ir_type_size[type]; @@ -13294,7 +13509,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_pro ir_emit_store(ctx, insn->type, def, def_reg); } } - } else if (ctx->use_lists[def].count > 1) { + } else if (ctx->vregs[def] > 0) { #if IR_X86_I64 if (insn->type == IR_I64 || insn->type == IR_U64) { IR_ASSERT(cc->int_ret2_reg != IR_REG_NONE); @@ -13322,14 +13537,14 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_pro if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } - } else if (ctx->use_lists[def].count > 1) { + } else if (ctx->vregs[def] > 0) { ir_emit_store(ctx, insn->type, def, ret_reg); } } #ifdef IR_TARGET_X86 if (insn->op == IR_TAILCALL) { /* pass */ - } else if (ctx->use_lists[def].count > 1 && ret_reg == IR_REG_NONE) { + } else if (ctx->vregs[def] > 0 && ret_reg == IR_REG_NONE) { int32_t offset; ir_reg fp; @@ -13407,7 +13622,7 @@ static void ir_emit_tailcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) bool is_spill_slot = op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg) - && ctx->vregs[func]; + && ctx->vregs[func] > 0; if (op2_reg != IR_REG_NONE && !is_spill_slot) { if (IR_REGSET_IN(preserved_regs, IR_REG_NUM(op2_reg))) { @@ -17660,7 +17875,7 @@ static void ir_emit_vector_extract(ir_ctx *ctx, ir_ref def, ir_insn *insn) } /* extract through stack memory */ - int offset = -width; + int offset = 0 - width; ir_mem mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1); ir_mem mem2 = IR_MEM(IR_REG_RSP, offset, op2_reg, ir_type_size[element_type]); @@ -17789,7 +18004,7 @@ static void ir_emit_vector_replace(ir_ctx *ctx, ir_ref def, ir_insn *insn) int offset = width; | movdqu [Ra(IR_REG_RSP)-offset], xmm(tmp_reg-IR_REG_FP_FIRST) - | mov [Ra(IR_REG_RSP)-(offset-(lane%16))], Rb(op3_reg) + | mov [Ra(IR_REG_RSP)+((lane%16)-offset)], Rb(op3_reg) | movdqu xmm(tmp_reg-IR_REG_FP_FIRST), [Ra(IR_REG_RSP)-offset] } } else if (element_type == IR_I16 || element_type == IR_U16) { @@ -17936,7 +18151,7 @@ static void ir_emit_vector_replace(ir_ctx *ctx, ir_ref def, ir_insn *insn) } /* modify through stack memory */ - int offset = -width; + int offset = 0 - width; ir_mem mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1); ir_mem mem2 = IR_MEM(IR_REG_RSP, offset, op2_reg, ir_type_size[element_type]); @@ -18790,7 +19005,7 @@ static void ir_emit_vector_shuffle(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_type mask_element_type = IR_VECTOR_BASE_TYPE(mask_type); uint32_t mask_element_size = ir_type_size[mask_element_type]; uint32_t mask_len = IR_VECTOR_LENGTH(mask_type); - int dst_offset = -width; + int dst_offset = 0 - width; int src_offset = dst_offset; uint32_t mod = 0; ir_reg tmp_reg = ctx->tmp_regs[def]; @@ -18818,22 +19033,22 @@ static void ir_emit_vector_shuffle(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_mem src, dst; if (mask_element_size == 1) { - j = *(uint8_t*)ptr; + j = *(uint8_t*)ptr % mod; ptr = ((uint8_t*)ptr) + 1; } else if (mask_element_size == 2) { - j = *(uint16_t*)ptr; + j = *(uint16_t*)ptr % mod; ptr = ((uint16_t*)ptr) + 1; } else if (mask_element_size == 4) { - j = *(uint32_t*)ptr; + j = *(uint32_t*)ptr % mod; ptr = ((uint32_t*)ptr) + 1; } else if (mask_element_size == 8) { - j = *(uint64_t*)ptr; + j = *(uint64_t*)ptr % mod; ptr = ((uint64_t*)ptr) + 1; } else { IR_ASSERT(0); j = 0; } - src = IR_MEM(IR_REG_RSP, src_offset + (j % mod) * element_size, IR_REG_NONE, 1); + src = IR_MEM(IR_REG_RSP, src_offset + j * element_size, IR_REG_NONE, 1); dst = IR_MEM(IR_REG_RSP, dst_offset + i * element_size, IR_REG_NONE, 1); #if IR_X86_I64 if (element_type == IR_I64 || element_type == IR_U64) { @@ -21978,7 +22193,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn) dst_mem = ir_ref_spill_slot(ctx, def); def_reg = IR_REG_NUM(def_reg); } else { - offset = -width; + offset = 0 - width; dst_mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1); } @@ -21991,7 +22206,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (offset != 0) { src1_mem = dst_mem; } else { - offset = -width; + offset = 0 - width; src1_mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1); } ir_emit_store_mem_fp(ctx, type, src1_mem, op1_reg); @@ -22144,7 +22359,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (!scalar_shift) { if (element_size == 8) { if (IR_IS_SIGNED_32BIT(((int64_t*)ptr2)[i])) { - val = ((int64_t*)ptr2)[i]; + val = ((int64_t*)ptr2)[i] & 0xffffffff; } else { #if IR_X86_I64 val_hi = ((uint64_t*)ptr2)[i] >> 32; @@ -25739,7 +25954,7 @@ static void ir_emit_load_params(ir_ctx *ctx) int_param_num++; } } - if (ctx->vregs[use]) { + if (ctx->vregs[use] > 0) { dst_reg = IR_REG_NUM(ctx->regs[use][0]); IR_ASSERT(src_reg != IR_REG_NONE || dst_reg != IR_REG_NONE || stack_start + stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos); @@ -25805,7 +26020,6 @@ static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_s if (param_store_insn->op == IR_VSTORE) { ir_ref var = param_store_insn->op2; - ir_insn *var_insn = &ctx->ir_base[var]; IR_ASSERT(param_store_insn->op3 == param); IR_ASSERT(ctx->ir_base[var].op == IR_VAR); @@ -25814,19 +26028,28 @@ static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_s ir_use_list *var_use_list = &ctx->use_lists[var]; ir_ref n = var_use_list->count; ir_ref *p = &ctx->use_edges[var_use_list->refs]; + int32_t old_slot = IR_VREG_TO_STACK_SLOT(ctx->vregs[var]); + for (; n > 0; p++, n--) { ir_ref use = *p; ir_insn *insn = &ctx->ir_base[use]; if (insn->op == IR_VADDR) { if (ctx->rules[use] == IR_STATIC_ALLOCA) { - IR_ASSERT(insn->op3 == var_insn->op3); - insn->op3 = spill_slot; + IR_ASSERT(ctx->vregs[use] == ctx->vregs[var]); + ctx->vregs[use] = IR_STACK_SLOT_TO_VREG(spill_slot); } - } else if (insn->op == IR_VLOAD) { - if (ctx->vregs[use]) { + } else if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { + if (ctx->vregs[use] > 0) { ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; - if (ival->stack_spill_pos == var_insn->op3) { + if (ival && ival->stack_spill_pos == old_slot) { + ival->stack_spill_pos = spill_slot; + } + } + } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { + if (insn->op3 > 0 && ctx->vregs[insn->op3] > 0) { + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op3]]; + if (ival && ival->stack_spill_pos == old_slot) { ival->stack_spill_pos = spill_slot; } } @@ -25834,7 +26057,7 @@ static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_s } /* Remap VAR itself */ - var_insn->op3 = spill_slot; + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(spill_slot); /* Avoid copying to itself */ ctx->rules[param_store] = IR_SKIPPED | IR_NOP; @@ -25877,7 +26100,7 @@ static void ir_fix_param_spills(ir_ctx *ctx) align = IR_MAX(sizeof(void*), align); stack_offset = IR_ALIGNED_SIZE(stack_offset, align); - ctx->value_params[insn->op3 - 1].offset = stack_start + stack_offset; + ctx->vregs[use] = IR_STACK_SLOT_TO_VREG(stack_start + stack_offset); stack_offset += ctx->value_params[insn->op3 - 1].size; stack_offset = IR_ALIGNED_SIZE(stack_offset, sizeof(void*)); continue; @@ -25923,7 +26146,7 @@ static void ir_fix_param_spills(ir_ctx *ctx) } } if (src_reg == IR_REG_NONE) { - if (ctx->vregs[use]) { + if (ctx->vregs[use] > 0) { ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) && ival->stack_spill_pos == -1) { @@ -26003,7 +26226,7 @@ void ir_fix_stack_frame(ir_ctx *ctx) #ifdef IR_TARGET_X86 if (ctx->flags2 & IR_HAS_MEMCPY) { - ir_regset tmp = ctx->used_preserved_regs; + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; IR_REGSET_INCL(tmp, IR_REG_RSI); IR_REGSET_INCL(tmp, IR_REG_RDI); ctx->used_preserved_regs = tmp; From 3e9fcac36630043f466bf088531609ec1ee3f163 Mon Sep 17 00:00:00 2001 From: Jorg Sowa Date: Mon, 19 Oct 2026 12:47:48 +0200 Subject: [PATCH 2/3] Update IR IR commit: 563b51a8c2c07aadb13db897abc04b5cf08d151b --- .github/scripts/download-bundled/jit-ir.sh | 2 +- ext/opcache/config.m4 | 1 + ext/opcache/config.w32 | 2 +- ext/opcache/jit/Makefile.frag.w32 | 5 + ext/opcache/jit/ir/ir.c | 5 + ext/opcache/jit/ir/ir.h | 12 +- ext/opcache/jit/ir/ir_aarch64.dasc | 267 ++++++-- ext/opcache/jit/ir/ir_disasm.c | 9 + ext/opcache/jit/ir/ir_emit.c | 707 +++------------------ ext/opcache/jit/ir/ir_fold.h | 6 +- ext/opcache/jit/ir/ir_perf.c | 6 + ext/opcache/jit/ir/ir_private.h | 8 +- ext/opcache/jit/ir/ir_ra.c | 75 ++- ext/opcache/jit/ir/ir_ra_simple.c | 662 +++++++++++++++++++ ext/opcache/jit/ir/ir_sccp.c | 52 +- ext/opcache/jit/ir/ir_x86.dasc | 492 ++++++++------ 16 files changed, 1382 insertions(+), 929 deletions(-) create mode 100644 ext/opcache/jit/ir/ir_ra_simple.c diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index f3c7242e9d9b..93f3aa7c27d4 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=ed32c881a0d7c14ce7ace6ac00dead75647b69c6 +revision=563b51a8c2c07aadb13db897abc04b5cf08d151b git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 2a1ac4e332fc..224e9e54a2e4 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -48,6 +48,7 @@ AS_VAR_IF([PHP_OPCACHE_JIT], [yes], [ jit/ir/ir_patch.c jit/ir/ir_perf.c jit/ir/ir_ra.c + jit/ir/ir_ra_simple.c jit/ir/ir_save.c jit/ir/ir_sccp.c jit/ir/ir_strtab.c diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index cd22b363db0b..f92b02e59671 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -25,7 +25,7 @@ if (PHP_OPCACHE_JIT == "yes") { } else if (CHECK_HEADER("ir/ir.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) { var dasm_flags = (X64 ? "-D X64=1" : "") + (X64 ? " -D X64WIN=1" : "") + " -D WIN=1"; var ir_target = (X64 ? "IR_TARGET_X64" : "IR_TARGET_X86"); - var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_save.c \ + var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_ra_simple.c ir_save.c \ ir_dump.c ir_check.c ir_patch.c"; DEFINE("IR_TARGET", ir_target); diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32 index 7caf81458870..b5273a1b86ea 100644 --- a/ext/opcache/jit/Makefile.frag.w32 +++ b/ext/opcache/jit/Makefile.frag.w32 @@ -19,6 +19,11 @@ $(BUILD_DIR)\ext\opcache\jit\ir\ir_ra.obj: \ ext\opcache\jit\ir\ir_private.h \ ext\opcache\jit\ir\ir_x86.h +$(BUILD_DIR)\ext\opcache\jit\ir\ir_ra_simple.obj: \ + ext\opcache\jit\ir\ir.h \ + ext\opcache\jit\ir\ir_private.h \ + ext\opcache\jit\ir\ir_x86.h + $(BUILD_DIR)\ext\opcache\jit\ir\ir_emit.obj: \ ext\opcache\jit\ir\ir.h \ ext\opcache\jit\ir\ir_private.h \ diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 5f9b30980518..fd56dd8067cc 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -1301,6 +1301,11 @@ ir_ref ir_fold(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3) if (UNEXPECTED(!(ctx->flags & IR_OPT_FOLDING))) { if ((opt & IR_OPT_OP_MASK) == IR_PHI) { opt |= (3 << IR_OPT_INPUTS_SHIFT); + } else if ((ir_op_flags[(opt & IR_OPT_OP_MASK)] & IR_OP_FLAG_COMMUTATIVE) + && op1 < op2 + && IR_IS_CONST_REF(op1)) { + /* Move constants to op2 for better instruction selection with (this is for -O0) */ + SWAP_REFS(op1, op2); } return ir_emit(ctx, opt, op1, op2, op3); } diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index ee0dc259dee4..7395b6863414 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -321,10 +321,10 @@ typedef enum _ir_type { _(NE, d2C, def, def, ___) /* not equal */ \ \ /* comparison ops (order matters, LT must be a modulo of 4 !!!) */ \ - _(LT, d2, def, def, ___) /* less */ \ - _(GE, d2, def, def, ___) /* greater or equal */ \ - _(LE, d2, def, def, ___) /* less or equal */ \ - _(GT, d2, def, def, ___) /* greater */ \ + _(LT, d2, def, def, ___) /* less (signed) */ \ + _(GE, d2, def, def, ___) /* greater or equal (signed) */ \ + _(LE, d2, def, def, ___) /* less or equal (signed) */ \ + _(GT, d2, def, def, ___) /* greater (signed) */ \ _(ULT, d2, def, def, ___) /* unsigned less */ \ _(UGE, d2, def, def, ___) /* unsigned greater or equal */ \ _(ULE, d2, def, def, ___) /* unsigned less or equal */ \ @@ -968,6 +968,8 @@ int ir_compute_dessa_moves(ir_ctx *ctx); int ir_reg_alloc(ir_ctx *ctx); int ir_reg_alloc_simple(ir_ctx *ctx); +#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_simple(_ctx) + int ir_regs_number(void); bool ir_reg_is_int(int32_t reg); const char *ir_reg_name(int8_t reg, ir_type type); @@ -1128,7 +1130,7 @@ IR_ALWAYS_INLINE void *ir_jit_compile(ir_ctx *ctx, int opt_level, size_t *size) || !ir_match(ctx) || !ir_assign_virtual_registers(ctx) || !ir_compute_dessa_moves(ctx) - || !ir_reg_alloc_simple(ctx)) { + || !ir_reg_alloc_O0(ctx)) { return NULL; } diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 43d918179118..b35797216a81 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -440,8 +440,6 @@ const ir_call_conv_dsc ir_call_conv_aarch64_preserve_none = { #define IR_RULE_ENUM(name) IR_ ## name, -#define IR_STATIC_ALLOCA (IR_SKIPPED | IR_FUSED | IR_SIMPLE | IR_ALLOCA) - enum _ir_rule { IR_FIRST_RULE = IR_LAST_OP, IR_RULES(IR_RULE_ENUM) @@ -525,7 +523,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co if (insn->op == IR_MOD) { constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; - } else if (insn->op == IR_MUL_OV && (ir_type_size[insn->type] == 8 || IR_IS_TYPE_SIGNED(insn->type))) { + } else if (insn->op == IR_MUL_OV && ir_type_size[insn->type] != 4) { constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n++; } @@ -603,6 +601,10 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co insn = &ctx->ir_base[ref]; constraints->tmp_regs[0] = IR_TMP_REG(2, IR_DOUBLE, IR_USE_SUB_REF, IR_SAVE_SUB_REF); n = 1; + if (IR_IS_CONST_REF(insn->op1)) { + constraints->tmp_regs[n] = IR_TMP_REG(1, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; + } break; case IR_BINOP_FP: case IR_MIN_MAX_INT: @@ -2890,13 +2892,73 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_REG_REG_REG_OP add, type, def_reg, op1_reg, op2_reg break; case IR_ADD_OV: - | ASM_REG_REG_REG_OP adds, type, def_reg, op1_reg, op2_reg + if (ir_type_size[type] >= 4) { + | ASM_REG_REG_REG_OP adds, type, def_reg, op1_reg, op2_reg + } else { + if (def_reg == op2_reg) { + SWAP_REGS(op1_reg, op2_reg); + } + if (ir_type_size[type] == 2) { + if (IR_IS_TYPE_SIGNED(type)) { + | sxth Rw(def_reg), Rw(op1_reg) + | add Rw(def_reg), Rw(def_reg), Rw(op2_reg), sxth + | cmp Rw(def_reg), Rw(def_reg), sxth + } else { + | and Rw(def_reg), Rw(op1_reg), #0xffff + | add Rw(def_reg), Rw(def_reg), Rw(op2_reg), uxth + | cmp Rw(def_reg), Rw(def_reg), uxth + } + | and Rw(def_reg), Rw(def_reg), #0xffff + } else { + IR_ASSERT(ir_type_size[type] == 1); + if (IR_IS_TYPE_SIGNED(type)) { + | sxtb Rw(def_reg), Rw(op1_reg) + | add Rw(def_reg), Rw(def_reg), Rw(op2_reg), sxtb + | cmp Rw(def_reg), Rw(def_reg), sxtb + } else { + | and Rw(def_reg), Rw(op1_reg), #0xff + | add Rw(def_reg), Rw(def_reg), Rw(op2_reg), uxtb + | cmp Rw(def_reg), Rw(def_reg), uxtb + } + | and Rw(def_reg), Rw(def_reg), #0xff + } + } break; case IR_SUB: | ASM_REG_REG_REG_OP sub, type, def_reg, op1_reg, op2_reg break; case IR_SUB_OV: - | ASM_REG_REG_REG_OP subs, type, def_reg, op1_reg, op2_reg + if (ir_type_size[type] >= 4) { + | ASM_REG_REG_REG_OP subs, type, def_reg, op1_reg, op2_reg + } else { + if (def_reg == op2_reg) { + SWAP_REGS(op1_reg, op2_reg); + } + if (ir_type_size[type] == 2) { + if (IR_IS_TYPE_SIGNED(type)) { + | sxth Rw(def_reg), Rw(op1_reg) + | sub Rw(def_reg), Rw(def_reg), Rw(op2_reg), sxth + | cmp Rw(def_reg), Rw(def_reg), sxth + } else { + | and Rw(def_reg), Rw(op1_reg), #0xffff + | sub Rw(def_reg), Rw(def_reg), Rw(op2_reg), uxth + | cmp Rw(def_reg), Rw(def_reg), uxth + } + | and Rw(def_reg), Rw(def_reg), #0xffff + } else { + IR_ASSERT(ir_type_size[type] == 1); + if (IR_IS_TYPE_SIGNED(type)) { + | sxtb Rw(def_reg), Rw(op1_reg) + | sub Rw(def_reg), Rw(def_reg), Rw(op2_reg), sxtb + | cmp Rw(def_reg), Rw(def_reg), sxtb + } else { + | and Rw(def_reg), Rw(op1_reg), #0xff + | sub Rw(def_reg), Rw(def_reg), Rw(op2_reg), uxtb + | cmp Rw(def_reg), Rw(def_reg), uxtb + } + | and Rw(def_reg), Rw(def_reg), #0xff + } + } break; case IR_MUL: | ASM_REG_REG_REG_OP mul, type, def_reg, op1_reg, op2_reg @@ -2916,17 +2978,51 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) | mul Rx(def_reg), Rx(op1_reg), Rx(op2_reg) | cmp Rx(tmp_reg), xzr } - } else { + } else if (ir_type_size[type] == 4) { if (IR_IS_TYPE_SIGNED(type)) { - tmp_reg = ctx->regs[def][3]; - IR_ASSERT(tmp_reg != IR_REG_NONE); | smull Rx(def_reg), Rw(op1_reg), Rw(op2_reg) - | asr Rx(tmp_reg), Rx(def_reg), #32 - | cmp Rx(tmp_reg), Rx(def_reg), asr #31 + | cmp Rx(def_reg), Rw(def_reg), sxtw } else { | umull Rx(def_reg), Rw(op1_reg), Rw(op2_reg) - | cmp xzr, Rx(def_reg), lsr #32 + | tst Rx(def_reg), #0xffffffff00000000 + } + } else if (ir_type_size[type] == 2) { + if (def_reg == op2_reg) { + SWAP_REGS(op1_reg, op2_reg); } + tmp_reg = ctx->regs[def][3]; + IR_ASSERT(tmp_reg != IR_REG_NONE && tmp_reg != def_reg); + if (IR_IS_TYPE_SIGNED(type)) { + | sxth Rw(def_reg), Rw(op1_reg) + | sxth Rw(tmp_reg), Rw(op2_reg) + | mul Rw(def_reg), Rw(def_reg), Rw(tmp_reg) + | cmp Rw(def_reg), Rw(def_reg), sxth + } else { + | and Rw(def_reg), Rw(op1_reg), #0xffff + | and Rw(tmp_reg), Rw(op2_reg), #0xffff + | mul Rw(def_reg), Rw(def_reg), Rw(tmp_reg) + | cmp Rw(def_reg), Rw(def_reg), uxth + } + | and Rw(def_reg), Rw(def_reg), #0xffff + } else { + IR_ASSERT(ir_type_size[type] == 1); + if (def_reg == op2_reg) { + SWAP_REGS(op1_reg, op2_reg); + } + tmp_reg = ctx->regs[def][3]; + IR_ASSERT(tmp_reg != IR_REG_NONE && tmp_reg != def_reg); + if (IR_IS_TYPE_SIGNED(type)) { + | sxtb Rw(def_reg), Rw(op1_reg) + | sxtb Rw(tmp_reg), Rw(op2_reg) + | mul Rw(def_reg), Rw(def_reg), Rw(tmp_reg) + | cmp Rw(def_reg), Rw(def_reg), sxtb + } else { + | and Rw(def_reg), Rw(op1_reg), #0xff + | and Rw(tmp_reg), Rw(op2_reg), #0xff + | mul Rw(def_reg), Rw(def_reg), Rw(tmp_reg) + | cmp Rw(def_reg), Rw(def_reg), uxtb + } + | and Rw(def_reg), Rw(def_reg), #0xff } break; case IR_DIV: @@ -2968,13 +3064,73 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_REG_REG_IMM_OP add, type, def_reg, op1_reg, val break; case IR_ADD_OV: - | ASM_REG_REG_IMM_OP adds, type, def_reg, op1_reg, val + if (ir_type_size[type] >= 4) { + | ASM_REG_REG_IMM_OP adds, type, def_reg, op1_reg, val + } else { + if (def_reg == op2_reg) { + SWAP_REGS(op1_reg, op2_reg); + } + if (ir_type_size[type] == 2) { + if (IR_IS_TYPE_SIGNED(type)) { + | sxth Rw(def_reg), Rw(op1_reg) + | add Rw(def_reg), Rw(def_reg), #((int32_t)(int16_t)val) + | cmp Rw(def_reg), Rw(def_reg), sxth + } else { + | and Rw(def_reg), Rw(op1_reg), #0xffff + | add Rw(def_reg), Rw(def_reg), #((uint32_t)(uint16_t)val) + | cmp Rw(def_reg), Rw(def_reg), uxth + } + | and Rw(def_reg), Rw(def_reg), #0xffff + } else { + IR_ASSERT(ir_type_size[type] == 1); + if (IR_IS_TYPE_SIGNED(type)) { + | sxtb Rw(def_reg), Rw(op1_reg) + | add Rw(def_reg), Rw(def_reg), #((int32_t)(int8_t)val) + | cmp Rw(def_reg), Rw(def_reg), sxtb + } else { + | and Rw(def_reg), Rw(op1_reg), #0xff + | add Rw(def_reg), Rw(def_reg), #((uint32_t)(uint8_t)val) + | cmp Rw(def_reg), Rw(def_reg), uxtb + } + | and Rw(def_reg), Rw(def_reg), #0xff + } + } break; case IR_SUB: | ASM_REG_REG_IMM_OP sub, type, def_reg, op1_reg, val break; case IR_SUB_OV: - | ASM_REG_REG_IMM_OP subs, type, def_reg, op1_reg, val + if (ir_type_size[type] >= 4) { + | ASM_REG_REG_IMM_OP subs, type, def_reg, op1_reg, val + } else { + if (def_reg == op2_reg) { + SWAP_REGS(op1_reg, op2_reg); + } + if (ir_type_size[type] == 2) { + if (IR_IS_TYPE_SIGNED(type)) { + | sxth Rw(def_reg), Rw(op1_reg) + | sub Rw(def_reg), Rw(def_reg), #((int32_t)(int16_t)val) + | cmp Rw(def_reg), Rw(def_reg), sxth + } else { + | and Rw(def_reg), Rw(op1_reg), #0xffff + | sub Rw(def_reg), Rw(def_reg), #((uint32_t)(uint16_t)val) + | cmp Rw(def_reg), Rw(def_reg), uxth + } + | and Rw(def_reg), Rw(def_reg), #0xffff + } else { + IR_ASSERT(ir_type_size[type] == 1); + if (IR_IS_TYPE_SIGNED(type)) { + | sxtb Rw(def_reg), Rw(op1_reg) + | sub Rw(def_reg), Rw(def_reg), #((int32_t)(int8_t)val) + | cmp Rw(def_reg), Rw(def_reg), sxtb + } else { + | and Rw(def_reg), Rw(op1_reg), #0xff + | sub Rw(def_reg), Rw(def_reg), #((uint32_t)(uint8_t)val) + | cmp Rw(def_reg), Rw(def_reg), uxtb + } + | and Rw(def_reg), Rw(def_reg), #0xff + } + } break; case IR_OR: if (ir_type_size[type] == 8) { @@ -3021,7 +3177,7 @@ static void ir_emit_bic(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(ctx->ir_base[not_ref].op == IR_NOT); if (UNEXPECTED(ctx->rules[not_ref] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, not_ref * sizeof(ir_ref) + 1); + op1_reg = ir_get_fused_reg(ctx, def, not_ref, 1); } else { op1_reg = ctx->regs[not_ref][1]; } @@ -3134,12 +3290,15 @@ static void ir_emit_overflow(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE); IR_ASSERT(IR_IS_TYPE_INT(type)); - if (math_insn->op == IR_MUL_OV) { + if (math_insn->op == IR_MUL_OV || ir_type_size[math_insn->type] < 4) { | cset Rw(def_reg), ne } else if (IR_IS_TYPE_SIGNED(type)) { | cset Rw(def_reg), vs - } else { + } else if (math_insn->op == IR_ADD_OV) { | cset Rw(def_reg), cs + } else { + IR_ASSERT(math_insn->op == IR_SUB_OV); + | cset Rw(def_reg), cc } if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); @@ -3165,7 +3324,7 @@ static void ir_emit_overflow_and_branch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_ false_block = 0; } - if (math_insn->op == IR_MUL_OV) { + if (math_insn->op == IR_MUL_OV || ir_type_size[math_insn->type] < 4) { if (reverse) { | beq =>true_block } else { @@ -3177,12 +3336,19 @@ static void ir_emit_overflow_and_branch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_ } else { | bvs =>true_block } - } else { + } else if (math_insn->op == IR_ADD_OV) { if (reverse) { | bcc =>true_block } else { | bcs =>true_block } + } else { + IR_ASSERT(math_insn->op == IR_SUB_OV); + if (reverse) { + | bcs =>true_block + } else { + | bcc =>true_block + } } if (false_block) { | b =>false_block @@ -3657,7 +3823,11 @@ static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_REG_REG_OP clz, src_type, def_reg, def_reg } else { IR_ASSERT(insn->op == IR_BSWAP); - | ASM_REG_REG_OP rev, src_type, def_reg, op1_reg + if (ir_type_size[src_type] == 2) { + | rev16 Rw(def_reg), Rw(op1_reg) + } else { + | ASM_REG_REG_OP rev, src_type, def_reg, op1_reg + } } if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); @@ -3973,8 +4143,8 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref root, ir_ref cmp_ref, ir_ op1 = cmp_insn->op1; op2 = cmp_insn->op2; if (UNEXPECTED(ctx->rules[cmp_ref] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, root, cmp_ref * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, root, cmp_ref * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, root, cmp_ref, 1); + op2_reg = ir_get_fused_reg(ctx, root, cmp_ref, 2); } else { op1_reg = ctx->regs[cmp_ref][1]; op2_reg = ctx->regs[cmp_ref][2]; @@ -4227,8 +4397,8 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i ir_reg op1_reg, op2_reg; if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1); + op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2); } else { op1_reg = ctx->regs[insn->op2][1]; op2_reg = ctx->regs[insn->op2][2]; @@ -5035,13 +5205,13 @@ static void ir_emit_vstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn) fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; slot = ir_local_slot(ctx, insn->op2); offset = IR_SPILL_POS_TO_OFFSET(slot); - IR_ASSERT(op3_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op3_reg) + if ((op3_reg == IR_REG_NONE || IR_REG_SPILLED(op3_reg)) && !IR_IS_CONST_REF(insn->op3) && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA && ir_is_same_mem_var(ctx, insn->op3, slot)) { return; // fake store } + IR_ASSERT(op3_reg != IR_REG_NONE); if (IR_REG_SPILLED(op3_reg)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); @@ -5066,7 +5236,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) return IR_MEM_BO(reg, offset); } else { if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { - reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1); + reg = ir_get_fused_reg(ctx, root, ref, 1); } else { reg = ctx->regs[ref][1]; } @@ -6689,8 +6859,8 @@ static void ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * ir_reg op1_reg, op2_reg; if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1); + op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2); } else { op1_reg = ctx->regs[insn->op2][1]; op2_reg = ctx->regs[insn->op2][2]; @@ -9013,36 +9183,6 @@ static void ir_emit_load_params(ir_ctx *ctx) } } -static int ir_fix_dessa_tmps(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to, void *dessa_from_block) -{ - ir_ref ref = ctx->cfg_blocks[(intptr_t)dessa_from_block].end; - - if (to == 0) { - if (IR_IS_TYPE_INT(type)) { - if (ctx->regs[ref][0] == IR_REG_NONE) { - ctx->regs[ref][0] = IR_REG_X0; - } - } else { - IR_ASSERT(IR_IS_TYPE_FP(type)); - if (ctx->regs[ref][1] == IR_REG_NONE) { - ctx->regs[ref][1] = IR_REG_V0; - } - } - } else if (from != 0) { - if (IR_IS_TYPE_INT(type)) { - if (ctx->regs[ref][0] == IR_REG_NONE) { - ctx->regs[ref][0] = IR_REG_X0; - } - } else { - IR_ASSERT(IR_IS_TYPE_FP(type)); - if (ctx->regs[ref][1] == IR_REG_NONE) { - ctx->regs[ref][1] = IR_REG_V0; - } - } - } - return 1; -} - static void ir_fix_param_spills(ir_ctx *ctx) { ir_use_list *use_list = &ctx->use_lists[1]; @@ -9079,10 +9219,13 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (src_reg == IR_REG_NONE) { if (ctx->vregs[use] > 0) { ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; - if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) - && ival->stack_spill_pos == -1 - && (ival->next || ival->reg == IR_REG_NONE)) { + if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { + IR_ASSERT(ival->stack_spill_pos == -1); ival->stack_spill_pos = stack_start + stack_offset; + if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { + /* Remap VAR to PARAM stack slot */ + ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); + } } } if (sizeof(void*) == 8) { @@ -9686,7 +9829,7 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) if (!c) { break; } - w |= c << (8 * j); + w |= (uint32_t)(uint8_t)c << (8 * j); i++; } | .long w diff --git a/ext/opcache/jit/ir/ir_disasm.c b/ext/opcache/jit/ir/ir_disasm.c index 30cfb694cff6..a4c847cb1e29 100644 --- a/ext/opcache/jit/ir/ir_disasm.c +++ b/ext/opcache/jit/ir/ir_disasm.c @@ -17,6 +17,9 @@ # if defined(__FreeBSD__) || defined(__DragonFly__) # include # endif +# if defined(__OpenBSD__) +# include +# endif #endif #include "ir.h" @@ -789,6 +792,12 @@ static void ir_elf_load_symbols(void) return; } int fd = open(path, O_RDONLY); +#elif defined(__OpenBSD__) && OpenBSD >= 202610 + char path[PATH_MAX]; + if (getexecpath(path, sizeof(path)) == -1) { + return; + } + int fd = open(path, O_RDONLY); #elif defined(__sun) int fd = open("/proc/self/path/a.out", O_RDONLY); #elif defined(__HAIKU__) diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index 71555ff5da5a..2a9336bc483b 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -176,6 +176,62 @@ static ir_reg ir_get_param_reg(const ir_ctx *ctx, ir_ref ref) return IR_REG_NONE; } +/* This remaps spill slot of the VAR, where PARAM is stored, to the PARAMs slot. + * This eliminates copying, but more important, this fixes va_start() compilation + * on systems, where it's defined through address calculation macro (e.g. MSVC x86). + */ +static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_slot) +{ + ir_use_list *param_use_list = &ctx->use_lists[param]; + + if (param_use_list->count >= 1) { + ir_ref param_store = ctx->use_edges[param_use_list->refs]; + ir_insn *param_store_insn = &ctx->ir_base[param_store]; + + if (param_store_insn->op == IR_VSTORE && (ctx->rules[param_store] & IR_SKIPPED)) { + ir_ref var = param_store_insn->op2; + + IR_ASSERT(param_store_insn->op3 == param); + IR_ASSERT(ctx->ir_base[var].op == IR_VAR); + + /* Remap all related VADDR and VLOAD */ + ir_use_list *var_use_list = &ctx->use_lists[var]; + ir_ref n = var_use_list->count; + ir_ref *p = &ctx->use_edges[var_use_list->refs]; + int32_t old_slot = IR_VREG_TO_STACK_SLOT(ctx->vregs[var]); + + for (; n > 0; p++, n--) { + ir_ref use = *p; + ir_insn *insn = &ctx->ir_base[use]; + + if (insn->op == IR_VADDR) { + if (ctx->rules[use] == IR_STATIC_ALLOCA) { + IR_ASSERT(ctx->vregs[use] == ctx->vregs[var]); + ctx->vregs[use] = IR_STACK_SLOT_TO_VREG(spill_slot); + } + } else if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { + if (ctx->vregs[use] > 0) { + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; + if (ival && ival->stack_spill_pos == old_slot) { + ival->stack_spill_pos = spill_slot; + } + } + } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { + if (insn->op3 > 0 && ctx->vregs[insn->op3] > 0) { + ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op3]]; + if (ival && ival->stack_spill_pos == old_slot) { + ival->stack_spill_pos = spill_slot; + } + } + } + } + + /* Remap VAR itself */ + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(spill_slot); + } + } +} + static int ir_get_args_regs(const ir_ctx *ctx, const ir_insn *insn, const ir_call_conv_dsc *cc, int8_t *regs) { int j, n; @@ -347,7 +403,21 @@ static void *ir_jmp_addr(ir_ctx *ctx, ir_insn *insn, ir_insn *addr_insn) return addr; } -static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op) +void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ref op, int8_t reg) +{ + char key[10]; + ir_ref ref_and_op = ref * sizeof(ir_ref) + op; + + if (!ctx->fused_regs) { + ctx->fused_regs = ir_mem_malloc(sizeof(ir_strtab)); + ir_strtab_init(ctx->fused_regs, 8, 128); + } + memcpy(key, &root, sizeof(ir_ref)); + memcpy(key + 4, &ref_and_op, sizeof(ir_ref)); + ir_strtab_lookup(ctx->fused_regs, key, 8, 0x10000000 | (uint8_t)reg); +} + +static int8_t ir_get_fused_reg_ex(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op) { if (ctx->fused_regs) { char key[10]; @@ -364,6 +434,11 @@ static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op) return ((int8_t*)ctx->regs)[ref_and_op]; } +static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ref op) +{ + return ir_get_fused_reg_ex(ctx, root, ref * sizeof(ir_ref) + op); +} + #if defined(__GNUC__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Warray-bounds" @@ -964,7 +1039,7 @@ static void ir_emit_dessa_moves(ir_ctx *ctx, int b, ir_block *bb) ir_ref ref = *p; ir_insn *insn = &ctx->ir_base[ref]; - if (insn->op == IR_PHI) { + if (insn->op == IR_PHI && ctx->vregs[ref] > 0) { ir_ref input = ir_insn_op(insn, k); ir_reg src = ir_get_alocated_reg(ctx, ref, k); ir_reg dst = ctx->regs[ref][0]; @@ -1427,631 +1502,3 @@ const ir_call_conv_dsc *ir_get_call_conv_dsc(uint32_t flags) IR_ASSERT((flags & IR_CALL_CONV_MASK) == IR_CC_DEFAULT || (flags & IR_CALL_CONV_MASK) == IR_CC_BUILTIN); return &ir_call_conv_default; } - -/* Simple Register Allocator */ -typedef struct { - int32_t num; - ir_regset preserved_regs; - ir_regset clobbered[IR_SUB_REFS_COUNT]; - struct { - uint8_t type; - int8_t start; - int8_t end; - int8_t hint; - int8_t flags; - ir_ref root; - ir_ref ref; - ir_ref op; - } regs[32]; -} ir_reg_alloc_simple_data; - -static void _add_scratch(ir_reg_alloc_simple_data *x, ir_reg reg, int8_t start, int8_t end) -{ - int8_t j; - - if (start < 0) start = 0; // TODO: ARGVAL support ??? - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT); - if (reg >= IR_REG_NUM) { - for (j = start; j < end; j++) { - x->clobbered[j] = IR_REGSET_UNION(x->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]); - } - } else { - for (j = start; j < end; j++) { - IR_REGSET_INCL(x->clobbered[j], reg); - } - } -} - -static void _add_reg(ir_reg_alloc_simple_data *x, ir_type type, - int8_t start, int8_t end, ir_reg hint, int8_t flags, - ir_ref root, ir_ref ref, ir_ref op) -{ - IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && x->num < 32); - x->regs[x->num].type = type; - x->regs[x->num].start = start; - x->regs[x->num].end = end; - x->regs[x->num].hint = hint; - x->regs[x->num].flags = flags; - x->regs[x->num].root = root; - x->regs[x->num].ref = ref; - x->regs[x->num].op = op; - x->num++; -} - -static ir_reg _get_free_reg(ir_type type, ir_regset available) -{ - if (IR_IS_TYPE_INT(type)) { - available = IR_REGSET_INTERSECTION(available, IR_REGSET_GP); - } else { - IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); - available = IR_REGSET_INTERSECTION(available, IR_REGSET_FP); - } - if (!IR_REGSET_IS_EMPTY(available)) { - return IR_REGSET_FIRST(available); - } else { - return IR_REG_NONE; - } -} - -static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data *x, int j) -{ - int n; - ir_regset available; - ir_reg reg; - - if (IR_IS_TYPE_INT(type)) { - available = IR_REGSET_GP; - if (ctx->flags & IR_USE_FRAME_POINTER) { - IR_REGSET_EXCL(available, IR_REG_FRAME_POINTER); - } - -#if defined(IR_TARGET_X86) - if (ir_type_size[type] == 1) { - /* TODO: if no registers avialivle, we may use of one this register for already allocated interval ??? */ - IR_REGSET_EXCL(available, IR_REG_RBP); - IR_REGSET_EXCL(available, IR_REG_RSI); - IR_REGSET_EXCL(available, IR_REG_RDI); - } -#endif - } else { - IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); - available = IR_REGSET_FP; - } - for (n = x->regs[j].start; n < x->regs[j].end; n++) { - available = IR_REGSET_DIFFERENCE(available, x->clobbered[n]); - } - if (IR_REGSET_IS_EMPTY(available)) { - fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n"); - IR_ASSERT(0); - exit(-1); - } - - reg = IR_REGSET_FIRST(available); - if (IR_REGSET_IN(x->preserved_regs, reg)) { - ir_regset tmp = (ir_regset)ctx->used_preserved_regs; - IR_REGSET_INCL(tmp, reg); - ctx->used_preserved_regs = tmp; - } - return reg; -} - -static void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op, int8_t reg) -{ - char key[10]; - - if (!ctx->fused_regs) { - ctx->fused_regs = ir_mem_malloc(sizeof(ir_strtab)); - ir_strtab_init(ctx->fused_regs, 8, 128); - } - memcpy(key, &root, sizeof(ir_ref)); - memcpy(key + 4, &ref_and_op, sizeof(ir_ref)); - ir_strtab_lookup(ctx->fused_regs, key, 8, 0x10000000 | (uint8_t)reg); -} - -static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load) -{ - ir_use_list *use_list = &ctx->use_lists[load]; - ir_ref *p, use, i, n = use_list->count; - ir_ref last_use = IR_UNUSED; - ir_insn *insn; - int address_taken = -1; /* unknown */ - - if (n) { - /* Check if all uses of the result of VLOAD are in the same basic block (after VLOAD) */ - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - use = *p; - if (use < load || use > bb->end) return 0; - if (use > last_use) last_use = use; - } - /* Check if the result of VLOAD can not be clobbered between VLOAD and the last use */ - for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) { - if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) { - return 0; - } else if (address_taken - && (insn->op == IR_STORE - || insn->op == IR_STORE_v - || insn->op == IR_CALL)) { - if (address_taken == -1) { - ir_use_list *use_list = &ctx->use_lists[var]; - ir_ref *p, n = use_list->count; - - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - if (ctx->ir_base[*p].op == IR_VADDR) { - return 0; - } - } - address_taken = 0; - } - } - n = ir_insn_len(insn); - i += n; - insn += n; - } - } - return 1; -} - -static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref store, ir_ref val) -{ - ir_ref i, n; - ir_insn *insn; - int address_taken = -1; /* unknown */ - - /* Check if the value stored by VSTORE is produced in the same basic block (before VSTORE) */ - if (val < bb->start || val > store || store > bb->end) return 0; - - /* Check if the instructions between the "producer" and VSTORE can not access the same VAR */ - for (i = val, insn = &ctx->ir_base[i]; i < store;) { - if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) - && insn->op2 == var) { - return 0; - } else if (address_taken - && (insn->op == IR_LOAD - || insn->op == IR_LOAD_v - || insn->op == IR_STORE - || insn->op == IR_STORE_v - || insn->op == IR_CALL)) { - if (address_taken == -1) { - ir_use_list *use_list = &ctx->use_lists[var]; - ir_ref *p, n = use_list->count; - - for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { - if (ctx->ir_base[*p].op == IR_VADDR) { - return 0; - } - } - address_taken = 0; - } - } - n = ir_insn_len(insn); - i += n; - insn += n; - } - return 1; -} - -static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_simple_data *x) -{ - ir_ref stack[4]; - int stack_pos = 0; - ir_target_constraints constraints; - ir_insn *insn; - uint32_t j, n, flags, def_flags; - ir_ref *p, child; - - while (1) { - IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED); - - if (!(ctx->rules[input] & IR_SIMPLE)) { - def_flags = ir_get_target_constraints(ctx, input, &constraints); - n = constraints.tmps_count; - while (n > 0) { - n--; - if (constraints.tmp_regs[n].type) { - ir_reg flags = 0; - ir_ref op = constraints.tmp_regs[n].num; - - if (op > 0 && op <= ctx->ir_base[input].inputs_count) { - ir_ref *ops = ctx->ir_base[input].ops; - - if (IR_IS_CONST_REF(ops[op])) { - /* rematerialization */ - flags = IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { - /* local address rematerialization */ - flags = IR_REG_SPILL_LOAD; - } - } - _add_reg(x, constraints.tmp_regs[n].type, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags, - IR_UNUSED, input, op); - } else { - _add_scratch(x, constraints.tmp_regs[n].reg, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); - } - } - } else { - def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; - constraints.hints_count = 0; - } - - insn = &ctx->ir_base[input]; - flags = ir_op_flags[insn->op]; - n = IR_INPUT_EDGES_COUNT(flags); - j = 1; - p = insn->ops + j; - if (flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) { - j++; - p++; - } - for (; j <= n; j++, p++) { - IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); - child = *p; - if (child > 0) { - if (ctx->vregs[child] > 0) { - if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) { - ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; - int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF; - - _add_reg(x, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD, - ref, input, j); - } - } else if (ctx->rules[child] & IR_FUSED) { - IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos))); - stack[stack_pos++] = child; - } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) { - ctx->regs[input][j] = ctx->ir_base[child].op2; - } - } - } - if (!stack_pos) { - break; - } - input = stack[--stack_pos]; - } -} - -static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn) -{ - size_t size = ir_get_type_size(var_insn->type); - size_t align = var_insn->op3; - int32_t offset; - -#if IR_SIMD - if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) { - size = 4; - align = IR_MAX(align, 4); - } -#endif - - offset = ir_allocate_spill_slot(ctx, size, align); - ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); -} - -int ir_reg_alloc_simple(ir_ctx *ctx) -{ - ir_reg_alloc_data data; - uint32_t b; - ir_block *bb; - ir_insn *insn; - ir_ref i, n, j, *p; - uint32_t *rule, insn_flags; - ir_target_constraints constraints; - uint32_t def_flags; - ir_reg reg; - ir_regset scratch; - ir_reg_alloc_simple_data x; - int32_t offset; - - memset(&data, 0, sizeof(data)); - data.cc = ir_get_call_conv_dsc(ctx->flags); - ctx->data = &data; - - ctx->stack_frame_size = 0; - ctx->call_stack_size = 0; - ctx->used_preserved_regs = 0; - ctx->used_preserved_regs = ctx->fixed_save_regset; - - scratch = ir_scratch_regset[data.cc->scratch_reg - IR_REG_NUM]; - x.preserved_regs = IR_REGSET_DIFFERENCE(data.cc->preserved_regs, ctx->fixed_save_regset); - - ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count); - memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count); - - /* vregs + tmp + fixed + SRATCH + ALL */ - ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*)); - - if (!ctx->arena) { - ctx->arena = ir_arena_create(16 * 1024); - } - - for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) { - IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE)); - for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) { - if (*rule & (IR_FUSED|IR_SKIPPED)) { - if ((*rule & IR_RULE_MASK) == IR_ALLOCA) { - if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) { - /* spill slot already allocated */ - } else if (insn->op == IR_VAR) { - if (ctx->use_lists[i].count > 0) { - ir_allocate_var_spill_slot(ctx, i, insn); - } - } else if (insn->op == IR_ALLOCA) { - if (ctx->use_lists[i].count > 0) { - ir_insn *val = &ctx->ir_base[insn->op2]; - - IR_ASSERT(IR_IS_CONST_REF(insn->op2)); - IR_ASSERT(IR_IS_TYPE_INT(val->type)); - IR_ASSERT(!IR_IS_SYM_CONST(val->op)); - IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); - IR_ASSERT(val->val.i64 < 0x7fffffff); - offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); - ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset); - } - } else if (insn->op == IR_VADDR) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) { - ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]); - } - ctx->vregs[i] = ctx->vregs[insn->op1]; - } - } - } else { - x.num = 0; - for (j = 0; j < IR_SUB_REFS_COUNT; j++) { - x.clobbered[j] = IR_REGSET_EMPTY; - } - - def_flags = ir_get_target_constraints(ctx, i, &constraints); - n = constraints.tmps_count; - while (n) { - n--; - - IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT); - if (constraints.tmp_regs[n].type) { - ir_reg flags = 0; - ir_ref op = constraints.tmp_regs[n].num; - - if (op > 0 && op <= insn->inputs_count) { - ir_ref *ops = insn->ops; - - if (IR_IS_CONST_REF(ops[op])) { - /* rematerialization */ - flags = IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { - /* local address rematerialization */ - flags = IR_REG_SPILL_LOAD; - } - } - _add_reg(&x, constraints.tmp_regs[n].type, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags, - IR_UNUSED, i, op); - } else { - _add_scratch(&x, constraints.tmp_regs[n].reg, - constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); - } - } - - if (ctx->vregs[i] > 0) { - reg = constraints.def_reg; - if (!ctx->live_intervals[ctx->vregs[i]]) { - do { - ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); - memset(ival, 0, sizeof(ir_live_interval)); - ctx->live_intervals[ctx->vregs[i]] = ival; - ival->type = insn->type; - ival->reg = IR_REG_NONE; - ival->vreg = ctx->vregs[i]; - ival->stack_spill_pos = -1; - - if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { - ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); - } - if (ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) { - ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]); - reg = IR_REG_NONE; - def_flags &= ~IR_USE_MUST_BE_IN_REG; - break; - } - } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) { - ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - break; - } - - if (ctx->use_lists[i].count == 1) { - ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; - ir_insn *use_insn = &ctx->ir_base[use]; - - if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) - && use_insn->op3 == i - && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[use_insn->op2])) { - ir_allocate_var_spill_slot(ctx, use_insn->op2, &ctx->ir_base[use_insn->op2]); - } - ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[use_insn->op2]); - break; - } - } - - size_t size = ir_get_type_size(ival->type); -#if IR_SIMD - if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { - size = 4; - } -#endif - ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); - } while (0); - } - - if (def_flags & IR_USE_MUST_BE_IN_REG) { - ir_live_pos def_pos; - - if (reg != IR_REG_NONE) { - def_pos = IR_SAVE_SUB_REF; - } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - def_pos = IR_USE_SUB_REF; - } else { - def_pos = IR_LOAD_SUB_REF; - } - } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - def_pos = IR_LOAD_SUB_REF; - } else { - if (insn->op == IR_PARAM) { - /* We may reuse parameter stack slot for spilling */ - ctx->live_intervals[ctx->vregs[i]]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - } - def_pos = IR_DEF_SUB_REF; - } - - _add_reg(&x, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE, - IR_UNUSED, i, 0); - } - } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { - if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { - ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); - } - } - - n = insn->inputs_count; - insn_flags = ir_op_flags[insn->op]; - j = 1; - p = insn->ops + 1; - if (insn_flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) { - j++; - p++; - } - for (; j <= n; j++, p++) { - ir_ref input = *p; - ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; - ir_live_pos use_pos; - uint32_t use_flags = IR_USE_FLAGS(def_flags, j); - - if (reg != IR_REG_NONE) { -#if IR_X86_I64 - if (use_flags & IR_HINT_TWO_REGS) { - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); - } else -#endif - IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], reg); - } - - if (input > 0) { - if (ctx->vregs[input] > 0) { - use_pos = IR_USE_SUB_REF; - if (reg != IR_REG_NONE) { - use_pos = IR_LOAD_SUB_REF; - } else if (def_flags & IR_DEF_REUSES_OP1_REG) { - if (j == 1) { - if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { - use_pos = IR_USE_SUB_REF; - } else { - use_pos = IR_LOAD_SUB_REF; - } - } else if (input == insn->op1) { - /* Input is the same as "op1" */ - use_pos = IR_LOAD_SUB_REF; - } - } - if (use_flags & IR_USE_MUST_BE_IN_REG) { - _add_reg(&x, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD, - IR_UNUSED, i, j); - } - } else { - if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) { - ir_add_fusion_data(ctx, i, input, &x); - } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { - ctx->regs[i][j] = ctx->ir_base[input].op2; - } - } - } - } - - for (j = 0; j < x.num; j++) { - ir_regset available = scratch; -#if IR_X86_I64 - ir_reg reg2; -#endif - - for (n = x.regs[j].start; n < x.regs[j].end; n++) { - available = IR_REGSET_DIFFERENCE(available, x.clobbered[n]); - } - reg = x.regs[j].hint; -#if IR_X86_I64 - reg2 = IR_REG_NONE; - if (reg != IR_REG_NONE && (x.regs[j].type == IR_I64 || x.regs[j].type == IR_U64)) { - reg2 = IR_REG_I64_HI(reg); - reg = IR_REG_I64_LO(reg); - } -#endif - if (reg == IR_REG_NONE || !IR_REGSET_IN(available, reg)) { - reg = _get_free_reg(x.regs[j].type, available); - if (UNEXPECTED(reg == IR_REG_NONE)) { - reg = _get_free_reg2(ctx, x.regs[j].type, &x, j); - } - } - for (n = x.regs[j].start; n < x.regs[j].end; n++) { - IR_REGSET_INCL(x.clobbered[n], reg); - } -#if IR_X86_I64 - if (x.regs[j].type == IR_I64 || x.regs[j].type == IR_U64) { - IR_REGSET_EXCL(available, reg); - if (reg2 == IR_REG_NONE || !IR_REGSET_IN(available, reg2)) { - reg2 = _get_free_reg(x.regs[j].type, available); - if (UNEXPECTED(reg2 == IR_REG_NONE)) { - reg2 = _get_free_reg2(ctx, x.regs[j].type, &x, j); - } - } - for (n = x.regs[j].start; n < x.regs[j].end; n++) { - IR_REGSET_INCL(x.clobbered[n], reg2); - } - if (reg > reg2) { - SWAP_REGS(reg, reg2); - } - reg = IR_REG_I64_PAIR(reg, reg2); - } -#endif - reg = reg | x.regs[j].flags; - if (x.regs[j].op == 4 && insn->inputs_count < 4) { - if (!ctx->tmp_regs) { - ctx->tmp_regs = ir_mem_malloc(ctx->insns_count); - memset(ctx->tmp_regs, -1, ctx->insns_count); - } - ctx->tmp_regs[x.regs[j].ref] = reg; - } else if (!x.regs[j].root || ctx->regs[x.regs[j].ref][x.regs[j].op] == IR_REG_NONE) { - ctx->regs[x.regs[j].ref][x.regs[j].op] = reg; - } else if (ctx->regs[x.regs[j].ref][x.regs[j].op] != reg) { - ctx->rules[x.regs[j].ref] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, x.regs[j].root, x.regs[j].ref * sizeof(ir_ref) + x.regs[j].op, reg); - } - } - } - - n = ir_insn_len(insn); - i += n; - insn += n; - rule += n; - } - if (bb->flags & IR_BB_DESSA_MOVES) { - ir_gen_dessa_moves(ctx, b, ir_fix_dessa_tmps, (void*)(intptr_t)b); - } - } - -#ifdef IR_TARGET_X86 - if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); - } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) - && data.cc->fp_ret_reg == IR_REG_NONE) { - ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); - } else { - ctx->ret_slot = -1; - } -#endif - - ctx->flags |= IR_NO_STACK_COMBINE; - ir_fix_stack_frame(ctx); - ctx->data = NULL; - - return 1; -} diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 85ed6c74b7e5..0ea607a66bc8 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -3945,7 +3945,9 @@ IR_FOLD(MUL(MUL, C_ADDR)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x * c1) * c2 => x * (c1 * c2) */ + uint32_t shift = (8 - ir_type_size[op2_insn->type]) * 8; val.u64 = ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64; + val.u64 = (val.u64 << shift) >> shift; op1 = op1_insn->op1; op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; @@ -3960,7 +3962,9 @@ IR_FOLD(MUL(MUL, C_I64)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x * c1) * c2 => x * (c1 * c2) */ - val.i64 = ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64; + uint32_t shift = (8 - ir_type_size[op2_insn->type]) * 8; + val.u64 = ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64; + val.i64 = (int64_t)(val.u64 << shift) >> shift; op1 = op1_insn->op1; op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; diff --git a/ext/opcache/jit/ir/ir_perf.c b/ext/opcache/jit/ir/ir_perf.c index 52be820088a6..dca2b0eef080 100644 --- a/ext/opcache/jit/ir/ir_perf.c +++ b/ext/opcache/jit/ir/ir_perf.c @@ -167,6 +167,7 @@ int ir_perf_jitdump_open(void) if (jitdump_mem == MAP_FAILED) { close(jitdump_fd); jitdump_fd = -1; + unlink(filename); return 0; } @@ -179,6 +180,11 @@ int ir_perf_jitdump_open(void) jit_hdr.time_stamp = ir_perf_timestamp(); jit_hdr.flags = 0; if (write(jitdump_fd, &jit_hdr, sizeof(jit_hdr)) != sizeof(jit_hdr)) { + munmap(jitdump_mem, sysconf(_SC_PAGESIZE)); + jitdump_mem = MAP_FAILED; + close(jitdump_fd); + jitdump_fd = -1; + unlink(filename); return 0; } return 1; diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index 950b6535f577..028acd66cb7d 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -1320,12 +1320,13 @@ struct _ir_live_range { #define IR_LIVE_INTERVAL_TEMP (1<<1) #define IR_LIVE_INTERVAL_HAS_HINT_REGS (1<<2) #define IR_LIVE_INTERVAL_HAS_HINT_REFS (1<<3) -#define IR_LIVE_INTERVAL_MEM_PARAM (1<<4) +#define IR_LIVE_INTERVAL_MEM_PARAM (1<<4) /* PARAM passed on stack */ #define IR_LIVE_INTERVAL_COALESCED (1<<5) #define IR_LIVE_INTERVAL_SPILL_SPECIAL (1<<6) /* spill slot is pre-allocated in a special area (see ir_ctx.spill_reserved_base) */ #define IR_LIVE_INTERVAL_SPILLED (1<<7) #define IR_LIVE_INTERVAL_SPLIT_CHILD (1<<8) #define IR_LIVE_INTERVAL_TWO_REGS (1<<9) +#define IR_LIVE_INTERVAL_VAR_MEM_PARAM (1<<10) /* PARAM stack slot reused by VAR used in VSTORE(_, VAR, PARAM) */ /* macros to encode and decode stack offset stored in ctx->vregs[] */ #define IR_VREG_IS_STACK_SLOT(vreg) ((vreg) < 0) @@ -1536,6 +1537,8 @@ IR_ALWAYS_INLINE int8_t ir_get_alocated_reg(const ir_ctx *ctx, ir_ref ref, int o #define IR_MAY_REUSE (1U<<25) /* Result may reuse register of the source */ #define IR_TWO_REGS (1U<<24) /* Result needs two registers (used for 64-bit integers on x86) */ +#define IR_STATIC_ALLOCA (IR_SKIPPED | IR_FUSED | IR_SIMPLE | IR_ALLOCA) + #define IR_RULE_MASK 0xffff #define IR_MAX_REG_ARGS 64 @@ -1565,8 +1568,9 @@ typedef struct { #define IR_SCRATCH_REG(_reg, _start, _end) \ (ir_tmp_reg){.reg=(_reg), .type=IR_VOID, .start=(_start), .end=(_end)} -int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *constraints); +int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *constraints); +void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ref op, int8_t reg); void ir_fix_stack_frame(ir_ctx *ctx); /* Utility */ diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index 6cfc779e6c3c..9589c9cbd15c 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -798,18 +798,25 @@ int ir_compute_live_ranges(ir_ctx *ctx) } else { if (insn->op == IR_PARAM) { /* We may reuse parameter stack slot for spilling */ - ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) - if (param_ival->use_pos) { - ir_use_pos *use_pos = param_ival->use_pos; - ir_ref use = IR_LIVE_POS_TO_REF(use_pos->pos); - ir_insn *use_insn = &ctx->ir_base[use]; - if (use_insn->op == IR_VSTORE) { - /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */ - param_ival->use_pos = use_pos->next; + ir_live_interval *param_ival = ctx->live_intervals[v]; + + param_ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; + if (ctx->rules && ctx->use_lists[ref].count > 0) { + ir_ref use = ctx->use_edges[ctx->use_lists[ref].refs]; + if (use < bb->end) { + ir_insn *use_insn = &ctx->ir_base[use]; + if (use_insn->op == IR_VSTORE + && !(ctx->rules[use] & IR_SKIPPED) + /* first use of VAR is VSTORE(_, VAR, PARAM) */ + && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) { + IR_ASSERT(use_insn->op3 == ref); + /* skip VSTORE (VAR is going to be remapped to PARAM) */ + param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + param_ival->use_pos = NULL; + ctx->rules[use] = IR_SKIPPED | IR_NOP; + } } } -#endif } def_pos = IR_DEF_LIVE_POS_FROM_REF(ref); } @@ -1512,17 +1519,22 @@ int ir_compute_live_ranges(ir_ctx *ctx) ir_live_interval *param_ival = ctx->live_intervals[v]; param_ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; -#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) - if (param_ival->use_pos) { - ir_use_pos *use_pos = param_ival->use_pos; - ir_ref use = IR_LIVE_POS_TO_REF(use_pos->pos); - ir_insn *use_insn = &ctx->ir_base[use]; - if (use_insn->op == IR_VSTORE) { - /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */ - param_ival->use_pos = use_pos->next; + if (ctx->rules && ctx->use_lists[ref].count > 0) { + ir_ref use = ctx->use_edges[ctx->use_lists[ref].refs]; + if (use < bb->end) { + ir_insn *use_insn = &ctx->ir_base[use]; + if (use_insn->op == IR_VSTORE + && !(ctx->rules[use] & IR_SKIPPED) + /* first use of VAR is VSTORE(_, VAR, PARAM) */ + && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) { + IR_ASSERT(use_insn->op3 == ref); + /* skip VSTORE (VAR is going to be remapped to PARAM) */ + param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + param_ival->use_pos = NULL; + ctx->rules[use] = IR_SKIPPED | IR_NOP; + } } } -#endif } def_pos = IR_DEF_LIVE_POS_FROM_REF(ref); } @@ -2267,6 +2279,12 @@ int ir_compute_dessa_moves(ir_ctx *ctx) use = *p; insn = &ctx->ir_base[use]; if (insn->op == IR_PHI) { + if (ctx->use_lists[use].count == 0) { + ctx->rules[use] = IR_SKIPPED | IR_NOP; + ctx->vregs[use] = 0; + continue; + } + IR_ASSERT(ctx->vregs[use] > 0); for (j = 2; j <= k; j++) { ir_ref input = ir_insn_op(insn, j); @@ -4355,19 +4373,6 @@ static bool needs_spill_load(ir_ctx *ctx, ir_live_interval *ival, ir_use_pos *us return use_pos->next && use_pos->next->op_num != 0; } -static void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op, int8_t reg) -{ - char key[10]; - - if (!ctx->fused_regs) { - ctx->fused_regs = ir_mem_malloc(sizeof(ir_strtab)); - ir_strtab_init(ctx->fused_regs, 8, 128); - } - memcpy(key, &root, sizeof(ir_ref)); - memcpy(key + 4, &ref_and_op, sizeof(ir_ref)); - ir_strtab_lookup(ctx->fused_regs, key, 8, 0x10000000 | (uint8_t)reg); -} - static void assign_regs(ir_ctx *ctx) { ir_ref i; @@ -4548,7 +4553,7 @@ static void assign_regs(ir_ctx *ctx) old_reg = ir_get_alocated_reg(ctx, -use_pos->hint_ref, use_pos->op_num); if ((old_reg != IR_REG_NONE && reg != old_reg) || reg == IR_REG_NONE) { ctx->rules[-use_pos->hint_ref] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, ref, -use_pos->hint_ref * sizeof(ir_ref) + use_pos->op_num, reg); + ir_set_fused_reg(ctx, ref, -use_pos->hint_ref, use_pos->op_num, reg); use_pos = use_pos->next; continue; } @@ -4574,7 +4579,7 @@ static void assign_regs(ir_ctx *ctx) } else if (use_pos->hint_ref < 0 && !(use_pos->flags & IR_PHI_USE)) { IR_ASSERT(ctx->rules[-use_pos->hint_ref] & IR_FUSED); ctx->rules[-use_pos->hint_ref] |= IR_FUSED_REG; - ir_set_fused_reg(ctx, ref, -use_pos->hint_ref * sizeof(ir_ref) + use_pos->op_num, IR_REG_NONE); + ir_set_fused_reg(ctx, ref, -use_pos->hint_ref, use_pos->op_num, IR_REG_NONE); } use_pos = use_pos->next; } @@ -4603,7 +4608,7 @@ static void assign_regs(ir_ctx *ctx) if (IR_IS_CONST_REF(ops[ival->tmp_op_num])) { /* constant rematerialization */ reg |= IR_REG_SPILL_LOAD; - } else if (ctx->rules[ops[ival->tmp_op_num]] == (IR_SKIPPED|IR_FUSED|IR_SIMPLE|IR_ALLOCA) + } else if (ctx->rules[ops[ival->tmp_op_num]] == IR_STATIC_ALLOCA && IR_OPND_KIND(ir_op_flags[insn->op], ival->tmp_op_num) == IR_OPND_DATA) { /* local address rematerialization */ reg |= IR_REG_SPILL_LOAD; diff --git a/ext/opcache/jit/ir/ir_ra_simple.c b/ext/opcache/jit/ir/ir_ra_simple.c new file mode 100644 index 000000000000..4ecdd79986b0 --- /dev/null +++ b/ext/opcache/jit/ir/ir_ra_simple.c @@ -0,0 +1,662 @@ +/* + * IR - Lightweight JIT Compilation Framework + * Simple Local Register Allocation (used for -O0) + * This file is part of the IR Project distributed under the MIT-style LICENSE. + * Authors: Dmitry Stogov + * + * Simple RA assign registers for each IR instruction separately and always pass + * values between instrutios in stack spill slots. + */ + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif + +#include "ir.h" + +#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) +# include "ir_x86.h" +#elif defined(IR_TARGET_AARCH64) +# include "ir_aarch64.h" +#else +# error "Unknown IR target" +#endif + +#include "ir_private.h" + +typedef struct { + ir_reg_alloc_data ra_data; + int32_t num; + ir_regset preserved_regs; + ir_regset clobbered[IR_SUB_REFS_COUNT]; + struct { + uint8_t type; + int8_t start; + int8_t end; + int8_t hint; + int8_t flags; + ir_ref root; + ir_ref ref; + ir_ref op; + } regs[32]; +} ir_reg_alloc_simple_data; + +static void _add_scratch(ir_reg_alloc_simple_data *data, ir_reg reg, int8_t start, int8_t end) +{ + int8_t j; + + if (start < 0) start = 0; // TODO: ARGVAL support ??? + IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT); + if (reg >= IR_REG_NUM) { + for (j = start; j < end; j++) { + data->clobbered[j] = IR_REGSET_UNION(data->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]); + } + } else { + for (j = start; j < end; j++) { + IR_REGSET_INCL(data->clobbered[j], reg); + } + } +} + +static void _add_reg(ir_reg_alloc_simple_data *data, ir_type type, + int8_t start, int8_t end, ir_reg hint, int8_t flags, + ir_ref root, ir_ref ref, ir_ref op) +{ + IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32); + data->regs[data->num].type = type; + data->regs[data->num].start = start; + data->regs[data->num].end = end; + data->regs[data->num].hint = hint; + data->regs[data->num].flags = flags; + data->regs[data->num].root = root; + data->regs[data->num].ref = ref; + data->regs[data->num].op = op; + data->num++; +} + +static ir_reg _get_free_reg(ir_type type, ir_regset available) +{ + if (IR_IS_TYPE_INT(type)) { + available = IR_REGSET_INTERSECTION(available, IR_REGSET_GP); + } else { + IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); + available = IR_REGSET_INTERSECTION(available, IR_REGSET_FP); + } + if (!IR_REGSET_IS_EMPTY(available)) { + return IR_REGSET_FIRST(available); + } else { + return IR_REG_NONE; + } +} + +static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data *data, int j) +{ + int n; + ir_regset available; + ir_reg reg; + + if (IR_IS_TYPE_INT(type)) { + available = IR_REGSET_GP; + if (ctx->flags & IR_USE_FRAME_POINTER) { + IR_REGSET_EXCL(available, IR_REG_FRAME_POINTER); + } + +#if defined(IR_TARGET_X86) + if (ir_type_size[type] == 1) { + /* TODO: if no registers avialivle, we may use of one this register for already allocated interval ??? */ + IR_REGSET_EXCL(available, IR_REG_RBP); + IR_REGSET_EXCL(available, IR_REG_RSI); + IR_REGSET_EXCL(available, IR_REG_RDI); + } +#endif + } else { + IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type)); + available = IR_REGSET_FP; + } + for (n = data->regs[j].start; n < data->regs[j].end; n++) { + available = IR_REGSET_DIFFERENCE(available, data->clobbered[n]); + } + if (IR_REGSET_IS_EMPTY(available)) { + fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n"); + IR_ASSERT(0); + exit(-1); + } + + reg = IR_REGSET_FIRST(available); + if (IR_REGSET_IN(data->preserved_regs, reg)) { + ir_regset tmp = (ir_regset)ctx->used_preserved_regs; + IR_REGSET_INCL(tmp, reg); + ctx->used_preserved_regs = tmp; + } + return reg; +} + +static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load) +{ + ir_use_list *use_list = &ctx->use_lists[load]; + ir_ref *p, use, i, n = use_list->count; + ir_ref last_use = IR_UNUSED; + ir_insn *insn; + int address_taken = -1; /* unknown */ + + if (n) { + /* Check if all uses of the result of VLOAD are in the same basic block (after VLOAD) */ + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + use = *p; + if (use < load || use > bb->end) return 0; + if (use > last_use) last_use = use; + } + /* Check if the result of VLOAD can not be clobbered between VLOAD and the last use */ + for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) { + if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) { + return 0; + } else if (address_taken + && (insn->op == IR_STORE + || insn->op == IR_STORE_v + || insn->op == IR_CALL)) { + if (address_taken == -1) { + ir_use_list *use_list = &ctx->use_lists[var]; + ir_ref *p, n = use_list->count; + + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + if (ctx->ir_base[*p].op == IR_VADDR) { + return 0; + } + } + address_taken = 0; + } + } + n = ir_insn_len(insn); + i += n; + insn += n; + } + } + return 1; +} + +static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref store, ir_ref val) +{ + ir_ref i, n; + ir_insn *insn; + int address_taken = -1; /* unknown */ + + /* Check if the value stored by VSTORE is produced in the same basic block (before VSTORE) */ + if (val < bb->start || val > store || store > bb->end) return 0; + + /* Check if the instructions between the "producer" and VSTORE can not access the same VAR */ + for (i = val, insn = &ctx->ir_base[i]; i < store;) { + if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) + && insn->op2 == var) { + return 0; + } else if (address_taken + && (insn->op == IR_LOAD + || insn->op == IR_LOAD_v + || insn->op == IR_STORE + || insn->op == IR_STORE_v + || insn->op == IR_CALL)) { + if (address_taken == -1) { + ir_use_list *use_list = &ctx->use_lists[var]; + ir_ref *p, n = use_list->count; + + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + if (ctx->ir_base[*p].op == IR_VADDR) { + return 0; + } + } + address_taken = 0; + } + } + n = ir_insn_len(insn); + i += n; + insn += n; + } + return 1; +} + +static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_simple_data *data) +{ + ir_ref stack[4]; + int stack_pos = 0; + ir_target_constraints constraints; + ir_insn *insn; + uint32_t j, n, flags, def_flags; + ir_ref *p, child; + + while (1) { + IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED); + + if (!(ctx->rules[input] & IR_SIMPLE)) { + def_flags = ir_get_target_constraints(ctx, input, &constraints); + n = constraints.tmps_count; + while (n > 0) { + n--; + if (constraints.tmp_regs[n].type) { + ir_reg flags = 0; + ir_ref op = constraints.tmp_regs[n].num; + + if (op > 0 && op <= ctx->ir_base[input].inputs_count) { + ir_ref *ops = ctx->ir_base[input].ops; + + if (IR_IS_CONST_REF(ops[op])) { + /* rematerialization */ + flags = IR_REG_SPILL_LOAD; + } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { + /* local address rematerialization */ + flags = IR_REG_SPILL_LOAD; + } + } + _add_reg(data, constraints.tmp_regs[n].type, + constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags, + IR_UNUSED, input, op); + } else { + _add_scratch(data, constraints.tmp_regs[n].reg, + constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); + } + } + } else { + def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; + constraints.hints_count = 0; + } + + insn = &ctx->ir_base[input]; + flags = ir_op_flags[insn->op]; + n = IR_INPUT_EDGES_COUNT(flags); + j = 1; + p = insn->ops + j; + if (flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) { + j++; + p++; + } + for (; j <= n; j++, p++) { + IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA); + child = *p; + if (child > 0) { + if (ctx->vregs[child] > 0) { + if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) { + ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; + int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF; + + _add_reg(data, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD, + ref, input, j); + } + } else if (ctx->rules[child] & IR_FUSED) { + IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos))); + stack[stack_pos++] = child; + } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) { + ctx->regs[input][j] = ctx->ir_base[child].op2; + } + } + } + if (!stack_pos) { + break; + } + input = stack[--stack_pos]; + } +} + +static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn) +{ + size_t size = ir_get_type_size(var_insn->type); + size_t align = var_insn->op3; + int32_t offset; + +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) { + size = 4; + align = IR_MAX(align, 4); + } +#endif + + offset = ir_allocate_spill_slot(ctx, size, align); + ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset); +} + +int ir_reg_alloc_simple(ir_ctx *ctx) +{ + uint32_t b; + ir_block *bb; + ir_insn *insn; + ir_ref i, n, j, *p; + uint32_t *rule, insn_flags; + ir_target_constraints constraints; + uint32_t def_flags; + ir_reg reg; + ir_regset scratch; + ir_reg_alloc_simple_data data; + int32_t offset; + + ir_compute_dessa_moves(ctx); + + memset(&data.ra_data, 0, sizeof(data.ra_data)); + data.ra_data.cc = ir_get_call_conv_dsc(ctx->flags); + ctx->data = &data.ra_data; + + ctx->stack_frame_size = 0; + ctx->call_stack_size = 0; + ctx->used_preserved_regs = 0; + ctx->used_preserved_regs = ctx->fixed_save_regset; + + scratch = ir_scratch_regset[data.ra_data.cc->scratch_reg - IR_REG_NUM]; + data.preserved_regs = IR_REGSET_DIFFERENCE(data.ra_data.cc->preserved_regs, ctx->fixed_save_regset); + + ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count); + memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count); + + /* vregs + tmp + fixed + SRATCH + ALL */ + ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*)); + + if (!ctx->arena) { + ctx->arena = ir_arena_create(16 * 1024); + } + + for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) { + IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE)); + for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) { + if (*rule & (IR_FUSED|IR_SKIPPED)) { + if ((*rule & IR_RULE_MASK) == IR_ALLOCA) { + if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) { + /* spill slot already allocated */ + } else if (insn->op == IR_VAR) { + if (ctx->use_lists[i].count > 0) { + ir_allocate_var_spill_slot(ctx, i, insn); + } + } else if (insn->op == IR_ALLOCA) { + if (ctx->use_lists[i].count > 0) { + ir_insn *val = &ctx->ir_base[insn->op2]; + + IR_ASSERT(IR_IS_CONST_REF(insn->op2)); + IR_ASSERT(IR_IS_TYPE_INT(val->type)); + IR_ASSERT(!IR_IS_SYM_CONST(val->op)); + IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); + IR_ASSERT(val->val.i64 < 0x7fffffff); + offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3); + ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset); + } + } else if (insn->op == IR_VADDR) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) { + ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]); + } + ctx->vregs[i] = ctx->vregs[insn->op1]; + } + } + } else { + data.num = 0; + for (j = 0; j < IR_SUB_REFS_COUNT; j++) { + data.clobbered[j] = IR_REGSET_EMPTY; + } + + def_flags = ir_get_target_constraints(ctx, i, &constraints); + n = constraints.tmps_count; + while (n) { + n--; + + IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT); + if (constraints.tmp_regs[n].type) { + ir_reg flags = 0; + ir_ref op = constraints.tmp_regs[n].num; + + if (op > 0 && op <= insn->inputs_count) { + ir_ref *ops = insn->ops; + + if (IR_IS_CONST_REF(ops[op])) { + /* rematerialization */ + flags = IR_REG_SPILL_LOAD; + } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) { + /* local address rematerialization */ + flags = IR_REG_SPILL_LOAD; + } + } + _add_reg(&data, constraints.tmp_regs[n].type, + constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags, + IR_UNUSED, i, op); + } else { + _add_scratch(&data, constraints.tmp_regs[n].reg, + constraints.tmp_regs[n].start, constraints.tmp_regs[n].end); + } + } + + if (ctx->vregs[i] > 0) { + reg = constraints.def_reg; + if (!ctx->live_intervals[ctx->vregs[i]]) { + do { + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); + ctx->live_intervals[ctx->vregs[i]] = ival; + ival->type = insn->type; + ival->reg = IR_REG_NONE; + ival->vreg = ctx->vregs[i]; + ival->stack_spill_pos = -1; + + if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { + ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); + } + if (ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) { + ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]); + reg = IR_REG_NONE; + def_flags &= ~IR_USE_MUST_BE_IN_REG; + break; + } + } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) { + /* We may reuse parameter stack slot for spilling */ + ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM; + if (ctx->use_lists[i].count > 0) { + ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; + if (use < bb->end) { + ir_insn *use_insn = &ctx->ir_base[use]; + if (use_insn->op == IR_VSTORE + && !(ctx->rules[use] & IR_SKIPPED) + /* first use of VAR is VSTORE(_, VAR, PARAM) */ + && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) { + IR_ASSERT(use_insn->op3 == i); + /* skip VSTORE (VAR is going to be remapped to PARAM) */ + ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; + ival->use_pos = NULL; + ctx->rules[use] = IR_SKIPPED | IR_NOP; + } + } + } + break; + } + + if (ctx->use_lists[i].count == 1) { + ir_ref use = ctx->use_edges[ctx->use_lists[i].refs]; + ir_insn *use_insn = &ctx->ir_base[use]; + + if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) + && use_insn->op3 == i + && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[use_insn->op2])) { + ir_allocate_var_spill_slot(ctx, use_insn->op2, &ctx->ir_base[use_insn->op2]); + } + ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[use_insn->op2]); + break; + } + } + + size_t size = ir_get_type_size(ival->type); +#if IR_SIMD + if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) { + size = 4; + } +#endif + ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0); + } while (0); + } + + if (def_flags & IR_USE_MUST_BE_IN_REG) { + ir_live_pos def_pos; + + if (reg != IR_REG_NONE) { + def_pos = IR_SAVE_SUB_REF; + } else if (def_flags & IR_DEF_REUSES_OP1_REG) { + if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { + def_pos = IR_USE_SUB_REF; + } else { + def_pos = IR_LOAD_SUB_REF; + } + } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { + def_pos = IR_LOAD_SUB_REF; + } else { + def_pos = IR_DEF_SUB_REF; + } + + _add_reg(&data, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE, + IR_UNUSED, i, 0); + } + } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { + if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) { + ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]); + } + } + + n = insn->inputs_count; + insn_flags = ir_op_flags[insn->op]; + j = 1; + p = insn->ops + 1; + if (insn_flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) { + j++; + p++; + } + for (; j <= n; j++, p++) { + ir_ref input = *p; + ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE; + ir_live_pos use_pos; + uint32_t use_flags = IR_USE_FLAGS(def_flags, j); + + if (reg != IR_REG_NONE) { +#if IR_X86_I64 + if (use_flags & IR_HINT_TWO_REGS) { + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg)); + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg)); + } else +#endif + IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg); + } + + if (input > 0) { + if (ctx->vregs[input] > 0) { + use_pos = IR_USE_SUB_REF; + if (reg != IR_REG_NONE) { + use_pos = IR_LOAD_SUB_REF; + } else if (def_flags & IR_DEF_REUSES_OP1_REG) { + if (j == 1) { + if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) { + use_pos = IR_USE_SUB_REF; + } else { + use_pos = IR_LOAD_SUB_REF; + } + } else if (input == insn->op1) { + /* Input is the same as "op1" */ + use_pos = IR_LOAD_SUB_REF; + } + } + if (use_flags & IR_USE_MUST_BE_IN_REG) { + _add_reg(&data, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD, + IR_UNUSED, i, j); + } + } else { + if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) { + ir_add_fusion_data(ctx, i, input, &data); + } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { + ctx->regs[i][j] = ctx->ir_base[input].op2; + } + } + } + } + + for (j = 0; j < data.num; j++) { + ir_regset available = scratch; +#if IR_X86_I64 + ir_reg reg2; +#endif + + for (n = data.regs[j].start; n < data.regs[j].end; n++) { + available = IR_REGSET_DIFFERENCE(available, data.clobbered[n]); + } + reg = data.regs[j].hint; +#if IR_X86_I64 + reg2 = IR_REG_NONE; + if (reg != IR_REG_NONE && (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64)) { + reg2 = IR_REG_I64_HI(reg); + reg = IR_REG_I64_LO(reg); + } +#endif + if (reg == IR_REG_NONE || !IR_REGSET_IN(available, reg)) { + reg = _get_free_reg(data.regs[j].type, available); + if (UNEXPECTED(reg == IR_REG_NONE)) { + reg = _get_free_reg2(ctx, data.regs[j].type, &data, j); + } + } + for (n = data.regs[j].start; n < data.regs[j].end; n++) { + IR_REGSET_INCL(data.clobbered[n], reg); + } +#if IR_X86_I64 + if (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64) { + IR_REGSET_EXCL(available, reg); + if (reg2 == IR_REG_NONE || !IR_REGSET_IN(available, reg2)) { + reg2 = _get_free_reg(data.regs[j].type, available); + if (UNEXPECTED(reg2 == IR_REG_NONE)) { + reg2 = _get_free_reg2(ctx, data.regs[j].type, &data, j); + } + } + for (n = data.regs[j].start; n < data.regs[j].end; n++) { + IR_REGSET_INCL(data.clobbered[n], reg2); + } + if (reg > reg2) { + SWAP_REGS(reg, reg2); + } + reg = IR_REG_I64_PAIR(reg, reg2); + } +#endif + reg = reg | data.regs[j].flags; + if (data.regs[j].op == 4 && insn->inputs_count < 4) { + if (!ctx->tmp_regs) { + ctx->tmp_regs = ir_mem_malloc(ctx->insns_count); + memset(ctx->tmp_regs, -1, ctx->insns_count); + } + ctx->tmp_regs[data.regs[j].ref] = reg; + } else if (!data.regs[j].root || ctx->regs[data.regs[j].ref][data.regs[j].op] == IR_REG_NONE) { + ctx->regs[data.regs[j].ref][data.regs[j].op] = reg; + } else if (ctx->regs[data.regs[j].ref][data.regs[j].op] != reg) { + ctx->rules[data.regs[j].ref] |= IR_FUSED_REG; + ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg); + } + } + } + + n = ir_insn_len(insn); + i += n; + insn += n; + rule += n; + } + if (bb->flags & IR_BB_DESSA_MOVES) { +#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) + ctx->regs[bb->end][0] = IR_REG_RAX; + ctx->regs[bb->end][1] = IR_REG_XMM0; +#elif defined(IR_TARGET_AARCH64) + ctx->regs[bb->end][0] = IR_REG_X0; + ctx->regs[bb->end][1] = IR_REG_V0; +#else +# error "Unknown IR target" +#endif + } + } + +#ifdef IR_TARGET_X86 + if (ctx->flags2 & IR_HAS_FP_RET_SLOT) { + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0); + } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE) + && data.ra_data.cc->fp_ret_reg == IR_REG_NONE) { + ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0); + } else { + ctx->ret_slot = -1; + } +#endif + + ctx->flags |= IR_NO_STACK_COMBINE; + ir_fix_stack_frame(ctx); + ctx->data = NULL; + + return 1; +} diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index c14d34b6c4d4..6ff106d4cffd 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -3028,26 +3028,70 @@ static bool ir_cmp_is_true(ir_op op, const ir_insn *op1, const ir_insn *op2) } else if (op == IR_LT) { if (IR_IS_TYPE_SIGNED(op1->type)) { return op1->val.i64 < op2->val.i64; + } else if (op1->type == IR_TYPE_ADDR) { + return op1->val.addr < op2->val.addr; + } else if (ir_type_size[op1->type] == 8) { + return op1->val.i64 < op2->val.i64; + } else if (ir_type_size[op1->type] == 4) { + return op1->val.i32 < op2->val.i32; + } else if (ir_type_size[op1->type] == 4) { + return op1->val.i16 < op2->val.i16; + } else if (ir_type_size[op1->type] == 1) { + return op1->val.i8 < op2->val.i8; } else { - return op1->val.u64 < op2->val.u64; + IR_ASSERT(0); + return 0; } } else if (op == IR_GE) { if (IR_IS_TYPE_SIGNED(op1->type)) { return op1->val.i64 >= op2->val.i64; + } else if (op1->type == IR_TYPE_ADDR) { + return op1->val.addr >= op2->val.addr; + } else if (ir_type_size[op1->type] == 8) { + return op1->val.i64 >= op2->val.i64; + } else if (ir_type_size[op1->type] == 4) { + return op1->val.i32 >= op2->val.i32; + } else if (ir_type_size[op1->type] == 4) { + return op1->val.i16 >= op2->val.i16; + } else if (ir_type_size[op1->type] == 1) { + return op1->val.i8 >= op2->val.i8; } else { - return op1->val.u64 >= op2->val.u64; + IR_ASSERT(0); + return 0; } } else if (op == IR_LE) { if (IR_IS_TYPE_SIGNED(op1->type)) { return op1->val.i64 <= op2->val.i64; + } else if (op1->type == IR_TYPE_ADDR) { + return op1->val.addr <= op2->val.addr; + } else if (ir_type_size[op1->type] == 8) { + return op1->val.i64 <= op2->val.i64; + } else if (ir_type_size[op1->type] == 4) { + return op1->val.i32 <= op2->val.i32; + } else if (ir_type_size[op1->type] == 4) { + return op1->val.i16 <= op2->val.i16; + } else if (ir_type_size[op1->type] == 1) { + return op1->val.i8 <= op2->val.i8; } else { - return op1->val.u64 <= op2->val.u64; + IR_ASSERT(0); + return 0; } } else if (op == IR_GT) { if (IR_IS_TYPE_SIGNED(op1->type)) { return op1->val.i64 > op2->val.i64; + } else if (op1->type == IR_TYPE_ADDR) { + return op1->val.addr > op2->val.addr; + } else if (ir_type_size[op1->type] == 8) { + return op1->val.i64 > op2->val.i64; + } else if (ir_type_size[op1->type] == 4) { + return op1->val.i32 > op2->val.i32; + } else if (ir_type_size[op1->type] == 4) { + return op1->val.i16 > op2->val.i16; + } else if (ir_type_size[op1->type] == 1) { + return op1->val.i8 > op2->val.i8; } else { - return op1->val.u64 > op2->val.u64; + IR_ASSERT(0); + return 0; } } else if (op == IR_ULT) { return op1->val.u64 < op2->val.u64; diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 3f1a3be180f8..e69a785621c8 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -28,7 +28,7 @@ typedef uint64_t ir_mem; # define IR_MEM_NONE 0 # define IR_MEM_VAL(loc) (loc) -# define IR_MEM_ADD(mem, offset) ((mem) + (offset)) +# define IR_MEM_ADD(mem, offset) (((mem) & 0xffffffff00000000) | (((mem)+ (offset)) & 0xffffffff)) #endif #define IR_MEM_OFFSET(loc) ((int32_t)(IR_MEM_VAL(loc) & 0xffffffff)) @@ -1711,7 +1711,8 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = { _(CMP_I64) \ _(BINOP_I64) \ _(MUL_I64) \ - _(MUL_OV_I64) \ + _(IMUL_OV_I64) \ + _(UMUL_OV_I64) \ _(BINOP_HELPER_I64) \ _(OP_I64) \ _(SHIFT_I64) \ @@ -1743,8 +1744,6 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = { #define IR_RULE_ENUM(name) IR_ ## name, -#define IR_STATIC_ALLOCA (IR_SKIPPED | IR_FUSED | IR_SIMPLE | IR_ALLOCA) - enum _ir_rule { IR_FIRST_RULE = IR_LAST_OP, IR_RULES(IR_RULE_ENUM) @@ -2495,14 +2494,14 @@ get_arg_hints: ctx->flags2 |= IR_HAS_FP_RET_SLOT; break; case IR_MUL_I64: - case IR_MUL_OV_I64: + case IR_IMUL_OV_I64: flags = IR_USE_SHOULD_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_HINT_TWO_REGS; constraints->def_reg = IR_REG_I64_PAIR(IR_REG_RAX, IR_REG_RDX); constraints->tmp_regs[0] = IR_SCRATCH_REG(IR_REG_RAX, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); constraints->tmp_regs[1] = IR_SCRATCH_REG(IR_REG_RDX, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); constraints->tmp_regs[2] = IR_TMP_REG(3, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n = 3; - if ((rule & IR_RULE_MASK) == IR_MUL_OV_I64) { + if ((rule & IR_RULE_MASK) == IR_IMUL_OV_I64) { constraints->tmp_regs[3] = IR_TMP_REG(4, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n = 4; } @@ -2514,6 +2513,15 @@ get_arg_hints: constraints->hints_count = 2; } break; + case IR_UMUL_OV_I64: + flags = IR_USE_SHOULD_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_HINT_TWO_REGS; + constraints->def_reg = IR_REG_I64_PAIR(IR_REG_RAX, IR_REG_RDX); + constraints->tmp_regs[0] = IR_SCRATCH_REG(IR_REG_RAX, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + constraints->tmp_regs[1] = IR_SCRATCH_REG(IR_REG_RDX, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + constraints->tmp_regs[2] = IR_TMP_REG(3, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + constraints->tmp_regs[3] = IR_TMP_REG(4, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n = 4; + break; case IR_BIT_COUNT_I64: flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG; constraints->tmp_regs[0] = IR_TMP_REG(2, IR_U32, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); @@ -4102,9 +4110,12 @@ binop_fp: case IR_MUL_OV: IR_ASSERT(IR_IS_TYPE_INT(insn->type)); #if IR_X86_I64 - if (insn->type == IR_I64 || insn->type == IR_U64) { + if (insn->type == IR_I64) { ir_match_fuse_load(ctx, insn->op2, ref); - return IR_TWO_REGS | IR_MUL_OV_I64; + return IR_TWO_REGS | IR_IMUL_OV_I64; + } else if (insn->type == IR_U64) { + ir_match_fuse_load(ctx, insn->op2, ref); + return IR_TWO_REGS | IR_UMUL_OV_I64; } #endif if (IR_IS_TYPE_SIGNED(insn->type) && ir_type_size[insn->type] != 1) { @@ -6580,7 +6591,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) if (base_reg_ref) { if (UNEXPECTED(ctx->rules[base_reg_ref / sizeof(ir_ref)] & IR_FUSED_REG)) { - base_reg = ir_get_fused_reg(ctx, root, base_reg_ref); + base_reg = ir_get_fused_reg_ex(ctx, root, base_reg_ref); } else { base_reg = ((int8_t*)ctx->regs)[base_reg_ref]; } @@ -6599,7 +6610,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) index_reg = base_reg; } else { if (UNEXPECTED(ctx->rules[index_reg_ref / sizeof(ir_ref)] & IR_FUSED_REG)) { - index_reg = ir_get_fused_reg(ctx, root, index_reg_ref); + index_reg = ir_get_fused_reg_ex(ctx, root, index_reg_ref); } else { index_reg = ((int8_t*)ctx->regs)[index_reg_ref]; } @@ -6637,7 +6648,7 @@ static ir_mem ir_fuse_load(ir_ctx *ctx, ir_ref root, ir_ref ref) IR_ASSERT(load_insn->op == IR_LOAD || load_insn->op == IR_LOAD_v || load_insn->op == IR_VLOAD || load_insn->op == IR_VLOAD_v); if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { - reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 2); + reg = ir_get_fused_reg(ctx, root, ref, 2); } else { reg = ctx->regs[ref][2]; } @@ -7868,6 +7879,9 @@ static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn, uint32_t rule switch (ir_type_size[insn->type]) { default: IR_ASSERT(0); + case 2: + | rol Rw(def_reg), 8 + break; case 4: | bswap Rd(def_reg) break; @@ -8808,8 +8822,8 @@ static void ir_emit_cmp_int_common2(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ins ir_reg op1_reg, op2_reg; if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, root, ref, 1); + op2_reg = ir_get_fused_reg(ctx, root, ref, 2); } else { op1_reg = ctx->regs[ref][1]; op2_reg = ctx->regs[ref][2]; @@ -8982,8 +8996,8 @@ static void ir_emit_test_int_common(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_op ir_reg op1_reg, op2_reg; if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, root, ref, 1); + op2_reg = ir_get_fused_reg(ctx, root, ref, 2); } else { op1_reg = ctx->regs[ref][1]; op2_reg = ctx->regs[ref][2]; @@ -9102,7 +9116,7 @@ static void ir_emit_test_bit_common(ir_ctx *ctx, ir_ref root, ir_ref ref) bit = IR_LOG2(ctx->ir_base[binop_insn->op2].val.u64); if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1); + op1_reg = ir_get_fused_reg(ctx, root, ref, 1); } else { op1_reg = ctx->regs[ref][1]; } @@ -9162,8 +9176,8 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref root, ir_ref cmp_ref, ir_ op1 = cmp_insn->op1; op2 = cmp_insn->op2; if (UNEXPECTED(ctx->rules[cmp_ref] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, root, cmp_ref * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, root, cmp_ref * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, root, cmp_ref, 1); + op2_reg = ir_get_fused_reg(ctx, root, cmp_ref, 2); } else { op1_reg = ctx->regs[cmp_ref][1]; op2_reg = ctx->regs[cmp_ref][2]; @@ -9443,8 +9457,8 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i ir_reg op1_reg, op2_reg; if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1); + op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2); } else { op1_reg = ctx->regs[insn->op2][1]; op2_reg = ctx->regs[insn->op2][2]; @@ -9638,6 +9652,30 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, op1_type, op1_reg, op1); } + if (IR_IS_CONST_REF(op1)) { + if (!ir_const_is_true(&ctx->ir_base[insn->op1])) { + op2 = op3; + op2_reg = op3_reg; + } + + if (op2_reg != IR_REG_NONE) { + if (def_reg != op2_reg) { + if (IR_IS_TYPE_INT(type)) { + ir_emit_mov(ctx, type, def_reg, op2_reg); + } else { + ir_emit_fp_mov(ctx, type, def_reg, op2_reg); + } + } + } else { + ir_emit_load_ex(ctx, type, def_reg, op2, def); + } + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, type, def, def_reg); + } + return; + } + if (IR_IS_TYPE_INT(op1_type)) { if (op1_reg != IR_REG_NONE) { | ASM_REG_REG_OP test, op1_type, op1_reg, op1_reg @@ -14163,8 +14201,8 @@ static bool ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * ir_reg op1_reg, op2_reg; if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1); + op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2); } else { op1_reg = ctx->regs[insn->op2][1]; op2_reg = ctx->regs[insn->op2][2]; @@ -14894,8 +14932,8 @@ static void ir_emit_cmp_and_branch_i64(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i ir_reg op1_reg_hi, op2_reg_hi; if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1); + op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2); } else { op1_reg = ctx->regs[insn->op2][1]; op2_reg = ctx->regs[insn->op2][2]; @@ -15297,11 +15335,10 @@ static void ir_emit_mul_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } -static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) +static void ir_emit_imul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; - ir_type type = insn->type; ir_ref op1 = insn->op1; ir_ref op2 = insn->op2; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); @@ -15342,14 +15379,9 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(op2_reg != IR_REG_RDX && tmp_reg != op2_reg_hi && tmp_reg != op2_reg); - if (type == IR_I64) { - | mov Rd(tmp2_reg), Rd(op2_reg_hi) // expected sign = sign A ^ sign B - | xor Rd(tmp2_reg), edx - | and Rd(tmp2_reg), 0x80000000 - } else { - | xor Rd(tmp2_reg), Rd(tmp2_reg) - } - + | mov Rd(tmp2_reg), Rd(op2_reg_hi) // expected sign = sign A ^ sign B + | xor Rd(tmp2_reg), edx + | and Rd(tmp2_reg), 0x80000000 | mov Rd(tmp_reg), edx | imul Rd(tmp_reg), Rd(op2_reg_hi) // hi(Ah * Bh) == 0 | adc Rd(tmp2_reg), 0 @@ -15364,31 +15396,22 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) | adc Rd(tmp2_reg), 0 | mul Rd(op2_reg) // Al * Bl + c | add edx, Rd(tmp_reg) - - if (type == IR_I64) { - | adc Rd(tmp2_reg), 0 - | mov Rd(tmp_reg), edx // check sign of result - | xor Rd(tmp_reg), Rd(tmp2_reg) - | and Rd(tmp2_reg), 0x7f - | and Rd(tmp_reg), 0x80000000 - | add Rd(tmp_reg), -1 - | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0 - } else { - | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow - } + | adc Rd(tmp2_reg), 0 + | mov Rd(tmp_reg), edx // check sign of result + | xor Rd(tmp_reg), Rd(tmp2_reg) + | and Rd(tmp2_reg), 0x7f + | and Rd(tmp_reg), 0x80000000 + | add Rd(tmp_reg), -1 + | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0 } else if (IR_IS_CONST_REF(op2)) { ir_insn *val = &ctx->ir_base[op2]; - if (type == IR_I64) { - | mov Rd(tmp2_reg), edx // expected sign = sign A ^ sign B - if (val->val.i64 >= 0) { - | and Rd(tmp2_reg), 0x80000000 - } else { - | not Rd(tmp2_reg) - | and Rd(tmp2_reg), 0x80000000 - } - } else { - | xor Rd(tmp2_reg), Rd(tmp2_reg) + | mov Rd(tmp2_reg), edx // expected sign = sign A ^ sign B + if (val->val.i64 >= 0) { + | and Rd(tmp2_reg), 0x80000000 + } else { + | not Rd(tmp2_reg) + | and Rd(tmp2_reg), 0x80000000 } if (val->val.u32_hi) { @@ -15409,22 +15432,18 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) | imul Rd(tmp_reg), val->val.u32 // hi(Ah * Bl) == 0, c = lo(Ah * Bl) | adc Rd(tmp2_reg), 0 } + | mov edx, val->val.u32 // Al * Bl + c | mul edx | add edx, Rd(tmp_reg) | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow - - if (type == IR_I64) { - | adc Rd(tmp2_reg), 0 - | mov Rd(tmp_reg), edx // check sign of result - | xor Rd(tmp_reg), Rd(tmp2_reg) - | and Rd(tmp2_reg), 0x7f - | and Rd(tmp_reg), 0x80000000 - | add Rd(tmp_reg), -1 - | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0 - } else { - | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow - } + | adc Rd(tmp2_reg), 0 + | mov Rd(tmp_reg), edx // check sign of result + | xor Rd(tmp_reg), Rd(tmp2_reg) + | and Rd(tmp2_reg), 0x7f + | and Rd(tmp_reg), 0x80000000 + | add Rd(tmp_reg), -1 + | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0 } else { ir_mem mem, mem_hi; @@ -15435,14 +15454,9 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) } mem_hi = IR_MEM_I64_HI(mem); - if (type == IR_I64) { - | ASM_TXT_TMEM_OP mov, Rd(tmp2_reg), dword, mem_hi // expected sign = sign A ^ sign B - | xor Rd(tmp2_reg), edx - | and Rd(tmp2_reg), 0x80000000 - } else { - | xor Rd(tmp2_reg), Rd(tmp2_reg) - } - + | ASM_TXT_TMEM_OP mov, Rd(tmp2_reg), dword, mem_hi // expected sign = sign A ^ sign B + | xor Rd(tmp2_reg), edx + | and Rd(tmp2_reg), 0x80000000 | mov Rd(tmp_reg), edx | ASM_TXT_TMEM_OP imul, Rd(tmp_reg), dword, mem_hi // hi(Ah * Bh) == 0 | adc Rd(tmp2_reg), 0 @@ -15457,18 +15471,204 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) | adc Rd(tmp2_reg), 0 | ASM_TMEM_OP mul, dword, mem // Al * Bl + c | add edx, Rd(tmp_reg) + | adc Rd(tmp2_reg), 0 + | mov Rd(tmp_reg), edx // check sign of result + | xor Rd(tmp_reg), Rd(tmp2_reg) + | and Rd(tmp2_reg), 0x7f + | and Rd(tmp_reg), 0x80000000 + | add Rd(tmp_reg), -1 + | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0 + } - if (type == IR_I64) { - | adc Rd(tmp2_reg), 0 - | mov Rd(tmp_reg), edx // check sign of result - | xor Rd(tmp_reg), Rd(tmp2_reg) - | and Rd(tmp2_reg), 0x7f - | and Rd(tmp_reg), 0x80000000 - | add Rd(tmp_reg), -1 - | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0 - } else { - | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow + if (def_reg != IR_REG_NONE) { + def_reg_hi = IR_REG_I64_HI(def_reg); + def_reg = IR_REG_I64_LO(def_reg); + if (def_reg != IR_REG_RDX) { + if (def_reg != IR_REG_RAX) { + | mov Rd(def_reg), eax + } + if (def_reg_hi != IR_REG_RDX) { + | mov Rd(def_reg_hi), edx + } + } else { + if (def_reg_hi != IR_REG_RDX) { + | mov Rd(def_reg_hi), edx + } + if (def_reg != IR_REG_RAX) { + IR_ASSERT(def_reg_hi != IR_REG_RAX); + | mov Rd(def_reg), eax + } + } + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store_i64_lo(ctx, def, def_reg); + ir_emit_store_i64_hi(ctx, def, def_reg_hi); } + } else { + ir_emit_store_i64_lo(ctx, def, IR_REG_RAX); + ir_emit_store_i64_hi(ctx, def, IR_REG_RDX); + } +} + +static void ir_emit_umul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_ref op1 = insn->op1; + ir_ref op2 = insn->op2; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op1_reg = ctx->regs[def][1]; + ir_reg op2_reg = ctx->regs[def][2]; + ir_reg tmp_reg = ctx->regs[def][3]; + ir_reg tmp2_reg = ctx->tmp_regs[def]; + ir_reg def_reg_hi, op1_reg_hi, op2_reg_hi; + ir_mem op1_mem = IR_MEM_NONE, op1_mem_hi = IR_MEM_NONE, op2_mem = IR_MEM_NONE, op2_mem_hi = IR_MEM_NONE; + ir_val op1_val = {0}; + + if (IR_IS_CONST_REF(op1) && IR_IS_CONST_REF(op2)) { + op1_val.u64 = ctx->ir_base[op1].val.u64 * ctx->ir_base[op2].val.u64; + | mov edx, op1_val.u32_hi + | mov eax, op1_val.u32 + if (ctx->ir_base[op1].val.u64 == 0 || (op1_val.u64 / ctx->ir_base[op1].val.u64) == ctx->ir_base[op2].val.u64) { + | clc + } else { + | stc + } + } else { + if (op1_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + op1_reg_hi = IR_REG_I64_HI(op1_reg); + op1_reg = IR_REG_I64_LO(op1_reg); + ir_emit_load_i64_lo(ctx, op1_reg, op1); + ir_emit_load_i64_hi(ctx, op1_reg_hi, op1); + } else { + op1_reg_hi = IR_REG_I64_HI(op1_reg); + op1_reg = IR_REG_I64_LO(op1_reg); + } + if (op1 == op2) { + op2_reg_hi = op1_reg_hi; + op2_reg = op1_reg; + } + } else { + op1_reg_hi = IR_REG_NONE; + } + + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + op2_reg_hi = IR_REG_I64_HI(op2_reg); + op2_reg = IR_REG_I64_LO(op2_reg); + ir_emit_load_i64_lo(ctx, op2_reg, op2); + ir_emit_load_i64_hi(ctx, op2_reg_hi, op2); + } else { + op2_reg_hi = IR_REG_I64_HI(op2_reg); + op2_reg = IR_REG_I64_LO(op2_reg); + } + } else { + op2_reg_hi = IR_REG_NONE; + } + + if (op2_reg == IR_REG_NONE && (IR_IS_CONST_REF(op2) || op1_reg != IR_REG_NONE)) { + IR_ASSERT(op1_reg != IR_REG_NONE || !IR_IS_CONST_REF(op1)); + SWAP_REFS(op1, op2); + SWAP_REGS(op1_reg, op2_reg); + SWAP_REGS(op1_reg_hi, op2_reg_hi); + } + + if (op1_reg == IR_REG_NONE) { + if (IR_IS_CONST_REF(op1)) { + op1_val = ctx->ir_base[op1].val; + } else { + if (ir_rule(ctx, op1) & IR_FUSED) { + op1_mem = ir_fuse_load(ctx, def, op1); + } else { + op1_mem = ir_ref_spill_slot(ctx, op1); + } + op1_mem_hi = IR_MEM_I64_HI(op1_mem); + } + } + + if (op2_reg == IR_REG_NONE) { + IR_ASSERT(!IR_IS_CONST_REF(op2)); + if (ir_rule(ctx, op2) & IR_FUSED) { + op2_mem = ir_fuse_load(ctx, def, op2); + } else { + op2_mem = ir_ref_spill_slot(ctx, op2); + } + op2_mem_hi = IR_MEM_I64_HI(op2_mem); + } + + | xor Rd(tmp2_reg), Rd(tmp2_reg) + + if (op1_reg != IR_REG_NONE) { + | mov eax, Rd(op1_reg_hi) + } else if (IR_IS_CONST_REF(op1)) { + | mov eax, op1_val.u32_hi + } else { + | ASM_TXT_TMEM_OP mov, eax, dword, op1_mem_hi + } + + if (op2_reg != IR_REG_NONE) { + | mul Rd(op2_reg_hi) // hi(Ah * Bh) == 0 + } else { + | ASM_TMEM_OP mul, dword, op2_mem_hi // hi(Ah * Bh) == 0 + } + + | adc Rd(tmp2_reg), 0 + | add eax, -1 // lo(Ah * Bh) == 0 ; set C flag if tmp_reg != 0 + | adc Rd(tmp2_reg), 0 + + if (op1_reg != IR_REG_NONE) { + | mov eax, Rd(op1_reg_hi) + } else if (IR_IS_CONST_REF(op1)) { + | mov eax, op1_val.u32_hi + } else { + | ASM_TXT_TMEM_OP mov, eax, dword, op1_mem_hi + } + + if (op2_reg != IR_REG_NONE) { + | mul Rd(op2_reg) // hi(Ah * Bl) == 0 + } else { + | ASM_TMEM_OP mul, dword, op2_mem // hi(Ah * Bl) == 0 + } + + | adc Rd(tmp2_reg), 0 + | mov Rd(tmp_reg), eax // c = lo(Ah * Bl) + + if (op1_reg != IR_REG_NONE) { + | mov eax, Rd(op1_reg) + } else if (IR_IS_CONST_REF(op1)) { + | mov eax, op1_val.u32 + } else { + | ASM_TXT_TMEM_OP mov, eax, dword, op1_mem + } + if (op2_reg != IR_REG_NONE) { + | mul Rd(op2_reg_hi) // hi(Al * Bh) == 0 + } else { + | ASM_TMEM_OP mul, dword, op2_mem_hi // hi(Al * Bh) == 0 + } + + | adc Rd(tmp2_reg), 0 + | add Rd(tmp_reg), eax // c = lo(Ah * Bl) + lo(Al * Bh) + | adc Rd(tmp2_reg), 0 + + if (op1_reg != IR_REG_NONE) { + | mov eax, Rd(op1_reg) + } else if (IR_IS_CONST_REF(op1)) { + | mov eax, op1_val.u32 + } else { + | ASM_TXT_TMEM_OP mov, eax, dword, op1_mem + } + + if (op2_reg != IR_REG_NONE) { + | mul Rd(op2_reg) // Al * Bl + } else { + | ASM_TMEM_OP mul, dword, op2_mem // Al * Bl + } + + | add edx, Rd(tmp_reg) // Al * Bl + c + | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow } if (def_reg != IR_REG_NONE) { @@ -16849,8 +17049,8 @@ static void ir_emit_cond_cmp_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(tmp_reg != IR_REG_NONE); if (!IR_IS_CONST_REF(insn->op2) && UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, insn->op1 * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, def, insn->op1 * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); + op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); } else { op1_reg = ctx->regs[insn->op1][1]; op2_reg = ctx->regs[insn->op1][2]; @@ -17011,8 +17211,8 @@ static void ir_emit_cond_i64_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(tmp_reg != IR_REG_NONE); if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, insn->op1 * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, def, insn->op1 * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1); + op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2); } else { op1_reg = ctx->regs[insn->op1][1]; op2_reg = ctx->regs[insn->op1][2]; @@ -17480,8 +17680,8 @@ static bool ir_emit_guard_cmp_i64(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]); if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) { - op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1); - op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2); + op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1); + op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2); } else { op1_reg = ctx->regs[insn->op2][1]; op2_reg = ctx->regs[insn->op2][2]; @@ -25976,95 +26176,6 @@ static void ir_emit_load_params(ir_ctx *ctx) } } -static int ir_fix_dessa_tmps(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to, void *dessa_from_block) -{ - ir_ref ref = ctx->cfg_blocks[(intptr_t)dessa_from_block].end; - - if (to == 0) { - if (IR_IS_TYPE_INT(type)) { - if (ctx->regs[ref][0] == IR_REG_NONE) { - ctx->regs[ref][0] = IR_REG_RAX; - } - } else { - IR_ASSERT(IR_IS_TYPE_FP(type)); - if (ctx->regs[ref][1] == IR_REG_NONE) { - ctx->regs[ref][1] = IR_REG_XMM0; - } - } - } else if (from != 0) { - if (IR_IS_TYPE_INT(type)) { - if (ctx->regs[ref][0] == IR_REG_NONE) { - ctx->regs[ref][0] = IR_REG_RAX; - } - } else { - IR_ASSERT(IR_IS_TYPE_FP(type)); - if (ctx->regs[ref][1] == IR_REG_NONE) { - ctx->regs[ref][1] = IR_REG_XMM0; - } - } - } - return 1; -} - -/* This remaps spill slot of the VAR, where PARAM is stored, to the PARAMs slot. - * This eliminates copying, but more important, this fixes va_start() compilation - * on systems, where it's defined through address calculation macro (e.g. MSVC x86). - */ -static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_slot) -{ - ir_use_list *param_use_list = &ctx->use_lists[param]; - - if (param_use_list->count >= 1) { - ir_ref param_store = ctx->use_edges[param_use_list->refs]; - ir_insn *param_store_insn = &ctx->ir_base[param_store]; - - if (param_store_insn->op == IR_VSTORE) { - ir_ref var = param_store_insn->op2; - - IR_ASSERT(param_store_insn->op3 == param); - IR_ASSERT(ctx->ir_base[var].op == IR_VAR); - - /* Remap all related VADDR and VLOAD */ - ir_use_list *var_use_list = &ctx->use_lists[var]; - ir_ref n = var_use_list->count; - ir_ref *p = &ctx->use_edges[var_use_list->refs]; - int32_t old_slot = IR_VREG_TO_STACK_SLOT(ctx->vregs[var]); - - for (; n > 0; p++, n--) { - ir_ref use = *p; - ir_insn *insn = &ctx->ir_base[use]; - - if (insn->op == IR_VADDR) { - if (ctx->rules[use] == IR_STATIC_ALLOCA) { - IR_ASSERT(ctx->vregs[use] == ctx->vregs[var]); - ctx->vregs[use] = IR_STACK_SLOT_TO_VREG(spill_slot); - } - } else if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) { - if (ctx->vregs[use] > 0) { - ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; - if (ival && ival->stack_spill_pos == old_slot) { - ival->stack_spill_pos = spill_slot; - } - } - } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { - if (insn->op3 > 0 && ctx->vregs[insn->op3] > 0) { - ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op3]]; - if (ival && ival->stack_spill_pos == old_slot) { - ival->stack_spill_pos = spill_slot; - } - } - } - } - - /* Remap VAR itself */ - ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(spill_slot); - - /* Avoid copying to itself */ - ctx->rules[param_store] = IR_SKIPPED | IR_NOP; - } - } -} - static void ir_fix_param_spills(ir_ctx *ctx) { ir_use_list *use_list = &ctx->use_lists[1]; @@ -26148,11 +26259,13 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (src_reg == IR_REG_NONE) { if (ctx->vregs[use] > 0) { ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]]; - if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) - && ival->stack_spill_pos == -1) { + if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { + IR_ASSERT(ival->stack_spill_pos == -1); ival->stack_spill_pos = stack_start + stack_offset; - /* Remap VAR to PARAM stack slot */ - ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); + if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) { + /* Remap VAR to PARAM stack slot */ + ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset); + } } } if (sizeof(void*) == 8) { @@ -26909,8 +27022,11 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_MUL_I64: ir_emit_mul_i64(ctx, i, insn); break; - case IR_MUL_OV_I64: - ir_emit_mul_ov_i64(ctx, i, insn); + case IR_IMUL_OV_I64: + ir_emit_imul_ov_i64(ctx, i, insn); + break; + case IR_UMUL_OV_I64: + ir_emit_umul_ov_i64(ctx, i, insn); break; case IR_BINOP_HELPER_I64: ir_emit_binop_helper_i64(ctx, i, insn); @@ -27033,7 +27149,7 @@ next_block:; |.align 8 |=>label: while (str[i]) { - char c = str[i]; + uint8_t c = (uint8_t)str[i]; |.byte c i++; From 183899979a27c4433f2b00dec8f5996b411a8f5f Mon Sep 17 00:00:00 2001 From: Jorg Sowa Date: Mon, 19 Oct 2026 14:42:53 +0200 Subject: [PATCH 3/3] Update IR IR commit: a4d13e263dd3010de06cd05235e006bfefa927a5 --- .github/scripts/download-bundled/jit-ir.sh | 2 +- ext/opcache/jit/ir/ir_aarch64.dasc | 10 +++++----- ext/opcache/jit/ir/ir_fold.h | 8 ++++---- ext/opcache/jit/ir/ir_gcm.c | 15 +++++++++------ ext/opcache/jit/ir/ir_ra.c | 4 ++-- ext/opcache/jit/ir/ir_sccp.c | 16 ++++------------ 6 files changed, 25 insertions(+), 30 deletions(-) diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index 93f3aa7c27d4..c4450f36b7b4 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=563b51a8c2c07aadb13db897abc04b5cf08d151b +revision=a4d13e263dd3010de06cd05235e006bfefa927a5 git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index b35797216a81..79468945e9de 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -6041,10 +6041,10 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) } /* Generate a table jmp or a sequence of calls */ - if (!has_case_range && count > 2 && (max.i64-min.i64) < count * 8) { - int *labels = ir_mem_malloc(sizeof(int) * (max.i64 - min.i64 + 1)); + if (!has_case_range && count > 2 && (max.u64 - min.u64) < (uint64_t)count * 8) { + int *labels = ir_mem_malloc(sizeof(int) * (size_t)(max.u64 - min.u64 + 1)); - for (i = 0; i <= (max.i64 - min.i64); i++) { + for (i = 0; i <= (int)(max.u64 - min.u64); i++) { labels[i] = default_label; } p = &ctx->cfg_edges[bb->successors]; @@ -6055,7 +6055,7 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) val = &ctx->ir_base[use_insn->op2]; IR_ASSERT(!IR_IS_SYM_CONST(val->op)); label = ir_skip_empty_target_blocks(ctx, use_block); - labels[val->val.i64 - min.i64] = label; + labels[val->val.u64 - min.u64] = label; } } @@ -6101,7 +6101,7 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) } |.align 8 |1: - for (i = 0; i <= (max.i64 - min.i64); i++) { + for (i = 0; i <= (int)(max.u64 - min.u64); i++) { int b = labels[i]; if (b) { ir_block *bb = &ctx->cfg_blocks[b]; diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 0ea607a66bc8..bd73a3cbf413 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -81,7 +81,7 @@ IR_FOLD(LT(C_U64, C_U64)) IR_FOLD(LT(C_ADDR, C_ADDR)) { - IR_FOLD_BOOL(op1_insn->val.addr < op2_insn->val.addr); + IR_FOLD_BOOL((intptr_t)op1_insn->val.addr < (intptr_t)op2_insn->val.addr); } IR_FOLD(LT(C_CHAR, C_CHAR)) @@ -126,7 +126,7 @@ IR_FOLD(GE(C_U64, C_U64)) IR_FOLD(GE(C_ADDR, C_ADDR)) { - IR_FOLD_BOOL(op1_insn->val.addr >= op2_insn->val.addr); + IR_FOLD_BOOL((intptr_t)op1_insn->val.addr >= (intptr_t)op2_insn->val.addr); } IR_FOLD(GE(C_CHAR, C_CHAR)) @@ -171,7 +171,7 @@ IR_FOLD(LE(C_U64, C_U64)) IR_FOLD(LE(C_ADDR, C_ADDR)) { - IR_FOLD_BOOL(op1_insn->val.addr <= op2_insn->val.addr); + IR_FOLD_BOOL((intptr_t)op1_insn->val.addr <= (intptr_t)op2_insn->val.addr); } IR_FOLD(LE(C_CHAR, C_CHAR)) @@ -217,7 +217,7 @@ IR_FOLD(GT(C_U64, C_U64)) IR_FOLD(GT(C_ADDR, C_ADDR)) { - IR_FOLD_BOOL(op1_insn->val.addr > op2_insn->val.addr); + IR_FOLD_BOOL((intptr_t)op1_insn->val.addr > (intptr_t)op2_insn->val.addr); } diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c index 7a0755b74774..712411774a47 100644 --- a/ext/opcache/jit/ir/ir_gcm.c +++ b/ext/opcache/jit/ir/ir_gcm.c @@ -1202,13 +1202,16 @@ int ir_schedule(ir_ctx *ctx) consts_count += ir_count_constant(ctx, _xlat, insn->op2); } } else if (insn->op == IR_CASE_VAL) { - IR_ASSERT(insn->op2 < IR_TRUE); - consts_count += ir_count_constant(ctx, _xlat, insn->op2); + if (insn->op2 < IR_TRUE) { + consts_count += ir_count_constant(ctx, _xlat, insn->op2); + } } else if (insn->op == IR_CASE_RANGE) { - IR_ASSERT(insn->op2 < IR_TRUE); - consts_count += ir_count_constant(ctx, _xlat, insn->op2); - IR_ASSERT(insn->op3 < IR_TRUE); - consts_count += ir_count_constant(ctx, _xlat, insn->op3); + if (insn->op2 < IR_TRUE) { + consts_count += ir_count_constant(ctx, _xlat, insn->op2); + } + if (insn->op3 < IR_TRUE) { + consts_count += ir_count_constant(ctx, _xlat, insn->op3); + } } n = insn->inputs_count; insns_count += ir_insn_inputs_to_len(n); diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index 9589c9cbd15c..53c3cbd8a047 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -812,7 +812,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) IR_ASSERT(use_insn->op3 == ref); /* skip VSTORE (VAR is going to be remapped to PARAM) */ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; - param_ival->use_pos = NULL; + param_ival->use_pos = param_ival->use_pos->next; ctx->rules[use] = IR_SKIPPED | IR_NOP; } } @@ -1530,7 +1530,7 @@ int ir_compute_live_ranges(ir_ctx *ctx) IR_ASSERT(use_insn->op3 == ref); /* skip VSTORE (VAR is going to be remapped to PARAM) */ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM; - param_ival->use_pos = NULL; + param_ival->use_pos = param_ival->use_pos->next; ctx->rules[use] = IR_SKIPPED | IR_NOP; } } diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 6ff106d4cffd..27bb9163dccf 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -3028,13 +3028,11 @@ static bool ir_cmp_is_true(ir_op op, const ir_insn *op1, const ir_insn *op2) } else if (op == IR_LT) { if (IR_IS_TYPE_SIGNED(op1->type)) { return op1->val.i64 < op2->val.i64; - } else if (op1->type == IR_TYPE_ADDR) { - return op1->val.addr < op2->val.addr; } else if (ir_type_size[op1->type] == 8) { return op1->val.i64 < op2->val.i64; } else if (ir_type_size[op1->type] == 4) { return op1->val.i32 < op2->val.i32; - } else if (ir_type_size[op1->type] == 4) { + } else if (ir_type_size[op1->type] == 2) { return op1->val.i16 < op2->val.i16; } else if (ir_type_size[op1->type] == 1) { return op1->val.i8 < op2->val.i8; @@ -3045,13 +3043,11 @@ static bool ir_cmp_is_true(ir_op op, const ir_insn *op1, const ir_insn *op2) } else if (op == IR_GE) { if (IR_IS_TYPE_SIGNED(op1->type)) { return op1->val.i64 >= op2->val.i64; - } else if (op1->type == IR_TYPE_ADDR) { - return op1->val.addr >= op2->val.addr; } else if (ir_type_size[op1->type] == 8) { return op1->val.i64 >= op2->val.i64; } else if (ir_type_size[op1->type] == 4) { return op1->val.i32 >= op2->val.i32; - } else if (ir_type_size[op1->type] == 4) { + } else if (ir_type_size[op1->type] == 2) { return op1->val.i16 >= op2->val.i16; } else if (ir_type_size[op1->type] == 1) { return op1->val.i8 >= op2->val.i8; @@ -3062,13 +3058,11 @@ static bool ir_cmp_is_true(ir_op op, const ir_insn *op1, const ir_insn *op2) } else if (op == IR_LE) { if (IR_IS_TYPE_SIGNED(op1->type)) { return op1->val.i64 <= op2->val.i64; - } else if (op1->type == IR_TYPE_ADDR) { - return op1->val.addr <= op2->val.addr; } else if (ir_type_size[op1->type] == 8) { return op1->val.i64 <= op2->val.i64; } else if (ir_type_size[op1->type] == 4) { return op1->val.i32 <= op2->val.i32; - } else if (ir_type_size[op1->type] == 4) { + } else if (ir_type_size[op1->type] == 2) { return op1->val.i16 <= op2->val.i16; } else if (ir_type_size[op1->type] == 1) { return op1->val.i8 <= op2->val.i8; @@ -3079,13 +3073,11 @@ static bool ir_cmp_is_true(ir_op op, const ir_insn *op1, const ir_insn *op2) } else if (op == IR_GT) { if (IR_IS_TYPE_SIGNED(op1->type)) { return op1->val.i64 > op2->val.i64; - } else if (op1->type == IR_TYPE_ADDR) { - return op1->val.addr > op2->val.addr; } else if (ir_type_size[op1->type] == 8) { return op1->val.i64 > op2->val.i64; } else if (ir_type_size[op1->type] == 4) { return op1->val.i32 > op2->val.i32; - } else if (ir_type_size[op1->type] == 4) { + } else if (ir_type_size[op1->type] == 2) { return op1->val.i16 > op2->val.i16; } else if (ir_type_size[op1->type] == 1) { return op1->val.i8 > op2->val.i8;