diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc index 4b486aeea90ea2afb9cdd96a4dbe15c5bb2abd7a..a08043e18d609e258ebfe033875201163d129aba 100644 --- a/gcc/config/aarch64/aarch64.cc +++ b/gcc/config/aarch64/aarch64.cc @@ -305,6 +305,7 @@ static machine_mode aarch64_simd_container_mode (scalar_mode, poly_int64); static bool aarch64_print_address_internal (FILE*, machine_mode, rtx, aarch64_addr_query_type); static HOST_WIDE_INT aarch64_clamp_to_uimm12_shift (HOST_WIDE_INT val); +static void aarch64_expand_vector_init (rtx, rtx_vector_builder&); /* The processor for which instructions should be scheduled. */ enum aarch64_processor aarch64_tune = cortexa53; @@ -21804,55 +21805,96 @@ aarch64_simd_make_constant (rtx vals) return NULL_RTX; } +static void +aarch64_vec_duplicate (rtx target, machine_mode mode, machine_mode element_mode, + int narrow_n_elts) +{ + poly_uint64 size = narrow_n_elts * GET_MODE_BITSIZE (element_mode); + scalar_mode i_mode = int_mode_for_size (size, 0).require (); + machine_mode o_mode; + if (aarch64_sve_mode_p (mode)) + o_mode = aarch64_full_sve_mode (i_mode).require (); + else + o_mode + = aarch64_simd_container_mode (i_mode, + GET_MODE_BITSIZE (mode)); + rtx input = simplify_gen_subreg (i_mode, target, mode, 0); + rtx output = simplify_gen_subreg (o_mode, target, mode, 0); + aarch64_emit_move (output, gen_vec_duplicate (o_mode, input)); +} + + /* Expand a vector initialisation sequence, such that TARGET is initialised to contain VALS. */ void aarch64_expand_vector_init (rtx target, rtx vals) { - machine_mode mode = GET_MODE (target); - scalar_mode inner_mode = GET_MODE_INNER (mode); /* The number of vector elements. */ int n_elts = XVECLEN (vals, 0); - /* The number of vector elements which are not constant. */ - int n_var = 0; - rtx any_const = NULL_RTX; + machine_mode mode = GET_MODE (target); + scalar_mode inner_mode = GET_MODE_INNER (mode); /* The first element of vals. */ rtx v0 = XVECEXP (vals, 0, 0); - bool all_same = true; /* This is a special vec_init where N is not an element mode but a vector mode with half the elements of M. We expect to find two entries of mode N in VALS and we must put their concatentation into TARGET. */ - if (XVECLEN (vals, 0) == 2 && VECTOR_MODE_P (GET_MODE (XVECEXP (vals, 0, 0)))) + if (n_elts == 2 + && VECTOR_MODE_P (GET_MODE (v0))) { - machine_mode narrow_mode = GET_MODE (XVECEXP (vals, 0, 0)); + machine_mode narrow_mode = GET_MODE (v0); gcc_assert (GET_MODE_INNER (narrow_mode) == inner_mode && known_eq (GET_MODE_SIZE (mode), 2 * GET_MODE_SIZE (narrow_mode))); - emit_insn (gen_aarch64_vec_concat (narrow_mode, target, - XVECEXP (vals, 0, 0), + emit_insn (gen_aarch64_vec_concat (narrow_mode, target, v0, XVECEXP (vals, 0, 1))); return; } - /* Count the number of variable elements to initialise. */ + rtx_vector_builder builder (mode, n_elts, 1); for (int i = 0; i < n_elts; ++i) + builder.quick_push (XVECEXP (vals, 0, i)); + builder.finalize (); + + aarch64_expand_vector_init (target, builder); +} + +static void +aarch64_expand_vector_init (rtx target, rtx_vector_builder &v) +{ + machine_mode mode = GET_MODE (target); + scalar_mode inner_mode = GET_MODE_INNER (mode); + /* The number of vector elements which are not constant. */ + unsigned n_var = 0; + rtx any_const = NULL_RTX; + /* The first element of vals. */ + rtx v0 = v.elt (0); + /* Get the number of elements to insert into an Advanced SIMD vector. + If we have more than one element per pattern then we use the constant + number of elements in a full vector. + If we only have one element per pattern we use the number of patterns as + this may be lower than the number of elements in a full vector, which + means they repeat and we should use a duplicate of the smaller vector. */ + unsigned n_elts + = v.nelts_per_pattern () == 1 ? v.npatterns () + : v.full_nelts ().coeffs[0]; + + /* Count the number of variable elements to initialise. */ + for (unsigned i = 0; i < n_elts ; ++i) { - rtx x = XVECEXP (vals, 0, i); + rtx x = v.elt (i); if (!(CONST_INT_P (x) || CONST_DOUBLE_P (x))) ++n_var; else any_const = x; - - all_same &= rtx_equal_p (x, v0); } /* No variable elements, hand off to aarch64_simd_make_constant which knows how best to handle this. */ if (n_var == 0) { - rtx constant = aarch64_simd_make_constant (vals); + rtx constant = aarch64_simd_make_constant (v.build ()); if (constant != NULL_RTX) { emit_move_insn (target, constant); @@ -21861,7 +21903,7 @@ aarch64_expand_vector_init (rtx target, rtx vals) } /* Splat a single non-constant element if we can. */ - if (all_same) + if (n_elts == 1) { rtx x = copy_to_mode_reg (inner_mode, v0); aarch64_emit_move (target, gen_vec_duplicate (mode, x)); @@ -21879,14 +21921,15 @@ aarch64_expand_vector_init (rtx target, rtx vals) and matches[X][1] with the count of duplicate elements (if X is the earliest element which has duplicates). */ - if (n_var == n_elts && n_elts <= 16) + if (n_var == n_elts) { - int matches[16][2] = {0}; - for (int i = 0; i < n_elts; i++) + gcc_assert (n_elts <= 16); + unsigned matches[16][2] = {0}; + for (unsigned i = 0; i < n_elts; i++) { - for (int j = 0; j <= i; j++) + for (unsigned j = 0; j <= i; j++) { - if (rtx_equal_p (XVECEXP (vals, 0, i), XVECEXP (vals, 0, j))) + if (rtx_equal_p (v.elt (i), v.elt (j))) { matches[i][0] = j; matches[j][1]++; @@ -21894,9 +21937,9 @@ aarch64_expand_vector_init (rtx target, rtx vals) } } } - int maxelement = 0; - int maxv = 0; - for (int i = 0; i < n_elts; i++) + unsigned maxelement = 0; + unsigned maxv = 0; + for (unsigned i = 0; i < n_elts; i++) if (matches[i][1] > maxv) { maxelement = i; @@ -21915,8 +21958,8 @@ aarch64_expand_vector_init (rtx target, rtx vals) || inner_mode == E_DFmode)) { - rtx x0 = XVECEXP (vals, 0, 0); - rtx x1 = XVECEXP (vals, 0, 1); + rtx x0 = v.elt (0); + rtx x1 = v.elt (1); /* Combine can pick up this case, but handling it directly here leaves clearer RTL. @@ -21939,24 +21982,26 @@ aarch64_expand_vector_init (rtx target, rtx vals) vector register. For big-endian we want that position to hold the last element of VALS. */ maxelement = BYTES_BIG_ENDIAN ? n_elts - 1 : 0; - rtx x = copy_to_mode_reg (inner_mode, XVECEXP (vals, 0, maxelement)); + rtx x = copy_to_mode_reg (inner_mode, v.elt (maxelement)); aarch64_emit_move (target, lowpart_subreg (mode, x, inner_mode)); } else { - rtx x = copy_to_mode_reg (inner_mode, XVECEXP (vals, 0, maxelement)); + rtx x = copy_to_mode_reg (inner_mode, v.elt (maxelement)); aarch64_emit_move (target, gen_vec_duplicate (mode, x)); } /* Insert the rest. */ - for (int i = 0; i < n_elts; i++) + for (unsigned i = 0; i < n_elts; i++) { - rtx x = XVECEXP (vals, 0, i); + rtx x = v.elt (i); if (matches[i][0] == maxelement) continue; x = copy_to_mode_reg (inner_mode, x); emit_insn (GEN_FCN (icode) (target, x, GEN_INT (i))); } + if (!known_eq (v.full_nelts (), n_elts)) + aarch64_vec_duplicate (target, mode, GET_MODE (v0), n_elts); return; } @@ -21965,19 +22010,19 @@ aarch64_expand_vector_init (rtx target, rtx vals) can. */ if (n_var != n_elts) { - rtx copy = copy_rtx (vals); + rtx copy = v.build (); /* Load constant part of vector. We really don't care what goes into the parts we will overwrite, but we're more likely to be able to load the constant efficiently if it has fewer, larger, repeating parts (see aarch64_simd_valid_immediate). */ - for (int i = 0; i < n_elts; i++) + for (unsigned i = 0; i < n_elts; i++) { - rtx x = XVECEXP (vals, 0, i); + rtx x = XVECEXP (copy, 0, i); if (CONST_INT_P (x) || CONST_DOUBLE_P (x)) continue; rtx subst = any_const; - for (int bit = n_elts / 2; bit > 0; bit /= 2) + for (unsigned bit = n_elts / 2; bit > 0; bit /= 2) { /* Look in the copied vector, as more elements are const. */ rtx test = XVECEXP (copy, 0, i ^ bit); @@ -21989,18 +22034,21 @@ aarch64_expand_vector_init (rtx target, rtx vals) } XVECEXP (copy, 0, i) = subst; } + gcc_assert (GET_MODE (target) == GET_MODE (copy)); aarch64_expand_vector_init (target, copy); } /* Insert the variable lanes directly. */ - for (int i = 0; i < n_elts; i++) + for (unsigned i = 0; i < n_elts; i++) { - rtx x = XVECEXP (vals, 0, i); + rtx x = v.elt (i); if (CONST_INT_P (x) || CONST_DOUBLE_P (x)) continue; x = copy_to_mode_reg (inner_mode, x); emit_insn (GEN_FCN (icode) (target, x, GEN_INT (i))); } + if (!known_eq (v.full_nelts (), n_elts)) + aarch64_vec_duplicate (target, mode, inner_mode, n_elts); } /* Emit RTL corresponding to: diff --git a/gcc/testsuite/gcc.target/aarch64/ldp_stp_16.c b/gcc/testsuite/gcc.target/aarch64/ldp_stp_16.c index 8ab117c4dcd7a731abc7e1b039e1faf0dfa09a5d..b307d2791824dd9c30200931452b2636708b5035 100644 --- a/gcc/testsuite/gcc.target/aarch64/ldp_stp_16.c +++ b/gcc/testsuite/gcc.target/aarch64/ldp_stp_16.c @@ -96,8 +96,8 @@ CONS2_FN (4, float); /* ** cons2_8_float: -** dup v([0-9]+)\.4s, .* -** ... +** ins v0\.s\[1\], v1\.s\[0\] +** dup v([0-9]+)\.2d, v0\.d\[0\] ** stp q\1, q\1, \[x0\] ** stp q\1, q\1, \[x0, #?32\] ** ret diff --git a/gcc/testsuite/gcc.target/aarch64/vect_init.c b/gcc/testsuite/gcc.target/aarch64/vect_init.c new file mode 100644 index 0000000000000000000000000000000000000000..546e44e96f4db60d289b4bc0ebfecbe18c81b4cc --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/vect_init.c @@ -0,0 +1,144 @@ +#include + +/* +** int32_0: +** fmov s0, w0 +** ins v0.s\[1\], w1 +** dup v0.2d, v0.d\[0\] +** ret +*/ + +int32x4_t int32_0 (int a, int b) +{ + int32x4_t v = {a, b, a, b}; + return v; +} +/* +** int32_1: +** dup v0.4s, w0 +** ret +*/ + +int32x4_t int32_1 (int a) +{ + int32x4_t v = {a, a, a, a}; + return v; +} + +/* +** int16_0: +** sxth w0, w0 +** fmov s0, w0 +** ins v0.h\[1\], w1 +** ins v0.h\[2\], w2 +** ins v0.h\[3\], w3 +** dup v0.2d, v0.d\[0\] +** ret +*/ + +int16x8_t int16_0 (int16_t a, int16_t b, int16_t c, int16_t d) +{ + int16x8_t v = {a, b, c, d, + a, b, c, d}; + return v; +} + +/* +** int16_1: +** sxth w0, w0 +** fmov s0, w0 +** ins v0.h\[1\], w1 +** dup v0.4s, v0.s\[0\] +** ret +*/ + +int16x8_t int16_1 (int16_t a, int16_t b) +{ + int16x8_t v = {a, b, a, b, + a, b, a, b}; + return v; +} + +/* +** int16_2: +** dup v0.8h, w0 +** ret +*/ + +int16x8_t int16_2 (int16_t a) +{ + int16x8_t v = {a, a, a, a, + a, a, a, a}; + return v; +} + +/* +** int8_0: +** sxtb w0, w0 +** fmov s0, w0 +** ins v0.b\[1\], w1 +** ins v0.b\[2\], w2 +** ins v0.b\[3\], w3 +** ins v0.b\[4\], w4 +** ins v0.b\[5\], w5 +** ins v0.b\[6\], w6 +** ins v0.b\[7\], w7 +** dup v0.2d, v0.d\[0\] +** ret +*/ + +int8x16_t int8_0 (int8_t a, int8_t b, int8_t c, int8_t d, int8_t e, int8_t f, + int8_t g, int8_t h) +{ + int8x16_t v = {a, b, c, d, e, f, g, h, + a, b, c, d, e, f, g, h}; + return v; +} + +/* +** int8_1: +** sxtb w0, w0 +** fmov s0, w0 +** ins v0.b\[1\], w1 +** ins v0.b\[2\], w2 +** ins v0.b\[3\], w3 +** dup v0.4s, v0.s\[0\] +** ret +*/ + +int8x16_t int8_1 (int8_t a, int8_t b, int8_t c, int8_t d) +{ + int8x16_t v = {a, b, c, d, a, b, c, d, + a, b, c, d, a, b, c, d}; + return v; +} + +/* +** int8_2: +** sxtb w0, w0 +** fmov s0, w0 +** ins v0.b\[1\], w1 +** dup v0.8h, v0.h\[0\] +** ret +*/ + +int8x16_t int8_2 (int8_t a, int8_t b) +{ + int8x16_t v = {a, b, a, b, a, b, a, b, + a, b, a, b, a, b, a, b}; + return v; +} + +/* +** int8_3: +** dup v0.16b, w0 +** ret +*/ + +int8x16_t int8_3 (int8_t a) +{ + int8x16_t v = {a, a, a, a, a, a, a, a, + a, a, a, a, a, a, a, a}; + return v; +} +