public inbox for gcc-cvs@sourceware.org
help / color / mirror / Atom feed
* [gcc(refs/users/clyon/heads/mve-autovec)] arm: Fix vcond_mask expander for MVE (PR target/100757)
@ 2022-01-12 8:28 Christophe Lyon
0 siblings, 0 replies; 6+ messages in thread
From: Christophe Lyon @ 2022-01-12 8:28 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:eeebb17b451bf19c73debd166314ac5b73a02f9b
commit eeebb17b451bf19c73debd166314ac5b73a02f9b
Author: Christophe Lyon <christophe.lyon@foss.st.com>
Date: Wed Oct 20 15:30:16 2021 +0000
arm: Fix vcond_mask expander for MVE (PR target/100757)
The problem in this PR is that we call VPSEL with a mask of vector
type instead of HImode. This happens because operand 3 in vcond_mask
is the pre-computed vector comparison and has vector type.
This patch fixes it by implementing TARGET_VECTORIZE_GET_MASK_MODE,
returning the appropriate VxBI mode when targeting MVE. In turn, this
implies implementing vec_cmp<mode><MVE_vpred>,
vec_cmpu<mode><MVE_vpred> and vcond_mask_<mode><MVE_vpred>, and we can
move vec_cmp<mode><v_cmp_result>, vec_cmpu<mode><mode> and
vcond_mask_<mode><v_cmp_result> back to neon.md since they are not
used by MVE anymore. The new *<MVE_vpred> patterns listed above are
implemented in mve.md since they are only valid for MVE. However this
may make maintenance/comparison more painful than having all of them
in vec-common.md.
In the process, we can get rid of the recently added vcond_mve
parameter of arm_expand_vector_compare.
Compared to neon.md's vcond_mask_<mode><v_cmp_result> before my "arm:
Auto-vectorization for MVE: vcmp" patch (r12-834), it keeps the VDQWH
iterator added in r12-835 (to have V4HF/V8HF support), as well as the
(!<Is_float_mode> || flag_unsafe_math_optimizations) condition which
was not present before r12-834 although SF modes were enabled by VDQW
(I think this was a bug).
Using TARGET_VECTORIZE_GET_MASK_MODE has the advantage that we no
longer need to generate vpsel with vectors of 0 and 1: the masks are
now merged via scalar 'ands' instructions operating on 16-bit masks
after converting the boolean vectors.
In addition, this patch fixes a problem in arm_expand_vcond() where
the result would be a vector of 0 or 1 instead of operand 1 or 2.
Reducing the number of iterations in pr100757-3.c from 32 to 8, we
generate the code below:
float a[32];
float fn1(int d) {
float c = 4.0f;
for (int b = 0; b < 8; b++)
if (a[b] != 2.0f)
c = 5.0f;
return c;
}
fn1:
ldr r3, .L3+48
vldr.64 d4, .L3 // q2=(2.0,2.0,2.0,2.0)
vldr.64 d5, .L3+8
vldrw.32 q0, [r3] // q0=a(0..3)
adds r3, r3, #16
vcmp.f32 eq, q0, q2 // cmp a(0..3) == (2.0,2.0,2.0,2.0)
vldrw.32 q1, [r3] // q1=a(4..7)
vmrs r3, P0
vcmp.f32 eq, q1, q2 // cmp a(4..7) == (2.0,2.0,2.0,2.0)
vmrs r2, P0 @ movhi
ands r3, r3, r2 // r3=select(a(0..3]) & select(a(4..7))
vldr.64 d4, .L3+16 // q2=(5.0,5.0,5.0,5.0)
vldr.64 d5, .L3+24
vmsr P0, r3
vldr.64 d6, .L3+32 // q3=(4.0,4.0,4.0,4.0)
vldr.64 d7, .L3+40
vpsel q3, q3, q2 // q3=vcond_mask(4.0,5.0)
vmov.32 r2, q3[1] // keep the scalar max
vmov.32 r0, q3[3]
vmov.32 r3, q3[2]
vmov.f32 s11, s12
vmov s15, r2
vmov s14, r3
vmaxnm.f32 s15, s11, s15
vmaxnm.f32 s15, s15, s14
vmov s14, r0
vmaxnm.f32 s15, s15, s14
vmov r0, s15
bx lr
.L4:
.align 3
.L3:
.word 1073741824 // 2.0f
.word 1073741824
.word 1073741824
.word 1073741824
.word 1084227584 // 5.0f
.word 1084227584
.word 1084227584
.word 1084227584
.word 1082130432 // 4.0f
.word 1082130432
.word 1082130432
.word 1082130432
2021-10-13 Christophe Lyon <christophe.lyon@foss.st.com>
PR target/100757
gcc/
* config/arm/arm-protos.h (arm_get_mask_mode): New prototype.
(arm_expand_vector_compare): Update prototype.
* config/arm/arm.c (TARGET_VECTORIZE_GET_MASK_MODE): New.
(arm_vector_mode_supported_p): Add support for VxBI modes.
(arm_expand_vector_compare): Remove useless generation of vpsel.
(arm_expand_vcond): Fix select operands.
(arm_get_mask_mode): New.
* config/arm/mve.md (vec_cmp<mode><MVE_vpred>): New.
(vec_cmpu<mode><MVE_vpred>): New.
(vcond_mask_<mode><MVE_vpred>): New.
* config/arm/vec-common.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): Move to ...
* config/arm/neon.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): ... here
and disable for MVE.
Diff:
---
gcc/config/arm/arm-protos.h | 3 +-
gcc/config/arm/arm.c | 118 +++++++++++++++----------------------------
gcc/config/arm/mve.md | 54 ++++++++++++++++++++
gcc/config/arm/neon.md | 39 ++++++++++++++
gcc/config/arm/vec-common.md | 52 -------------------
5 files changed, 136 insertions(+), 130 deletions(-)
diff --git a/gcc/config/arm/arm-protos.h b/gcc/config/arm/arm-protos.h
index 9b1f61394ad..9e3d71e0c29 100644
--- a/gcc/config/arm/arm-protos.h
+++ b/gcc/config/arm/arm-protos.h
@@ -201,6 +201,7 @@ extern void arm_init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree);
extern bool arm_pad_reg_upward (machine_mode, tree, int);
#endif
extern int arm_apply_result_size (void);
+extern opt_machine_mode arm_get_mask_mode (machine_mode mode);
#endif /* RTX_CODE */
@@ -372,7 +373,7 @@ extern void arm_emit_coreregs_64bit_shift (enum rtx_code, rtx, rtx, rtx, rtx,
extern bool arm_fusion_enabled_p (tune_params::fuse_ops);
extern bool arm_valid_symbolic_address_p (rtx);
extern bool arm_validize_comparison (rtx *, rtx *, rtx *);
-extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool, bool);
+extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool);
#endif /* RTX_CODE */
extern bool arm_gen_setmem (rtx *);
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index 1c76262fa31..bc18d3c888d 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -829,6 +829,10 @@ static const struct attribute_spec arm_attribute_table[] =
#undef TARGET_MD_ASM_ADJUST
#define TARGET_MD_ASM_ADJUST arm_md_asm_adjust
+
+#undef TARGET_VECTORIZE_GET_MASK_MODE
+#define TARGET_VECTORIZE_GET_MASK_MODE arm_get_mask_mode
+
\f
/* Obstack for minipool constant handling. */
static struct obstack minipool_obstack;
@@ -29206,7 +29210,8 @@ arm_vector_mode_supported_p (machine_mode mode)
if (TARGET_HAVE_MVE
&& (mode == V2DImode || mode == V4SImode || mode == V8HImode
- || mode == V16QImode))
+ || mode == V16QImode
+ || mode == V16BImode || mode == V8BImode || mode == V4BImode))
return true;
if (TARGET_HAVE_MVE_FLOAT
@@ -31005,7 +31010,7 @@ arm_split_atomic_op (enum rtx_code code, rtx old_out, rtx new_out, rtx mem,
}
\f
/* Return the mode for the MVE vector of predicates corresponding to MODE. */
-machine_mode
+opt_machine_mode
arm_mode_to_pred_mode (machine_mode mode)
{
switch (GET_MODE_NUNITS (mode))
@@ -31014,7 +31019,7 @@ arm_mode_to_pred_mode (machine_mode mode)
case 8: return V8BImode;
case 4: return V4BImode;
}
- gcc_unreachable ();
+ return opt_machine_mode ();
}
/* Expand code to compare vectors OP0 and OP1 using condition CODE.
@@ -31022,16 +31027,12 @@ arm_mode_to_pred_mode (machine_mode mode)
and return true if TARGET contains the inverse. If !CAN_INVERT,
always store the result in TARGET, never its inverse.
- If VCOND_MVE, do not emit the vpsel instruction here, let arm_expand_vcond do
- it with the right destination type to avoid emiting two vpsel, one here and
- one in arm_expand_vcond.
-
Note that the handling of floating-point comparisons is not
IEEE compliant. */
bool
arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
- bool can_invert, bool vcond_mve)
+ bool can_invert)
{
machine_mode cmp_result_mode = GET_MODE (target);
machine_mode cmp_mode = GET_MODE (op0);
@@ -31060,7 +31061,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
and then store its inverse in TARGET. This avoids reusing
TARGET (which for integer NE could be one of the inputs). */
rtx tmp = gen_reg_rtx (cmp_result_mode);
- if (arm_expand_vector_compare (tmp, code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (tmp, code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_NOT (cmp_result_mode, tmp)));
return false;
@@ -31096,36 +31097,22 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case NE:
if (TARGET_HAVE_MVE)
{
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vcmpq_f (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq_f (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
else
gcc_unreachable ();
break;
default:
gcc_unreachable ();
}
-
- /* If we are not expanding a vcond, build the result here. */
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
}
else
emit_insn (gen_neon_vc (code, cmp_mode, target, op0, op1));
@@ -31137,23 +31124,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case GEU:
case GTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
else
emit_insn (gen_neon_vc (code, cmp_mode, target,
op0, force_reg (cmp_mode, op1)));
@@ -31164,23 +31136,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case LEU:
case LTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, vpr_p0, force_reg (cmp_mode, op1), op0));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, target,
+ force_reg (cmp_mode, op1), op0));
else
emit_insn (gen_neon_vc (swap_condition (code), cmp_mode,
target, force_reg (cmp_mode, op1), op0));
@@ -31195,8 +31152,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
rtx gt_res = gen_reg_rtx (cmp_result_mode);
rtx alt_res = gen_reg_rtx (cmp_result_mode);
rtx_code alt_code = (code == LTGT ? LT : LE);
- if (arm_expand_vector_compare (gt_res, GT, op0, op1, true, vcond_mve)
- || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (gt_res, GT, op0, op1, true)
+ || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_IOR (cmp_result_mode,
gt_res, alt_res)));
@@ -31216,19 +31173,15 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
{
/* When expanding for MVE, we do not want to emit a (useless) vpsel in
arm_expand_vector_compare, and another one here. */
- bool vcond_mve=false;
rtx mask;
if (TARGET_HAVE_MVE)
- {
- vcond_mve=true;
- mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
- }
+ mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode).require ());
else
mask = gen_reg_rtx (cmp_result_mode);
bool inverted = arm_expand_vector_compare (mask, GET_CODE (operands[3]),
- operands[4], operands[5], true, vcond_mve);
+ operands[4], operands[5], true);
if (inverted)
std::swap (operands[1], operands[2]);
if (TARGET_NEON)
@@ -31236,20 +31189,20 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
mask, operands[1], operands[2]));
else
{
- machine_mode cmp_mode = GET_MODE (operands[4]);
- rtx vpr_p0 = mask;
- rtx zero = gen_reg_rtx (cmp_mode);
- rtx one = gen_reg_rtx (cmp_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_mode));
- emit_move_insn (one, CONST1_RTX (cmp_mode));
+ machine_mode cmp_mode = GET_MODE (operands[0]);
+
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_mode, operands[0],
+ operands[1], operands[2], mask));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0],
+ operands[1], operands[2], mask));
+ else
+ gcc_unreachable ();
break;
default:
gcc_unreachable ();
@@ -34159,4 +34112,15 @@ arm_mode_base_reg_class (machine_mode mode)
struct gcc_target targetm = TARGET_INITIALIZER;
+/* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
+
+opt_machine_mode
+arm_get_mask_mode (machine_mode mode)
+{
+ if (TARGET_HAVE_MVE)
+ return arm_mode_to_pred_mode (mode);
+
+ return default_get_mask_mode (mode);
+}
+
#include "gt-arm.h"
diff --git a/gcc/config/arm/mve.md b/gcc/config/arm/mve.md
index e70d92fa1ee..3268be528d6 100644
--- a/gcc/config/arm/mve.md
+++ b/gcc/config/arm/mve.md
@@ -10527,3 +10527,57 @@
operands[1] = force_reg (<MODE>mode, operands[1]);
}
)
+
+;; Expanders for vec_cmp and vcond
+
+(define_expand "vec_cmp<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_VLD_ST 2 "s_register_operand")
+ (match_operand:MVE_VLD_ST 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_2 2 "s_register_operand")
+ (match_operand:MVE_2 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><MVE_vpred>"
+ [(set (match_operand:MVE_VLD_ST 0 "s_register_operand")
+ (if_then_else:MVE_VLD_ST
+ (match_operand:<MVE_VPRED> 3 "s_register_operand")
+ (match_operand:MVE_VLD_ST 1 "s_register_operand")
+ (match_operand:MVE_VLD_ST 2 "s_register_operand")))]
+ "TARGET_HAVE_MVE"
+{
+ switch (GET_MODE_CLASS (<MODE>mode))
+ {
+ case MODE_VECTOR_INT:
+ emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ break;
+ case MODE_VECTOR_FLOAT:
+ if (TARGET_HAVE_MVE_FLOAT)
+ emit_insn (gen_mve_vpselq_f (<MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ else
+ gcc_unreachable ();
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ DONE;
+})
diff --git a/gcc/config/arm/neon.md b/gcc/config/arm/neon.md
index 8b0a396947c..28310d93a4e 100644
--- a/gcc/config/arm/neon.md
+++ b/gcc/config/arm/neon.md
@@ -1394,6 +1394,45 @@
[(set_attr "type" "neon_qsub<q>")]
)
+(define_expand "vec_cmp<mode><v_cmp_result>"
+ [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
+ (match_operator:<V_cmp_result> 1 "comparison_operator"
+ [(match_operand:VDQWH 2 "s_register_operand")
+ (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><mode>"
+ [(set (match_operand:VDQIW 0 "s_register_operand")
+ (match_operator:VDQIW 1 "comparison_operator"
+ [(match_operand:VDQIW 2 "s_register_operand")
+ (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><v_cmp_result>"
+ [(set (match_operand:VDQWH 0 "s_register_operand")
+ (if_then_else:VDQWH
+ (match_operand:<V_cmp_result> 3 "s_register_operand")
+ (match_operand:VDQWH 1 "s_register_operand")
+ (match_operand:VDQWH 2 "s_register_operand")))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ emit_insn (gen_neon_vbsl<mode> (operands[0], operands[3], operands[1],
+ operands[2]));
+ DONE;
+})
+
;; Patterns for builtins.
; good for plain vadd, vaddq.
diff --git a/gcc/config/arm/vec-common.md b/gcc/config/arm/vec-common.md
index e71d9b3811f..a95d8f3d6b4 100644
--- a/gcc/config/arm/vec-common.md
+++ b/gcc/config/arm/vec-common.md
@@ -363,33 +363,6 @@
}
})
-(define_expand "vec_cmp<mode><v_cmp_result>"
- [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
- (match_operator:<V_cmp_result> 1 "comparison_operator"
- [(match_operand:VDQWH 2 "s_register_operand")
- (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
-(define_expand "vec_cmpu<mode><mode>"
- [(set (match_operand:VDQIW 0 "s_register_operand")
- (match_operator:VDQIW 1 "comparison_operator"
- [(match_operand:VDQIW 2 "s_register_operand")
- (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
;; Conditional instructions. These are comparisons with conditional moves for
;; vectors. They perform the assignment:
;;
@@ -461,31 +434,6 @@
DONE;
})
-(define_expand "vcond_mask_<mode><v_cmp_result>"
- [(set (match_operand:VDQWH 0 "s_register_operand")
- (if_then_else:VDQWH
- (match_operand:<V_cmp_result> 3 "s_register_operand")
- (match_operand:VDQWH 1 "s_register_operand")
- (match_operand:VDQWH 2 "s_register_operand")))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- if (TARGET_NEON)
- {
- emit_insn (gen_neon_vbsl (<MODE>mode, operands[0], operands[3],
- operands[1], operands[2]));
- }
- else if (TARGET_HAVE_MVE)
- {
- emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
- operands[1], operands[2], operands[3]));
- }
- else
- gcc_unreachable ();
- DONE;
-})
-
(define_expand "vec_load_lanesoi<mode>"
[(set (match_operand:OI 0 "s_register_operand")
(unspec:OI [(match_operand:OI 1 "neon_struct_operand")
^ permalink raw reply [flat|nested] 6+ messages in thread
* [gcc(refs/users/clyon/heads/mve-autovec)] arm: Fix vcond_mask expander for MVE (PR target/100757)
@ 2022-02-22 9:08 Christophe Lyon
0 siblings, 0 replies; 6+ messages in thread
From: Christophe Lyon @ 2022-02-22 9:08 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:f3232e0d6ab43cc8b88d1ce3b56b99fad3a61c2d
commit f3232e0d6ab43cc8b88d1ce3b56b99fad3a61c2d
Author: Christophe Lyon <christophe.lyon@foss.st.com>
Date: Wed Oct 20 15:30:16 2021 +0000
arm: Fix vcond_mask expander for MVE (PR target/100757)
The problem in this PR is that we call VPSEL with a mask of vector
type instead of HImode. This happens because operand 3 in vcond_mask
is the pre-computed vector comparison and has vector type.
This patch fixes it by implementing TARGET_VECTORIZE_GET_MASK_MODE,
returning the appropriate VxBI mode when targeting MVE. In turn, this
implies implementing vec_cmp<mode><MVE_vpred>,
vec_cmpu<mode><MVE_vpred> and vcond_mask_<mode><MVE_vpred>, and we can
move vec_cmp<mode><v_cmp_result>, vec_cmpu<mode><mode> and
vcond_mask_<mode><v_cmp_result> back to neon.md since they are not
used by MVE anymore. The new *<MVE_vpred> patterns listed above are
implemented in mve.md since they are only valid for MVE. However this
may make maintenance/comparison more painful than having all of them
in vec-common.md.
In the process, we can get rid of the recently added vcond_mve
parameter of arm_expand_vector_compare.
Compared to neon.md's vcond_mask_<mode><v_cmp_result> before my "arm:
Auto-vectorization for MVE: vcmp" patch (r12-834), it keeps the VDQWH
iterator added in r12-835 (to have V4HF/V8HF support), as well as the
(!<Is_float_mode> || flag_unsafe_math_optimizations) condition which
was not present before r12-834 although SF modes were enabled by VDQW
(I think this was a bug).
Using TARGET_VECTORIZE_GET_MASK_MODE has the advantage that we no
longer need to generate vpsel with vectors of 0 and 1: the masks are
now merged via scalar 'ands' instructions operating on 16-bit masks
after converting the boolean vectors.
In addition, this patch fixes a problem in arm_expand_vcond() where
the result would be a vector of 0 or 1 instead of operand 1 or 2.
Since we want to skip gcc.dg/signbit-2.c for MVE, we also add a new
arm_mve effective target.
Reducing the number of iterations in pr100757-3.c from 32 to 8, we
generate the code below:
float a[32];
float fn1(int d) {
float c = 4.0f;
for (int b = 0; b < 8; b++)
if (a[b] != 2.0f)
c = 5.0f;
return c;
}
fn1:
ldr r3, .L3+48
vldr.64 d4, .L3 // q2=(2.0,2.0,2.0,2.0)
vldr.64 d5, .L3+8
vldrw.32 q0, [r3] // q0=a(0..3)
adds r3, r3, #16
vcmp.f32 eq, q0, q2 // cmp a(0..3) == (2.0,2.0,2.0,2.0)
vldrw.32 q1, [r3] // q1=a(4..7)
vmrs r3, P0
vcmp.f32 eq, q1, q2 // cmp a(4..7) == (2.0,2.0,2.0,2.0)
vmrs r2, P0 @ movhi
ands r3, r3, r2 // r3=select(a(0..3]) & select(a(4..7))
vldr.64 d4, .L3+16 // q2=(5.0,5.0,5.0,5.0)
vldr.64 d5, .L3+24
vmsr P0, r3
vldr.64 d6, .L3+32 // q3=(4.0,4.0,4.0,4.0)
vldr.64 d7, .L3+40
vpsel q3, q3, q2 // q3=vcond_mask(4.0,5.0)
vmov.32 r2, q3[1] // keep the scalar max
vmov.32 r0, q3[3]
vmov.32 r3, q3[2]
vmov.f32 s11, s12
vmov s15, r2
vmov s14, r3
vmaxnm.f32 s15, s11, s15
vmaxnm.f32 s15, s15, s14
vmov s14, r0
vmaxnm.f32 s15, s15, s14
vmov r0, s15
bx lr
.L4:
.align 3
.L3:
.word 1073741824 // 2.0f
.word 1073741824
.word 1073741824
.word 1073741824
.word 1084227584 // 5.0f
.word 1084227584
.word 1084227584
.word 1084227584
.word 1082130432 // 4.0f
.word 1082130432
.word 1082130432
.word 1082130432
2022-01-13 Christophe Lyon <christophe.lyon@foss.st.com>
PR target/100757
gcc/
* config/arm/arm-protos.h (arm_get_mask_mode): New prototype.
(arm_expand_vector_compare): Update prototype.
* config/arm/arm.c (TARGET_VECTORIZE_GET_MASK_MODE): New.
(arm_vector_mode_supported_p): Add support for VxBI modes.
(arm_expand_vector_compare): Remove useless generation of vpsel.
(arm_expand_vcond): Fix select operands.
(arm_get_mask_mode): New.
* config/arm/mve.md (vec_cmp<mode><MVE_vpred>): New.
(vec_cmpu<mode><MVE_vpred>): New.
(vcond_mask_<mode><MVE_vpred>): New.
* config/arm/vec-common.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): Move to ...
* config/arm/neon.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): ... here
and disable for MVE.
* doc/sourcebuild.texi (arm_mve): Document new effective-target.
gcc/testsuite/
* gcc.dg/signbit-2.c: Skip when targeting ARM/MVE.
* lib/target-supports.exp (check_effective_target_arm_mve): New.
Diff:
---
gcc/config/arm/arm-protos.h | 3 +-
gcc/config/arm/arm.c | 118 ++++++++++++----------------------
gcc/config/arm/mve.md | 54 ++++++++++++++++
gcc/config/arm/neon.md | 39 +++++++++++
gcc/config/arm/vec-common.md | 52 ---------------
gcc/doc/sourcebuild.texi | 4 ++
gcc/testsuite/gcc.dg/signbit-2.c | 1 +
gcc/testsuite/lib/target-supports.exp | 12 ++++
8 files changed, 153 insertions(+), 130 deletions(-)
diff --git a/gcc/config/arm/arm-protos.h b/gcc/config/arm/arm-protos.h
index b978adf2038..a84613104b1 100644
--- a/gcc/config/arm/arm-protos.h
+++ b/gcc/config/arm/arm-protos.h
@@ -202,6 +202,7 @@ extern void arm_init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree);
extern bool arm_pad_reg_upward (machine_mode, tree, int);
#endif
extern int arm_apply_result_size (void);
+extern opt_machine_mode arm_get_mask_mode (machine_mode mode);
#endif /* RTX_CODE */
@@ -378,7 +379,7 @@ extern void arm_emit_coreregs_64bit_shift (enum rtx_code, rtx, rtx, rtx, rtx,
extern bool arm_fusion_enabled_p (tune_params::fuse_ops);
extern bool arm_valid_symbolic_address_p (rtx);
extern bool arm_validize_comparison (rtx *, rtx *, rtx *);
-extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool, bool);
+extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool);
#endif /* RTX_CODE */
extern bool arm_gen_setmem (rtx *);
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index fa18c7bd3fe..7d56fa71806 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -829,6 +829,10 @@ static const struct attribute_spec arm_attribute_table[] =
#undef TARGET_MD_ASM_ADJUST
#define TARGET_MD_ASM_ADJUST arm_md_asm_adjust
+
+#undef TARGET_VECTORIZE_GET_MASK_MODE
+#define TARGET_VECTORIZE_GET_MASK_MODE arm_get_mask_mode
+
\f
/* Obstack for minipool constant handling. */
static struct obstack minipool_obstack;
@@ -29234,7 +29238,8 @@ arm_vector_mode_supported_p (machine_mode mode)
if (TARGET_HAVE_MVE
&& (mode == V2DImode || mode == V4SImode || mode == V8HImode
- || mode == V16QImode))
+ || mode == V16QImode
+ || mode == V16BImode || mode == V8BImode || mode == V4BImode))
return true;
if (TARGET_HAVE_MVE_FLOAT
@@ -31033,7 +31038,7 @@ arm_split_atomic_op (enum rtx_code code, rtx old_out, rtx new_out, rtx mem,
}
\f
/* Return the mode for the MVE vector of predicates corresponding to MODE. */
-machine_mode
+opt_machine_mode
arm_mode_to_pred_mode (machine_mode mode)
{
switch (GET_MODE_NUNITS (mode))
@@ -31042,7 +31047,7 @@ arm_mode_to_pred_mode (machine_mode mode)
case 8: return V8BImode;
case 4: return V4BImode;
}
- gcc_unreachable ();
+ return opt_machine_mode ();
}
/* Expand code to compare vectors OP0 and OP1 using condition CODE.
@@ -31050,16 +31055,12 @@ arm_mode_to_pred_mode (machine_mode mode)
and return true if TARGET contains the inverse. If !CAN_INVERT,
always store the result in TARGET, never its inverse.
- If VCOND_MVE, do not emit the vpsel instruction here, let arm_expand_vcond do
- it with the right destination type to avoid emiting two vpsel, one here and
- one in arm_expand_vcond.
-
Note that the handling of floating-point comparisons is not
IEEE compliant. */
bool
arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
- bool can_invert, bool vcond_mve)
+ bool can_invert)
{
machine_mode cmp_result_mode = GET_MODE (target);
machine_mode cmp_mode = GET_MODE (op0);
@@ -31088,7 +31089,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
and then store its inverse in TARGET. This avoids reusing
TARGET (which for integer NE could be one of the inputs). */
rtx tmp = gen_reg_rtx (cmp_result_mode);
- if (arm_expand_vector_compare (tmp, code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (tmp, code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_NOT (cmp_result_mode, tmp)));
return false;
@@ -31124,36 +31125,22 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case NE:
if (TARGET_HAVE_MVE)
{
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vcmpq_f (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq_f (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
else
gcc_unreachable ();
break;
default:
gcc_unreachable ();
}
-
- /* If we are not expanding a vcond, build the result here. */
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
}
else
emit_insn (gen_neon_vc (code, cmp_mode, target, op0, op1));
@@ -31165,23 +31152,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case GEU:
case GTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
else
emit_insn (gen_neon_vc (code, cmp_mode, target,
op0, force_reg (cmp_mode, op1)));
@@ -31192,23 +31164,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case LEU:
case LTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, vpr_p0, force_reg (cmp_mode, op1), op0));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, target,
+ force_reg (cmp_mode, op1), op0));
else
emit_insn (gen_neon_vc (swap_condition (code), cmp_mode,
target, force_reg (cmp_mode, op1), op0));
@@ -31223,8 +31180,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
rtx gt_res = gen_reg_rtx (cmp_result_mode);
rtx alt_res = gen_reg_rtx (cmp_result_mode);
rtx_code alt_code = (code == LTGT ? LT : LE);
- if (arm_expand_vector_compare (gt_res, GT, op0, op1, true, vcond_mve)
- || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (gt_res, GT, op0, op1, true)
+ || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_IOR (cmp_result_mode,
gt_res, alt_res)));
@@ -31244,19 +31201,15 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
{
/* When expanding for MVE, we do not want to emit a (useless) vpsel in
arm_expand_vector_compare, and another one here. */
- bool vcond_mve=false;
rtx mask;
if (TARGET_HAVE_MVE)
- {
- vcond_mve=true;
- mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
- }
+ mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode).require ());
else
mask = gen_reg_rtx (cmp_result_mode);
bool inverted = arm_expand_vector_compare (mask, GET_CODE (operands[3]),
- operands[4], operands[5], true, vcond_mve);
+ operands[4], operands[5], true);
if (inverted)
std::swap (operands[1], operands[2]);
if (TARGET_NEON)
@@ -31264,20 +31217,20 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
mask, operands[1], operands[2]));
else
{
- machine_mode cmp_mode = GET_MODE (operands[4]);
- rtx vpr_p0 = mask;
- rtx zero = gen_reg_rtx (cmp_mode);
- rtx one = gen_reg_rtx (cmp_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_mode));
- emit_move_insn (one, CONST1_RTX (cmp_mode));
+ machine_mode cmp_mode = GET_MODE (operands[0]);
+
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_mode, operands[0],
+ operands[1], operands[2], mask));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0],
+ operands[1], operands[2], mask));
+ else
+ gcc_unreachable ();
break;
default:
gcc_unreachable ();
@@ -34187,4 +34140,15 @@ arm_mode_base_reg_class (machine_mode mode)
struct gcc_target targetm = TARGET_INITIALIZER;
+/* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
+
+opt_machine_mode
+arm_get_mask_mode (machine_mode mode)
+{
+ if (TARGET_HAVE_MVE)
+ return arm_mode_to_pred_mode (mode);
+
+ return default_get_mask_mode (mode);
+}
+
#include "gt-arm.h"
diff --git a/gcc/config/arm/mve.md b/gcc/config/arm/mve.md
index 983aa10e652..35564e870bc 100644
--- a/gcc/config/arm/mve.md
+++ b/gcc/config/arm/mve.md
@@ -10527,3 +10527,57 @@
operands[1] = force_reg (<MODE>mode, operands[1]);
}
)
+
+;; Expanders for vec_cmp and vcond
+
+(define_expand "vec_cmp<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_VLD_ST 2 "s_register_operand")
+ (match_operand:MVE_VLD_ST 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_2 2 "s_register_operand")
+ (match_operand:MVE_2 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><MVE_vpred>"
+ [(set (match_operand:MVE_VLD_ST 0 "s_register_operand")
+ (if_then_else:MVE_VLD_ST
+ (match_operand:<MVE_VPRED> 3 "s_register_operand")
+ (match_operand:MVE_VLD_ST 1 "s_register_operand")
+ (match_operand:MVE_VLD_ST 2 "s_register_operand")))]
+ "TARGET_HAVE_MVE"
+{
+ switch (GET_MODE_CLASS (<MODE>mode))
+ {
+ case MODE_VECTOR_INT:
+ emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ break;
+ case MODE_VECTOR_FLOAT:
+ if (TARGET_HAVE_MVE_FLOAT)
+ emit_insn (gen_mve_vpselq_f (<MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ else
+ gcc_unreachable ();
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ DONE;
+})
diff --git a/gcc/config/arm/neon.md b/gcc/config/arm/neon.md
index e06c8245672..20e9f11ec81 100644
--- a/gcc/config/arm/neon.md
+++ b/gcc/config/arm/neon.md
@@ -1394,6 +1394,45 @@
[(set_attr "type" "neon_qsub<q>")]
)
+(define_expand "vec_cmp<mode><v_cmp_result>"
+ [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
+ (match_operator:<V_cmp_result> 1 "comparison_operator"
+ [(match_operand:VDQWH 2 "s_register_operand")
+ (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><mode>"
+ [(set (match_operand:VDQIW 0 "s_register_operand")
+ (match_operator:VDQIW 1 "comparison_operator"
+ [(match_operand:VDQIW 2 "s_register_operand")
+ (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><v_cmp_result>"
+ [(set (match_operand:VDQWH 0 "s_register_operand")
+ (if_then_else:VDQWH
+ (match_operand:<V_cmp_result> 3 "s_register_operand")
+ (match_operand:VDQWH 1 "s_register_operand")
+ (match_operand:VDQWH 2 "s_register_operand")))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ emit_insn (gen_neon_vbsl<mode> (operands[0], operands[3], operands[1],
+ operands[2]));
+ DONE;
+})
+
;; Patterns for builtins.
; good for plain vadd, vaddq.
diff --git a/gcc/config/arm/vec-common.md b/gcc/config/arm/vec-common.md
index cef358e44f5..20586973ed9 100644
--- a/gcc/config/arm/vec-common.md
+++ b/gcc/config/arm/vec-common.md
@@ -363,33 +363,6 @@
}
})
-(define_expand "vec_cmp<mode><v_cmp_result>"
- [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
- (match_operator:<V_cmp_result> 1 "comparison_operator"
- [(match_operand:VDQWH 2 "s_register_operand")
- (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
-(define_expand "vec_cmpu<mode><mode>"
- [(set (match_operand:VDQIW 0 "s_register_operand")
- (match_operator:VDQIW 1 "comparison_operator"
- [(match_operand:VDQIW 2 "s_register_operand")
- (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
;; Conditional instructions. These are comparisons with conditional moves for
;; vectors. They perform the assignment:
;;
@@ -461,31 +434,6 @@
DONE;
})
-(define_expand "vcond_mask_<mode><v_cmp_result>"
- [(set (match_operand:VDQWH 0 "s_register_operand")
- (if_then_else:VDQWH
- (match_operand:<V_cmp_result> 3 "s_register_operand")
- (match_operand:VDQWH 1 "s_register_operand")
- (match_operand:VDQWH 2 "s_register_operand")))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- if (TARGET_NEON)
- {
- emit_insn (gen_neon_vbsl (<MODE>mode, operands[0], operands[3],
- operands[1], operands[2]));
- }
- else if (TARGET_HAVE_MVE)
- {
- emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
- operands[1], operands[2], operands[3]));
- }
- else
- gcc_unreachable ();
- DONE;
-})
-
(define_expand "vec_load_lanesoi<mode>"
[(set (match_operand:OI 0 "s_register_operand")
(unspec:OI [(match_operand:OI 1 "neon_struct_operand")
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index 6095a35cd45..8d369935396 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -2236,6 +2236,10 @@ ARM target supports the @code{-mfloat-abi=softfp} option.
@anchor{arm_hard_ok}
ARM target supports the @code{-mfloat-abi=hard} option.
+@item arm_mve
+@anchor{arm_mve}
+ARM target supports generating MVE instructions.
+
@item arm_v8_1_lob_ok
@anchor{arm_v8_1_lob_ok}
ARM Target supports executing the Armv8.1-M Mainline Low Overhead Loop
diff --git a/gcc/testsuite/gcc.dg/signbit-2.c b/gcc/testsuite/gcc.dg/signbit-2.c
index b609f67dc9f..2f2dc448286 100644
--- a/gcc/testsuite/gcc.dg/signbit-2.c
+++ b/gcc/testsuite/gcc.dg/signbit-2.c
@@ -4,6 +4,7 @@
/* This test does not work when the truth type does not match vector type. */
/* { dg-additional-options "-mno-avx512f" { target { i?86-*-* x86_64-*-* } } } */
/* { dg-additional-options "-march=armv8-a" { target aarch64_sve } } */
+/* { dg-skip-if "no fallback for MVE" { arm_mve } } */
#include <stdint.h>
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index 0fe1e1e077a..8dac516ec12 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -5234,6 +5234,18 @@ proc check_effective_target_arm_hard_ok { } {
} "-mfloat-abi=hard"]
}
+# Return 1 if this is an ARM target supporting MVE.
+proc check_effective_target_arm_mve { } {
+ if { ![istarget arm*-*-*] } {
+ return 0
+ }
+ return [check_no_compiler_messages arm_mve assembly {
+ #if !defined (__ARM_FEATURE_MVE)
+ #error FOO
+ #endif
+ }]
+}
+
# Return 1 if the target supports ARMv8.1-M MVE with floating point
# instructions, 0 otherwise. The test is valid for ARM.
# Record the command line options needed.
^ permalink raw reply [flat|nested] 6+ messages in thread
* [gcc(refs/users/clyon/heads/mve-autovec)] arm: Fix vcond_mask expander for MVE (PR target/100757)
@ 2021-11-16 14:07 Christophe Lyon
0 siblings, 0 replies; 6+ messages in thread
From: Christophe Lyon @ 2021-11-16 14:07 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:9c2d8285be52f1bbc2c6fd49b6cbda61797278dc
commit 9c2d8285be52f1bbc2c6fd49b6cbda61797278dc
Author: Christophe Lyon <christophe.lyon@foss.st.com>
Date: Wed Oct 20 15:30:16 2021 +0000
arm: Fix vcond_mask expander for MVE (PR target/100757)
The problem in this PR is that we call VPSEL with a mask of vector
type instead of HImode. This happens because operand 3 in vcond_mask
is the pre-computed vector comparison and has vector type.
This patch fixes it by implementing TARGET_VECTORIZE_GET_MASK_MODE,
returning the appropriate VxBI mode when targeting MVE. In turn, this
implies implementing vec_cmp<mode><MVE_vpred>,
vec_cmpu<mode><MVE_vpred> and vcond_mask_<mode><MVE_vpred>, and we can
move vec_cmp<mode><v_cmp_result>, vec_cmpu<mode><mode> and
vcond_mask_<mode><v_cmp_result> back to neon.md since they are not
used by MVE anymore. The new *<MVE_vpred> patterns listed above are
implemented in mve.md since they are only valid for MVE. However this
may make maintenance/comparison more painful than having all of them
in vec-common.md.
In the process, we can get rid of the recently added vcond_mve
parameter of arm_expand_vector_compare.
Compared to neon.md's vcond_mask_<mode><v_cmp_result> before my "arm:
Auto-vectorization for MVE: vcmp" patch (r12-834), it keeps the VDQWH
iterator added in r12-835 (to have V4HF/V8HF support), as well as the
(!<Is_float_mode> || flag_unsafe_math_optimizations) condition which
was not present before r12-834 although SF modes were enabled by VDQW
(I think this was a bug).
Using TARGET_VECTORIZE_GET_MASK_MODE has the advantage that we no
longer need to generate vpsel with vectors of 0 and 1: the masks are
now merged via scalar 'ands' instructions operating on 16-bit masks
after converting the boolean vectors.
In addition, this patch fixes a problem in arm_expand_vcond() where
the result would be a vector of 0 or 1 instead of operand 1 or 2.
Reducing the number of iterations in pr100757-3.c from 32 to 8, we
generate the code below:
float a[32];
float fn1(int d) {
float c = 4.0f;
for (int b = 0; b < 8; b++)
if (a[b] != 2.0f)
c = 5.0f;
return c;
}
fn1:
ldr r3, .L3+48
vldr.64 d4, .L3 // q2=(2.0,2.0,2.0,2.0)
vldr.64 d5, .L3+8
vldrw.32 q0, [r3] // q0=a(0..3)
adds r3, r3, #16
vcmp.f32 eq, q0, q2 // cmp a(0..3) == (2.0,2.0,2.0,2.0)
vldrw.32 q1, [r3] // q1=a(4..7)
vmrs r3, P0
vcmp.f32 eq, q1, q2 // cmp a(4..7) == (2.0,2.0,2.0,2.0)
vmrs r2, P0 @ movhi
ands r3, r3, r2 // r3=select(a(0..3]) & select(a(4..7))
vldr.64 d4, .L3+16 // q2=(5.0,5.0,5.0,5.0)
vldr.64 d5, .L3+24
vmsr P0, r3
vldr.64 d6, .L3+32 // q3=(4.0,4.0,4.0,4.0)
vldr.64 d7, .L3+40
vpsel q3, q3, q2 // q3=vcond_mask(4.0,5.0)
vmov.32 r2, q3[1] // keep the scalar max
vmov.32 r0, q3[3]
vmov.32 r3, q3[2]
vmov.f32 s11, s12
vmov s15, r2
vmov s14, r3
vmaxnm.f32 s15, s11, s15
vmaxnm.f32 s15, s15, s14
vmov s14, r0
vmaxnm.f32 s15, s15, s14
vmov r0, s15
bx lr
.L4:
.align 3
.L3:
.word 1073741824 // 2.0f
.word 1073741824
.word 1073741824
.word 1073741824
.word 1084227584 // 5.0f
.word 1084227584
.word 1084227584
.word 1084227584
.word 1082130432 // 4.0f
.word 1082130432
.word 1082130432
.word 1082130432
2021-10-13 Christophe Lyon <christophe.lyon@foss.st.com>
PR target/100757
gcc/
* config/arm/arm-protos.h (arm_get_mask_mode): New prototype.
(arm_expand_vector_compare): Update prototype.
* config/arm/arm.c (TARGET_VECTORIZE_GET_MASK_MODE): New.
(arm_vector_mode_supported_p): Add support for VxBI modes.
(arm_expand_vector_compare): Remove useless generation of vpsel.
(arm_expand_vcond): Fix select operands.
(arm_get_mask_mode): New.
* config/arm/mve.md (vec_cmp<mode><MVE_vpred>): New.
(vec_cmpu<mode><MVE_vpred>): New.
(vcond_mask_<mode><MVE_vpred>): New.
* config/arm/vec-common.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): Move to ...
* config/arm/neon.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): ... here
and disable for MVE.
Diff:
---
gcc/config/arm/arm-protos.h | 3 +-
gcc/config/arm/arm.c | 118 +++++++++++++++----------------------------
gcc/config/arm/mve.md | 54 ++++++++++++++++++++
gcc/config/arm/neon.md | 39 ++++++++++++++
gcc/config/arm/vec-common.md | 52 -------------------
5 files changed, 136 insertions(+), 130 deletions(-)
diff --git a/gcc/config/arm/arm-protos.h b/gcc/config/arm/arm-protos.h
index 9b1f61394ad..9e3d71e0c29 100644
--- a/gcc/config/arm/arm-protos.h
+++ b/gcc/config/arm/arm-protos.h
@@ -201,6 +201,7 @@ extern void arm_init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree);
extern bool arm_pad_reg_upward (machine_mode, tree, int);
#endif
extern int arm_apply_result_size (void);
+extern opt_machine_mode arm_get_mask_mode (machine_mode mode);
#endif /* RTX_CODE */
@@ -372,7 +373,7 @@ extern void arm_emit_coreregs_64bit_shift (enum rtx_code, rtx, rtx, rtx, rtx,
extern bool arm_fusion_enabled_p (tune_params::fuse_ops);
extern bool arm_valid_symbolic_address_p (rtx);
extern bool arm_validize_comparison (rtx *, rtx *, rtx *);
-extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool, bool);
+extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool);
#endif /* RTX_CODE */
extern bool arm_gen_setmem (rtx *);
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index 94696d528f8..37b6f230ba4 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -835,6 +835,10 @@ static const struct attribute_spec arm_attribute_table[] =
#undef TARGET_MD_ASM_ADJUST
#define TARGET_MD_ASM_ADJUST arm_md_asm_adjust
+
+#undef TARGET_VECTORIZE_GET_MASK_MODE
+#define TARGET_VECTORIZE_GET_MASK_MODE arm_get_mask_mode
+
\f
/* Obstack for minipool constant handling. */
static struct obstack minipool_obstack;
@@ -29196,7 +29200,8 @@ arm_vector_mode_supported_p (machine_mode mode)
if (TARGET_HAVE_MVE
&& (mode == V2DImode || mode == V4SImode || mode == V8HImode
- || mode == V16QImode))
+ || mode == V16QImode
+ || mode == V16BImode || mode == V8BImode || mode == V4BImode))
return true;
if (TARGET_HAVE_MVE_FLOAT
@@ -30995,7 +31000,7 @@ arm_split_atomic_op (enum rtx_code code, rtx old_out, rtx new_out, rtx mem,
}
\f
/* Return the mode for the MVE vector of predicates corresponding to MODE. */
-machine_mode
+opt_machine_mode
arm_mode_to_pred_mode (machine_mode mode)
{
switch (GET_MODE_NUNITS (mode))
@@ -31004,7 +31009,7 @@ arm_mode_to_pred_mode (machine_mode mode)
case 8: return V8BImode;
case 4: return V4BImode;
}
- gcc_unreachable ();
+ return opt_machine_mode ();
}
/* Expand code to compare vectors OP0 and OP1 using condition CODE.
@@ -31012,16 +31017,12 @@ arm_mode_to_pred_mode (machine_mode mode)
and return true if TARGET contains the inverse. If !CAN_INVERT,
always store the result in TARGET, never its inverse.
- If VCOND_MVE, do not emit the vpsel instruction here, let arm_expand_vcond do
- it with the right destination type to avoid emiting two vpsel, one here and
- one in arm_expand_vcond.
-
Note that the handling of floating-point comparisons is not
IEEE compliant. */
bool
arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
- bool can_invert, bool vcond_mve)
+ bool can_invert)
{
machine_mode cmp_result_mode = GET_MODE (target);
machine_mode cmp_mode = GET_MODE (op0);
@@ -31050,7 +31051,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
and then store its inverse in TARGET. This avoids reusing
TARGET (which for integer NE could be one of the inputs). */
rtx tmp = gen_reg_rtx (cmp_result_mode);
- if (arm_expand_vector_compare (tmp, code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (tmp, code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_NOT (cmp_result_mode, tmp)));
return false;
@@ -31086,36 +31087,22 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case NE:
if (TARGET_HAVE_MVE)
{
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vcmpq_f (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq_f (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
else
gcc_unreachable ();
break;
default:
gcc_unreachable ();
}
-
- /* If we are not expanding a vcond, build the result here. */
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
}
else
emit_insn (gen_neon_vc (code, cmp_mode, target, op0, op1));
@@ -31127,23 +31114,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case GEU:
case GTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target,
+ op0, force_reg (cmp_mode, op1)));
else
emit_insn (gen_neon_vc (code, cmp_mode, target,
op0, force_reg (cmp_mode, op1)));
@@ -31154,23 +31126,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case LEU:
case LTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, vpr_p0, force_reg (cmp_mode, op1), op0));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, target,
+ force_reg (cmp_mode, op1), op0));
else
emit_insn (gen_neon_vc (swap_condition (code), cmp_mode,
target, force_reg (cmp_mode, op1), op0));
@@ -31185,8 +31142,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
rtx gt_res = gen_reg_rtx (cmp_result_mode);
rtx alt_res = gen_reg_rtx (cmp_result_mode);
rtx_code alt_code = (code == LTGT ? LT : LE);
- if (arm_expand_vector_compare (gt_res, GT, op0, op1, true, vcond_mve)
- || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (gt_res, GT, op0, op1, true)
+ || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_IOR (cmp_result_mode,
gt_res, alt_res)));
@@ -31206,19 +31163,15 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
{
/* When expanding for MVE, we do not want to emit a (useless) vpsel in
arm_expand_vector_compare, and another one here. */
- bool vcond_mve=false;
rtx mask;
if (TARGET_HAVE_MVE)
- {
- vcond_mve=true;
- mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
- }
+ mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode).require ());
else
mask = gen_reg_rtx (cmp_result_mode);
bool inverted = arm_expand_vector_compare (mask, GET_CODE (operands[3]),
- operands[4], operands[5], true, vcond_mve);
+ operands[4], operands[5], true);
if (inverted)
std::swap (operands[1], operands[2]);
if (TARGET_NEON)
@@ -31226,20 +31179,20 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
mask, operands[1], operands[2]));
else
{
- machine_mode cmp_mode = GET_MODE (operands[4]);
- rtx vpr_p0 = mask;
- rtx zero = gen_reg_rtx (cmp_mode);
- rtx one = gen_reg_rtx (cmp_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_mode));
- emit_move_insn (one, CONST1_RTX (cmp_mode));
+ machine_mode cmp_mode = GET_MODE (operands[0]);
+
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_mode, operands[0],
+ operands[1], operands[2], mask));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0],
+ operands[1], operands[2], mask));
+ else
+ gcc_unreachable ();
break;
default:
gcc_unreachable ();
@@ -34149,4 +34102,15 @@ arm_mode_base_reg_class (machine_mode mode)
struct gcc_target targetm = TARGET_INITIALIZER;
+/* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
+
+opt_machine_mode
+arm_get_mask_mode (machine_mode mode)
+{
+ if (TARGET_HAVE_MVE)
+ return arm_mode_to_pred_mode (mode);
+
+ return default_get_mask_mode (mode);
+}
+
#include "gt-arm.h"
diff --git a/gcc/config/arm/mve.md b/gcc/config/arm/mve.md
index ccca7f8d2c0..67b537a5c7e 100644
--- a/gcc/config/arm/mve.md
+++ b/gcc/config/arm/mve.md
@@ -10527,3 +10527,57 @@
operands[1] = force_reg (<MODE>mode, operands[1]);
}
)
+
+;; Expanders for vec_cmp and vcond
+
+(define_expand "vec_cmp<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_VLD_ST 2 "s_register_operand")
+ (match_operand:MVE_VLD_ST 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_2 2 "s_register_operand")
+ (match_operand:MVE_2 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><MVE_vpred>"
+ [(set (match_operand:MVE_VLD_ST 0 "s_register_operand")
+ (if_then_else:MVE_VLD_ST
+ (match_operand:<MVE_VPRED> 3 "s_register_operand")
+ (match_operand:MVE_VLD_ST 1 "s_register_operand")
+ (match_operand:MVE_VLD_ST 2 "s_register_operand")))]
+ "TARGET_HAVE_MVE"
+{
+ switch (GET_MODE_CLASS (<MODE>mode))
+ {
+ case MODE_VECTOR_INT:
+ emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ break;
+ case MODE_VECTOR_FLOAT:
+ if (TARGET_HAVE_MVE_FLOAT)
+ emit_insn (gen_mve_vpselq_f (<MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ else
+ gcc_unreachable ();
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ DONE;
+})
diff --git a/gcc/config/arm/neon.md b/gcc/config/arm/neon.md
index 8b0a396947c..28310d93a4e 100644
--- a/gcc/config/arm/neon.md
+++ b/gcc/config/arm/neon.md
@@ -1394,6 +1394,45 @@
[(set_attr "type" "neon_qsub<q>")]
)
+(define_expand "vec_cmp<mode><v_cmp_result>"
+ [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
+ (match_operator:<V_cmp_result> 1 "comparison_operator"
+ [(match_operand:VDQWH 2 "s_register_operand")
+ (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><mode>"
+ [(set (match_operand:VDQIW 0 "s_register_operand")
+ (match_operator:VDQIW 1 "comparison_operator"
+ [(match_operand:VDQIW 2 "s_register_operand")
+ (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><v_cmp_result>"
+ [(set (match_operand:VDQWH 0 "s_register_operand")
+ (if_then_else:VDQWH
+ (match_operand:<V_cmp_result> 3 "s_register_operand")
+ (match_operand:VDQWH 1 "s_register_operand")
+ (match_operand:VDQWH 2 "s_register_operand")))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ emit_insn (gen_neon_vbsl<mode> (operands[0], operands[3], operands[1],
+ operands[2]));
+ DONE;
+})
+
;; Patterns for builtins.
; good for plain vadd, vaddq.
diff --git a/gcc/config/arm/vec-common.md b/gcc/config/arm/vec-common.md
index 68de4f0f943..9b461a76155 100644
--- a/gcc/config/arm/vec-common.md
+++ b/gcc/config/arm/vec-common.md
@@ -363,33 +363,6 @@
}
})
-(define_expand "vec_cmp<mode><v_cmp_result>"
- [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
- (match_operator:<V_cmp_result> 1 "comparison_operator"
- [(match_operand:VDQWH 2 "s_register_operand")
- (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
-(define_expand "vec_cmpu<mode><mode>"
- [(set (match_operand:VDQIW 0 "s_register_operand")
- (match_operator:VDQIW 1 "comparison_operator"
- [(match_operand:VDQIW 2 "s_register_operand")
- (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
;; Conditional instructions. These are comparisons with conditional moves for
;; vectors. They perform the assignment:
;;
@@ -461,31 +434,6 @@
DONE;
})
-(define_expand "vcond_mask_<mode><v_cmp_result>"
- [(set (match_operand:VDQWH 0 "s_register_operand")
- (if_then_else:VDQWH
- (match_operand:<V_cmp_result> 3 "s_register_operand")
- (match_operand:VDQWH 1 "s_register_operand")
- (match_operand:VDQWH 2 "s_register_operand")))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- if (TARGET_NEON)
- {
- emit_insn (gen_neon_vbsl (<MODE>mode, operands[0], operands[3],
- operands[1], operands[2]));
- }
- else if (TARGET_HAVE_MVE)
- {
- emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
- operands[1], operands[2], operands[3]));
- }
- else
- gcc_unreachable ();
- DONE;
-})
-
(define_expand "vec_load_lanesoi<mode>"
[(set (match_operand:OI 0 "s_register_operand")
(unspec:OI [(match_operand:OI 1 "neon_struct_operand")
^ permalink raw reply [flat|nested] 6+ messages in thread
* [gcc(refs/users/clyon/heads/mve-autovec)] arm: Fix vcond_mask expander for MVE (PR target/100757)
@ 2021-10-01 14:37 Christophe Lyon
0 siblings, 0 replies; 6+ messages in thread
From: Christophe Lyon @ 2021-10-01 14:37 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:24f07d42ba15a55df09b1de618f9b3c230fa54ce
commit 24f07d42ba15a55df09b1de618f9b3c230fa54ce
Author: Christophe Lyon <christophe.lyon@linaro.org>
Date: Tue Jun 8 21:32:28 2021 +0000
arm: Fix vcond_mask expander for MVE (PR target/100757)
The problem in this PR is that we call VPSEL with a mask of vector
type instead of HImode. This happens because operand 3 in vcond_mask
is the pre-computed vector comparison and has vector type.
This patch fixes it by implementing TARGET_VECTORIZE_GET_MASK_MODE,
returning the appropriate VxBI mode when targeting MVE. In turn, this
implies implementing vec_cmp<mode><MVE_vpred>,
vec_cmpu<mode><MVE_vpred> and vcond_mask_<mode><MVE_vpred>, and we can
move vec_cmp<mode><v_cmp_result>, vec_cmpu<mode><mode> and
vcond_mask_<mode><v_cmp_result> back to neon.md since they are not
used by MVE anymore. The new *<MVE_vpred> patterns listed above are
implemented in mve.md since they are only valid for MVE. However this
may make maintenance/comparison more painful than having all of them
in vec-common.md.
In the process, we can get rid of the recently added vcond_mve
parameter of arm_expand_vector_compare.
Compared to neon.md's vcond_mask_<mode><v_cmp_result> before my "arm:
Auto-vectorization for MVE: vcmp" patch (r12-834), it keeps the VDQWH
iterator added in r12-835 (to have V4HF/V8HF support), as well as the
(!<Is_float_mode> || flag_unsafe_math_optimizations) condition which
was not present before r12-834 although SF modes were enabled by VDQW
(I think this was a bug).
Using TARGET_VECTORIZE_GET_MASK_MODE has the advantage that we no
longer need to generate vpsel with vectors of 0 and 1: the masks are
now merged via scalar 'ands' instructions operating on 16-bit masks
after converting the boolean vectors.
In addition, this patch fixes a problem in arm_expand_vcond() where
the result would be a vector of 0 or 1 instead of operand 1 or 2.
Reducing the number of iterations in pr100757-3.c from 32 to 8, we
generate the code below:
float a[32];
float fn1(int d) {
float c = 4.0f;
for (int b = 0; b < 8; b++)
if (a[b] != 2.0f)
c = 5.0f;
return c;
}
fn1:
ldr r3, .L3+48
vldr.64 d4, .L3 // q2=(2.0,2.0,2.0,2.0)
vldr.64 d5, .L3+8
vldrw.32 q0, [r3] // q0=a(0..3)
adds r3, r3, #16
vcmp.f32 eq, q0, q2 // cmp a(0..3) == (2.0,2.0,2.0,2.0)
vldrw.32 q1, [r3] // q1=a(4..7)
vmrs r3, P0
vcmp.f32 eq, q1, q2 // cmp a(4..7) == (2.0,2.0,2.0,2.0)
vmrs r2, P0 @ movhi
ands r3, r3, r2 // r3=select(a(0..3]) & select(a(4..7))
vldr.64 d4, .L3+16 // q2=(5.0,5.0,5.0,5.0)
vldr.64 d5, .L3+24
vmsr P0, r3
vldr.64 d6, .L3+32 // q3=(4.0,4.0,4.0,4.0)
vldr.64 d7, .L3+40
vpsel q3, q3, q2 // q3=vcond_mask(4.0,5.0)
vmov.32 r2, q3[1] // keep the scalar max
vmov.32 r0, q3[3]
vmov.32 r3, q3[2]
vmov.f32 s11, s12
vmov s15, r2
vmov s14, r3
vmaxnm.f32 s15, s11, s15
vmaxnm.f32 s15, s15, s14
vmov s14, r0
vmaxnm.f32 s15, s15, s14
vmov r0, s15
bx lr
.L4:
.align 3
.L3:
.word 1073741824 // 2.0f
.word 1073741824
.word 1073741824
.word 1073741824
.word 1084227584 // 5.0f
.word 1084227584
.word 1084227584
.word 1084227584
.word 1082130432 // 4.0f
.word 1082130432
.word 1082130432
.word 1082130432
2021-09-02 Christophe Lyon <christophe.lyon@linaro.org>
PR target/100757
gcc/
* config/arm/arm-protos.h (arm_get_mask_mode): New prototype.
(arm_expand_vector_compare): Update prototype.
* config/arm/arm.c (TARGET_VECTORIZE_GET_MASK_MODE): New.
(arm_vector_mode_supported_p): Add support for VxBI modes.
(arm_expand_vector_compare): Remove useless generation of vpsel.
(arm_expand_vcond): Fix select operands.
(arm_get_mask_mode): New.
* config/arm/mve.md (vec_cmp<mode><MVE_vpred>): New.
(vec_cmpu<mode><MVE_vpred>): New.
(vcond_mask_<mode><MVE_vpred>): New.
* config/arm/vec-common.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): Move to ...
* config/arm/neon.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): ... here
and disable for MVE.
Diff:
---
gcc/config/arm/arm-protos.h | 3 +-
gcc/config/arm/arm.c | 110 ++++++++++++++-----------------------------
gcc/config/arm/mve.md | 55 ++++++++++++++++++++++
gcc/config/arm/neon.md | 39 +++++++++++++++
gcc/config/arm/vec-common.md | 52 --------------------
5 files changed, 131 insertions(+), 128 deletions(-)
diff --git a/gcc/config/arm/arm-protos.h b/gcc/config/arm/arm-protos.h
index 9b1f61394ad..9e3d71e0c29 100644
--- a/gcc/config/arm/arm-protos.h
+++ b/gcc/config/arm/arm-protos.h
@@ -201,6 +201,7 @@ extern void arm_init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree);
extern bool arm_pad_reg_upward (machine_mode, tree, int);
#endif
extern int arm_apply_result_size (void);
+extern opt_machine_mode arm_get_mask_mode (machine_mode mode);
#endif /* RTX_CODE */
@@ -372,7 +373,7 @@ extern void arm_emit_coreregs_64bit_shift (enum rtx_code, rtx, rtx, rtx, rtx,
extern bool arm_fusion_enabled_p (tune_params::fuse_ops);
extern bool arm_valid_symbolic_address_p (rtx);
extern bool arm_validize_comparison (rtx *, rtx *, rtx *);
-extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool, bool);
+extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool);
#endif /* RTX_CODE */
extern bool arm_gen_setmem (rtx *);
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index 5f6637d9a5f..3326cd163a2 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -835,6 +835,10 @@ static const struct attribute_spec arm_attribute_table[] =
#undef TARGET_MD_ASM_ADJUST
#define TARGET_MD_ASM_ADJUST arm_md_asm_adjust
+
+#undef TARGET_VECTORIZE_GET_MASK_MODE
+#define TARGET_VECTORIZE_GET_MASK_MODE arm_get_mask_mode
+
\f
/* Obstack for minipool constant handling. */
static struct obstack minipool_obstack;
@@ -29193,7 +29197,8 @@ arm_vector_mode_supported_p (machine_mode mode)
if (TARGET_HAVE_MVE
&& (mode == V2DImode || mode == V4SImode || mode == V8HImode
- || mode == V16QImode))
+ || mode == V16QImode
+ || mode == V16BImode || mode == V8BImode || mode == V4BImode))
return true;
if (TARGET_HAVE_MVE_FLOAT
@@ -31012,16 +31017,12 @@ arm_mode_to_pred_mode (machine_mode mode)
and return true if TARGET contains the inverse. If !CAN_INVERT,
always store the result in TARGET, never its inverse.
- If VCOND_MVE, do not emit the vpsel instruction here, let arm_expand_vcond do
- it with the right destination type to avoid emiting two vpsel, one here and
- one in arm_expand_vcond.
-
Note that the handling of floating-point comparisons is not
IEEE compliant. */
bool
arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
- bool can_invert, bool vcond_mve)
+ bool can_invert)
{
machine_mode cmp_result_mode = GET_MODE (target);
machine_mode cmp_mode = GET_MODE (op0);
@@ -31050,7 +31051,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
and then store its inverse in TARGET. This avoids reusing
TARGET (which for integer NE could be one of the inputs). */
rtx tmp = gen_reg_rtx (cmp_result_mode);
- if (arm_expand_vector_compare (tmp, code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (tmp, code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_NOT (cmp_result_mode, tmp)));
return false;
@@ -31086,36 +31087,20 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case NE:
if (TARGET_HAVE_MVE)
{
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vcmpq_f (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq_f (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
else
gcc_unreachable ();
break;
default:
gcc_unreachable ();
}
-
- /* If we are not expanding a vcond, build the result here. */
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
}
else
emit_insn (gen_neon_vc (code, cmp_mode, target, op0, op1));
@@ -31127,23 +31112,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case GEU:
case GTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
else
emit_insn (gen_neon_vc (code, cmp_mode, target,
op0, force_reg (cmp_mode, op1)));
@@ -31154,23 +31123,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case LEU:
case LTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, vpr_p0, force_reg (cmp_mode, op1), op0));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, target, force_reg (cmp_mode, op1), op0));
else
emit_insn (gen_neon_vc (swap_condition (code), cmp_mode,
target, force_reg (cmp_mode, op1), op0));
@@ -31185,8 +31138,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
rtx gt_res = gen_reg_rtx (cmp_result_mode);
rtx alt_res = gen_reg_rtx (cmp_result_mode);
rtx_code alt_code = (code == LTGT ? LT : LE);
- if (arm_expand_vector_compare (gt_res, GT, op0, op1, true, vcond_mve)
- || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (gt_res, GT, op0, op1, true)
+ || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_IOR (cmp_result_mode,
gt_res, alt_res)));
@@ -31206,19 +31159,15 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
{
/* When expanding for MVE, we do not want to emit a (useless) vpsel in
arm_expand_vector_compare, and another one here. */
- bool vcond_mve=false;
rtx mask;
if (TARGET_HAVE_MVE)
- {
- vcond_mve=true;
- mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
- }
+ mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
else
mask = gen_reg_rtx (cmp_result_mode);
bool inverted = arm_expand_vector_compare (mask, GET_CODE (operands[3]),
- operands[4], operands[5], true, vcond_mve);
+ operands[4], operands[5], true);
if (inverted)
std::swap (operands[1], operands[2]);
if (TARGET_NEON)
@@ -31226,20 +31175,20 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
mask, operands[1], operands[2]));
else
{
- machine_mode cmp_mode = GET_MODE (operands[4]);
- rtx vpr_p0 = mask;
- rtx zero = gen_reg_rtx (cmp_mode);
- rtx one = gen_reg_rtx (cmp_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_mode));
- emit_move_insn (one, CONST1_RTX (cmp_mode));
+ machine_mode cmp_mode = GET_MODE (operands[0]);
+
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_mode, operands[0],
+ operands[1], operands[2], mask));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0],
+ operands[1], operands[2], mask));
+ else
+ gcc_unreachable ();
break;
default:
gcc_unreachable ();
@@ -34149,4 +34098,15 @@ arm_mode_base_reg_class (machine_mode mode)
struct gcc_target targetm = TARGET_INITIALIZER;
+/* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
+
+opt_machine_mode
+arm_get_mask_mode (machine_mode mode)
+{
+ if (TARGET_HAVE_MVE)
+ return arm_mode_to_pred_mode (mode);
+
+ return default_get_mask_mode (mode);
+}
+
#include "gt-arm.h"
diff --git a/gcc/config/arm/mve.md b/gcc/config/arm/mve.md
index c9c8e2c13fe..d663c698cfb 100644
--- a/gcc/config/arm/mve.md
+++ b/gcc/config/arm/mve.md
@@ -10550,3 +10550,58 @@
vmsr%?\t P0, %1
vmrs%?\t %0, P0"
)
+
+;; Expanders for vec_cmp and vcond
+
+(define_expand "vec_cmp<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_VLD_ST 2 "s_register_operand")
+ (match_operand:MVE_VLD_ST 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_2 2 "s_register_operand")
+ (match_operand:MVE_2 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><MVE_vpred>"
+ [(set (match_operand:MVE_VLD_ST 0 "s_register_operand")
+ (if_then_else:MVE_VLD_ST
+ (match_operand:<MVE_VPRED> 3 "s_register_operand")
+ (match_operand:MVE_VLD_ST 1 "s_register_operand")
+ (match_operand:MVE_VLD_ST 2 "s_register_operand")))]
+ "TARGET_HAVE_MVE"
+{
+ switch (GET_MODE_CLASS (<MODE>mode))
+ {
+ case MODE_VECTOR_INT:
+ emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ break;
+ case MODE_VECTOR_FLOAT:
+ if (TARGET_HAVE_MVE_FLOAT)
+ emit_insn (gen_mve_vpselq_f (<MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ else
+ gcc_unreachable ();
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ DONE;
+})
diff --git a/gcc/config/arm/neon.md b/gcc/config/arm/neon.md
index 8b0a396947c..28310d93a4e 100644
--- a/gcc/config/arm/neon.md
+++ b/gcc/config/arm/neon.md
@@ -1394,6 +1394,45 @@
[(set_attr "type" "neon_qsub<q>")]
)
+(define_expand "vec_cmp<mode><v_cmp_result>"
+ [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
+ (match_operator:<V_cmp_result> 1 "comparison_operator"
+ [(match_operand:VDQWH 2 "s_register_operand")
+ (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><mode>"
+ [(set (match_operand:VDQIW 0 "s_register_operand")
+ (match_operator:VDQIW 1 "comparison_operator"
+ [(match_operand:VDQIW 2 "s_register_operand")
+ (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><v_cmp_result>"
+ [(set (match_operand:VDQWH 0 "s_register_operand")
+ (if_then_else:VDQWH
+ (match_operand:<V_cmp_result> 3 "s_register_operand")
+ (match_operand:VDQWH 1 "s_register_operand")
+ (match_operand:VDQWH 2 "s_register_operand")))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ emit_insn (gen_neon_vbsl<mode> (operands[0], operands[3], operands[1],
+ operands[2]));
+ DONE;
+})
+
;; Patterns for builtins.
; good for plain vadd, vaddq.
diff --git a/gcc/config/arm/vec-common.md b/gcc/config/arm/vec-common.md
index 68de4f0f943..9b461a76155 100644
--- a/gcc/config/arm/vec-common.md
+++ b/gcc/config/arm/vec-common.md
@@ -363,33 +363,6 @@
}
})
-(define_expand "vec_cmp<mode><v_cmp_result>"
- [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
- (match_operator:<V_cmp_result> 1 "comparison_operator"
- [(match_operand:VDQWH 2 "s_register_operand")
- (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
-(define_expand "vec_cmpu<mode><mode>"
- [(set (match_operand:VDQIW 0 "s_register_operand")
- (match_operator:VDQIW 1 "comparison_operator"
- [(match_operand:VDQIW 2 "s_register_operand")
- (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
;; Conditional instructions. These are comparisons with conditional moves for
;; vectors. They perform the assignment:
;;
@@ -461,31 +434,6 @@
DONE;
})
-(define_expand "vcond_mask_<mode><v_cmp_result>"
- [(set (match_operand:VDQWH 0 "s_register_operand")
- (if_then_else:VDQWH
- (match_operand:<V_cmp_result> 3 "s_register_operand")
- (match_operand:VDQWH 1 "s_register_operand")
- (match_operand:VDQWH 2 "s_register_operand")))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- if (TARGET_NEON)
- {
- emit_insn (gen_neon_vbsl (<MODE>mode, operands[0], operands[3],
- operands[1], operands[2]));
- }
- else if (TARGET_HAVE_MVE)
- {
- emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
- operands[1], operands[2], operands[3]));
- }
- else
- gcc_unreachable ();
- DONE;
-})
-
(define_expand "vec_load_lanesoi<mode>"
[(set (match_operand:OI 0 "s_register_operand")
(unspec:OI [(match_operand:OI 1 "neon_struct_operand")
^ permalink raw reply [flat|nested] 6+ messages in thread
* [gcc(refs/users/clyon/heads/mve-autovec)] arm: Fix vcond_mask expander for MVE (PR target/100757)
@ 2021-09-29 7:30 Christophe Lyon
0 siblings, 0 replies; 6+ messages in thread
From: Christophe Lyon @ 2021-09-29 7:30 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:dd34ed8ae5187440b5aad860ade75de08b7b8323
commit dd34ed8ae5187440b5aad860ade75de08b7b8323
Author: Christophe Lyon <christophe.lyon@linaro.org>
Date: Tue Jun 8 21:32:28 2021 +0000
arm: Fix vcond_mask expander for MVE (PR target/100757)
The problem in this PR is that we call VPSEL with a mask of vector
type instead of HImode. This happens because operand 3 in vcond_mask
is the pre-computed vector comparison and has vector type.
This patch fixes it by implementing TARGET_VECTORIZE_GET_MASK_MODE,
returning the appropriate VxBI mode when targeting MVE. In turn, this
implies implementing vec_cmp<mode><MVE_vpred>,
vec_cmpu<mode><MVE_vpred> and vcond_mask_<mode><MVE_vpred>, and we can
move vec_cmp<mode><v_cmp_result>, vec_cmpu<mode><mode> and
vcond_mask_<mode><v_cmp_result> back to neon.md since they are not
used by MVE anymore. The new *<MVE_vpred> patterns listed above are
implemented in mve.md since they are only valid for MVE. However this
may make maintenance/comparison more painful than having all of them
in vec-common.md.
In the process, we can get rid of the recently added vcond_mve
parameter of arm_expand_vector_compare.
Compared to neon.md's vcond_mask_<mode><v_cmp_result> before my "arm:
Auto-vectorization for MVE: vcmp" patch (r12-834), it keeps the VDQWH
iterator added in r12-835 (to have V4HF/V8HF support), as well as the
(!<Is_float_mode> || flag_unsafe_math_optimizations) condition which
was not present before r12-834 although SF modes were enabled by VDQW
(I think this was a bug).
Using TARGET_VECTORIZE_GET_MASK_MODE has the advantage that we no
longer need to generate vpsel with vectors of 0 and 1: the masks are
now merged via scalar 'ands' instructions operating on 16-bit masks
after converting the boolean vectors.
In addition, this patch fixes a problem in arm_expand_vcond() where
the result would be a vector of 0 or 1 instead of operand 1 or 2.
Reducing the number of iterations in pr100757-3.c from 32 to 8, we
generate the code below:
float a[32];
float fn1(int d) {
float c = 4.0f;
for (int b = 0; b < 8; b++)
if (a[b] != 2.0f)
c = 5.0f;
return c;
}
fn1:
ldr r3, .L3+48
vldr.64 d4, .L3 // q2=(2.0,2.0,2.0,2.0)
vldr.64 d5, .L3+8
vldrw.32 q0, [r3] // q0=a(0..3)
adds r3, r3, #16
vcmp.f32 eq, q0, q2 // cmp a(0..3) == (2.0,2.0,2.0,2.0)
vldrw.32 q1, [r3] // q1=a(4..7)
vmrs r3, P0
vcmp.f32 eq, q1, q2 // cmp a(4..7) == (2.0,2.0,2.0,2.0)
vmrs r2, P0 @ movhi
ands r3, r3, r2 // r3=select(a(0..3]) & select(a(4..7))
vldr.64 d4, .L3+16 // q2=(5.0,5.0,5.0,5.0)
vldr.64 d5, .L3+24
vmsr P0, r3
vldr.64 d6, .L3+32 // q3=(4.0,4.0,4.0,4.0)
vldr.64 d7, .L3+40
vpsel q3, q3, q2 // q3=vcond_mask(4.0,5.0)
vmov.32 r2, q3[1] // keep the scalar max
vmov.32 r0, q3[3]
vmov.32 r3, q3[2]
vmov.f32 s11, s12
vmov s15, r2
vmov s14, r3
vmaxnm.f32 s15, s11, s15
vmaxnm.f32 s15, s15, s14
vmov s14, r0
vmaxnm.f32 s15, s15, s14
vmov r0, s15
bx lr
.L4:
.align 3
.L3:
.word 1073741824 // 2.0f
.word 1073741824
.word 1073741824
.word 1073741824
.word 1084227584 // 5.0f
.word 1084227584
.word 1084227584
.word 1084227584
.word 1082130432 // 4.0f
.word 1082130432
.word 1082130432
.word 1082130432
2021-09-02 Christophe Lyon <christophe.lyon@linaro.org>
PR target/100757
gcc/
* config/arm/arm-protos.h (arm_get_mask_mode): New prototype.
(arm_expand_vector_compare): Update prototype.
* config/arm/arm.c (TARGET_VECTORIZE_GET_MASK_MODE): New.
(arm_vector_mode_supported_p): Add support for VxBI modes.
(arm_expand_vector_compare): Remove useless generation of vpsel.
(arm_expand_vcond): Fix select operands.
(arm_get_mask_mode): New.
* config/arm/mve.md (vec_cmp<mode><MVE_vpred>): New.
(vec_cmpu<mode><MVE_vpred>): New.
(vcond_mask_<mode><MVE_vpred>): New.
* config/arm/vec-common.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): Move to ...
* config/arm/neon.md (vec_cmp<mode><v_cmp_result>)
(vec_cmpu<mode><mode, vcond_mask_<mode><v_cmp_result>): ... here
and disable for MVE.
Diff:
---
gcc/config/arm/arm-protos.h | 3 +-
gcc/config/arm/arm.c | 110 ++++++++++++++-----------------------------
gcc/config/arm/mve.md | 55 ++++++++++++++++++++++
gcc/config/arm/neon.md | 39 +++++++++++++++
gcc/config/arm/vec-common.md | 52 --------------------
5 files changed, 131 insertions(+), 128 deletions(-)
diff --git a/gcc/config/arm/arm-protos.h b/gcc/config/arm/arm-protos.h
index 9b1f61394ad..9e3d71e0c29 100644
--- a/gcc/config/arm/arm-protos.h
+++ b/gcc/config/arm/arm-protos.h
@@ -201,6 +201,7 @@ extern void arm_init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree);
extern bool arm_pad_reg_upward (machine_mode, tree, int);
#endif
extern int arm_apply_result_size (void);
+extern opt_machine_mode arm_get_mask_mode (machine_mode mode);
#endif /* RTX_CODE */
@@ -372,7 +373,7 @@ extern void arm_emit_coreregs_64bit_shift (enum rtx_code, rtx, rtx, rtx, rtx,
extern bool arm_fusion_enabled_p (tune_params::fuse_ops);
extern bool arm_valid_symbolic_address_p (rtx);
extern bool arm_validize_comparison (rtx *, rtx *, rtx *);
-extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool, bool);
+extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool);
#endif /* RTX_CODE */
extern bool arm_gen_setmem (rtx *);
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index 5f6637d9a5f..3326cd163a2 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -835,6 +835,10 @@ static const struct attribute_spec arm_attribute_table[] =
#undef TARGET_MD_ASM_ADJUST
#define TARGET_MD_ASM_ADJUST arm_md_asm_adjust
+
+#undef TARGET_VECTORIZE_GET_MASK_MODE
+#define TARGET_VECTORIZE_GET_MASK_MODE arm_get_mask_mode
+
\f
/* Obstack for minipool constant handling. */
static struct obstack minipool_obstack;
@@ -29193,7 +29197,8 @@ arm_vector_mode_supported_p (machine_mode mode)
if (TARGET_HAVE_MVE
&& (mode == V2DImode || mode == V4SImode || mode == V8HImode
- || mode == V16QImode))
+ || mode == V16QImode
+ || mode == V16BImode || mode == V8BImode || mode == V4BImode))
return true;
if (TARGET_HAVE_MVE_FLOAT
@@ -31012,16 +31017,12 @@ arm_mode_to_pred_mode (machine_mode mode)
and return true if TARGET contains the inverse. If !CAN_INVERT,
always store the result in TARGET, never its inverse.
- If VCOND_MVE, do not emit the vpsel instruction here, let arm_expand_vcond do
- it with the right destination type to avoid emiting two vpsel, one here and
- one in arm_expand_vcond.
-
Note that the handling of floating-point comparisons is not
IEEE compliant. */
bool
arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
- bool can_invert, bool vcond_mve)
+ bool can_invert)
{
machine_mode cmp_result_mode = GET_MODE (target);
machine_mode cmp_mode = GET_MODE (op0);
@@ -31050,7 +31051,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
and then store its inverse in TARGET. This avoids reusing
TARGET (which for integer NE could be one of the inputs). */
rtx tmp = gen_reg_rtx (cmp_result_mode);
- if (arm_expand_vector_compare (tmp, code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (tmp, code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_NOT (cmp_result_mode, tmp)));
return false;
@@ -31086,36 +31087,20 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case NE:
if (TARGET_HAVE_MVE)
{
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vcmpq_f (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq_f (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
else
gcc_unreachable ();
break;
default:
gcc_unreachable ();
}
-
- /* If we are not expanding a vcond, build the result here. */
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
}
else
emit_insn (gen_neon_vc (code, cmp_mode, target, op0, op1));
@@ -31127,23 +31112,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case GEU:
case GTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
else
emit_insn (gen_neon_vc (code, cmp_mode, target,
op0, force_reg (cmp_mode, op1)));
@@ -31154,23 +31123,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case LEU:
case LTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, vpr_p0, force_reg (cmp_mode, op1), op0));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, target, force_reg (cmp_mode, op1), op0));
else
emit_insn (gen_neon_vc (swap_condition (code), cmp_mode,
target, force_reg (cmp_mode, op1), op0));
@@ -31185,8 +31138,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
rtx gt_res = gen_reg_rtx (cmp_result_mode);
rtx alt_res = gen_reg_rtx (cmp_result_mode);
rtx_code alt_code = (code == LTGT ? LT : LE);
- if (arm_expand_vector_compare (gt_res, GT, op0, op1, true, vcond_mve)
- || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (gt_res, GT, op0, op1, true)
+ || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_IOR (cmp_result_mode,
gt_res, alt_res)));
@@ -31206,19 +31159,15 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
{
/* When expanding for MVE, we do not want to emit a (useless) vpsel in
arm_expand_vector_compare, and another one here. */
- bool vcond_mve=false;
rtx mask;
if (TARGET_HAVE_MVE)
- {
- vcond_mve=true;
- mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
- }
+ mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
else
mask = gen_reg_rtx (cmp_result_mode);
bool inverted = arm_expand_vector_compare (mask, GET_CODE (operands[3]),
- operands[4], operands[5], true, vcond_mve);
+ operands[4], operands[5], true);
if (inverted)
std::swap (operands[1], operands[2]);
if (TARGET_NEON)
@@ -31226,20 +31175,20 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
mask, operands[1], operands[2]));
else
{
- machine_mode cmp_mode = GET_MODE (operands[4]);
- rtx vpr_p0 = mask;
- rtx zero = gen_reg_rtx (cmp_mode);
- rtx one = gen_reg_rtx (cmp_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_mode));
- emit_move_insn (one, CONST1_RTX (cmp_mode));
+ machine_mode cmp_mode = GET_MODE (operands[0]);
+
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_mode, operands[0],
+ operands[1], operands[2], mask));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0],
+ operands[1], operands[2], mask));
+ else
+ gcc_unreachable ();
break;
default:
gcc_unreachable ();
@@ -34149,4 +34098,15 @@ arm_mode_base_reg_class (machine_mode mode)
struct gcc_target targetm = TARGET_INITIALIZER;
+/* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
+
+opt_machine_mode
+arm_get_mask_mode (machine_mode mode)
+{
+ if (TARGET_HAVE_MVE)
+ return arm_mode_to_pred_mode (mode);
+
+ return default_get_mask_mode (mode);
+}
+
#include "gt-arm.h"
diff --git a/gcc/config/arm/mve.md b/gcc/config/arm/mve.md
index c9c8e2c13fe..d663c698cfb 100644
--- a/gcc/config/arm/mve.md
+++ b/gcc/config/arm/mve.md
@@ -10550,3 +10550,58 @@
vmsr%?\t P0, %1
vmrs%?\t %0, P0"
)
+
+;; Expanders for vec_cmp and vcond
+
+(define_expand "vec_cmp<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_VLD_ST 2 "s_register_operand")
+ (match_operand:MVE_VLD_ST 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_2 2 "s_register_operand")
+ (match_operand:MVE_2 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><MVE_vpred>"
+ [(set (match_operand:MVE_VLD_ST 0 "s_register_operand")
+ (if_then_else:MVE_VLD_ST
+ (match_operand:<MVE_VPRED> 3 "s_register_operand")
+ (match_operand:MVE_VLD_ST 1 "s_register_operand")
+ (match_operand:MVE_VLD_ST 2 "s_register_operand")))]
+ "TARGET_HAVE_MVE"
+{
+ switch (GET_MODE_CLASS (<MODE>mode))
+ {
+ case MODE_VECTOR_INT:
+ emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ break;
+ case MODE_VECTOR_FLOAT:
+ if (TARGET_HAVE_MVE_FLOAT)
+ emit_insn (gen_mve_vpselq_f (<MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ else
+ gcc_unreachable ();
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ DONE;
+})
diff --git a/gcc/config/arm/neon.md b/gcc/config/arm/neon.md
index 8b0a396947c..28310d93a4e 100644
--- a/gcc/config/arm/neon.md
+++ b/gcc/config/arm/neon.md
@@ -1394,6 +1394,45 @@
[(set_attr "type" "neon_qsub<q>")]
)
+(define_expand "vec_cmp<mode><v_cmp_result>"
+ [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
+ (match_operator:<V_cmp_result> 1 "comparison_operator"
+ [(match_operand:VDQWH 2 "s_register_operand")
+ (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><mode>"
+ [(set (match_operand:VDQIW 0 "s_register_operand")
+ (match_operator:VDQIW 1 "comparison_operator"
+ [(match_operand:VDQIW 2 "s_register_operand")
+ (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
+ "TARGET_NEON"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><v_cmp_result>"
+ [(set (match_operand:VDQWH 0 "s_register_operand")
+ (if_then_else:VDQWH
+ (match_operand:<V_cmp_result> 3 "s_register_operand")
+ (match_operand:VDQWH 1 "s_register_operand")
+ (match_operand:VDQWH 2 "s_register_operand")))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ emit_insn (gen_neon_vbsl<mode> (operands[0], operands[3], operands[1],
+ operands[2]));
+ DONE;
+})
+
;; Patterns for builtins.
; good for plain vadd, vaddq.
diff --git a/gcc/config/arm/vec-common.md b/gcc/config/arm/vec-common.md
index 68de4f0f943..9b461a76155 100644
--- a/gcc/config/arm/vec-common.md
+++ b/gcc/config/arm/vec-common.md
@@ -363,33 +363,6 @@
}
})
-(define_expand "vec_cmp<mode><v_cmp_result>"
- [(set (match_operand:<V_cmp_result> 0 "s_register_operand")
- (match_operator:<V_cmp_result> 1 "comparison_operator"
- [(match_operand:VDQWH 2 "s_register_operand")
- (match_operand:VDQWH 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
-(define_expand "vec_cmpu<mode><mode>"
- [(set (match_operand:VDQIW 0 "s_register_operand")
- (match_operator:VDQIW 1 "comparison_operator"
- [(match_operand:VDQIW 2 "s_register_operand")
- (match_operand:VDQIW 3 "reg_or_zero_operand")]))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT"
-{
- arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
- DONE;
-})
-
;; Conditional instructions. These are comparisons with conditional moves for
;; vectors. They perform the assignment:
;;
@@ -461,31 +434,6 @@
DONE;
})
-(define_expand "vcond_mask_<mode><v_cmp_result>"
- [(set (match_operand:VDQWH 0 "s_register_operand")
- (if_then_else:VDQWH
- (match_operand:<V_cmp_result> 3 "s_register_operand")
- (match_operand:VDQWH 1 "s_register_operand")
- (match_operand:VDQWH 2 "s_register_operand")))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- if (TARGET_NEON)
- {
- emit_insn (gen_neon_vbsl (<MODE>mode, operands[0], operands[3],
- operands[1], operands[2]));
- }
- else if (TARGET_HAVE_MVE)
- {
- emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
- operands[1], operands[2], operands[3]));
- }
- else
- gcc_unreachable ();
- DONE;
-})
-
(define_expand "vec_load_lanesoi<mode>"
[(set (match_operand:OI 0 "s_register_operand")
(unspec:OI [(match_operand:OI 1 "neon_struct_operand")
^ permalink raw reply [flat|nested] 6+ messages in thread
* [gcc(refs/users/clyon/heads/mve-autovec)] arm: Fix vcond_mask expander for MVE (PR target/100757)
@ 2021-08-27 16:30 Christophe Lyon
0 siblings, 0 replies; 6+ messages in thread
From: Christophe Lyon @ 2021-08-27 16:30 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:02a2f59fb98cddb0b5b2e16413344510f3a3268d
commit 02a2f59fb98cddb0b5b2e16413344510f3a3268d
Author: Christophe Lyon <christophe.lyon@linaro.org>
Date: Tue Jun 8 21:32:28 2021 +0000
arm: Fix vcond_mask expander for MVE (PR target/100757)
The problem in this PR is that we call VPSEL with a mask of vector
type instead of HImode. This happens because operand 3 in vcond_mask
is the pre-computed vector comparison and has vector type.
This patch fixes it by implementing TARGET_VECTORIZE_GET_MASK_MODE,
returning HImode when targeting MVE. In turn, this implies
implementing vec_cmp<mode>hi, vec_cmpu<mode>hi and
vcond_mask_<mode>hi, but we still need to keep the non-HI versions
even for MVE, which was not obvious. Indeed, only defining
TARGET_VECTORIZE_GET_MASK_MODE causes regressions (no vectorization),
but also ICEs. At some point I also added vcond<V_cvtto>hi, but it
seems useless (it would help if the vectorizer traces included the
list of optabs it is trying to use)..
The patch moves vcond_mask_<mode><v_cmp_result> back to neon.md since
it is not used by MVE anymore. The new *hi patterns listed above are
implemented in mve.md since they are only valid for MVE. However this
may make maintenance/comparison more painful than having all of them
in vec-common.md.
Compared to neon.md's vcond_mask_<mode><v_cmp_result> before my
"arm: Auto-vectorization for MVE: vcmp" patch (r12-834
a6eacbf1055520e968d1a25f6d30d6ff4b66272d), it keeps the VDQWH iterator
added in r12-835 7606865198b241b4c944f66761d6506b02ead951 (to have
V4HF/V8HF support), as well as the
(!<Is_float_mode> || flag_unsafe_math_optimizations) condition which
was not present before r12-834 although SF modes were enabled by VDQW
(I think this was a bug).
Using TARGET_VECTORIZE_GET_MASK_MODE has the advantage that we no
longer need to generate vpsel with vectors of 0 and 1: the masks are
now merged via scalar 'and' instructions operating on 16-bit masks
(HImode).
In addition, this patch fixes a problem in arm_expand_vcond() where
the result would be a vector of 0 or 1 instead of operand 1 or 2. The
mve-vcmp-f32-2.c testcase is an update from mve-vcmp-f32.c using a
conditional with 2.0f and 3.0f constants to help scan-assembler-times.
The patch adds several Neon tests similar to the MVE ones to make sure
the fix for MVE does not break Neon support.
The pr100757*.c testcases are derived from
gcc.c-torture/compile/20160205-1.c, forcing the use of MVE, and using
various types and return values different from 0 and 1 to avoid
commonalization with boolean masks. In addition, since we no longer
need these masks unlike the previous version of this patch, the tests
make sure they are not present.
Reducing the number of iterations in pr100757-3.c from 32 to 8, we
generate the code below:
float a[32];
float fn1(int d) {
int c = 4;
for (int b = 0; b < 8; b++)
if (a[b] != 2.0f)
c = 5;
return c;
}
fn1:
ldr r3, .L3+48
vldr.64 d4, .L3 // q2=(2.0,2.0,2.0,2.0)
vldr.64 d5, .L3+8
vldrw.32 q0, [r3] // q0=a[0..3]
adds r3, r3, #16
vcmp.f32 eq, q0, q2 // cmp a[0..3] == (2.0,2.0,2.0,2.0)
vldrw.32 q1, [r3] // q1=a[4..7]
vmrs r3, P0 @ movhi // r3=mask[0..3]
vcmp.f32 eq, q1, q2 // cmp a[4..7] == (2.0,2.0,2.0,2.0)
vmrs r2, P0 @ movhi // r2=mask[4..7]
ands r3, r3, r2 // r3=mask[0..7]
vmsr P0, r3 @ movhi
vldr.64 d4, .L3+16 // q2=(4.0,4.0,4.0,4.0)
vldr.64 d5, .L3+24
vldr.64 d6, .L3+32 // q3=(5.0,5.0,5.0,5.0)
vldr.64 d7, .L3+40
vpsel q3, q3, q2 // q3=select (a[0..7])
vmov.32 r3, q3[0]
vmov.32 r1, q3[1]
vmov.32 r0, q3[3]
vmov.32 r2, q3[2]
vmov s14, r1
vmov s15, r3
vmaxnm.f32 s15, s15, s14
vmov s14, r2
vmaxnm.f32 s15, s15, s14
vmov s14, r0
vmaxnm.f32 s15, s15, s14
vmov r0, s15
bx lr
.L4:
.align 3
.L3:
.word 1073741824
.word 1073741824
.word 1073741824
.word 1073741824
.word 1084227584
.word 1084227584
.word 1084227584
.word 1084227584
.word 1082130432
.word 1082130432
.word 1082130432
.word 1082130432
2021-07-02 Christophe Lyon <christophe.lyon@linaro.org>
PR target/100757
gcc/
* config/arm/arm-protos.h (arm_get_mask_mode): New prototype.
* config/arm/arm.c (TARGET_VECTORIZE_GET_MASK_MODE): New.
(arm_expand_vector_compare): Remove useless generation of vpsel.
(arm_expand_vcond): Fix select operands.
(arm_get_mask_mode): New.
* config/arm/mve.md (vec_cmp<mode>hi): New.
(vec_cmpu<mode>hi): New.
(vcond_mask_<mode>hi): New.
* config/arm/vec-common.md (vcond_mask_<mode><v_cmp_result>): Move to ...
* config/arm/neon.md (vcond_mask_<mode><v_cmp_result>): ... here
and disable for MVE.
gcc/testsuite/
* gcc.target/arm/simd/mve-vcmp-f32-2.c: New test.
* gcc.target/arm/simd/neon-compare-1.c: New test.
* gcc.target/arm/simd/neon-compare-2.c: New test.
* gcc.target/arm/simd/neon-compare-3.c: New test.
* gcc.target/arm/simd/neon-compare-scalar-1.c: New test.
* gcc.target/arm/simd/neon-vcmp-f16.c: New test.
* gcc.target/arm/simd/neon-vcmp-f32-2.c: New test.
* gcc.target/arm/simd/neon-vcmp-f32-3.c: New test.
* gcc.target/arm/simd/neon-vcmp-f32.c: New test.
* gcc.target/arm/simd/neon-vcmp.c: New test.
* gcc.target/arm/simd/pr100757.c: New test.
* gcc.target/arm/simd/pr100757-2.c: New test.
* gcc.target/arm/simd/pr100757-3.c: New test.
* gcc.target/arm/simd/pr100757-4.c: New test.
Diff:
---
gcc/config/arm/arm-protos.h | 3 +-
gcc/config/arm/arm.c | 110 ++++++++++++++-----------------------------
gcc/config/arm/iterators.md | 3 ++
gcc/config/arm/mve.md | 74 +++++++++++++++++++++++++++++
gcc/config/arm/neon.md | 14 ++++++
gcc/config/arm/vec-common.md | 29 +-----------
6 files changed, 130 insertions(+), 103 deletions(-)
diff --git a/gcc/config/arm/arm-protos.h b/gcc/config/arm/arm-protos.h
index 9b1f61394ad..9e3d71e0c29 100644
--- a/gcc/config/arm/arm-protos.h
+++ b/gcc/config/arm/arm-protos.h
@@ -201,6 +201,7 @@ extern void arm_init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree);
extern bool arm_pad_reg_upward (machine_mode, tree, int);
#endif
extern int arm_apply_result_size (void);
+extern opt_machine_mode arm_get_mask_mode (machine_mode mode);
#endif /* RTX_CODE */
@@ -372,7 +373,7 @@ extern void arm_emit_coreregs_64bit_shift (enum rtx_code, rtx, rtx, rtx, rtx,
extern bool arm_fusion_enabled_p (tune_params::fuse_ops);
extern bool arm_valid_symbolic_address_p (rtx);
extern bool arm_validize_comparison (rtx *, rtx *, rtx *);
-extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool, bool);
+extern bool arm_expand_vector_compare (rtx, rtx_code, rtx, rtx, bool);
#endif /* RTX_CODE */
extern bool arm_gen_setmem (rtx *);
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index 9fc68e2b57b..8fd8a4d3909 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -835,6 +835,10 @@ static const struct attribute_spec arm_attribute_table[] =
#undef TARGET_MD_ASM_ADJUST
#define TARGET_MD_ASM_ADJUST arm_md_asm_adjust
+
+#undef TARGET_VECTORIZE_GET_MASK_MODE
+#define TARGET_VECTORIZE_GET_MASK_MODE arm_get_mask_mode
+
\f
/* Obstack for minipool constant handling. */
static struct obstack minipool_obstack;
@@ -29193,7 +29197,8 @@ arm_vector_mode_supported_p (machine_mode mode)
if (TARGET_HAVE_MVE
&& (mode == V2DImode || mode == V4SImode || mode == V8HImode
- || mode == V16QImode))
+ || mode == V16QImode
+ || mode == V16BImode || mode == V8BImode || mode == V4BImode))
return true;
if (TARGET_HAVE_MVE_FLOAT
@@ -31009,16 +31014,12 @@ arm_mode_to_pred_mode (machine_mode mode)
and return true if TARGET contains the inverse. If !CAN_INVERT,
always store the result in TARGET, never its inverse.
- If VCOND_MVE, do not emit the vpsel instruction here, let arm_expand_vcond do
- it with the right destination type to avoid emiting two vpsel, one here and
- one in arm_expand_vcond.
-
Note that the handling of floating-point comparisons is not
IEEE compliant. */
bool
arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
- bool can_invert, bool vcond_mve)
+ bool can_invert)
{
machine_mode cmp_result_mode = GET_MODE (target);
machine_mode cmp_mode = GET_MODE (op0);
@@ -31047,7 +31048,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
and then store its inverse in TARGET. This avoids reusing
TARGET (which for integer NE could be one of the inputs). */
rtx tmp = gen_reg_rtx (cmp_result_mode);
- if (arm_expand_vector_compare (tmp, code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (tmp, code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_NOT (cmp_result_mode, tmp)));
return false;
@@ -31083,36 +31084,20 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case NE:
if (TARGET_HAVE_MVE)
{
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vcmpq_f (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
+ emit_insn (gen_mve_vcmpq_f (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
else
gcc_unreachable ();
break;
default:
gcc_unreachable ();
}
-
- /* If we are not expanding a vcond, build the result here. */
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
}
else
emit_insn (gen_neon_vc (code, cmp_mode, target, op0, op1));
@@ -31124,23 +31109,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case GEU:
case GTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (code, cmp_mode, vpr_p0, op0, force_reg (cmp_mode, op1)));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (code, cmp_mode, target, op0, force_reg (cmp_mode, op1)));
else
emit_insn (gen_neon_vc (code, cmp_mode, target,
op0, force_reg (cmp_mode, op1)));
@@ -31151,23 +31120,7 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
case LEU:
case LTU:
if (TARGET_HAVE_MVE)
- {
- rtx vpr_p0;
- if (vcond_mve)
- vpr_p0 = target;
- else
- vpr_p0 = gen_reg_rtx (arm_mode_to_pred_mode (cmp_mode));
-
- emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, vpr_p0, force_reg (cmp_mode, op1), op0));
- if (!vcond_mve)
- {
- rtx zero = gen_reg_rtx (cmp_result_mode);
- rtx one = gen_reg_rtx (cmp_result_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_result_mode));
- emit_move_insn (one, CONST1_RTX (cmp_result_mode));
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, target, one, zero, vpr_p0));
- }
- }
+ emit_insn (gen_mve_vcmpq (swap_condition (code), cmp_mode, target, force_reg (cmp_mode, op1), op0));
else
emit_insn (gen_neon_vc (swap_condition (code), cmp_mode,
target, force_reg (cmp_mode, op1), op0));
@@ -31182,8 +31135,8 @@ arm_expand_vector_compare (rtx target, rtx_code code, rtx op0, rtx op1,
rtx gt_res = gen_reg_rtx (cmp_result_mode);
rtx alt_res = gen_reg_rtx (cmp_result_mode);
rtx_code alt_code = (code == LTGT ? LT : LE);
- if (arm_expand_vector_compare (gt_res, GT, op0, op1, true, vcond_mve)
- || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true, vcond_mve))
+ if (arm_expand_vector_compare (gt_res, GT, op0, op1, true)
+ || arm_expand_vector_compare (alt_res, alt_code, op0, op1, true))
gcc_unreachable ();
emit_insn (gen_rtx_SET (target, gen_rtx_IOR (cmp_result_mode,
gt_res, alt_res)));
@@ -31203,19 +31156,15 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
{
/* When expanding for MVE, we do not want to emit a (useless) vpsel in
arm_expand_vector_compare, and another one here. */
- bool vcond_mve=false;
rtx mask;
if (TARGET_HAVE_MVE)
- {
- vcond_mve=true;
- mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
- }
+ mask = gen_reg_rtx (arm_mode_to_pred_mode (cmp_result_mode));
else
mask = gen_reg_rtx (cmp_result_mode);
bool inverted = arm_expand_vector_compare (mask, GET_CODE (operands[3]),
- operands[4], operands[5], true, vcond_mve);
+ operands[4], operands[5], true);
if (inverted)
std::swap (operands[1], operands[2]);
if (TARGET_NEON)
@@ -31223,20 +31172,20 @@ arm_expand_vcond (rtx *operands, machine_mode cmp_result_mode)
mask, operands[1], operands[2]));
else
{
- machine_mode cmp_mode = GET_MODE (operands[4]);
- rtx vpr_p0 = mask;
- rtx zero = gen_reg_rtx (cmp_mode);
- rtx one = gen_reg_rtx (cmp_mode);
- emit_move_insn (zero, CONST0_RTX (cmp_mode));
- emit_move_insn (one, CONST1_RTX (cmp_mode));
+ machine_mode cmp_mode = GET_MODE (operands[0]);
+
switch (GET_MODE_CLASS (cmp_mode))
{
case MODE_VECTOR_INT:
- emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_result_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq (VPSELQ_S, cmp_mode, operands[0],
+ operands[1], operands[2], mask));
break;
case MODE_VECTOR_FLOAT:
if (TARGET_HAVE_MVE_FLOAT)
- emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0], one, zero, vpr_p0));
+ emit_insn (gen_mve_vpselq_f (cmp_mode, operands[0],
+ operands[1], operands[2], mask));
+ else
+ gcc_unreachable ();
break;
default:
gcc_unreachable ();
@@ -34146,4 +34095,15 @@ arm_mode_base_reg_class (machine_mode mode)
struct gcc_target targetm = TARGET_INITIALIZER;
+/* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
+
+opt_machine_mode
+arm_get_mask_mode (machine_mode mode)
+{
+ if (TARGET_HAVE_MVE)
+ return arm_mode_to_pred_mode (mode);
+
+ return default_get_mask_mode (mode);
+}
+
#include "gt-arm.h"
diff --git a/gcc/config/arm/iterators.md b/gcc/config/arm/iterators.md
index 0fe6f5afd3e..b40480d3a74 100644
--- a/gcc/config/arm/iterators.md
+++ b/gcc/config/arm/iterators.md
@@ -272,6 +272,7 @@
(define_mode_iterator MVE_2 [V16QI V8HI V4SI])
(define_mode_iterator MVE_5 [V8HI V4SI])
(define_mode_iterator MVE_6 [V8HI V4SI])
+(define_mode_iterator MVE_7 [V16BI V8BI V4BI])
;;----------------------------------------------------------------------------
;; Code iterators
@@ -948,6 +949,8 @@
(V8HF "=w") (V4SF "=&w")])
(define_mode_attr MVE_VPRED [(V16QI "V16BI") (V8HI "V8BI") (V4SI "V4BI")
(V8HF "V8BI") (V4SF "V4BI")])
+(define_mode_attr MVE_vpred [(V16QI "v16bi") (V8HI "v8bi") (V4SI "v4bi")
+ (V8HF "v8bi") (V4SF "v4bi")])
;;----------------------------------------------------------------------------
;; Code attributes
diff --git a/gcc/config/arm/mve.md b/gcc/config/arm/mve.md
index 3a7939f8a7b..a61f950d489 100644
--- a/gcc/config/arm/mve.md
+++ b/gcc/config/arm/mve.md
@@ -10531,3 +10531,77 @@
"vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1"
[(set_attr "type" "mve_load")]
)
+
+;; Expanders for vec_cmp and vcond
+
+(define_expand "vec_cmp<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_VLD_ST 2 "s_register_operand")
+ (match_operand:MVE_VLD_ST 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vec_cmpu<mode><MVE_vpred>"
+ [(set (match_operand:<MVE_VPRED> 0 "s_register_operand")
+ (match_operator:<MVE_VPRED> 1 "comparison_operator"
+ [(match_operand:MVE_2 2 "s_register_operand")
+ (match_operand:MVE_2 3 "reg_or_zero_operand")]))]
+ "TARGET_HAVE_MVE
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
+ operands[2], operands[3], false);
+ DONE;
+})
+
+(define_expand "vcond_mask_<mode><MVE_vpred>"
+ [(set (match_operand:MVE_VLD_ST 0 "s_register_operand")
+ (if_then_else:MVE_VLD_ST
+ (match_operand:<MVE_VPRED> 3 "s_register_operand")
+ (match_operand:MVE_VLD_ST 1 "s_register_operand")
+ (match_operand:MVE_VLD_ST 2 "s_register_operand")))]
+ "TARGET_HAVE_MVE"
+{
+ switch (GET_MODE_CLASS (<MODE>mode))
+ {
+ case MODE_VECTOR_INT:
+ emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ break;
+ case MODE_VECTOR_FLOAT:
+ if (TARGET_HAVE_MVE_FLOAT)
+ emit_insn (gen_mve_vpselq_f (<MODE>mode, operands[0],
+ operands[1], operands[2], operands[3]));
+ else
+ gcc_unreachable ();
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ DONE;
+})
+
+(define_expand "mov<mode>"
+ [(set (match_operand:MVE_7 0 "s_register_operand")
+ (match_operand:MVE_7 1 "s_register_operand"))]
+ "TARGET_HAVE_MVE"
+ {
+ }
+)
+
+(define_insn "*mve_mov<mode>"
+ [(set (match_operand:MVE_7 0 "s_register_operand" "=Up, r")
+ (match_operand:MVE_7 1 "s_register_operand" "r, Up"))]
+ "TARGET_HAVE_MVE
+ && (register_operand (operands[0], <MODE>mode)
+ || register_operand (operands[1], <MODE>mode))"
+ "@
+ vmsr%?\t P0, %1
+ vmrs%?\t %0, P0"
+)
diff --git a/gcc/config/arm/neon.md b/gcc/config/arm/neon.md
index 8b0a396947c..a1781be3005 100644
--- a/gcc/config/arm/neon.md
+++ b/gcc/config/arm/neon.md
@@ -1394,6 +1394,20 @@
[(set_attr "type" "neon_qsub<q>")]
)
+(define_expand "vcond_mask_<mode><v_cmp_result>"
+ [(set (match_operand:VDQWH 0 "s_register_operand")
+ (if_then_else:VDQWH
+ (match_operand:<V_cmp_result> 3 "s_register_operand")
+ (match_operand:VDQWH 1 "s_register_operand")
+ (match_operand:VDQWH 2 "s_register_operand")))]
+ "TARGET_NEON
+ && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
+{
+ emit_insn (gen_neon_vbsl<mode> (operands[0], operands[3], operands[1],
+ operands[2]));
+ DONE;
+})
+
;; Patterns for builtins.
; good for plain vadd, vaddq.
diff --git a/gcc/config/arm/vec-common.md b/gcc/config/arm/vec-common.md
index 68de4f0f943..83b43073e60 100644
--- a/gcc/config/arm/vec-common.md
+++ b/gcc/config/arm/vec-common.md
@@ -373,7 +373,7 @@
&& (!<Is_float_mode> || flag_unsafe_math_optimizations)"
{
arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
+ operands[2], operands[3], false);
DONE;
})
@@ -386,7 +386,7 @@
&& !TARGET_REALLY_IWMMXT"
{
arm_expand_vector_compare (operands[0], GET_CODE (operands[1]),
- operands[2], operands[3], false, false);
+ operands[2], operands[3], false);
DONE;
})
@@ -461,31 +461,6 @@
DONE;
})
-(define_expand "vcond_mask_<mode><v_cmp_result>"
- [(set (match_operand:VDQWH 0 "s_register_operand")
- (if_then_else:VDQWH
- (match_operand:<V_cmp_result> 3 "s_register_operand")
- (match_operand:VDQWH 1 "s_register_operand")
- (match_operand:VDQWH 2 "s_register_operand")))]
- "ARM_HAVE_<MODE>_ARITH
- && !TARGET_REALLY_IWMMXT
- && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
-{
- if (TARGET_NEON)
- {
- emit_insn (gen_neon_vbsl (<MODE>mode, operands[0], operands[3],
- operands[1], operands[2]));
- }
- else if (TARGET_HAVE_MVE)
- {
- emit_insn (gen_mve_vpselq (VPSELQ_S, <MODE>mode, operands[0],
- operands[1], operands[2], operands[3]));
- }
- else
- gcc_unreachable ();
- DONE;
-})
-
(define_expand "vec_load_lanesoi<mode>"
[(set (match_operand:OI 0 "s_register_operand")
(unspec:OI [(match_operand:OI 1 "neon_struct_operand")
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2022-02-22 9:08 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-01-12 8:28 [gcc(refs/users/clyon/heads/mve-autovec)] arm: Fix vcond_mask expander for MVE (PR target/100757) Christophe Lyon
-- strict thread matches above, loose matches on Subject: below --
2022-02-22 9:08 Christophe Lyon
2021-11-16 14:07 Christophe Lyon
2021-10-01 14:37 Christophe Lyon
2021-09-29 7:30 Christophe Lyon
2021-08-27 16:30 Christophe Lyon
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).