* [PATCH] RISC-V: Finish all integer C/C++ intrinsics
@ 2023-02-14 9:24 juzhe.zhong
0 siblings, 0 replies; only message in thread
From: juzhe.zhong @ 2023-02-14 9:24 UTC (permalink / raw)
To: gcc-patches; +Cc: kito.cheng, Ju-Zhe Zhong
From: Ju-Zhe Zhong <juzhe.zhong@rivai.ai>
gcc/ChangeLog:
* config/riscv/predicates.md: Refine codes.
* config/riscv/riscv-protos.h (RVV_VUNDEF): New macro.
* config/riscv/riscv-v.cc: Refine codes.
* config/riscv/riscv-vector-builtins-bases.cc (enum ternop_type): New enum.
(class imac): New class.
(enum widen_ternop_type): New enum.
(class iwmac): New class.
(BASE): New class.
* config/riscv/riscv-vector-builtins-bases.h: Ditto.
* config/riscv/riscv-vector-builtins-functions.def (vmacc): Ditto.
(vnmsac): Ditto.
(vmadd): Ditto.
(vnmsub): Ditto.
(vwmacc): Ditto.
(vwmaccu): Ditto.
(vwmaccsu): Ditto.
(vwmaccus): Ditto.
* config/riscv/riscv-vector-builtins.cc (function_builder::apply_predication): Adjust for multiply-add support.
(function_expander::add_vundef_operand): Refine codes.
(function_expander::use_ternop_insn): New function.
(function_expander::use_widen_ternop_insn): Ditto.
* config/riscv/riscv-vector-builtins.h: New function.
* config/riscv/vector.md (@pred_mul_<optab><mode>): New pattern.
(pred_mul_<optab><mode>_undef_merge): Ditto.
(*pred_<madd_nmsub><mode>): Ditto.
(*pred_<macc_nmsac><mode>): Ditto.
(*pred_mul_<optab><mode>): Ditto.
(@pred_mul_<optab><mode>_scalar): Ditto.
(*pred_mul_<optab><mode>_undef_merge_scalar): Ditto.
(*pred_<madd_nmsub><mode>_scalar): Ditto.
(*pred_<macc_nmsac><mode>_scalar): Ditto.
(*pred_mul_<optab><mode>_scalar): Ditto.
(*pred_mul_<optab><mode>_undef_merge_extended_scalar): Ditto.
(*pred_<madd_nmsub><mode>_extended_scalar): Ditto.
(*pred_<macc_nmsac><mode>_extended_scalar): Ditto.
(*pred_mul_<optab><mode>_extended_scalar): Ditto.
(@pred_widen_mul_plus<su><mode>): Ditto.
(@pred_widen_mul_plus<su><mode>_scalar): Ditto.
(@pred_widen_mul_plussu<mode>): Ditto.
(@pred_widen_mul_plussu<mode>_scalar): Ditto.
(@pred_widen_mul_plusus<mode>_scalar): Ditto.
---
gcc/config/riscv/predicates.md | 3 +-
gcc/config/riscv/riscv-protos.h | 2 +
gcc/config/riscv/riscv-v.cc | 4 +-
.../riscv/riscv-vector-builtins-bases.cc | 142 ++++
.../riscv/riscv-vector-builtins-bases.h | 8 +
.../riscv/riscv-vector-builtins-functions.def | 15 +
gcc/config/riscv/riscv-vector-builtins.cc | 243 ++++++-
gcc/config/riscv/riscv-vector-builtins.h | 2 +
gcc/config/riscv/vector.md | 680 +++++++++++++++++-
9 files changed, 1079 insertions(+), 20 deletions(-)
diff --git a/gcc/config/riscv/predicates.md b/gcc/config/riscv/predicates.md
index bbbf523d588..7bc7c0b4f4d 100644
--- a/gcc/config/riscv/predicates.md
+++ b/gcc/config/riscv/predicates.md
@@ -292,8 +292,7 @@
(match_operand 0 "vector_all_trues_mask_operand")))
(define_predicate "vector_undef_operand"
- (match_test "GET_CODE (op) == UNSPEC
- && (XINT (op, 1) == UNSPEC_VUNDEF)"))
+ (match_test "rtx_equal_p (op, RVV_VUNDEF (GET_MODE (op)))"))
(define_predicate "vector_merge_operand"
(ior (match_operand 0 "register_operand")
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index a4476e6235f..9d8b0b78a06 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -121,6 +121,8 @@ extern void riscv_run_selftests (void);
namespace riscv_vector {
#define RVV_VLMAX gen_rtx_REG (Pmode, X0_REGNUM)
+#define RVV_VUNDEF(MODE) \
+ gen_rtx_UNSPEC (MODE, gen_rtvec (1, const0_rtx), UNSPEC_VUNDEF)
enum vlmul_type
{
LMUL_1 = 0,
diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
index cc26888d58b..600b2e6ecad 100644
--- a/gcc/config/riscv/riscv-v.cc
+++ b/gcc/config/riscv/riscv-v.cc
@@ -67,9 +67,7 @@ public:
}
void add_vundef_operand (machine_mode mode)
{
- add_input_operand (gen_rtx_UNSPEC (mode, gen_rtvec (1, const0_rtx),
- UNSPEC_VUNDEF),
- mode);
+ add_input_operand (RVV_VUNDEF (mode), mode);
}
void add_policy_operand (enum tail_policy vta, enum mask_policy vma)
{
diff --git a/gcc/config/riscv/riscv-vector-builtins-bases.cc b/gcc/config/riscv/riscv-vector-builtins-bases.cc
index 4f3531d4486..ba701482728 100644
--- a/gcc/config/riscv/riscv-vector-builtins-bases.cc
+++ b/gcc/config/riscv/riscv-vector-builtins-bases.cc
@@ -539,6 +539,132 @@ public:
}
};
+/* Enumerates types of ternary operations.
+ We have 2 types ternop:
+ - 1. accumulator is vd:
+ vmacc.vv vd,vs1,vs2 # vd = vs1 * vs2 + vd.
+ - 2. accumulator is vs2:
+ vmadd.vv vd,vs1,vs2 # vd = vs1 * vd + vs2. */
+enum ternop_type
+{
+ TERNOP_VMACC,
+ TERNOP_VNMSAC,
+ TERNOP_VMADD,
+ TERNOP_VNMSUB,
+};
+
+/* Implements vmacc/vnmsac/vmadd/vnmsub. */
+template<ternop_type TERNOP_TYPE>
+class imac : public function_base
+{
+public:
+ bool has_merge_operand_p () const override { return false; }
+
+ rtx expand (function_expander &e) const override
+ {
+ switch (TERNOP_TYPE)
+ {
+ case TERNOP_VMACC:
+ if (e.op_info->op == OP_TYPE_vx)
+ return e.use_ternop_insn (
+ true, code_for_pred_mul_scalar (PLUS, e.vector_mode ()));
+ if (e.op_info->op == OP_TYPE_vv)
+ return e.use_ternop_insn (true,
+ code_for_pred_mul (PLUS, e.vector_mode ()));
+ break;
+ case TERNOP_VNMSAC:
+ if (e.op_info->op == OP_TYPE_vx)
+ return e.use_ternop_insn (
+ true, code_for_pred_mul_scalar (MINUS, e.vector_mode ()));
+ if (e.op_info->op == OP_TYPE_vv)
+ return e.use_ternop_insn (true, code_for_pred_mul (MINUS,
+ e.vector_mode ()));
+ break;
+ case TERNOP_VMADD:
+ if (e.op_info->op == OP_TYPE_vx)
+ return e.use_ternop_insn (
+ false, code_for_pred_mul_scalar (PLUS, e.vector_mode ()));
+ if (e.op_info->op == OP_TYPE_vv)
+ return e.use_ternop_insn (false,
+ code_for_pred_mul (PLUS, e.vector_mode ()));
+ break;
+ case TERNOP_VNMSUB:
+ if (e.op_info->op == OP_TYPE_vx)
+ return e.use_ternop_insn (
+ false, code_for_pred_mul_scalar (MINUS, e.vector_mode ()));
+ if (e.op_info->op == OP_TYPE_vv)
+ return e.use_ternop_insn (false,
+ code_for_pred_mul (MINUS,
+ e.vector_mode ()));
+ break;
+ default:
+ break;
+ }
+ gcc_unreachable ();
+ }
+};
+
+/* Enumerates types of widen ternary operations.
+ We have 4 types ternop:
+ - 1. vwmacc.
+ - 2. vwmaccu.
+ - 3. vwmaccsu.
+ - 4. vwmaccus. */
+enum widen_ternop_type
+{
+ WIDEN_TERNOP_VWMACC,
+ WIDEN_TERNOP_VWMACCU,
+ WIDEN_TERNOP_VWMACCSU,
+ WIDEN_TERNOP_VWMACCUS,
+};
+
+/* Implements vwmacc<su><su>. */
+template<widen_ternop_type WIDEN_TERNOP_TYPE>
+class iwmac : public function_base
+{
+public:
+ bool has_merge_operand_p () const override { return false; }
+
+ rtx expand (function_expander &e) const override
+ {
+ switch (WIDEN_TERNOP_TYPE)
+ {
+ case WIDEN_TERNOP_VWMACC:
+ if (e.op_info->op == OP_TYPE_vx)
+ return e.use_widen_ternop_insn (
+ code_for_pred_widen_mul_plus_scalar (SIGN_EXTEND,
+ e.vector_mode ()));
+ if (e.op_info->op == OP_TYPE_vv)
+ return e.use_widen_ternop_insn (
+ code_for_pred_widen_mul_plus (SIGN_EXTEND, e.vector_mode ()));
+ break;
+ case WIDEN_TERNOP_VWMACCU:
+ if (e.op_info->op == OP_TYPE_vx)
+ return e.use_widen_ternop_insn (
+ code_for_pred_widen_mul_plus_scalar (ZERO_EXTEND,
+ e.vector_mode ()));
+ if (e.op_info->op == OP_TYPE_vv)
+ return e.use_widen_ternop_insn (
+ code_for_pred_widen_mul_plus (ZERO_EXTEND, e.vector_mode ()));
+ break;
+ case WIDEN_TERNOP_VWMACCSU:
+ if (e.op_info->op == OP_TYPE_vx)
+ return e.use_widen_ternop_insn (
+ code_for_pred_widen_mul_plussu_scalar (e.vector_mode ()));
+ if (e.op_info->op == OP_TYPE_vv)
+ return e.use_widen_ternop_insn (
+ code_for_pred_widen_mul_plussu (e.vector_mode ()));
+ break;
+ case WIDEN_TERNOP_VWMACCUS:
+ return e.use_widen_ternop_insn (
+ code_for_pred_widen_mul_plusus_scalar (e.vector_mode ()));
+ default:
+ break;
+ }
+ gcc_unreachable ();
+ }
+};
+
static CONSTEXPR const vsetvl<false> vsetvl_obj;
static CONSTEXPR const vsetvl<true> vsetvlmax_obj;
static CONSTEXPR const loadstore<false, LST_UNIT_STRIDE, false> vle_obj;
@@ -616,6 +742,14 @@ static CONSTEXPR const icmp<LTU> vmsltu_obj;
static CONSTEXPR const icmp<GTU> vmsgtu_obj;
static CONSTEXPR const icmp<LEU> vmsleu_obj;
static CONSTEXPR const icmp<GEU> vmsgeu_obj;
+static CONSTEXPR const imac<TERNOP_VMACC> vmacc_obj;
+static CONSTEXPR const imac<TERNOP_VNMSAC> vnmsac_obj;
+static CONSTEXPR const imac<TERNOP_VMADD> vmadd_obj;
+static CONSTEXPR const imac<TERNOP_VNMSUB> vnmsub_obj;
+static CONSTEXPR const iwmac<WIDEN_TERNOP_VWMACC> vwmacc_obj;
+static CONSTEXPR const iwmac<WIDEN_TERNOP_VWMACCU> vwmaccu_obj;
+static CONSTEXPR const iwmac<WIDEN_TERNOP_VWMACCSU> vwmaccsu_obj;
+static CONSTEXPR const iwmac<WIDEN_TERNOP_VWMACCUS> vwmaccus_obj;
static CONSTEXPR const binop<SS_PLUS> vsadd_obj;
static CONSTEXPR const binop<SS_MINUS> vssub_obj;
static CONSTEXPR const binop<US_PLUS> vsaddu_obj;
@@ -712,6 +846,14 @@ BASE (vmsltu)
BASE (vmsgtu)
BASE (vmsleu)
BASE (vmsgeu)
+BASE (vmacc)
+BASE (vnmsac)
+BASE (vmadd)
+BASE (vnmsub)
+BASE (vwmacc)
+BASE (vwmaccu)
+BASE (vwmaccsu)
+BASE (vwmaccus)
BASE (vsadd)
BASE (vssub)
BASE (vsaddu)
diff --git a/gcc/config/riscv/riscv-vector-builtins-bases.h b/gcc/config/riscv/riscv-vector-builtins-bases.h
index 6233b4122f8..cb36f1f2699 100644
--- a/gcc/config/riscv/riscv-vector-builtins-bases.h
+++ b/gcc/config/riscv/riscv-vector-builtins-bases.h
@@ -101,6 +101,14 @@ extern const function_base *const vmsltu;
extern const function_base *const vmsgtu;
extern const function_base *const vmsleu;
extern const function_base *const vmsgeu;
+extern const function_base *const vmacc;
+extern const function_base *const vnmsac;
+extern const function_base *const vmadd;
+extern const function_base *const vnmsub;
+extern const function_base *const vwmacc;
+extern const function_base *const vwmaccu;
+extern const function_base *const vwmaccsu;
+extern const function_base *const vwmaccus;
extern const function_base *const vsadd;
extern const function_base *const vssub;
extern const function_base *const vsaddu;
diff --git a/gcc/config/riscv/riscv-vector-builtins-functions.def b/gcc/config/riscv/riscv-vector-builtins-functions.def
index f61f48d36a0..22271273655 100644
--- a/gcc/config/riscv/riscv-vector-builtins-functions.def
+++ b/gcc/config/riscv/riscv-vector-builtins-functions.def
@@ -177,6 +177,21 @@ DEF_RVV_FUNCTION (vmsltu, return_mask, none_m_mu_preds, u_mvx_ops)
DEF_RVV_FUNCTION (vmsgtu, return_mask, none_m_mu_preds, u_mvx_ops)
DEF_RVV_FUNCTION (vmsleu, return_mask, none_m_mu_preds, u_mvx_ops)
DEF_RVV_FUNCTION (vmsgeu, return_mask, none_m_mu_preds, u_mvx_ops)
+DEF_RVV_FUNCTION (vmacc, alu, full_preds, iu_vvvv_ops)
+DEF_RVV_FUNCTION (vnmsac, alu, full_preds, iu_vvvv_ops)
+DEF_RVV_FUNCTION (vmadd, alu, full_preds, iu_vvvv_ops)
+DEF_RVV_FUNCTION (vnmsub, alu, full_preds, iu_vvvv_ops)
+DEF_RVV_FUNCTION (vmacc, alu, full_preds, iu_vvxv_ops)
+DEF_RVV_FUNCTION (vnmsac, alu, full_preds, iu_vvxv_ops)
+DEF_RVV_FUNCTION (vmadd, alu, full_preds, iu_vvxv_ops)
+DEF_RVV_FUNCTION (vnmsub, alu, full_preds, iu_vvxv_ops)
+DEF_RVV_FUNCTION (vwmacc, alu, full_preds, i_wwvv_ops)
+DEF_RVV_FUNCTION (vwmacc, alu, full_preds, i_wwxv_ops)
+DEF_RVV_FUNCTION (vwmaccu, alu, full_preds, u_wwvv_ops)
+DEF_RVV_FUNCTION (vwmaccu, alu, full_preds, u_wwxv_ops)
+DEF_RVV_FUNCTION (vwmaccsu, alu, full_preds, i_su_wwvv_ops)
+DEF_RVV_FUNCTION (vwmaccsu, alu, full_preds, i_su_wwxv_ops)
+DEF_RVV_FUNCTION (vwmaccus, alu, full_preds, i_us_wwxv_ops)
/* 12. Vector Fixed-Point Arithmetic Instructions. */
DEF_RVV_FUNCTION (vsadd, alu, full_preds, i_vvv_ops)
DEF_RVV_FUNCTION (vssub, alu, full_preds, i_vvv_ops)
diff --git a/gcc/config/riscv/riscv-vector-builtins.cc b/gcc/config/riscv/riscv-vector-builtins.cc
index 3d842c6e3a3..54681bab3ea 100644
--- a/gcc/config/riscv/riscv-vector-builtins.cc
+++ b/gcc/config/riscv/riscv-vector-builtins.cc
@@ -314,6 +314,18 @@ static CONSTEXPR const rvv_arg_type_info vv_args[]
= {rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info (RVV_BASE_vector),
rvv_arg_type_info_end};
+/* A list of args for vector_type func (vector_type, vector_type, vector_type)
+ * function. */
+static CONSTEXPR const rvv_arg_type_info vvv_args[]
+ = {rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info (RVV_BASE_vector),
+ rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end};
+
+/* A list of args for vector_type func (vector_type, vector_type, vector_type)
+ * function. */
+static CONSTEXPR const rvv_arg_type_info vxv_args[]
+ = {rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info (RVV_BASE_scalar),
+ rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end};
+
/* A list of args for vector_type func (vector_type, vector_type, mask_type)
* function. */
static CONSTEXPR const rvv_arg_type_info vvm_args[]
@@ -378,6 +390,44 @@ static CONSTEXPR const rvv_arg_type_info wvv_args[]
= {rvv_arg_type_info (RVV_BASE_double_trunc_vector),
rvv_arg_type_info (RVV_BASE_double_trunc_vector), rvv_arg_type_info_end};
+/* A list of args for vector_type func (vector_type, double demote type, double
+ * demote type) function. */
+static CONSTEXPR const rvv_arg_type_info wwvv_args[]
+ = {rvv_arg_type_info (RVV_BASE_vector),
+ rvv_arg_type_info (RVV_BASE_double_trunc_vector),
+ rvv_arg_type_info (RVV_BASE_double_trunc_vector), rvv_arg_type_info_end};
+
+/* A list of args for vector_type func (vector_type, double demote type, double
+ * demote type) function. */
+static CONSTEXPR const rvv_arg_type_info wwxv_args[]
+ = {rvv_arg_type_info (RVV_BASE_vector),
+ rvv_arg_type_info (RVV_BASE_double_trunc_scalar),
+ rvv_arg_type_info (RVV_BASE_double_trunc_vector), rvv_arg_type_info_end};
+
+/* A list of args for vector_type func (vector_type, double demote type, double
+ * demote type) function. */
+static CONSTEXPR const rvv_arg_type_info su_wwvv_args[]
+ = {rvv_arg_type_info (RVV_BASE_vector),
+ rvv_arg_type_info (RVV_BASE_double_trunc_vector),
+ rvv_arg_type_info (RVV_BASE_double_trunc_unsigned_vector),
+ rvv_arg_type_info_end};
+
+/* A list of args for vector_type func (vector_type, double demote type, double
+ * demote type) function. */
+static CONSTEXPR const rvv_arg_type_info su_wwxv_args[]
+ = {rvv_arg_type_info (RVV_BASE_vector),
+ rvv_arg_type_info (RVV_BASE_double_trunc_scalar),
+ rvv_arg_type_info (RVV_BASE_double_trunc_unsigned_vector),
+ rvv_arg_type_info_end};
+
+/* A list of args for vector_type func (vector_type, double demote type, double
+ * demote type) function. */
+static CONSTEXPR const rvv_arg_type_info us_wwxv_args[]
+ = {rvv_arg_type_info (RVV_BASE_vector),
+ rvv_arg_type_info (RVV_BASE_double_trunc_unsigned_scalar),
+ rvv_arg_type_info (RVV_BASE_double_trunc_vector),
+ rvv_arg_type_info_end};
+
/* A list of args for vector_type func (signed double demote type,
* unsigneddouble demote type) function. */
static CONSTEXPR const rvv_arg_type_info su_wvv_args[]
@@ -577,6 +627,22 @@ static CONSTEXPR const rvv_op_info iu_vvv_ops
rvv_arg_type_info (RVV_BASE_vector), /* Return type */
vv_args /* Args */};
+/* A static operand information for vector_type func (vector_type, vector_type,
+ * vector_type) function registration. */
+static CONSTEXPR const rvv_op_info iu_vvvv_ops
+ = {iu_ops, /* Types */
+ OP_TYPE_vv, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ vvv_args /* Args */};
+
+/* A static operand information for vector_type func (vector_type, scalar_type,
+ * vector_type) function registration. */
+static CONSTEXPR const rvv_op_info iu_vvxv_ops
+ = {iu_ops, /* Types */
+ OP_TYPE_vx, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ vxv_args /* Args */};
+
/* A static operand information for vector_type func (vector_type, vector_type,
* mask_type) function registration. */
static CONSTEXPR const rvv_op_info iu_vvvm_ops
@@ -882,6 +948,62 @@ static CONSTEXPR const rvv_op_info i_wvv_ops
rvv_arg_type_info (RVV_BASE_vector), /* Return type */
wvv_args /* Args */};
+/* A static operand information for vector_type func (vector_type, double demote
+ * type, double demote type) function registration. */
+static CONSTEXPR const rvv_op_info i_wwvv_ops
+ = {wexti_ops, /* Types */
+ OP_TYPE_vv, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ wwvv_args /* Args */};
+
+/* A static operand information for vector_type func (vector_type, double demote
+ * scalar_type, double demote type) function registration. */
+static CONSTEXPR const rvv_op_info i_wwxv_ops
+ = {wexti_ops, /* Types */
+ OP_TYPE_vx, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ wwxv_args /* Args */};
+
+/* A static operand information for vector_type func (vector_type, double demote
+ * type, double demote type) function registration. */
+static CONSTEXPR const rvv_op_info u_wwvv_ops
+ = {wextu_ops, /* Types */
+ OP_TYPE_vv, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ wwvv_args /* Args */};
+
+/* A static operand information for vector_type func (vector_type, double demote
+ * scalar_type, double demote type) function registration. */
+static CONSTEXPR const rvv_op_info u_wwxv_ops
+ = {wextu_ops, /* Types */
+ OP_TYPE_vx, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ wwxv_args /* Args */};
+
+/* A static operand information for vector_type func (vector_type, double demote
+ * type, double demote type) function registration. */
+static CONSTEXPR const rvv_op_info i_su_wwvv_ops
+ = {wexti_ops, /* Types */
+ OP_TYPE_vv, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ su_wwvv_args /* Args */};
+
+/* A static operand information for vector_type func (vector_type, double demote
+ * scalar_type, double demote type) function registration. */
+static CONSTEXPR const rvv_op_info i_su_wwxv_ops
+ = {wexti_ops, /* Types */
+ OP_TYPE_vx, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ su_wwxv_args /* Args */};
+
+/* A static operand information for vector_type func (vector_type, double demote
+ * scalar_type, double demote type) function registration. */
+static CONSTEXPR const rvv_op_info i_us_wwxv_ops
+ = {wexti_ops, /* Types */
+ OP_TYPE_vx, /* Suffix */
+ rvv_arg_type_info (RVV_BASE_vector), /* Return type */
+ us_wwxv_args /* Args */};
+
/* A static operand information for vector_type func (signed double demote type,
* unsigned double demote type) function registration. */
static CONSTEXPR const rvv_op_info i_su_wvv_ops
@@ -1617,9 +1739,10 @@ function_builder::apply_predication (const function_instance &instance,
vec<tree> &argument_types) const
{
/* These predication types need to apply merge type. */
- if (instance.pred == PRED_TYPE_tu || instance.pred == PRED_TYPE_tum
- || instance.pred == PRED_TYPE_tumu || instance.pred == PRED_TYPE_mu)
- argument_types.quick_insert (0, return_type);
+ if (instance.base->has_merge_operand_p ())
+ if (instance.pred == PRED_TYPE_tu || instance.pred == PRED_TYPE_tum
+ || instance.pred == PRED_TYPE_tumu || instance.pred == PRED_TYPE_mu)
+ argument_types.quick_insert (0, return_type);
/* These predication types need to apply mask type. */
tree mask_type = builtin_types[mask_types[instance.type.index]].vector;
@@ -1816,8 +1939,7 @@ function_expander::add_input_operand (unsigned argno)
void
function_expander::add_vundef_operand (machine_mode mode)
{
- rtx vundef = gen_rtx_UNSPEC (mode, gen_rtvec (1, const0_rtx), UNSPEC_VUNDEF);
- add_input_operand (mode, vundef);
+ add_input_operand (mode, RVV_VUNDEF (mode));
}
/* Add a memory operand with mode MODE and address ADDR. */
@@ -1979,6 +2101,117 @@ function_expander::use_compare_insn (rtx_code rcode, insn_code icode)
return generate_insn (icode);
}
+/* Implement the call using instruction ICODE, with a 1:1 mapping between
+ arguments and input operands. */
+rtx
+function_expander::use_ternop_insn (bool vd_accum_p, insn_code icode)
+{
+ machine_mode mode = TYPE_MODE (builtin_types[type.index].vector);
+ tree mask_type = builtin_types[mask_types[type.index]].vector;
+ machine_mode mask_mode = TYPE_MODE (mask_type);
+
+ /* Record the offset to get the argument. */
+ int arg_offset = 0;
+
+ if (use_real_mask_p (pred))
+ add_input_operand (arg_offset++);
+ else
+ add_all_one_mask_operand (mask_mode);
+
+ rtx vd = expand_normal (CALL_EXPR_ARG (exp, arg_offset++));
+ rtx vs1 = expand_normal (CALL_EXPR_ARG (exp, arg_offset++));
+ rtx vs2 = expand_normal (CALL_EXPR_ARG (exp, arg_offset++));
+ rtx merge = use_real_merge_p (pred) ? vd : RVV_VUNDEF (mode);
+
+ if (VECTOR_MODE_P (GET_MODE (vs1)))
+ {
+ if (!vd_accum_p)
+ add_input_operand (mode, vd);
+ add_input_operand (mode, vs1);
+ add_input_operand (mode, vs2);
+ if (vd_accum_p)
+ add_input_operand (mode, vd);
+ add_input_operand (mode, merge);
+ }
+ else
+ {
+ add_input_operand (GET_MODE_INNER (mode), vs1);
+ if (vd_accum_p)
+ {
+ add_input_operand (mode, vs2);
+ add_input_operand (mode, vd);
+ }
+ else
+ {
+ add_input_operand (mode, vd);
+ add_input_operand (mode, vs2);
+ }
+ add_input_operand (mode, merge);
+ }
+
+ for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++)
+ add_input_operand (argno);
+
+ add_input_operand (Pmode, get_tail_policy_for_pred (pred));
+ add_input_operand (Pmode, get_mask_policy_for_pred (pred));
+ add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX));
+
+ /* See optabs.cc, the maximum nops is 9 for using 'maybe_gen_insn'.
+ We temporarily use GCN directly. We will change it back it we
+ can support nops >= 10. */
+ gcc_assert (maybe_legitimize_operands (icode, 0, opno, m_ops));
+ rtx_insn *pat = GEN_FCN (
+ icode) (m_ops[0].value, m_ops[1].value, m_ops[2].value, m_ops[3].value,
+ m_ops[4].value, m_ops[5].value, m_ops[6].value, m_ops[7].value,
+ m_ops[8].value, m_ops[9].value);
+ emit_insn (pat);
+ return m_ops[0].value;
+}
+
+/* Implement the call using instruction ICODE, with a 1:1 mapping between
+ arguments and input operands. */
+rtx
+function_expander::use_widen_ternop_insn (insn_code icode)
+{
+ machine_mode mode = TYPE_MODE (builtin_types[type.index].vector);
+ tree mask_type = builtin_types[mask_types[type.index]].vector;
+ machine_mode mask_mode = TYPE_MODE (mask_type);
+
+ /* Record the offset to get the argument. */
+ int arg_offset = 0;
+
+ if (use_real_mask_p (pred))
+ add_input_operand (arg_offset++);
+ else
+ add_all_one_mask_operand (mask_mode);
+
+ rtx merge = RVV_VUNDEF (mode);
+ if (use_real_merge_p (pred))
+ merge = expand_normal (CALL_EXPR_ARG (exp, arg_offset));
+
+ for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++)
+ {
+ if (argno == call_expr_nargs (exp) - 1)
+ add_input_operand (mode, merge);
+ add_input_operand (argno);
+ }
+
+ add_input_operand (Pmode, get_tail_policy_for_pred (pred));
+ add_input_operand (Pmode, get_mask_policy_for_pred (pred));
+ add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX));
+
+ /* See optabs.cc, the maximum nops is 9 for using 'maybe_gen_insn'.
+ We temporarily use GCN directly. We will change it back it we
+ can support nops >= 10. */
+ gcc_assert (maybe_legitimize_operands (icode, 0, opno, m_ops));
+ rtx_insn *pat = GEN_FCN (
+ icode) (m_ops[0].value, m_ops[1].value, m_ops[2].value, m_ops[3].value,
+ m_ops[4].value, m_ops[5].value, m_ops[6].value, m_ops[7].value,
+ m_ops[8].value, m_ops[9].value);
+ emit_insn (pat);
+ return m_ops[0].value;
+}
+
/* Generate instruction ICODE, given that its operands have already
been added to M_OPS. Return the value of the first operand. */
rtx
diff --git a/gcc/config/riscv/riscv-vector-builtins.h b/gcc/config/riscv/riscv-vector-builtins.h
index 9807fdf2938..b9d73670789 100644
--- a/gcc/config/riscv/riscv-vector-builtins.h
+++ b/gcc/config/riscv/riscv-vector-builtins.h
@@ -348,6 +348,8 @@ public:
rtx use_contiguous_load_insn (insn_code);
rtx use_contiguous_store_insn (insn_code);
rtx use_compare_insn (rtx_code, insn_code);
+ rtx use_ternop_insn (bool, insn_code);
+ rtx use_widen_ternop_insn (insn_code);
rtx generate_insn (insn_code);
/* The function call expression. */
diff --git a/gcc/config/riscv/vector.md b/gcc/config/riscv/vector.md
index 239ffc851e3..dc79aa230bd 100644
--- a/gcc/config/riscv/vector.md
+++ b/gcc/config/riscv/vector.md
@@ -146,7 +146,7 @@
vialu,vshift,vicmp,vimul,vidiv,vsalu,\
vext,viwalu,viwmul,vicalu,vnshift,\
vimuladd,vimerge,vaalu,vsmul,vsshift,\
- vnclip,viminmax")
+ vnclip,viminmax,viwmuladd")
(const_int INVALID_ATTRIBUTE)
(eq_attr "mode" "VNx1QI,VNx1BI")
(symbol_ref "riscv_vector::get_ratio(E_VNx1QImode)")
@@ -198,11 +198,14 @@
(define_attr "merge_op_idx" ""
(cond [(eq_attr "type" "vlde,vimov,vfmov,vldm,vlds,vmalu,vldux,vldox,vicmp,\
vialu,vshift,viminmax,vimul,vidiv,vsalu,vext,viwalu,\
- viwmul,vnshift,vimuladd,vaalu,vsmul,vsshift,vnclip")
+ viwmul,vnshift,vaalu,vsmul,vsshift,vnclip")
(const_int 2)
(eq_attr "type" "vimerge")
- (const_int 1)]
+ (const_int 1)
+
+ (eq_attr "type" "vimuladd,viwmuladd")
+ (const_int 5)]
(const_int INVALID_ATTRIBUTE)))
;; The index of operand[] to get the avl op.
@@ -219,11 +222,14 @@
(const_int 4))
(eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
- viwalu,viwmul,vnshift,vimuladd,vimerge,vaalu,vsmul,\
+ viwalu,viwmul,vnshift,vimerge,vaalu,vsmul,\
vsshift,vnclip")
(const_int 5)
(eq_attr "type" "vicmp")
+ (const_int 6)
+
+ (eq_attr "type" "vimuladd,viwmuladd")
(const_int 6)]
(const_int INVALID_ATTRIBUTE)))
@@ -240,9 +246,12 @@
(symbol_ref "riscv_vector::get_ta(operands[5])"))
(eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
- viwalu,viwmul,vnshift,vimuladd,vimerge,vaalu,vsmul,\
+ viwalu,viwmul,vnshift,vimerge,vaalu,vsmul,\
vsshift,vnclip")
- (symbol_ref "riscv_vector::get_ta(operands[6])")]
+ (symbol_ref "riscv_vector::get_ta(operands[6])")
+
+ (eq_attr "type" "vimuladd,viwmuladd")
+ (symbol_ref "riscv_vector::get_ta(operands[7])")]
(const_int INVALID_ATTRIBUTE)))
;; The mask policy op value.
@@ -258,9 +267,12 @@
(symbol_ref "riscv_vector::get_ma(operands[6])"))
(eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
- viwalu,viwmul,vnshift,vimuladd,vaalu,vsmul,vsshift,\
+ viwalu,viwmul,vnshift,vaalu,vsmul,vsshift,\
vnclip,vicmp")
- (symbol_ref "riscv_vector::get_ma(operands[7])")]
+ (symbol_ref "riscv_vector::get_ma(operands[7])")
+
+ (eq_attr "type" "vimuladd,viwmuladd")
+ (symbol_ref "riscv_vector::get_ma(operands[8])")]
(const_int INVALID_ATTRIBUTE)))
;; The avl type value.
@@ -282,7 +294,10 @@
vnclip,vicmp")
(symbol_ref "INTVAL (operands[8])")
(eq_attr "type" "vstux,vstox")
- (symbol_ref "INTVAL (operands[5])")]
+ (symbol_ref "INTVAL (operands[5])")
+
+ (eq_attr "type" "vimuladd,viwmuladd")
+ (symbol_ref "INTVAL (operands[9])")]
(const_int INVALID_ATTRIBUTE)))
;; -----------------------------------------------------------------
@@ -3948,7 +3963,7 @@
"TARGET_VECTOR"
{
enum rtx_code code = GET_CODE (operands[3]);
- rtx undef = gen_rtx_UNSPEC (<VM>mode, gen_rtvec (1, const0_rtx), UNSPEC_VUNDEF);
+ rtx undef = RVV_VUNDEF (<VM>mode);
if (code == GEU && rtx_equal_p (operands[5], const0_rtx))
{
/* If vmsgeu with 0 immediate, expand it to vmset. */
@@ -4035,6 +4050,651 @@
DONE;
})
+;; -------------------------------------------------------------------------------
+;; ---- Predicated integer ternary operations
+;; -------------------------------------------------------------------------------
+;; Includes:
+;; - 11.13 Vector Single-Width Integer Multiply-Add Instructions
+;; -------------------------------------------------------------------------------
+
+(define_expand "@pred_mul_<optab><mode>"
+ [(set (match_operand:VI 0 "register_operand")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand")
+ (match_operand 6 "vector_length_operand")
+ (match_operand 7 "const_int_operand")
+ (match_operand 8 "const_int_operand")
+ (match_operand 9 "const_int_operand")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (match_operand:VI 2 "register_operand")
+ (match_operand:VI 3 "register_operand"))
+ (match_operand:VI 4 "register_operand"))
+ (match_operand:VI 5 "vector_merge_operand")))]
+ "TARGET_VECTOR"
+{
+ /* Swap the multiplication operands if the fallback value is the
+ second of the two. */
+ if (rtx_equal_p (operands[3], operands[5]))
+ std::swap (operands[2], operands[3]);
+})
+
+(define_insn "pred_mul_<optab><mode>_undef_merge"
+ [(set (match_operand:VI 0 "register_operand" "=vd, vr, vd, vr, ?&vr")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1, vm,Wc1,vmWc1")
+ (match_operand 6 "vector_length_operand" " rK, rK, rK, rK, rK")
+ (match_operand 7 "const_int_operand" " i, i, i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i, i, i")
+ (match_operand 9 "const_int_operand" " i, i, i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (match_operand:VI 2 "register_operand" " %0, 0, vr, vr, vr")
+ (match_operand:VI 3 "register_operand" " vr, vr, vr, vr, vr"))
+ (match_operand:VI 4 "register_operand" " vr, vr, 0, 0, vr"))
+ (match_operand:VI 5 "vector_undef_operand" " vu, vu, vu, vu, vu")))]
+ "TARGET_VECTOR"
+ "@
+ v<madd_nmsub>.vv\t%0,%3,%4%p1
+ v<madd_nmsub>.vv\t%0,%3,%4%p1
+ v<macc_nmsac>.vv\t%0,%2,%3%p1
+ v<macc_nmsac>.vv\t%0,%2,%3%p1
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vv\t%0,%2,%3%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "*pred_<madd_nmsub><mode>"
+ [(set (match_operand:VI 0 "register_operand" "=vd, vr, ?&vr")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1,vmWc1")
+ (match_operand 5 "vector_length_operand" " rK, rK, rK")
+ (match_operand 6 "const_int_operand" " i, i, i")
+ (match_operand 7 "const_int_operand" " i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (match_operand:VI 2 "register_operand" " 0, 0, vr")
+ (match_operand:VI 3 "register_operand" " vr, vr, vr"))
+ (match_operand:VI 4 "register_operand" " vr, vr, vr"))
+ (match_dup 2)))]
+ "TARGET_VECTOR"
+ "@
+ v<madd_nmsub>.vv\t%0,%3,%4%p1
+ v<madd_nmsub>.vv\t%0,%3,%4%p1
+ vmv.v.v\t%0,%2\;v<madd_nmsub>.vv\t%0,%3,%4%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")
+ (set_attr "merge_op_idx" "4")
+ (set_attr "vl_op_idx" "5")
+ (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
+ (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
+ (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
+
+(define_insn "*pred_<macc_nmsac><mode>"
+ [(set (match_operand:VI 0 "register_operand" "=vd, vr, ?&vr")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1,vmWc1")
+ (match_operand 5 "vector_length_operand" " rK, rK, rK")
+ (match_operand 6 "const_int_operand" " i, i, i")
+ (match_operand 7 "const_int_operand" " i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (match_operand:VI 2 "register_operand" " vr, vr, vr")
+ (match_operand:VI 3 "register_operand" " vr, vr, vr"))
+ (match_operand:VI 4 "register_operand" " 0, 0, vr"))
+ (match_dup 4)))]
+ "TARGET_VECTOR"
+ "@
+ v<macc_nmsac>.vv\t%0,%2,%3%p1
+ v<macc_nmsac>.vv\t%0,%2,%3%p1
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vv\t%0,%2,%3%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")
+ (set_attr "merge_op_idx" "2")
+ (set_attr "vl_op_idx" "5")
+ (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
+ (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
+ (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
+
+(define_insn_and_rewrite "*pred_mul_<optab><mode>"
+ [(set (match_operand:VI 0 "register_operand" "=&vr,?&vr, ?&vr, ?&vr, ?&vr")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1,vmWc1")
+ (match_operand 6 "vector_length_operand" " rK, rK, rK, rK, rK")
+ (match_operand 7 "const_int_operand" " i, i, i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i, i, i")
+ (match_operand 9 "const_int_operand" " i, i, i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (match_operand:VI 2 "register_operand" " vr, vr, vi, vr, vr")
+ (match_operand:VI 3 "register_operand" " vr, vr, vr, vi, vr"))
+ (match_operand:VI 4 "vector_arith_operand" " vr, vi, vr, vr, vr"))
+ (match_operand:VI 5 "register_operand" " 0, vr, vr, vr, vr")))]
+ "TARGET_VECTOR
+ && !rtx_equal_p (operands[2], operands[5])
+ && !rtx_equal_p (operands[3], operands[5])
+ && !rtx_equal_p (operands[4], operands[5])"
+ "@
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vv\t%0,%2,%3%p1
+ #
+ #
+ #
+ #"
+ "&& reload_completed
+ && !rtx_equal_p (operands[0], operands[5])"
+ {
+ if (satisfies_constraint_vi (operands[3]))
+ std::swap (operands[2], operands[3]);
+
+ if (satisfies_constraint_vi (operands[2]))
+ {
+ emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
+ operands[5], operands[2], operands[1], operands[6],
+ operands[7], operands[9]));
+ operands[5] = operands[2] = operands[0];
+ }
+ else
+ {
+ emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
+ operands[5], operands[4], operands[1], operands[6],
+ operands[7], operands[9]));
+ operands[5] = operands[4] = operands[0];
+ }
+ }
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")])
+
+(define_expand "@pred_mul_<optab><mode>_scalar"
+ [(set (match_operand:VI_QHS 0 "register_operand")
+ (if_then_else:VI_QHS
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand")
+ (match_operand 6 "vector_length_operand")
+ (match_operand 7 "const_int_operand")
+ (match_operand 8 "const_int_operand")
+ (match_operand 9 "const_int_operand")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI_QHS
+ (mult:VI_QHS
+ (vec_duplicate:VI_QHS
+ (match_operand:<VEL> 2 "reg_or_int_operand"))
+ (match_operand:VI_QHS 3 "register_operand"))
+ (match_operand:VI_QHS 4 "register_operand"))
+ (match_operand:VI_QHS 5 "vector_merge_operand")))]
+ "TARGET_VECTOR"
+{
+ operands[2] = force_reg (<VEL>mode, operands[2]);
+})
+
+(define_insn "*pred_mul_<optab><mode>_undef_merge_scalar"
+ [(set (match_operand:VI 0 "register_operand" "=vd, vr, vd, vr, ?&vr")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1, vm,Wc1,vmWc1")
+ (match_operand 6 "vector_length_operand" " rK, rK, rK, rK, rK")
+ (match_operand 7 "const_int_operand" " i, i, i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i, i, i")
+ (match_operand 9 "const_int_operand" " i, i, i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (vec_duplicate:VI
+ (match_operand:<VEL> 2 "register_operand" " r, r, r, r, r"))
+ (match_operand:VI 3 "register_operand" " 0, 0, vr, vr, vr"))
+ (match_operand:VI 4 "register_operand" " vr, vr, 0, 0, vr"))
+ (match_operand:VI 5 "vector_undef_operand" " vu, vu, vu, vu, vu")))]
+ "TARGET_VECTOR"
+ "@
+ v<madd_nmsub>.vx\t%0,%2,%4%p1
+ v<madd_nmsub>.vx\t%0,%2,%4%p1
+ v<macc_nmsac>.vx\t%0,%2,%3%p1
+ v<macc_nmsac>.vx\t%0,%2,%3%p1
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vx\t%0,%2,%3%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "*pred_<madd_nmsub><mode>_scalar"
+ [(set (match_operand:VI 0 "register_operand" "=vd, vr, ?&vr")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1,vmWc1")
+ (match_operand 5 "vector_length_operand" " rK, rK, rK")
+ (match_operand 6 "const_int_operand" " i, i, i")
+ (match_operand 7 "const_int_operand" " i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (vec_duplicate:VI
+ (match_operand:<VEL> 2 "register_operand" " r, r, vr"))
+ (match_operand:VI 3 "register_operand" " 0, 0, vr"))
+ (match_operand:VI 4 "register_operand" " vr, vr, vr"))
+ (match_dup 3)))]
+ "TARGET_VECTOR"
+ "@
+ v<madd_nmsub>.vx\t%0,%2,%4%p1
+ v<madd_nmsub>.vx\t%0,%2,%4%p1
+ vmv.v.v\t%0,%2\;v<madd_nmsub>.vx\t%0,%2,%4%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")
+ (set_attr "merge_op_idx" "4")
+ (set_attr "vl_op_idx" "5")
+ (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
+ (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
+ (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
+
+(define_insn "*pred_<macc_nmsac><mode>_scalar"
+ [(set (match_operand:VI 0 "register_operand" "=vd, vr, ?&vr")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1,vmWc1")
+ (match_operand 5 "vector_length_operand" " rK, rK, rK")
+ (match_operand 6 "const_int_operand" " i, i, i")
+ (match_operand 7 "const_int_operand" " i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (vec_duplicate:VI
+ (match_operand:<VEL> 2 "register_operand" " r, r, vr"))
+ (match_operand:VI 3 "register_operand" " vr, vr, vr"))
+ (match_operand:VI 4 "register_operand" " 0, 0, vr"))
+ (match_dup 4)))]
+ "TARGET_VECTOR"
+ "@
+ v<macc_nmsac>.vx\t%0,%2,%3%p1
+ v<macc_nmsac>.vx\t%0,%2,%3%p1
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vx\t%0,%2,%3%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")
+ (set_attr "merge_op_idx" "2")
+ (set_attr "vl_op_idx" "5")
+ (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
+ (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
+ (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
+
+(define_insn_and_rewrite "*pred_mul_<optab><mode>_scalar"
+ [(set (match_operand:VI 0 "register_operand" "=&vr, ?&vr, ?&vr, ?&vr")
+ (if_then_else:VI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1")
+ (match_operand 6 "vector_length_operand" " rK, rK, rK, rK")
+ (match_operand 7 "const_int_operand" " i, i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i, i")
+ (match_operand 9 "const_int_operand" " i, i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI
+ (mult:VI
+ (vec_duplicate:VI
+ (match_operand:<VEL> 2 "register_operand" " r, r, r, r"))
+ (match_operand:VI 3 "register_operand" " vr, vr, vi, vr"))
+ (match_operand:VI 4 "vector_arith_operand" " vr, vi, vr, vr"))
+ (match_operand:VI 5 "register_operand" " 0, vr, vr, vr")))]
+ "TARGET_VECTOR
+ && !rtx_equal_p (operands[3], operands[5])
+ && !rtx_equal_p (operands[4], operands[5])"
+ "@
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vx\t%0,%2,%3%p1
+ #
+ #
+ #"
+ "&& reload_completed
+ && !rtx_equal_p (operands[0], operands[5])"
+ {
+ if (satisfies_constraint_vi (operands[3]))
+ {
+ emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
+ operands[5], operands[3], operands[1], operands[6],
+ operands[7], operands[9]));
+ operands[5] = operands[3] = operands[0];
+ }
+ else
+ {
+ emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
+ operands[5], operands[4], operands[1], operands[6],
+ operands[7], operands[9]));
+ operands[5] = operands[4] = operands[0];
+ }
+ }
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")])
+
+(define_expand "@pred_mul_<optab><mode>_scalar"
+ [(set (match_operand:VI_D 0 "register_operand")
+ (if_then_else:VI_D
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand")
+ (match_operand 6 "vector_length_operand")
+ (match_operand 7 "const_int_operand")
+ (match_operand 8 "const_int_operand")
+ (match_operand 9 "const_int_operand")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI_D
+ (mult:VI_D
+ (vec_duplicate:VI_D
+ (match_operand:<VEL> 2 "reg_or_int_operand"))
+ (match_operand:VI_D 3 "register_operand"))
+ (match_operand:VI_D 4 "register_operand"))
+ (match_operand:VI_D 5 "vector_merge_operand")))]
+ "TARGET_VECTOR"
+{
+ if (!TARGET_64BIT)
+ {
+ rtx v = gen_reg_rtx (<MODE>mode);
+
+ if (riscv_vector::simm32_p (operands[2]))
+ operands[2] = gen_rtx_SIGN_EXTEND (<VEL>mode,
+ force_reg (Pmode, operands[2]));
+ else
+ {
+ if (CONST_INT_P (operands[2]))
+ operands[2] = force_reg (<VEL>mode, operands[2]);
+
+ riscv_vector::emit_nonvlmax_op (code_for_pred_broadcast (<MODE>mode),
+ v, operands[2], operands[6], <VM>mode);
+ emit_insn (gen_pred_mul_<optab><mode> (operands[0], operands[1],
+ v, operands[3], operands[4], operands[5], operands[6],
+ operands[7], operands[8], operands[9]));
+ DONE;
+ }
+ }
+ else
+ operands[2] = force_reg (<VEL>mode, operands[2]);
+})
+
+(define_insn "*pred_mul_<optab><mode>_undef_merge_extended_scalar"
+ [(set (match_operand:VI_D 0 "register_operand" "=vd, vr, vd, vr, ?&vr")
+ (if_then_else:VI_D
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1, vm,Wc1,vmWc1")
+ (match_operand 6 "vector_length_operand" " rK, rK, rK, rK, rK")
+ (match_operand 7 "const_int_operand" " i, i, i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i, i, i")
+ (match_operand 9 "const_int_operand" " i, i, i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI_D
+ (mult:VI_D
+ (vec_duplicate:VI_D
+ (sign_extend:<VEL>
+ (match_operand:<VSUBEL> 2 "register_operand" " r, r, r, r, r")))
+ (match_operand:VI_D 3 "register_operand" " 0, 0, vr, vr, vr"))
+ (match_operand:VI_D 4 "register_operand" " vr, vr, 0, 0, vr"))
+ (match_operand:VI_D 5 "vector_undef_operand" " vu, vu, vu, vu, vu")))]
+ "TARGET_VECTOR"
+ "@
+ v<madd_nmsub>.vx\t%0,%2,%4%p1
+ v<madd_nmsub>.vx\t%0,%2,%4%p1
+ v<macc_nmsac>.vx\t%0,%2,%3%p1
+ v<macc_nmsac>.vx\t%0,%2,%3%p1
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vx\t%0,%2,%3%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "*pred_<madd_nmsub><mode>_extended_scalar"
+ [(set (match_operand:VI_D 0 "register_operand" "=vd, vr, ?&vr")
+ (if_then_else:VI_D
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1,vmWc1")
+ (match_operand 5 "vector_length_operand" " rK, rK, rK")
+ (match_operand 6 "const_int_operand" " i, i, i")
+ (match_operand 7 "const_int_operand" " i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI_D
+ (mult:VI_D
+ (vec_duplicate:VI_D
+ (sign_extend:<VEL>
+ (match_operand:<VSUBEL> 2 "register_operand" " r, r, vr")))
+ (match_operand:VI_D 3 "register_operand" " 0, 0, vr"))
+ (match_operand:VI_D 4 "register_operand" " vr, vr, vr"))
+ (match_dup 3)))]
+ "TARGET_VECTOR"
+ "@
+ v<madd_nmsub>.vx\t%0,%2,%4%p1
+ v<madd_nmsub>.vx\t%0,%2,%4%p1
+ vmv.v.v\t%0,%2\;v<madd_nmsub>.vx\t%0,%2,%4%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")
+ (set_attr "merge_op_idx" "4")
+ (set_attr "vl_op_idx" "5")
+ (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
+ (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
+ (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
+
+(define_insn "*pred_<macc_nmsac><mode>_extended_scalar"
+ [(set (match_operand:VI_D 0 "register_operand" "=vd, vr, ?&vr")
+ (if_then_else:VI_D
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" " vm,Wc1,vmWc1")
+ (match_operand 5 "vector_length_operand" " rK, rK, rK")
+ (match_operand 6 "const_int_operand" " i, i, i")
+ (match_operand 7 "const_int_operand" " i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI_D
+ (mult:VI_D
+ (vec_duplicate:VI_D
+ (sign_extend:<VEL>
+ (match_operand:<VSUBEL> 2 "register_operand" " r, r, vr")))
+ (match_operand:VI_D 3 "register_operand" " vr, vr, vr"))
+ (match_operand:VI_D 4 "register_operand" " 0, 0, vr"))
+ (match_dup 4)))]
+ "TARGET_VECTOR"
+ "@
+ v<macc_nmsac>.vx\t%0,%2,%3%p1
+ v<macc_nmsac>.vx\t%0,%2,%3%p1
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vx\t%0,%2,%3%p1"
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")
+ (set_attr "merge_op_idx" "2")
+ (set_attr "vl_op_idx" "5")
+ (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
+ (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
+ (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
+
+(define_insn_and_rewrite "*pred_mul_<optab><mode>_extended_scalar"
+ [(set (match_operand:VI_D 0 "register_operand" "=&vr, ?&vr, ?&vr, ?&vr")
+ (if_then_else:VI_D
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1")
+ (match_operand 6 "vector_length_operand" " rK, rK, rK, rK")
+ (match_operand 7 "const_int_operand" " i, i, i, i")
+ (match_operand 8 "const_int_operand" " i, i, i, i")
+ (match_operand 9 "const_int_operand" " i, i, i, i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus_minus:VI_D
+ (mult:VI_D
+ (vec_duplicate:VI_D
+ (sign_extend:<VEL>
+ (match_operand:<VSUBEL> 2 "register_operand" " r, r, r, r")))
+ (match_operand:VI_D 3 "register_operand" " vr, vr, vr, vr"))
+ (match_operand:VI_D 4 "vector_arith_operand" " vr, vr, vr, vr"))
+ (match_operand:VI_D 5 "register_operand" " 0, vr, vr, vr")))]
+ "TARGET_VECTOR
+ && !rtx_equal_p (operands[3], operands[5])
+ && !rtx_equal_p (operands[4], operands[5])"
+ "@
+ vmv.v.v\t%0,%4\;v<macc_nmsac>.vx\t%0,%2,%3%p1
+ #
+ #
+ #"
+ "&& reload_completed
+ && !rtx_equal_p (operands[0], operands[5])"
+ {
+ if (satisfies_constraint_vi (operands[3]))
+ {
+ emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
+ operands[5], operands[3], operands[1], operands[6],
+ operands[7], operands[9]));
+ operands[5] = operands[3] = operands[0];
+ }
+ else
+ {
+ emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
+ operands[5], operands[4], operands[1], operands[6],
+ operands[7], operands[9]));
+ operands[5] = operands[4] = operands[0];
+ }
+ }
+ [(set_attr "type" "vimuladd")
+ (set_attr "mode" "<MODE>")])
+
+;; -------------------------------------------------------------------------------
+;; ---- Predicated integer ternary operations
+;; -------------------------------------------------------------------------------
+;; Includes:
+;; - 11.14 Vector Widening Integer Multiply-Add Instructions
+;; -------------------------------------------------------------------------------
+
+(define_insn "@pred_widen_mul_plus<su><mode>"
+ [(set (match_operand:VWEXTI 0 "register_operand" "=&vr")
+ (if_then_else:VWEXTI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
+ (match_operand 6 "vector_length_operand" " rK")
+ (match_operand 7 "const_int_operand" " i")
+ (match_operand 8 "const_int_operand" " i")
+ (match_operand 9 "const_int_operand" " i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus:VWEXTI
+ (mult:VWEXTI
+ (any_extend:VWEXTI
+ (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" " vr"))
+ (any_extend:VWEXTI
+ (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" " vr")))
+ (match_operand:VWEXTI 2 "register_operand" " 0"))
+ (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))]
+ "TARGET_VECTOR"
+ "vwmacc<u>.vv\t%0,%3,%4%p1"
+ [(set_attr "type" "viwmuladd")
+ (set_attr "mode" "<V_DOUBLE_TRUNC>")])
+
+(define_insn "@pred_widen_mul_plus<su><mode>_scalar"
+ [(set (match_operand:VWEXTI 0 "register_operand" "=&vr")
+ (if_then_else:VWEXTI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
+ (match_operand 6 "vector_length_operand" " rK")
+ (match_operand 7 "const_int_operand" " i")
+ (match_operand 8 "const_int_operand" " i")
+ (match_operand 9 "const_int_operand" " i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus:VWEXTI
+ (mult:VWEXTI
+ (any_extend:VWEXTI
+ (vec_duplicate:<V_DOUBLE_TRUNC>
+ (match_operand:<VSUBEL> 3 "register_operand" " r")))
+ (any_extend:VWEXTI
+ (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" " vr")))
+ (match_operand:VWEXTI 2 "register_operand" " 0"))
+ (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))]
+ "TARGET_VECTOR"
+ "vwmacc<u>.vx\t%0,%3,%4%p1"
+ [(set_attr "type" "viwmuladd")
+ (set_attr "mode" "<V_DOUBLE_TRUNC>")])
+
+(define_insn "@pred_widen_mul_plussu<mode>"
+ [(set (match_operand:VWEXTI 0 "register_operand" "=&vr")
+ (if_then_else:VWEXTI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
+ (match_operand 6 "vector_length_operand" " rK")
+ (match_operand 7 "const_int_operand" " i")
+ (match_operand 8 "const_int_operand" " i")
+ (match_operand 9 "const_int_operand" " i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus:VWEXTI
+ (mult:VWEXTI
+ (sign_extend:VWEXTI
+ (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" " vr"))
+ (zero_extend:VWEXTI
+ (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" " vr")))
+ (match_operand:VWEXTI 2 "register_operand" " 0"))
+ (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))]
+ "TARGET_VECTOR"
+ "vwmaccsu.vv\t%0,%3,%4%p1"
+ [(set_attr "type" "viwmuladd")
+ (set_attr "mode" "<V_DOUBLE_TRUNC>")])
+
+(define_insn "@pred_widen_mul_plussu<mode>_scalar"
+ [(set (match_operand:VWEXTI 0 "register_operand" "=&vr")
+ (if_then_else:VWEXTI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
+ (match_operand 6 "vector_length_operand" " rK")
+ (match_operand 7 "const_int_operand" " i")
+ (match_operand 8 "const_int_operand" " i")
+ (match_operand 9 "const_int_operand" " i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus:VWEXTI
+ (mult:VWEXTI
+ (sign_extend:VWEXTI
+ (vec_duplicate:<V_DOUBLE_TRUNC>
+ (match_operand:<VSUBEL> 3 "register_operand" " r")))
+ (zero_extend:VWEXTI
+ (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" " vr")))
+ (match_operand:VWEXTI 2 "register_operand" " 0"))
+ (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))]
+ "TARGET_VECTOR"
+ "vwmaccsu.vx\t%0,%3,%4%p1"
+ [(set_attr "type" "viwmuladd")
+ (set_attr "mode" "<V_DOUBLE_TRUNC>")])
+
+(define_insn "@pred_widen_mul_plusus<mode>_scalar"
+ [(set (match_operand:VWEXTI 0 "register_operand" "=&vr")
+ (if_then_else:VWEXTI
+ (unspec:<VM>
+ [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
+ (match_operand 6 "vector_length_operand" " rK")
+ (match_operand 7 "const_int_operand" " i")
+ (match_operand 8 "const_int_operand" " i")
+ (match_operand 9 "const_int_operand" " i")
+ (reg:SI VL_REGNUM)
+ (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+ (plus:VWEXTI
+ (mult:VWEXTI
+ (zero_extend:VWEXTI
+ (vec_duplicate:<V_DOUBLE_TRUNC>
+ (match_operand:<VSUBEL> 3 "register_operand" " r")))
+ (sign_extend:VWEXTI
+ (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" " vr")))
+ (match_operand:VWEXTI 2 "register_operand" " 0"))
+ (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))]
+ "TARGET_VECTOR"
+ "vwmaccus.vx\t%0,%3,%4%p1"
+ [(set_attr "type" "viwmuladd")
+ (set_attr "mode" "<V_DOUBLE_TRUNC>")])
+
;; -------------------------------------------------------------------------------
;; ---- Predicated BOOL mask operations
;; -------------------------------------------------------------------------------
--
2.36.3
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2023-02-14 9:24 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-02-14 9:24 [PATCH] RISC-V: Finish all integer C/C++ intrinsics juzhe.zhong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).