public inbox for gcc-cvs@sourceware.org
help / color / mirror / Atom feed
* [gcc(refs/vendors/riscv/heads/gcc-13-with-riscv-opts)] RISC-V: Fix bug of VLA SLP auto-vectorization
@ 2023-06-19 11:41 Jeff Law
0 siblings, 0 replies; 2+ messages in thread
From: Jeff Law @ 2023-06-19 11:41 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:fe5722dd6a0b153db2ed854c9ac80d0b8990181b
commit fe5722dd6a0b153db2ed854c9ac80d0b8990181b
Author: Juzhe-Zhong <juzhe.zhong@rivai.ai>
Date: Tue Jun 13 17:30:55 2023 +0800
RISC-V: Fix bug of VLA SLP auto-vectorization
Sorry for producing bugs in the previous VLA SLP patch.
Consider this following permutation:
_85 = VEC_PERM_EXPR <{ 99, 17, ... }, { 11, 80, ... }, { 0, POLY_INT_CST [4, 4], 1, POLY_INT_CST [5, 4], 2, POLY_INT_CST [6, 4], ... }>;
The correct result should be:
_85 = { 99, 11, 17, 80, ... }
However, I did wrong in the previous patch.
Code sequence before this patch:
set mask = { 0, 1, 0, 1, ... }
set v0 = { 99, 17, 99, 17, ... }
set v1 = { 11, 80, 11, 80, ... }
set index = viota (mask) = { 0, 0, 1, 1, 2, 2, ... }
set result = vrgather_mu (v0, v1, index, mask) = { 99, 11, 99, 80 }
The result is incorrect.
After this patch:
set mask = { 0, 1, 0, 1, ... }
set index = viota (mask) = { 0, 0, 1, 1, 2, 2, ... }
set v0 = vrgather ({ 99, 17, 99, 17, ... }, index) = { 99, 99, 17, 17, ... }
set v1 = { 11, 80, 11, 80, ... }
set result = vrgather_mu (v0, v1, index, mask) = { 99, 11, 17, 80 }
The result is what we expected.
This issue was discovered in the test I appended in this patch with --param=riscv-autovec-lmul=2.
gcc/ChangeLog:
* config/riscv/riscv-v.cc (emit_vlmax_decompress_insn): Fix bug.
(shuffle_decompress_patterns): Ditto.
gcc/testsuite/ChangeLog:
* gcc.target/riscv/rvv/autovec/partial/slp-12.c: New test.
* gcc.target/riscv/rvv/autovec/partial/slp_run-12.c: New test.
Diff:
---
gcc/config/riscv/riscv-v.cc | 8 ++---
.../gcc.target/riscv/rvv/autovec/partial/slp-12.c | 35 ++++++++++++++++++++++
.../riscv/rvv/autovec/partial/slp_run-12.c | 32 ++++++++++++++++++++
3 files changed, 71 insertions(+), 4 deletions(-)
diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
index fb970344521..34fdb53c140 100644
--- a/gcc/config/riscv/riscv-v.cc
+++ b/gcc/config/riscv/riscv-v.cc
@@ -863,7 +863,7 @@ emit_vlmax_masked_gather_mu_insn (rtx target, rtx op, rtx sel, rtx mask)
e q r d c b v a # v11 destination after vrgather using viota.m under mask
*/
static void
-emit_vlmax_decompress_insn (rtx target, rtx op, rtx mask)
+emit_vlmax_decompress_insn (rtx target, rtx op0, rtx op1, rtx mask)
{
machine_mode data_mode = GET_MODE (target);
machine_mode sel_mode = related_int_vector_mode (data_mode).require ();
@@ -873,7 +873,8 @@ emit_vlmax_decompress_insn (rtx target, rtx op, rtx mask)
rtx sel = gen_reg_rtx (sel_mode);
rtx iota_ops[] = {sel, mask};
emit_vlmax_insn (code_for_pred_iota (sel_mode), RVV_UNOP, iota_ops);
- emit_vlmax_masked_gather_mu_insn (target, op, sel, mask);
+ emit_vlmax_gather_insn (target, op0, sel);
+ emit_vlmax_masked_gather_mu_insn (target, op1, sel, mask);
}
/* Emit merge instruction. */
@@ -2441,8 +2442,7 @@ shuffle_decompress_patterns (struct expand_vec_perm_d *d)
rtx const_vec = gen_const_vector_dup (sel_mode, 1);
rtx mask = gen_reg_rtx (mask_mode);
expand_vec_cmp (mask, EQ, vid_repeat, const_vec);
- emit_move_insn (d->target, op0);
- emit_vlmax_decompress_insn (d->target, op1, mask);
+ emit_vlmax_decompress_insn (d->target, op0, op1, mask);
return true;
}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp-12.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp-12.c
new file mode 100644
index 00000000000..03529f4643a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp-12.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fno-vect-cost-model" } */
+
+#include <stdint-gcc.h>
+
+#define VEC_PERM(TYPE) \
+ TYPE __attribute__ ((noinline, noclone)) \
+ vec_slp_##TYPE (TYPE *restrict a, int n) \
+ { \
+ for (int i = 0; i < n; ++i) \
+ { \
+ a[i * 8] += 99; \
+ a[i * 8 + 1] += 11; \
+ a[i * 8 + 2] += 17; \
+ a[i * 8 + 3] += 80; \
+ a[i * 8 + 4] += 63; \
+ a[i * 8 + 5] += 37; \
+ a[i * 8 + 6] += 24; \
+ a[i * 8 + 7] += 81; \
+ } \
+ }
+
+#define TEST_ALL(T) \
+ T (int8_t) \
+ T (uint8_t) \
+ T (int16_t) \
+ T (uint16_t) \
+ T (int32_t) \
+ T (uint32_t) \
+ T (int64_t) \
+ T (uint64_t)
+
+TEST_ALL (VEC_PERM)
+/* This testcase is from aarch64 and floating-point operations are removed.
+ TODO: We will add floating-point operations back and make them as common test in the future. */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp_run-12.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp_run-12.c
new file mode 100644
index 00000000000..af892ad7ac4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp_run-12.c
@@ -0,0 +1,32 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-additional-options "--param riscv-autovec-preference=scalable -fno-vect-cost-model" } */
+
+#include "slp-12.c"
+
+#define N (59 * 8)
+
+#define HARNESS(TYPE) \
+ { \
+ TYPE a[N], b[8] = { 99, 11, 17, 80, 63, 37, 24, 81 }; \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ a[i] = i * 2 + i % 5; \
+ asm volatile ("" ::: "memory"); \
+ } \
+ vec_slp_##TYPE (a, N / 8); \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ TYPE orig = i * 2 + i % 5; \
+ TYPE expected = orig + b[i % 8]; \
+ if (a[i] != expected) \
+ __builtin_abort (); \
+ } \
+ }
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ TEST_ALL (HARNESS)
+}
+/* This testcase is from aarch64 and floating-point operations are removed.
+ TODO: We will add floating-point operations back and make them as common test in the future. */
^ permalink raw reply [flat|nested] 2+ messages in thread
* [gcc(refs/vendors/riscv/heads/gcc-13-with-riscv-opts)] RISC-V: Fix bug of VLA SLP auto-vectorization
@ 2023-07-14 2:45 Jeff Law
0 siblings, 0 replies; 2+ messages in thread
From: Jeff Law @ 2023-07-14 2:45 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:c57a7b4ef97e78da025c01cc7454628bdeb18d97
commit c57a7b4ef97e78da025c01cc7454628bdeb18d97
Author: Juzhe-Zhong <juzhe.zhong@rivai.ai>
Date: Tue Jun 13 17:30:55 2023 +0800
RISC-V: Fix bug of VLA SLP auto-vectorization
Sorry for producing bugs in the previous VLA SLP patch.
Consider this following permutation:
_85 = VEC_PERM_EXPR <{ 99, 17, ... }, { 11, 80, ... }, { 0, POLY_INT_CST [4, 4], 1, POLY_INT_CST [5, 4], 2, POLY_INT_CST [6, 4], ... }>;
The correct result should be:
_85 = { 99, 11, 17, 80, ... }
However, I did wrong in the previous patch.
Code sequence before this patch:
set mask = { 0, 1, 0, 1, ... }
set v0 = { 99, 17, 99, 17, ... }
set v1 = { 11, 80, 11, 80, ... }
set index = viota (mask) = { 0, 0, 1, 1, 2, 2, ... }
set result = vrgather_mu (v0, v1, index, mask) = { 99, 11, 99, 80 }
The result is incorrect.
After this patch:
set mask = { 0, 1, 0, 1, ... }
set index = viota (mask) = { 0, 0, 1, 1, 2, 2, ... }
set v0 = vrgather ({ 99, 17, 99, 17, ... }, index) = { 99, 99, 17, 17, ... }
set v1 = { 11, 80, 11, 80, ... }
set result = vrgather_mu (v0, v1, index, mask) = { 99, 11, 17, 80 }
The result is what we expected.
This issue was discovered in the test I appended in this patch with --param=riscv-autovec-lmul=2.
gcc/ChangeLog:
* config/riscv/riscv-v.cc (emit_vlmax_decompress_insn): Fix bug.
(shuffle_decompress_patterns): Ditto.
gcc/testsuite/ChangeLog:
* gcc.target/riscv/rvv/autovec/partial/slp-12.c: New test.
* gcc.target/riscv/rvv/autovec/partial/slp_run-12.c: New test.
Diff:
---
gcc/config/riscv/riscv-v.cc | 8 ++---
.../gcc.target/riscv/rvv/autovec/partial/slp-12.c | 35 ++++++++++++++++++++++
.../riscv/rvv/autovec/partial/slp_run-12.c | 32 ++++++++++++++++++++
3 files changed, 71 insertions(+), 4 deletions(-)
diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
index fb970344521..34fdb53c140 100644
--- a/gcc/config/riscv/riscv-v.cc
+++ b/gcc/config/riscv/riscv-v.cc
@@ -863,7 +863,7 @@ emit_vlmax_masked_gather_mu_insn (rtx target, rtx op, rtx sel, rtx mask)
e q r d c b v a # v11 destination after vrgather using viota.m under mask
*/
static void
-emit_vlmax_decompress_insn (rtx target, rtx op, rtx mask)
+emit_vlmax_decompress_insn (rtx target, rtx op0, rtx op1, rtx mask)
{
machine_mode data_mode = GET_MODE (target);
machine_mode sel_mode = related_int_vector_mode (data_mode).require ();
@@ -873,7 +873,8 @@ emit_vlmax_decompress_insn (rtx target, rtx op, rtx mask)
rtx sel = gen_reg_rtx (sel_mode);
rtx iota_ops[] = {sel, mask};
emit_vlmax_insn (code_for_pred_iota (sel_mode), RVV_UNOP, iota_ops);
- emit_vlmax_masked_gather_mu_insn (target, op, sel, mask);
+ emit_vlmax_gather_insn (target, op0, sel);
+ emit_vlmax_masked_gather_mu_insn (target, op1, sel, mask);
}
/* Emit merge instruction. */
@@ -2441,8 +2442,7 @@ shuffle_decompress_patterns (struct expand_vec_perm_d *d)
rtx const_vec = gen_const_vector_dup (sel_mode, 1);
rtx mask = gen_reg_rtx (mask_mode);
expand_vec_cmp (mask, EQ, vid_repeat, const_vec);
- emit_move_insn (d->target, op0);
- emit_vlmax_decompress_insn (d->target, op1, mask);
+ emit_vlmax_decompress_insn (d->target, op0, op1, mask);
return true;
}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp-12.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp-12.c
new file mode 100644
index 00000000000..03529f4643a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp-12.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fno-vect-cost-model" } */
+
+#include <stdint-gcc.h>
+
+#define VEC_PERM(TYPE) \
+ TYPE __attribute__ ((noinline, noclone)) \
+ vec_slp_##TYPE (TYPE *restrict a, int n) \
+ { \
+ for (int i = 0; i < n; ++i) \
+ { \
+ a[i * 8] += 99; \
+ a[i * 8 + 1] += 11; \
+ a[i * 8 + 2] += 17; \
+ a[i * 8 + 3] += 80; \
+ a[i * 8 + 4] += 63; \
+ a[i * 8 + 5] += 37; \
+ a[i * 8 + 6] += 24; \
+ a[i * 8 + 7] += 81; \
+ } \
+ }
+
+#define TEST_ALL(T) \
+ T (int8_t) \
+ T (uint8_t) \
+ T (int16_t) \
+ T (uint16_t) \
+ T (int32_t) \
+ T (uint32_t) \
+ T (int64_t) \
+ T (uint64_t)
+
+TEST_ALL (VEC_PERM)
+/* This testcase is from aarch64 and floating-point operations are removed.
+ TODO: We will add floating-point operations back and make them as common test in the future. */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp_run-12.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp_run-12.c
new file mode 100644
index 00000000000..af892ad7ac4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/slp_run-12.c
@@ -0,0 +1,32 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-additional-options "--param riscv-autovec-preference=scalable -fno-vect-cost-model" } */
+
+#include "slp-12.c"
+
+#define N (59 * 8)
+
+#define HARNESS(TYPE) \
+ { \
+ TYPE a[N], b[8] = { 99, 11, 17, 80, 63, 37, 24, 81 }; \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ a[i] = i * 2 + i % 5; \
+ asm volatile ("" ::: "memory"); \
+ } \
+ vec_slp_##TYPE (a, N / 8); \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ TYPE orig = i * 2 + i % 5; \
+ TYPE expected = orig + b[i % 8]; \
+ if (a[i] != expected) \
+ __builtin_abort (); \
+ } \
+ }
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ TEST_ALL (HARNESS)
+}
+/* This testcase is from aarch64 and floating-point operations are removed.
+ TODO: We will add floating-point operations back and make them as common test in the future. */
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2023-07-14 2:45 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-06-19 11:41 [gcc(refs/vendors/riscv/heads/gcc-13-with-riscv-opts)] RISC-V: Fix bug of VLA SLP auto-vectorization Jeff Law
2023-07-14 2:45 Jeff Law
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).