* [PATCH] x86: Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER
@ 2022-02-17 4:26 H.J. Lu
2022-02-17 5:33 ` Hongtao Liu
0 siblings, 1 reply; 10+ messages in thread
From: H.J. Lu @ 2022-02-17 4:26 UTC (permalink / raw)
To: gcc-patches; +Cc: liuhongt, Uros Bizjak
Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
generate vzeroupper instruction after loading all-zero YMM/YMM registers
and enable it by default.
gcc/
PR target/101456
* config/i386/i386.cc (ix86_avx_u128_mode_needed): Skip the
vzeroupper optimization if target needs vzeroupper after reading
all-zero YMM/YMM registers.
* config/i386/i386.h (TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER):
New.
* config/i386/x86-tune.def
(X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER): New.
gcc/testsuite/
PR target/101456
* gcc.target/i386/pr101456-1.c (dg-options): Add
-mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper.
* gcc.target/i386/pr101456-2.c: Likewise.
* gcc.target/i386/pr101456-3.c: New test.
* gcc.target/i386/pr101456-4.c: Likewise.
---
gcc/config/i386/i386.cc | 51 ++++++++++++----------
gcc/config/i386/i386.h | 2 +
gcc/config/i386/x86-tune.def | 5 +++
gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +-
gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +-
gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++
gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++
7 files changed, 103 insertions(+), 25 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c
create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index cf246e74e57..1f8b4caf24c 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn)
subrtx_iterator::array_type array;
- rtx set = single_set (insn);
- if (set)
+ if (!TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER)
{
- rtx dest = SET_DEST (set);
- rtx src = SET_SRC (set);
- if (ix86_check_avx_upper_register (dest))
+ /* Perform this vzeroupper optimization if target doesn't need
+ vzeroupper after reading all-zero YMM/YMM registers. */
+ rtx set = single_set (insn);
+ if (set)
{
- /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
- source isn't zero. */
- if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
- return AVX_U128_DIRTY;
+ rtx dest = SET_DEST (set);
+ rtx src = SET_SRC (set);
+ if (ix86_check_avx_upper_register (dest))
+ {
+ /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
+ source isn't zero. */
+ if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
+ return AVX_U128_DIRTY;
+ else
+ return AVX_U128_ANY;
+ }
else
- return AVX_U128_ANY;
- }
- else
- {
- FOR_EACH_SUBRTX (iter, array, src, NONCONST)
- if (ix86_check_avx_upper_register (*iter))
- {
- int status = ix86_avx_u128_mode_source (insn, *iter);
- if (status == AVX_U128_DIRTY)
- return status;
- }
- }
+ {
+ FOR_EACH_SUBRTX (iter, array, src, NONCONST)
+ if (ix86_check_avx_upper_register (*iter))
+ {
+ int status = ix86_avx_u128_mode_source (insn, *iter);
+ if (status == AVX_U128_DIRTY)
+ return status;
+ }
+ }
- /* This isn't YMM/ZMM load/store. */
- return AVX_U128_ANY;
+ /* This isn't YMM/ZMM load/store. */
+ return AVX_U128_ANY;
+ }
}
/* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index f41e0908250..98c2e200027 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
#define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE]
#define TARGET_EMIT_VZEROUPPER \
ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER]
+#define TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER \
+ ix86_tune_features[X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER]
#define TARGET_EXPAND_ABS \
ix86_tune_features[X86_TUNE_EXPAND_ABS]
#define TARGET_V2DF_REDUCTION_PREFER_HADDPD \
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index 82ca0ae63ac..0a068c09202 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE)
/* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion
before a transfer of control flow out of the function. */
DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL)
+
+/* X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER: This knob generates
+ vzeroupper instruction after reading all-zero YMM/YMM registers. */
+DEF_TUNE (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER,
+ "read_zero_ymm_zmm_need_vzeroupper", HOST_WIDE_INT_M1U)
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c
index 803fc6e0207..7eb74d21439 100644
--- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -march=skylake" } */
+/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */
#include <x86intrin.h>
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c
index 554a0f1702c..9fdc9bd6eb1 100644
--- a/gcc/testsuite/gcc.target/i386/pr101456-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -march=skylake" } */
+/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */
#include <x86intrin.h>
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c
new file mode 100644
index 00000000000..8389d18ed6c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */
+
+#include <x86intrin.h>
+
+extern __m256 x1;
+extern __m256d x2;
+extern __m256i x3;
+
+extern void bar (void);
+
+void
+foo1 (void)
+{
+ x1 = _mm256_setzero_ps ();
+ bar ();
+}
+
+void
+foo2 (void)
+{
+ x2 = _mm256_setzero_pd ();
+ bar ();
+}
+
+void
+foo3 (void)
+{
+ x3 = _mm256_setzero_si256 ();
+ bar ();
+}
+
+/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c
new file mode 100644
index 00000000000..3e4cdcc4d28
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=haswell" } */
+
+#include <x86intrin.h>
+
+extern __m256 x1;
+extern __m256d x2;
+extern __m256i x3;
+
+extern void bar (void);
+
+void
+foo1 (void)
+{
+ x1 = _mm256_setzero_ps ();
+ bar ();
+}
+
+void
+foo2 (void)
+{
+ x2 = _mm256_setzero_pd ();
+ bar ();
+}
+
+void
+foo3 (void)
+{
+ x3 = _mm256_setzero_si256 ();
+ bar ();
+}
+
+/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
--
2.35.1
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH] x86: Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER
2022-02-17 4:26 [PATCH] x86: Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER H.J. Lu
@ 2022-02-17 5:33 ` Hongtao Liu
2022-02-17 7:51 ` Uros Bizjak
0 siblings, 1 reply; 10+ messages in thread
From: Hongtao Liu @ 2022-02-17 5:33 UTC (permalink / raw)
To: H.J. Lu; +Cc: GCC Patches, liuhongt
On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> generate vzeroupper instruction after loading all-zero YMM/YMM registers
> and enable it by default.
Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
Because originally we needed to add vzeroupper to all avx<->sse cases,
now it's a tune to indicate that we don't need to add it in some
cases.
>
> gcc/
>
> PR target/101456
> * config/i386/i386.cc (ix86_avx_u128_mode_needed): Skip the
> vzeroupper optimization if target needs vzeroupper after reading
> all-zero YMM/YMM registers.
> * config/i386/i386.h (TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER):
> New.
> * config/i386/x86-tune.def
> (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER): New.
>
> gcc/testsuite/
>
> PR target/101456
> * gcc.target/i386/pr101456-1.c (dg-options): Add
> -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper.
> * gcc.target/i386/pr101456-2.c: Likewise.
> * gcc.target/i386/pr101456-3.c: New test.
> * gcc.target/i386/pr101456-4.c: Likewise.
> ---
> gcc/config/i386/i386.cc | 51 ++++++++++++----------
> gcc/config/i386/i386.h | 2 +
> gcc/config/i386/x86-tune.def | 5 +++
> gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +-
> gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +-
> gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++
> gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++
> 7 files changed, 103 insertions(+), 25 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c
> create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c
>
> diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> index cf246e74e57..1f8b4caf24c 100644
> --- a/gcc/config/i386/i386.cc
> +++ b/gcc/config/i386/i386.cc
> @@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn)
>
> subrtx_iterator::array_type array;
>
> - rtx set = single_set (insn);
> - if (set)
> + if (!TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER)
> {
> - rtx dest = SET_DEST (set);
> - rtx src = SET_SRC (set);
> - if (ix86_check_avx_upper_register (dest))
> + /* Perform this vzeroupper optimization if target doesn't need
> + vzeroupper after reading all-zero YMM/YMM registers. */
> + rtx set = single_set (insn);
> + if (set)
> {
> - /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> - source isn't zero. */
> - if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> - return AVX_U128_DIRTY;
> + rtx dest = SET_DEST (set);
> + rtx src = SET_SRC (set);
> + if (ix86_check_avx_upper_register (dest))
> + {
> + /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> + source isn't zero. */
> + if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> + return AVX_U128_DIRTY;
> + else
> + return AVX_U128_ANY;
> + }
> else
> - return AVX_U128_ANY;
> - }
> - else
> - {
> - FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> - if (ix86_check_avx_upper_register (*iter))
> - {
> - int status = ix86_avx_u128_mode_source (insn, *iter);
> - if (status == AVX_U128_DIRTY)
> - return status;
> - }
> - }
> + {
> + FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> + if (ix86_check_avx_upper_register (*iter))
> + {
> + int status = ix86_avx_u128_mode_source (insn, *iter);
> + if (status == AVX_U128_DIRTY)
> + return status;
> + }
> + }
>
> - /* This isn't YMM/ZMM load/store. */
> - return AVX_U128_ANY;
> + /* This isn't YMM/ZMM load/store. */
> + return AVX_U128_ANY;
> + }
> }
>
> /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
> diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
> index f41e0908250..98c2e200027 100644
> --- a/gcc/config/i386/i386.h
> +++ b/gcc/config/i386/i386.h
> @@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
> #define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE]
> #define TARGET_EMIT_VZEROUPPER \
> ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER]
> +#define TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER \
> + ix86_tune_features[X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER]
> #define TARGET_EXPAND_ABS \
> ix86_tune_features[X86_TUNE_EXPAND_ABS]
> #define TARGET_V2DF_REDUCTION_PREFER_HADDPD \
> diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> index 82ca0ae63ac..0a068c09202 100644
> --- a/gcc/config/i386/x86-tune.def
> +++ b/gcc/config/i386/x86-tune.def
> @@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE)
> /* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion
> before a transfer of control flow out of the function. */
> DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL)
> +
> +/* X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER: This knob generates
> + vzeroupper instruction after reading all-zero YMM/YMM registers. */
> +DEF_TUNE (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER,
> + "read_zero_ymm_zmm_need_vzeroupper", HOST_WIDE_INT_M1U)
> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> index 803fc6e0207..7eb74d21439 100644
> --- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
> +++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> @@ -1,5 +1,5 @@
> /* { dg-do compile } */
> -/* { dg-options "-O2 -march=skylake" } */
> +/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */
>
> #include <x86intrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> index 554a0f1702c..9fdc9bd6eb1 100644
> --- a/gcc/testsuite/gcc.target/i386/pr101456-2.c
> +++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> @@ -1,5 +1,5 @@
> /* { dg-do compile } */
> -/* { dg-options "-O2 -march=skylake" } */
> +/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */
>
> #include <x86intrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> new file mode 100644
> index 00000000000..8389d18ed6c
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> @@ -0,0 +1,33 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */
> +
> +#include <x86intrin.h>
> +
> +extern __m256 x1;
> +extern __m256d x2;
> +extern __m256i x3;
> +
> +extern void bar (void);
> +
> +void
> +foo1 (void)
> +{
> + x1 = _mm256_setzero_ps ();
> + bar ();
> +}
> +
> +void
> +foo2 (void)
> +{
> + x2 = _mm256_setzero_pd ();
> + bar ();
> +}
> +
> +void
> +foo3 (void)
> +{
> + x3 = _mm256_setzero_si256 ();
> + bar ();
> +}
> +
> +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> new file mode 100644
> index 00000000000..3e4cdcc4d28
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> @@ -0,0 +1,33 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -march=haswell" } */
> +
> +#include <x86intrin.h>
> +
> +extern __m256 x1;
> +extern __m256d x2;
> +extern __m256i x3;
> +
> +extern void bar (void);
> +
> +void
> +foo1 (void)
> +{
> + x1 = _mm256_setzero_ps ();
> + bar ();
> +}
> +
> +void
> +foo2 (void)
> +{
> + x2 = _mm256_setzero_pd ();
> + bar ();
> +}
> +
> +void
> +foo3 (void)
> +{
> + x3 = _mm256_setzero_si256 ();
> + bar ();
> +}
> +
> +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> --
> 2.35.1
>
--
BR,
Hongtao
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH] x86: Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER
2022-02-17 5:33 ` Hongtao Liu
@ 2022-02-17 7:51 ` Uros Bizjak
2022-02-17 9:49 ` Richard Biener
2022-02-17 13:56 ` [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO H.J. Lu
0 siblings, 2 replies; 10+ messages in thread
From: Uros Bizjak @ 2022-02-17 7:51 UTC (permalink / raw)
To: Hongtao Liu; +Cc: H.J. Lu, liuhongt, GCC Patches
On Thu, Feb 17, 2022 at 6:25 AM Hongtao Liu via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
> <gcc-patches@gcc.gnu.org> wrote:
> >
> > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> > generate vzeroupper instruction after loading all-zero YMM/YMM registers
> > and enable it by default.
> Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
> Because originally we needed to add vzeroupper to all avx<->sse cases,
> now it's a tune to indicate that we don't need to add it in some
Perhaps we should go from the other side and use
X86_TUNE_OPTIMIZE_AVX_READ for new processors?
Uros.
> cases.
> >
> > gcc/
> >
> > PR target/101456
> > * config/i386/i386.cc (ix86_avx_u128_mode_needed): Skip the
> > vzeroupper optimization if target needs vzeroupper after reading
> > all-zero YMM/YMM registers.
> > * config/i386/i386.h (TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER):
> > New.
> > * config/i386/x86-tune.def
> > (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER): New.
> >
> > gcc/testsuite/
> >
> > PR target/101456
> > * gcc.target/i386/pr101456-1.c (dg-options): Add
> > -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper.
> > * gcc.target/i386/pr101456-2.c: Likewise.
> > * gcc.target/i386/pr101456-3.c: New test.
> > * gcc.target/i386/pr101456-4.c: Likewise.
> > ---
> > gcc/config/i386/i386.cc | 51 ++++++++++++----------
> > gcc/config/i386/i386.h | 2 +
> > gcc/config/i386/x86-tune.def | 5 +++
> > gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +-
> > gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +-
> > gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++
> > gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++
> > 7 files changed, 103 insertions(+), 25 deletions(-)
> > create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c
> > create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c
> >
> > diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> > index cf246e74e57..1f8b4caf24c 100644
> > --- a/gcc/config/i386/i386.cc
> > +++ b/gcc/config/i386/i386.cc
> > @@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn)
> >
> > subrtx_iterator::array_type array;
> >
> > - rtx set = single_set (insn);
> > - if (set)
> > + if (!TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER)
> > {
> > - rtx dest = SET_DEST (set);
> > - rtx src = SET_SRC (set);
> > - if (ix86_check_avx_upper_register (dest))
> > + /* Perform this vzeroupper optimization if target doesn't need
> > + vzeroupper after reading all-zero YMM/YMM registers. */
> > + rtx set = single_set (insn);
> > + if (set)
> > {
> > - /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > - source isn't zero. */
> > - if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > - return AVX_U128_DIRTY;
> > + rtx dest = SET_DEST (set);
> > + rtx src = SET_SRC (set);
> > + if (ix86_check_avx_upper_register (dest))
> > + {
> > + /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > + source isn't zero. */
> > + if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > + return AVX_U128_DIRTY;
> > + else
> > + return AVX_U128_ANY;
> > + }
> > else
> > - return AVX_U128_ANY;
> > - }
> > - else
> > - {
> > - FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> > - if (ix86_check_avx_upper_register (*iter))
> > - {
> > - int status = ix86_avx_u128_mode_source (insn, *iter);
> > - if (status == AVX_U128_DIRTY)
> > - return status;
> > - }
> > - }
> > + {
> > + FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> > + if (ix86_check_avx_upper_register (*iter))
> > + {
> > + int status = ix86_avx_u128_mode_source (insn, *iter);
> > + if (status == AVX_U128_DIRTY)
> > + return status;
> > + }
> > + }
> >
> > - /* This isn't YMM/ZMM load/store. */
> > - return AVX_U128_ANY;
> > + /* This isn't YMM/ZMM load/store. */
> > + return AVX_U128_ANY;
> > + }
> > }
> >
> > /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
> > diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
> > index f41e0908250..98c2e200027 100644
> > --- a/gcc/config/i386/i386.h
> > +++ b/gcc/config/i386/i386.h
> > @@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
> > #define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE]
> > #define TARGET_EMIT_VZEROUPPER \
> > ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER]
> > +#define TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER \
> > + ix86_tune_features[X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER]
> > #define TARGET_EXPAND_ABS \
> > ix86_tune_features[X86_TUNE_EXPAND_ABS]
> > #define TARGET_V2DF_REDUCTION_PREFER_HADDPD \
> > diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> > index 82ca0ae63ac..0a068c09202 100644
> > --- a/gcc/config/i386/x86-tune.def
> > +++ b/gcc/config/i386/x86-tune.def
> > @@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE)
> > /* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion
> > before a transfer of control flow out of the function. */
> > DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL)
> > +
> > +/* X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER: This knob generates
> > + vzeroupper instruction after reading all-zero YMM/YMM registers. */
> > +DEF_TUNE (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER,
> > + "read_zero_ymm_zmm_need_vzeroupper", HOST_WIDE_INT_M1U)
> > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > index 803fc6e0207..7eb74d21439 100644
> > --- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > +++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > @@ -1,5 +1,5 @@
> > /* { dg-do compile } */
> > -/* { dg-options "-O2 -march=skylake" } */
> > +/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */
> >
> > #include <x86intrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > index 554a0f1702c..9fdc9bd6eb1 100644
> > --- a/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > +++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > @@ -1,5 +1,5 @@
> > /* { dg-do compile } */
> > -/* { dg-options "-O2 -march=skylake" } */
> > +/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */
> >
> > #include <x86intrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> > new file mode 100644
> > index 00000000000..8389d18ed6c
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> > @@ -0,0 +1,33 @@
> > +/* { dg-do compile } */
> > +/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */
> > +
> > +#include <x86intrin.h>
> > +
> > +extern __m256 x1;
> > +extern __m256d x2;
> > +extern __m256i x3;
> > +
> > +extern void bar (void);
> > +
> > +void
> > +foo1 (void)
> > +{
> > + x1 = _mm256_setzero_ps ();
> > + bar ();
> > +}
> > +
> > +void
> > +foo2 (void)
> > +{
> > + x2 = _mm256_setzero_pd ();
> > + bar ();
> > +}
> > +
> > +void
> > +foo3 (void)
> > +{
> > + x3 = _mm256_setzero_si256 ();
> > + bar ();
> > +}
> > +
> > +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> > new file mode 100644
> > index 00000000000..3e4cdcc4d28
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> > @@ -0,0 +1,33 @@
> > +/* { dg-do compile } */
> > +/* { dg-options "-O2 -march=haswell" } */
> > +
> > +#include <x86intrin.h>
> > +
> > +extern __m256 x1;
> > +extern __m256d x2;
> > +extern __m256i x3;
> > +
> > +extern void bar (void);
> > +
> > +void
> > +foo1 (void)
> > +{
> > + x1 = _mm256_setzero_ps ();
> > + bar ();
> > +}
> > +
> > +void
> > +foo2 (void)
> > +{
> > + x2 = _mm256_setzero_pd ();
> > + bar ();
> > +}
> > +
> > +void
> > +foo3 (void)
> > +{
> > + x3 = _mm256_setzero_si256 ();
> > + bar ();
> > +}
> > +
> > +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> > --
> > 2.35.1
> >
>
>
> --
> BR,
> Hongtao
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH] x86: Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER
2022-02-17 7:51 ` Uros Bizjak
@ 2022-02-17 9:49 ` Richard Biener
2022-02-17 13:57 ` H.J. Lu
2022-02-17 13:56 ` [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO H.J. Lu
1 sibling, 1 reply; 10+ messages in thread
From: Richard Biener @ 2022-02-17 9:49 UTC (permalink / raw)
To: Uros Bizjak; +Cc: Hongtao Liu, liuhongt, GCC Patches
On Thu, Feb 17, 2022 at 8:52 AM Uros Bizjak via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> On Thu, Feb 17, 2022 at 6:25 AM Hongtao Liu via Gcc-patches
> <gcc-patches@gcc.gnu.org> wrote:
> >
> > On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
> > <gcc-patches@gcc.gnu.org> wrote:
> > >
> > > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > > transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> > > generate vzeroupper instruction after loading all-zero YMM/YMM registers
> > > and enable it by default.
> > Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
> > Because originally we needed to add vzeroupper to all avx<->sse cases,
> > now it's a tune to indicate that we don't need to add it in some
>
> Perhaps we should go from the other side and use
> X86_TUNE_OPTIMIZE_AVX_READ for new processors?
Btw, do you have a micro-benchmark to test this on AMD archs?
Thanks,
Richard.
> Uros.
>
> > cases.
> > >
> > > gcc/
> > >
> > > PR target/101456
> > > * config/i386/i386.cc (ix86_avx_u128_mode_needed): Skip the
> > > vzeroupper optimization if target needs vzeroupper after reading
> > > all-zero YMM/YMM registers.
> > > * config/i386/i386.h (TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER):
> > > New.
> > > * config/i386/x86-tune.def
> > > (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER): New.
> > >
> > > gcc/testsuite/
> > >
> > > PR target/101456
> > > * gcc.target/i386/pr101456-1.c (dg-options): Add
> > > -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper.
> > > * gcc.target/i386/pr101456-2.c: Likewise.
> > > * gcc.target/i386/pr101456-3.c: New test.
> > > * gcc.target/i386/pr101456-4.c: Likewise.
> > > ---
> > > gcc/config/i386/i386.cc | 51 ++++++++++++----------
> > > gcc/config/i386/i386.h | 2 +
> > > gcc/config/i386/x86-tune.def | 5 +++
> > > gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +-
> > > gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +-
> > > gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++
> > > gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++
> > > 7 files changed, 103 insertions(+), 25 deletions(-)
> > > create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c
> > > create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c
> > >
> > > diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> > > index cf246e74e57..1f8b4caf24c 100644
> > > --- a/gcc/config/i386/i386.cc
> > > +++ b/gcc/config/i386/i386.cc
> > > @@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn)
> > >
> > > subrtx_iterator::array_type array;
> > >
> > > - rtx set = single_set (insn);
> > > - if (set)
> > > + if (!TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER)
> > > {
> > > - rtx dest = SET_DEST (set);
> > > - rtx src = SET_SRC (set);
> > > - if (ix86_check_avx_upper_register (dest))
> > > + /* Perform this vzeroupper optimization if target doesn't need
> > > + vzeroupper after reading all-zero YMM/YMM registers. */
> > > + rtx set = single_set (insn);
> > > + if (set)
> > > {
> > > - /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > > - source isn't zero. */
> > > - if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > > - return AVX_U128_DIRTY;
> > > + rtx dest = SET_DEST (set);
> > > + rtx src = SET_SRC (set);
> > > + if (ix86_check_avx_upper_register (dest))
> > > + {
> > > + /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > > + source isn't zero. */
> > > + if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > > + return AVX_U128_DIRTY;
> > > + else
> > > + return AVX_U128_ANY;
> > > + }
> > > else
> > > - return AVX_U128_ANY;
> > > - }
> > > - else
> > > - {
> > > - FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> > > - if (ix86_check_avx_upper_register (*iter))
> > > - {
> > > - int status = ix86_avx_u128_mode_source (insn, *iter);
> > > - if (status == AVX_U128_DIRTY)
> > > - return status;
> > > - }
> > > - }
> > > + {
> > > + FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> > > + if (ix86_check_avx_upper_register (*iter))
> > > + {
> > > + int status = ix86_avx_u128_mode_source (insn, *iter);
> > > + if (status == AVX_U128_DIRTY)
> > > + return status;
> > > + }
> > > + }
> > >
> > > - /* This isn't YMM/ZMM load/store. */
> > > - return AVX_U128_ANY;
> > > + /* This isn't YMM/ZMM load/store. */
> > > + return AVX_U128_ANY;
> > > + }
> > > }
> > >
> > > /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
> > > diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
> > > index f41e0908250..98c2e200027 100644
> > > --- a/gcc/config/i386/i386.h
> > > +++ b/gcc/config/i386/i386.h
> > > @@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
> > > #define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE]
> > > #define TARGET_EMIT_VZEROUPPER \
> > > ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER]
> > > +#define TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER \
> > > + ix86_tune_features[X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER]
> > > #define TARGET_EXPAND_ABS \
> > > ix86_tune_features[X86_TUNE_EXPAND_ABS]
> > > #define TARGET_V2DF_REDUCTION_PREFER_HADDPD \
> > > diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> > > index 82ca0ae63ac..0a068c09202 100644
> > > --- a/gcc/config/i386/x86-tune.def
> > > +++ b/gcc/config/i386/x86-tune.def
> > > @@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE)
> > > /* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion
> > > before a transfer of control flow out of the function. */
> > > DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL)
> > > +
> > > +/* X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER: This knob generates
> > > + vzeroupper instruction after reading all-zero YMM/YMM registers. */
> > > +DEF_TUNE (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER,
> > > + "read_zero_ymm_zmm_need_vzeroupper", HOST_WIDE_INT_M1U)
> > > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > > index 803fc6e0207..7eb74d21439 100644
> > > --- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > > +++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > > @@ -1,5 +1,5 @@
> > > /* { dg-do compile } */
> > > -/* { dg-options "-O2 -march=skylake" } */
> > > +/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */
> > >
> > > #include <x86intrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > > index 554a0f1702c..9fdc9bd6eb1 100644
> > > --- a/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > > +++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > > @@ -1,5 +1,5 @@
> > > /* { dg-do compile } */
> > > -/* { dg-options "-O2 -march=skylake" } */
> > > +/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */
> > >
> > > #include <x86intrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> > > new file mode 100644
> > > index 00000000000..8389d18ed6c
> > > --- /dev/null
> > > +++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> > > @@ -0,0 +1,33 @@
> > > +/* { dg-do compile } */
> > > +/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */
> > > +
> > > +#include <x86intrin.h>
> > > +
> > > +extern __m256 x1;
> > > +extern __m256d x2;
> > > +extern __m256i x3;
> > > +
> > > +extern void bar (void);
> > > +
> > > +void
> > > +foo1 (void)
> > > +{
> > > + x1 = _mm256_setzero_ps ();
> > > + bar ();
> > > +}
> > > +
> > > +void
> > > +foo2 (void)
> > > +{
> > > + x2 = _mm256_setzero_pd ();
> > > + bar ();
> > > +}
> > > +
> > > +void
> > > +foo3 (void)
> > > +{
> > > + x3 = _mm256_setzero_si256 ();
> > > + bar ();
> > > +}
> > > +
> > > +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> > > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> > > new file mode 100644
> > > index 00000000000..3e4cdcc4d28
> > > --- /dev/null
> > > +++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> > > @@ -0,0 +1,33 @@
> > > +/* { dg-do compile } */
> > > +/* { dg-options "-O2 -march=haswell" } */
> > > +
> > > +#include <x86intrin.h>
> > > +
> > > +extern __m256 x1;
> > > +extern __m256d x2;
> > > +extern __m256i x3;
> > > +
> > > +extern void bar (void);
> > > +
> > > +void
> > > +foo1 (void)
> > > +{
> > > + x1 = _mm256_setzero_ps ();
> > > + bar ();
> > > +}
> > > +
> > > +void
> > > +foo2 (void)
> > > +{
> > > + x2 = _mm256_setzero_pd ();
> > > + bar ();
> > > +}
> > > +
> > > +void
> > > +foo3 (void)
> > > +{
> > > + x3 = _mm256_setzero_si256 ();
> > > + bar ();
> > > +}
> > > +
> > > +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> > > --
> > > 2.35.1
> > >
> >
> >
> > --
> > BR,
> > Hongtao
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO
2022-02-17 7:51 ` Uros Bizjak
2022-02-17 9:49 ` Richard Biener
@ 2022-02-17 13:56 ` H.J. Lu
2022-02-21 2:01 ` Hongtao Liu
1 sibling, 1 reply; 10+ messages in thread
From: H.J. Lu @ 2022-02-17 13:56 UTC (permalink / raw)
To: Uros Bizjak; +Cc: Hongtao Liu, liuhongt, GCC Patches
On Thu, Feb 17, 2022 at 08:51:31AM +0100, Uros Bizjak wrote:
> On Thu, Feb 17, 2022 at 6:25 AM Hongtao Liu via Gcc-patches
> <gcc-patches@gcc.gnu.org> wrote:
> >
> > On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
> > <gcc-patches@gcc.gnu.org> wrote:
> > >
> > > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > > transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> > > generate vzeroupper instruction after loading all-zero YMM/YMM registers
> > > and enable it by default.
> > Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
> > Because originally we needed to add vzeroupper to all avx<->sse cases,
> > now it's a tune to indicate that we don't need to add it in some
>
> Perhaps we should go from the other side and use
> X86_TUNE_OPTIMIZE_AVX_READ for new processors?
>
Here is the v2 patch to add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
H.J.
---
Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
transition penalty. Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO to
omit vzeroupper instruction after loading all-zero YMM/ZMM registers.
gcc/
PR target/101456
* config/i386/i386.cc (ix86_avx_u128_mode_needed): Omit
vzeroupper after reading all-zero YMM/ZMM registers for
TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
* config/i386/i386.h (TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO):
New.
* config/i386/x86-tune.def
(X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO): New.
gcc/testsuite/
PR target/101456
* gcc.target/i386/pr101456-1.c (dg-options): Add
-mtune-ctrl=-mtune-ctrl=omit_vzeroupper_after_avx_read_zero.
* gcc.target/i386/pr101456-2.c: Likewise.
* gcc.target/i386/pr101456-3.c: New test.
* gcc.target/i386/pr101456-4.c: Likewise.
---
gcc/config/i386/i386.cc | 51 ++++++++++++----------
gcc/config/i386/i386.h | 2 +
gcc/config/i386/x86-tune.def | 5 +++
gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +-
gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +-
gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++
gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++
7 files changed, 103 insertions(+), 25 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c
create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index cf246e74e57..60c72ceb72d 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn)
subrtx_iterator::array_type array;
- rtx set = single_set (insn);
- if (set)
+ if (TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO)
{
- rtx dest = SET_DEST (set);
- rtx src = SET_SRC (set);
- if (ix86_check_avx_upper_register (dest))
+ /* Perform this vzeroupper optimization if target doesn't need
+ vzeroupper after reading all-zero YMM/YMM registers. */
+ rtx set = single_set (insn);
+ if (set)
{
- /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
- source isn't zero. */
- if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
- return AVX_U128_DIRTY;
+ rtx dest = SET_DEST (set);
+ rtx src = SET_SRC (set);
+ if (ix86_check_avx_upper_register (dest))
+ {
+ /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
+ source isn't zero. */
+ if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
+ return AVX_U128_DIRTY;
+ else
+ return AVX_U128_ANY;
+ }
else
- return AVX_U128_ANY;
- }
- else
- {
- FOR_EACH_SUBRTX (iter, array, src, NONCONST)
- if (ix86_check_avx_upper_register (*iter))
- {
- int status = ix86_avx_u128_mode_source (insn, *iter);
- if (status == AVX_U128_DIRTY)
- return status;
- }
- }
+ {
+ FOR_EACH_SUBRTX (iter, array, src, NONCONST)
+ if (ix86_check_avx_upper_register (*iter))
+ {
+ int status = ix86_avx_u128_mode_source (insn, *iter);
+ if (status == AVX_U128_DIRTY)
+ return status;
+ }
+ }
- /* This isn't YMM/ZMM load/store. */
- return AVX_U128_ANY;
+ /* This isn't YMM/ZMM load/store. */
+ return AVX_U128_ANY;
+ }
}
/* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index f41e0908250..46379d2231b 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
#define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE]
#define TARGET_EMIT_VZEROUPPER \
ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER]
+#define TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO \
+ ix86_tune_features[X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO]
#define TARGET_EXPAND_ABS \
ix86_tune_features[X86_TUNE_EXPAND_ABS]
#define TARGET_V2DF_REDUCTION_PREFER_HADDPD \
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index 82ca0ae63ac..132de2db2eb 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE)
/* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion
before a transfer of control flow out of the function. */
DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL)
+
+/* X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO: This omits vzeroupper
+ instruction after reading all-zero YMM/ZMM registers. */
+DEF_TUNE (X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO,
+ "omit_vzeroupper_after_avx_read_zero", 0)
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c
index 803fc6e0207..f653197da7c 100644
--- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -march=skylake" } */
+/* { dg-options "-O2 -march=skylake -mtune-ctrl=omit_vzeroupper_after_avx_read_zero" } */
#include <x86intrin.h>
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c
index 554a0f1702c..9aac3ece14d 100644
--- a/gcc/testsuite/gcc.target/i386/pr101456-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -march=skylake" } */
+/* { dg-options "-O2 -march=skylake -mtune-ctrl=omit_vzeroupper_after_avx_read_zero" } */
#include <x86intrin.h>
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c
new file mode 100644
index 00000000000..8389d18ed6c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */
+
+#include <x86intrin.h>
+
+extern __m256 x1;
+extern __m256d x2;
+extern __m256i x3;
+
+extern void bar (void);
+
+void
+foo1 (void)
+{
+ x1 = _mm256_setzero_ps ();
+ bar ();
+}
+
+void
+foo2 (void)
+{
+ x2 = _mm256_setzero_pd ();
+ bar ();
+}
+
+void
+foo3 (void)
+{
+ x3 = _mm256_setzero_si256 ();
+ bar ();
+}
+
+/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c
new file mode 100644
index 00000000000..3e4cdcc4d28
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=haswell" } */
+
+#include <x86intrin.h>
+
+extern __m256 x1;
+extern __m256d x2;
+extern __m256i x3;
+
+extern void bar (void);
+
+void
+foo1 (void)
+{
+ x1 = _mm256_setzero_ps ();
+ bar ();
+}
+
+void
+foo2 (void)
+{
+ x2 = _mm256_setzero_pd ();
+ bar ();
+}
+
+void
+foo3 (void)
+{
+ x3 = _mm256_setzero_si256 ();
+ bar ();
+}
+
+/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
--
2.35.1
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH] x86: Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER
2022-02-17 9:49 ` Richard Biener
@ 2022-02-17 13:57 ` H.J. Lu
0 siblings, 0 replies; 10+ messages in thread
From: H.J. Lu @ 2022-02-17 13:57 UTC (permalink / raw)
To: Richard Biener; +Cc: Uros Bizjak, GCC Patches, liuhongt
On Thu, Feb 17, 2022 at 10:49:48AM +0100, Richard Biener via Gcc-patches wrote:
> On Thu, Feb 17, 2022 at 8:52 AM Uros Bizjak via Gcc-patches
> <gcc-patches@gcc.gnu.org> wrote:
> >
> > On Thu, Feb 17, 2022 at 6:25 AM Hongtao Liu via Gcc-patches
> > <gcc-patches@gcc.gnu.org> wrote:
> > >
> > > On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
> > > <gcc-patches@gcc.gnu.org> wrote:
> > > >
> > > > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > > > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > > > transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> > > > generate vzeroupper instruction after loading all-zero YMM/YMM registers
> > > > and enable it by default.
> > > Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
> > > Because originally we needed to add vzeroupper to all avx<->sse cases,
> > > now it's a tune to indicate that we don't need to add it in some
> >
> > Perhaps we should go from the other side and use
> > X86_TUNE_OPTIMIZE_AVX_READ for new processors?
>
> Btw, do you have a micro-benchmark to test this on AMD archs?
>
I don't believe AMD CPUs needs vzeroupper.
H.J.
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO
2022-02-17 13:56 ` [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO H.J. Lu
@ 2022-02-21 2:01 ` Hongtao Liu
2022-02-21 18:35 ` H.J. Lu
0 siblings, 1 reply; 10+ messages in thread
From: Hongtao Liu @ 2022-02-21 2:01 UTC (permalink / raw)
To: H.J. Lu; +Cc: Uros Bizjak, liuhongt, GCC Patches
On Thu, Feb 17, 2022 at 9:56 PM H.J. Lu <hjl.tools@gmail.com> wrote:
>
> On Thu, Feb 17, 2022 at 08:51:31AM +0100, Uros Bizjak wrote:
> > On Thu, Feb 17, 2022 at 6:25 AM Hongtao Liu via Gcc-patches
> > <gcc-patches@gcc.gnu.org> wrote:
> > >
> > > On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
> > > <gcc-patches@gcc.gnu.org> wrote:
> > > >
> > > > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > > > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > > > transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> > > > generate vzeroupper instruction after loading all-zero YMM/YMM registers
> > > > and enable it by default.
> > > Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
> > > Because originally we needed to add vzeroupper to all avx<->sse cases,
> > > now it's a tune to indicate that we don't need to add it in some
> >
> > Perhaps we should go from the other side and use
> > X86_TUNE_OPTIMIZE_AVX_READ for new processors?
> >
>
> Here is the v2 patch to add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
>
The patch LGTM in general, but please rebase against
https://gcc.gnu.org/pipermail/gcc-patches/2022-February/590541.html
and resend the patch, also wait a couple days in case Uros(and others)
have any comments.
>
> H.J.
> ---
> Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> transition penalty. Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO to
> omit vzeroupper instruction after loading all-zero YMM/ZMM registers.
>
> gcc/
>
> PR target/101456
> * config/i386/i386.cc (ix86_avx_u128_mode_needed): Omit
> vzeroupper after reading all-zero YMM/ZMM registers for
> TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
> * config/i386/i386.h (TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO):
> New.
> * config/i386/x86-tune.def
> (X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO): New.
>
> gcc/testsuite/
>
> PR target/101456
> * gcc.target/i386/pr101456-1.c (dg-options): Add
> -mtune-ctrl=-mtune-ctrl=omit_vzeroupper_after_avx_read_zero.
> * gcc.target/i386/pr101456-2.c: Likewise.
> * gcc.target/i386/pr101456-3.c: New test.
> * gcc.target/i386/pr101456-4.c: Likewise.
> ---
> gcc/config/i386/i386.cc | 51 ++++++++++++----------
> gcc/config/i386/i386.h | 2 +
> gcc/config/i386/x86-tune.def | 5 +++
> gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +-
> gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +-
> gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++
> gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++
> 7 files changed, 103 insertions(+), 25 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c
> create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c
>
> diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> index cf246e74e57..60c72ceb72d 100644
> --- a/gcc/config/i386/i386.cc
> +++ b/gcc/config/i386/i386.cc
> @@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn)
>
> subrtx_iterator::array_type array;
>
> - rtx set = single_set (insn);
> - if (set)
> + if (TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO)
> {
> - rtx dest = SET_DEST (set);
> - rtx src = SET_SRC (set);
> - if (ix86_check_avx_upper_register (dest))
> + /* Perform this vzeroupper optimization if target doesn't need
> + vzeroupper after reading all-zero YMM/YMM registers. */
> + rtx set = single_set (insn);
> + if (set)
> {
> - /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> - source isn't zero. */
> - if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> - return AVX_U128_DIRTY;
> + rtx dest = SET_DEST (set);
> + rtx src = SET_SRC (set);
> + if (ix86_check_avx_upper_register (dest))
> + {
> + /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> + source isn't zero. */
> + if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> + return AVX_U128_DIRTY;
> + else
> + return AVX_U128_ANY;
> + }
> else
> - return AVX_U128_ANY;
> - }
> - else
> - {
> - FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> - if (ix86_check_avx_upper_register (*iter))
> - {
> - int status = ix86_avx_u128_mode_source (insn, *iter);
> - if (status == AVX_U128_DIRTY)
> - return status;
> - }
> - }
> + {
> + FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> + if (ix86_check_avx_upper_register (*iter))
> + {
> + int status = ix86_avx_u128_mode_source (insn, *iter);
> + if (status == AVX_U128_DIRTY)
> + return status;
> + }
> + }
>
> - /* This isn't YMM/ZMM load/store. */
> - return AVX_U128_ANY;
> + /* This isn't YMM/ZMM load/store. */
> + return AVX_U128_ANY;
> + }
> }
>
> /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
> diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
> index f41e0908250..46379d2231b 100644
> --- a/gcc/config/i386/i386.h
> +++ b/gcc/config/i386/i386.h
> @@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
> #define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE]
> #define TARGET_EMIT_VZEROUPPER \
> ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER]
> +#define TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO \
> + ix86_tune_features[X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO]
> #define TARGET_EXPAND_ABS \
> ix86_tune_features[X86_TUNE_EXPAND_ABS]
> #define TARGET_V2DF_REDUCTION_PREFER_HADDPD \
> diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> index 82ca0ae63ac..132de2db2eb 100644
> --- a/gcc/config/i386/x86-tune.def
> +++ b/gcc/config/i386/x86-tune.def
> @@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE)
> /* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion
> before a transfer of control flow out of the function. */
> DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL)
> +
> +/* X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO: This omits vzeroupper
> + instruction after reading all-zero YMM/ZMM registers. */
> +DEF_TUNE (X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO,
> + "omit_vzeroupper_after_avx_read_zero", 0)
> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> index 803fc6e0207..f653197da7c 100644
> --- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
> +++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> @@ -1,5 +1,5 @@
> /* { dg-do compile } */
> -/* { dg-options "-O2 -march=skylake" } */
> +/* { dg-options "-O2 -march=skylake -mtune-ctrl=omit_vzeroupper_after_avx_read_zero" } */
>
> #include <x86intrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> index 554a0f1702c..9aac3ece14d 100644
> --- a/gcc/testsuite/gcc.target/i386/pr101456-2.c
> +++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> @@ -1,5 +1,5 @@
> /* { dg-do compile } */
> -/* { dg-options "-O2 -march=skylake" } */
> +/* { dg-options "-O2 -march=skylake -mtune-ctrl=omit_vzeroupper_after_avx_read_zero" } */
>
> #include <x86intrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> new file mode 100644
> index 00000000000..8389d18ed6c
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> @@ -0,0 +1,33 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */
> +
> +#include <x86intrin.h>
> +
> +extern __m256 x1;
> +extern __m256d x2;
> +extern __m256i x3;
> +
> +extern void bar (void);
> +
> +void
> +foo1 (void)
> +{
> + x1 = _mm256_setzero_ps ();
> + bar ();
> +}
> +
> +void
> +foo2 (void)
> +{
> + x2 = _mm256_setzero_pd ();
> + bar ();
> +}
> +
> +void
> +foo3 (void)
> +{
> + x3 = _mm256_setzero_si256 ();
> + bar ();
> +}
> +
> +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> new file mode 100644
> index 00000000000..3e4cdcc4d28
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> @@ -0,0 +1,33 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -march=haswell" } */
> +
> +#include <x86intrin.h>
> +
> +extern __m256 x1;
> +extern __m256d x2;
> +extern __m256i x3;
> +
> +extern void bar (void);
> +
> +void
> +foo1 (void)
> +{
> + x1 = _mm256_setzero_ps ();
> + bar ();
> +}
> +
> +void
> +foo2 (void)
> +{
> + x2 = _mm256_setzero_pd ();
> + bar ();
> +}
> +
> +void
> +foo3 (void)
> +{
> + x3 = _mm256_setzero_si256 ();
> + bar ();
> +}
> +
> +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> --
> 2.35.1
>
--
BR,
Hongtao
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO
2022-02-21 2:01 ` Hongtao Liu
@ 2022-02-21 18:35 ` H.J. Lu
2022-02-22 2:43 ` Hongtao Liu
0 siblings, 1 reply; 10+ messages in thread
From: H.J. Lu @ 2022-02-21 18:35 UTC (permalink / raw)
To: Hongtao Liu; +Cc: Uros Bizjak, liuhongt, GCC Patches
On Sun, Feb 20, 2022 at 6:01 PM Hongtao Liu <crazylht@gmail.com> wrote:
>
> On Thu, Feb 17, 2022 at 9:56 PM H.J. Lu <hjl.tools@gmail.com> wrote:
> >
> > On Thu, Feb 17, 2022 at 08:51:31AM +0100, Uros Bizjak wrote:
> > > On Thu, Feb 17, 2022 at 6:25 AM Hongtao Liu via Gcc-patches
> > > <gcc-patches@gcc.gnu.org> wrote:
> > > >
> > > > On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
> > > > <gcc-patches@gcc.gnu.org> wrote:
> > > > >
> > > > > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > > > > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > > > > transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> > > > > generate vzeroupper instruction after loading all-zero YMM/YMM registers
> > > > > and enable it by default.
> > > > Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
> > > > Because originally we needed to add vzeroupper to all avx<->sse cases,
> > > > now it's a tune to indicate that we don't need to add it in some
> > >
> > > Perhaps we should go from the other side and use
> > > X86_TUNE_OPTIMIZE_AVX_READ for new processors?
> > >
> >
> > Here is the v2 patch to add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
> >
> The patch LGTM in general, but please rebase against
> https://gcc.gnu.org/pipermail/gcc-patches/2022-February/590541.html
> and resend the patch, also wait a couple days in case Uros(and others)
> have any comments.
I am dropping my patch since it causes the compile-time regression.
> >
> > H.J.
> > ---
> > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > transition penalty. Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO to
> > omit vzeroupper instruction after loading all-zero YMM/ZMM registers.
> >
> > gcc/
> >
> > PR target/101456
> > * config/i386/i386.cc (ix86_avx_u128_mode_needed): Omit
> > vzeroupper after reading all-zero YMM/ZMM registers for
> > TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
> > * config/i386/i386.h (TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO):
> > New.
> > * config/i386/x86-tune.def
> > (X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO): New.
> >
> > gcc/testsuite/
> >
> > PR target/101456
> > * gcc.target/i386/pr101456-1.c (dg-options): Add
> > -mtune-ctrl=-mtune-ctrl=omit_vzeroupper_after_avx_read_zero.
> > * gcc.target/i386/pr101456-2.c: Likewise.
> > * gcc.target/i386/pr101456-3.c: New test.
> > * gcc.target/i386/pr101456-4.c: Likewise.
> > ---
> > gcc/config/i386/i386.cc | 51 ++++++++++++----------
> > gcc/config/i386/i386.h | 2 +
> > gcc/config/i386/x86-tune.def | 5 +++
> > gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +-
> > gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +-
> > gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++
> > gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++
> > 7 files changed, 103 insertions(+), 25 deletions(-)
> > create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c
> > create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c
> >
> > diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> > index cf246e74e57..60c72ceb72d 100644
> > --- a/gcc/config/i386/i386.cc
> > +++ b/gcc/config/i386/i386.cc
> > @@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn)
> >
> > subrtx_iterator::array_type array;
> >
> > - rtx set = single_set (insn);
> > - if (set)
> > + if (TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO)
> > {
> > - rtx dest = SET_DEST (set);
> > - rtx src = SET_SRC (set);
> > - if (ix86_check_avx_upper_register (dest))
> > + /* Perform this vzeroupper optimization if target doesn't need
> > + vzeroupper after reading all-zero YMM/YMM registers. */
> > + rtx set = single_set (insn);
> > + if (set)
> > {
> > - /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > - source isn't zero. */
> > - if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > - return AVX_U128_DIRTY;
> > + rtx dest = SET_DEST (set);
> > + rtx src = SET_SRC (set);
> > + if (ix86_check_avx_upper_register (dest))
> > + {
> > + /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > + source isn't zero. */
> > + if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > + return AVX_U128_DIRTY;
> > + else
> > + return AVX_U128_ANY;
> > + }
> > else
> > - return AVX_U128_ANY;
> > - }
> > - else
> > - {
> > - FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> > - if (ix86_check_avx_upper_register (*iter))
> > - {
> > - int status = ix86_avx_u128_mode_source (insn, *iter);
> > - if (status == AVX_U128_DIRTY)
> > - return status;
> > - }
> > - }
> > + {
> > + FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> > + if (ix86_check_avx_upper_register (*iter))
> > + {
> > + int status = ix86_avx_u128_mode_source (insn, *iter);
> > + if (status == AVX_U128_DIRTY)
> > + return status;
> > + }
> > + }
> >
> > - /* This isn't YMM/ZMM load/store. */
> > - return AVX_U128_ANY;
> > + /* This isn't YMM/ZMM load/store. */
> > + return AVX_U128_ANY;
> > + }
> > }
> >
> > /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
> > diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
> > index f41e0908250..46379d2231b 100644
> > --- a/gcc/config/i386/i386.h
> > +++ b/gcc/config/i386/i386.h
> > @@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
> > #define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE]
> > #define TARGET_EMIT_VZEROUPPER \
> > ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER]
> > +#define TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO \
> > + ix86_tune_features[X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO]
> > #define TARGET_EXPAND_ABS \
> > ix86_tune_features[X86_TUNE_EXPAND_ABS]
> > #define TARGET_V2DF_REDUCTION_PREFER_HADDPD \
> > diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> > index 82ca0ae63ac..132de2db2eb 100644
> > --- a/gcc/config/i386/x86-tune.def
> > +++ b/gcc/config/i386/x86-tune.def
> > @@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE)
> > /* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion
> > before a transfer of control flow out of the function. */
> > DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL)
> > +
> > +/* X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO: This omits vzeroupper
> > + instruction after reading all-zero YMM/ZMM registers. */
> > +DEF_TUNE (X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO,
> > + "omit_vzeroupper_after_avx_read_zero", 0)
> > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > index 803fc6e0207..f653197da7c 100644
> > --- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > +++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > @@ -1,5 +1,5 @@
> > /* { dg-do compile } */
> > -/* { dg-options "-O2 -march=skylake" } */
> > +/* { dg-options "-O2 -march=skylake -mtune-ctrl=omit_vzeroupper_after_avx_read_zero" } */
> >
> > #include <x86intrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > index 554a0f1702c..9aac3ece14d 100644
> > --- a/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > +++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > @@ -1,5 +1,5 @@
> > /* { dg-do compile } */
> > -/* { dg-options "-O2 -march=skylake" } */
> > +/* { dg-options "-O2 -march=skylake -mtune-ctrl=omit_vzeroupper_after_avx_read_zero" } */
> >
> > #include <x86intrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> > new file mode 100644
> > index 00000000000..8389d18ed6c
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> > @@ -0,0 +1,33 @@
> > +/* { dg-do compile } */
> > +/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */
> > +
> > +#include <x86intrin.h>
> > +
> > +extern __m256 x1;
> > +extern __m256d x2;
> > +extern __m256i x3;
> > +
> > +extern void bar (void);
> > +
> > +void
> > +foo1 (void)
> > +{
> > + x1 = _mm256_setzero_ps ();
> > + bar ();
> > +}
> > +
> > +void
> > +foo2 (void)
> > +{
> > + x2 = _mm256_setzero_pd ();
> > + bar ();
> > +}
> > +
> > +void
> > +foo3 (void)
> > +{
> > + x3 = _mm256_setzero_si256 ();
> > + bar ();
> > +}
> > +
> > +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> > new file mode 100644
> > index 00000000000..3e4cdcc4d28
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> > @@ -0,0 +1,33 @@
> > +/* { dg-do compile } */
> > +/* { dg-options "-O2 -march=haswell" } */
> > +
> > +#include <x86intrin.h>
> > +
> > +extern __m256 x1;
> > +extern __m256d x2;
> > +extern __m256i x3;
> > +
> > +extern void bar (void);
> > +
> > +void
> > +foo1 (void)
> > +{
> > + x1 = _mm256_setzero_ps ();
> > + bar ();
> > +}
> > +
> > +void
> > +foo2 (void)
> > +{
> > + x2 = _mm256_setzero_pd ();
> > + bar ();
> > +}
> > +
> > +void
> > +foo3 (void)
> > +{
> > + x3 = _mm256_setzero_si256 ();
> > + bar ();
> > +}
> > +
> > +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> > --
> > 2.35.1
> >
>
>
> --
> BR,
> Hongtao
--
H.J.
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO
2022-02-21 18:35 ` H.J. Lu
@ 2022-02-22 2:43 ` Hongtao Liu
2022-02-22 14:21 ` H.J. Lu
0 siblings, 1 reply; 10+ messages in thread
From: Hongtao Liu @ 2022-02-22 2:43 UTC (permalink / raw)
To: H.J. Lu; +Cc: Uros Bizjak, liuhongt, GCC Patches
On Tue, Feb 22, 2022 at 2:35 AM H.J. Lu <hjl.tools@gmail.com> wrote:
>
> On Sun, Feb 20, 2022 at 6:01 PM Hongtao Liu <crazylht@gmail.com> wrote:
> >
> > On Thu, Feb 17, 2022 at 9:56 PM H.J. Lu <hjl.tools@gmail.com> wrote:
> > >
> > > On Thu, Feb 17, 2022 at 08:51:31AM +0100, Uros Bizjak wrote:
> > > > On Thu, Feb 17, 2022 at 6:25 AM Hongtao Liu via Gcc-patches
> > > > <gcc-patches@gcc.gnu.org> wrote:
> > > > >
> > > > > On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
> > > > > <gcc-patches@gcc.gnu.org> wrote:
> > > > > >
> > > > > > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > > > > > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > > > > > transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> > > > > > generate vzeroupper instruction after loading all-zero YMM/YMM registers
> > > > > > and enable it by default.
> > > > > Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
> > > > > Because originally we needed to add vzeroupper to all avx<->sse cases,
> > > > > now it's a tune to indicate that we don't need to add it in some
> > > >
> > > > Perhaps we should go from the other side and use
> > > > X86_TUNE_OPTIMIZE_AVX_READ for new processors?
> > > >
> > >
> > > Here is the v2 patch to add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
> > >
> > The patch LGTM in general, but please rebase against
> > https://gcc.gnu.org/pipermail/gcc-patches/2022-February/590541.html
> > and resend the patch, also wait a couple days in case Uros(and others)
> > have any comments.
>
> I am dropping my patch since it causes the compile-time regression.
I think only vextractif128 part is reverted, but we still have
vmovdqu(below) which should also cause penalty?
> > > + if (ix86_check_avx_upper_register (dest))
> > > + {
> > > + /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > > + source isn't zero. */
> > > + if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > > + return AVX_U128_DIRTY;
> > > + else
> > > + return AVX_U128_ANY;
> > > + }
>
> > >
> > > H.J.
> > > ---
> > > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > > transition penalty. Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO to
> > > omit vzeroupper instruction after loading all-zero YMM/ZMM registers.
> > >
> > > gcc/
> > >
> > > PR target/101456
> > > * config/i386/i386.cc (ix86_avx_u128_mode_needed): Omit
> > > vzeroupper after reading all-zero YMM/ZMM registers for
> > > TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
> > > * config/i386/i386.h (TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO):
> > > New.
> > > * config/i386/x86-tune.def
> > > (X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO): New.
> > >
> > > gcc/testsuite/
> > >
> > > PR target/101456
> > > * gcc.target/i386/pr101456-1.c (dg-options): Add
> > > -mtune-ctrl=-mtune-ctrl=omit_vzeroupper_after_avx_read_zero.
> > > * gcc.target/i386/pr101456-2.c: Likewise.
> > > * gcc.target/i386/pr101456-3.c: New test.
> > > * gcc.target/i386/pr101456-4.c: Likewise.
> > > ---
> > > gcc/config/i386/i386.cc | 51 ++++++++++++----------
> > > gcc/config/i386/i386.h | 2 +
> > > gcc/config/i386/x86-tune.def | 5 +++
> > > gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +-
> > > gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +-
> > > gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++
> > > gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++
> > > 7 files changed, 103 insertions(+), 25 deletions(-)
> > > create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c
> > > create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c
> > >
> > > diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> > > index cf246e74e57..60c72ceb72d 100644
> > > --- a/gcc/config/i386/i386.cc
> > > +++ b/gcc/config/i386/i386.cc
> > > @@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn)
> > >
> > > subrtx_iterator::array_type array;
> > >
> > > - rtx set = single_set (insn);
> > > - if (set)
> > > + if (TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO)
> > > {
> > > - rtx dest = SET_DEST (set);
> > > - rtx src = SET_SRC (set);
> > > - if (ix86_check_avx_upper_register (dest))
> > > + /* Perform this vzeroupper optimization if target doesn't need
> > > + vzeroupper after reading all-zero YMM/YMM registers. */
> > > + rtx set = single_set (insn);
> > > + if (set)
> > > {
> > > - /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > > - source isn't zero. */
> > > - if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > > - return AVX_U128_DIRTY;
> > > + rtx dest = SET_DEST (set);
> > > + rtx src = SET_SRC (set);
> > > + if (ix86_check_avx_upper_register (dest))
> > > + {
> > > + /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
> > > + source isn't zero. */
> > > + if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
> > > + return AVX_U128_DIRTY;
> > > + else
> > > + return AVX_U128_ANY;
> > > + }
> > > else
> > > - return AVX_U128_ANY;
> > > - }
> > > - else
> > > - {
> > > - FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> > > - if (ix86_check_avx_upper_register (*iter))
> > > - {
> > > - int status = ix86_avx_u128_mode_source (insn, *iter);
> > > - if (status == AVX_U128_DIRTY)
> > > - return status;
> > > - }
> > > - }
> > > + {
> > > + FOR_EACH_SUBRTX (iter, array, src, NONCONST)
> > > + if (ix86_check_avx_upper_register (*iter))
> > > + {
> > > + int status = ix86_avx_u128_mode_source (insn, *iter);
> > > + if (status == AVX_U128_DIRTY)
> > > + return status;
> > > + }
> > > + }
> > >
> > > - /* This isn't YMM/ZMM load/store. */
> > > - return AVX_U128_ANY;
> > > + /* This isn't YMM/ZMM load/store. */
> > > + return AVX_U128_ANY;
> > > + }
> > > }
> > >
> > > /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
> > > diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
> > > index f41e0908250..46379d2231b 100644
> > > --- a/gcc/config/i386/i386.h
> > > +++ b/gcc/config/i386/i386.h
> > > @@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
> > > #define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE]
> > > #define TARGET_EMIT_VZEROUPPER \
> > > ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER]
> > > +#define TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO \
> > > + ix86_tune_features[X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO]
> > > #define TARGET_EXPAND_ABS \
> > > ix86_tune_features[X86_TUNE_EXPAND_ABS]
> > > #define TARGET_V2DF_REDUCTION_PREFER_HADDPD \
> > > diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> > > index 82ca0ae63ac..132de2db2eb 100644
> > > --- a/gcc/config/i386/x86-tune.def
> > > +++ b/gcc/config/i386/x86-tune.def
> > > @@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE)
> > > /* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion
> > > before a transfer of control flow out of the function. */
> > > DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL)
> > > +
> > > +/* X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO: This omits vzeroupper
> > > + instruction after reading all-zero YMM/ZMM registers. */
> > > +DEF_TUNE (X86_TUNE_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO,
> > > + "omit_vzeroupper_after_avx_read_zero", 0)
> > > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > > index 803fc6e0207..f653197da7c 100644
> > > --- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > > +++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
> > > @@ -1,5 +1,5 @@
> > > /* { dg-do compile } */
> > > -/* { dg-options "-O2 -march=skylake" } */
> > > +/* { dg-options "-O2 -march=skylake -mtune-ctrl=omit_vzeroupper_after_avx_read_zero" } */
> > >
> > > #include <x86intrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > > index 554a0f1702c..9aac3ece14d 100644
> > > --- a/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > > +++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c
> > > @@ -1,5 +1,5 @@
> > > /* { dg-do compile } */
> > > -/* { dg-options "-O2 -march=skylake" } */
> > > +/* { dg-options "-O2 -march=skylake -mtune-ctrl=omit_vzeroupper_after_avx_read_zero" } */
> > >
> > > #include <x86intrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> > > new file mode 100644
> > > index 00000000000..8389d18ed6c
> > > --- /dev/null
> > > +++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c
> > > @@ -0,0 +1,33 @@
> > > +/* { dg-do compile } */
> > > +/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */
> > > +
> > > +#include <x86intrin.h>
> > > +
> > > +extern __m256 x1;
> > > +extern __m256d x2;
> > > +extern __m256i x3;
> > > +
> > > +extern void bar (void);
> > > +
> > > +void
> > > +foo1 (void)
> > > +{
> > > + x1 = _mm256_setzero_ps ();
> > > + bar ();
> > > +}
> > > +
> > > +void
> > > +foo2 (void)
> > > +{
> > > + x2 = _mm256_setzero_pd ();
> > > + bar ();
> > > +}
> > > +
> > > +void
> > > +foo3 (void)
> > > +{
> > > + x3 = _mm256_setzero_si256 ();
> > > + bar ();
> > > +}
> > > +
> > > +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> > > diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> > > new file mode 100644
> > > index 00000000000..3e4cdcc4d28
> > > --- /dev/null
> > > +++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c
> > > @@ -0,0 +1,33 @@
> > > +/* { dg-do compile } */
> > > +/* { dg-options "-O2 -march=haswell" } */
> > > +
> > > +#include <x86intrin.h>
> > > +
> > > +extern __m256 x1;
> > > +extern __m256d x2;
> > > +extern __m256i x3;
> > > +
> > > +extern void bar (void);
> > > +
> > > +void
> > > +foo1 (void)
> > > +{
> > > + x1 = _mm256_setzero_ps ();
> > > + bar ();
> > > +}
> > > +
> > > +void
> > > +foo2 (void)
> > > +{
> > > + x2 = _mm256_setzero_pd ();
> > > + bar ();
> > > +}
> > > +
> > > +void
> > > +foo3 (void)
> > > +{
> > > + x3 = _mm256_setzero_si256 ();
> > > + bar ();
> > > +}
> > > +
> > > +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */
> > > --
> > > 2.35.1
> > >
> >
> >
> > --
> > BR,
> > Hongtao
>
>
>
> --
> H.J.
--
BR,
Hongtao
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO
2022-02-22 2:43 ` Hongtao Liu
@ 2022-02-22 14:21 ` H.J. Lu
0 siblings, 0 replies; 10+ messages in thread
From: H.J. Lu @ 2022-02-22 14:21 UTC (permalink / raw)
To: Hongtao Liu; +Cc: Uros Bizjak, liuhongt, GCC Patches
On Mon, Feb 21, 2022 at 6:43 PM Hongtao Liu <crazylht@gmail.com> wrote:
>
> On Tue, Feb 22, 2022 at 2:35 AM H.J. Lu <hjl.tools@gmail.com> wrote:
> >
> > On Sun, Feb 20, 2022 at 6:01 PM Hongtao Liu <crazylht@gmail.com> wrote:
> > >
> > > On Thu, Feb 17, 2022 at 9:56 PM H.J. Lu <hjl.tools@gmail.com> wrote:
> > > >
> > > > On Thu, Feb 17, 2022 at 08:51:31AM +0100, Uros Bizjak wrote:
> > > > > On Thu, Feb 17, 2022 at 6:25 AM Hongtao Liu via Gcc-patches
> > > > > <gcc-patches@gcc.gnu.org> wrote:
> > > > > >
> > > > > > On Thu, Feb 17, 2022 at 12:26 PM H.J. Lu via Gcc-patches
> > > > > > <gcc-patches@gcc.gnu.org> wrote:
> > > > > > >
> > > > > > > Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride,
> > > > > > > Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX
> > > > > > > transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to
> > > > > > > generate vzeroupper instruction after loading all-zero YMM/YMM registers
> > > > > > > and enable it by default.
> > > > > > Shouldn't TARGET_READ_ZERO_YMM_ZMM_NONEED_VZEROUPPER sounds a bit smoother?
> > > > > > Because originally we needed to add vzeroupper to all avx<->sse cases,
> > > > > > now it's a tune to indicate that we don't need to add it in some
> > > > >
> > > > > Perhaps we should go from the other side and use
> > > > > X86_TUNE_OPTIMIZE_AVX_READ for new processors?
> > > > >
> > > >
> > > > Here is the v2 patch to add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO.
> > > >
> > > The patch LGTM in general, but please rebase against
> > > https://gcc.gnu.org/pipermail/gcc-patches/2022-February/590541.html
> > > and resend the patch, also wait a couple days in case Uros(and others)
> > > have any comments.
> >
> > I am dropping my patch since it causes the compile-time regression.
> I think only vextractif128 part is reverted, but we still have
> vmovdqu(below) which should also cause penalty?
commit fe79d652c96b53384ddfa43e312cb0010251391b
Author: Richard Biener <rguenther@suse.de>
Date: Thu Feb 17 14:40:16 2022 +0100
target/104581 - compile-time regression in mode-switching
has
diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c
b/gcc/testsuite/gcc.target/i386/pr101456-1.c
index 803fc6e0207..7fb3a3f055c 100644
--- a/gcc/testsuite/gcc.target/i386/pr101456-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c
@@ -30,4 +30,5 @@ foo3 (void)
bar ();
}
-/* { dg-final { scan-assembler-not "vzeroupper" } } */
+/* See PR104581 for the XFAIL reason. */
+/* { dg-final { scan-assembler-not "vzeroupper" { xfail *-*-* } } } */
and I checked in:
commit 1931cbad498e625b1e24452dcfffe02539b12224
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Fri Feb 18 10:36:53 2022 -0800
pieces-memset-21.c: Expect vzeroupper for ia32
Update gcc.target/i386/pieces-memset-21.c to expect vzeroupper for ia32
caused by
commit fe79d652c96b53384ddfa43e312cb0010251391b
Author: Richard Biener <rguenther@suse.de>
Date: Thu Feb 17 14:40:16 2022 +0100
target/104581 - compile-time regression in mode-switching
PR target/104581
* gcc.target/i386/pieces-memset-21.c: Expect vzeroupper for ia32.
I believe that vmovdqu is also covered.
--
H.J.
^ permalink raw reply [flat|nested] 10+ messages in thread
end of thread, other threads:[~2022-02-22 14:22 UTC | newest]
Thread overview: 10+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-02-17 4:26 [PATCH] x86: Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER H.J. Lu
2022-02-17 5:33 ` Hongtao Liu
2022-02-17 7:51 ` Uros Bizjak
2022-02-17 9:49 ` Richard Biener
2022-02-17 13:57 ` H.J. Lu
2022-02-17 13:56 ` [PATCH v2] x86: Add TARGET_OMIT_VZEROUPPER_AFTER_AVX_READ_ZERO H.J. Lu
2022-02-21 2:01 ` Hongtao Liu
2022-02-21 18:35 ` H.J. Lu
2022-02-22 2:43 ` Hongtao Liu
2022-02-22 14:21 ` H.J. Lu
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).