From: Haochen Jiang <haochen.jiang@intel.com>
To: gcc-patches@gcc.gnu.org
Cc: hongtao.liu@intel.com, ubizjak@gmail.com
Subject: [PATCH] i386: Auto vectorize sdot_prod, udot_prod with VNNIINT8 instruction.
Date: Tue, 18 Oct 2022 17:17:27 +0800 [thread overview]
Message-ID: <20221018091727.82856-1-haochen.jiang@intel.com> (raw)
In-Reply-To: <20221014075445.7938-1-haochen.jiang@intel.com>
Hi all,
We would like to add one more patch to enhance the codegen with avxvnniint8.
Also renamed two awkward named mode_attr to make them more aligned with others.
Regtested on x86_64-pc-linux-gnu. Ok for trunk?
BRs,
Haochen
gcc/ChangeLog:
* config/i386/sse.md (ssedvecmode): Rename from VI1SI.
(ssedvecmodelower): Rename from vi1si.
(sdot_prod<mode>): New define_expand.
(udot_prod<mode>): Ditto.
gcc/testsuite/ChangeLog:
* gcc.target/i386/vnniint8-auto-vectorize-1.c: New test.
* gcc.target/i386/vnniint8-auto-vectorize-2.c: Ditto.
---
gcc/config/i386/sse.md | 61 ++++++++++++---
.../i386/vnniint8-auto-vectorize-1.c | 28 +++++++
.../i386/vnniint8-auto-vectorize-2.c | 75 +++++++++++++++++++
3 files changed, 153 insertions(+), 11 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/vnniint8-auto-vectorize-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/vnniint8-auto-vectorize-2.c
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 29cf6fa090b..fc17b5193dc 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -1043,6 +1043,13 @@
(V16HI "v16hi") (V8HI "v8hi")
(V32QI "v32qi") (V16QI "v16qi")])
+;; Mapping of vector modes to an V*SImode of the same size
+(define_mode_attr ssedvecmode
+ [(V64QI "V16SI") (V32QI "V8SI") (V16QI "V4SI")])
+
+(define_mode_attr ssedvecmodelower
+ [(V64QI "v16si") (V32QI "v8si") (V16QI "v4si")])
+
;; Mapping of vector modes to a vector mode of double size
(define_mode_attr ssedoublevecmode
[(V64QI "V128QI") (V32HI "V64HI") (V16SI "V32SI") (V8DI "V16DI")
@@ -28523,29 +28530,23 @@
[(set_attr ("prefix") ("evex"))
(set_attr "mode" "<sseinsnmode>")])
-(define_mode_attr VI1SI
- [(V64QI "V16SI") (V32QI "V8SI") (V16QI "V4SI")])
-
-(define_mode_attr vi1si
- [(V64QI "v16si") (V32QI "v8si") (V16QI "v4si")])
-
(define_expand "usdot_prod<mode>"
- [(match_operand:<VI1SI> 0 "register_operand")
+ [(match_operand:<ssedvecmode> 0 "register_operand")
(match_operand:VI1_AVX512VNNI 1 "register_operand")
(match_operand:VI1_AVX512VNNI 2 "register_operand")
- (match_operand:<VI1SI> 3 "register_operand")]
+ (match_operand:<ssedvecmode> 3 "register_operand")]
"(<MODE_SIZE> == 64
||((TARGET_AVX512VNNI && TARGET_AVX512VL)
|| TARGET_AVXVNNI))"
{
- operands[1] = lowpart_subreg (<VI1SI>mode,
+ operands[1] = lowpart_subreg (<ssedvecmode>mode,
force_reg (<MODE>mode, operands[1]),
<MODE>mode);
- operands[2] = lowpart_subreg (<VI1SI>mode,
+ operands[2] = lowpart_subreg (<ssedvecmode>mode,
force_reg (<MODE>mode, operands[2]),
<MODE>mode);
emit_insn (gen_rtx_SET (operands[0], operands[3]));
- emit_insn (gen_vpdpbusd_<vi1si> (operands[0], operands[3],
+ emit_insn (gen_vpdpbusd_<ssedvecmodelower> (operands[0], operands[3],
operands[1], operands[2]));
DONE;
})
@@ -29358,6 +29359,44 @@
(UNSPEC_VPDPBSUD "bsud") (UNSPEC_VPDPBSUDS "bsuds")
(UNSPEC_VPDPBUUD "buud") (UNSPEC_VPDPBUUDS "buuds")])
+(define_expand "sdot_prod<mode>"
+ [(match_operand:<ssedvecmode> 0 "register_operand")
+ (match_operand:VI1 1 "register_operand")
+ (match_operand:VI1 2 "register_operand")
+ (match_operand:<ssedvecmode> 3 "register_operand")]
+ "TARGET_AVXVNNIINT8"
+{
+ operands[1] = lowpart_subreg (<ssedvecmode>mode,
+ force_reg (<MODE>mode, operands[1]),
+ <MODE>mode);
+ operands[2] = lowpart_subreg (<ssedvecmode>mode,
+ force_reg (<MODE>mode, operands[2]),
+ <MODE>mode);
+ emit_insn (gen_rtx_SET (operands[0], operands[3]));
+ emit_insn (gen_vpdpbssd_<ssedvecmodelower> (operands[0], operands[3],
+ operands[1], operands[2]));
+ DONE;
+})
+
+(define_expand "udot_prod<mode>"
+ [(match_operand:<ssedvecmode> 0 "register_operand")
+ (match_operand:VI1 1 "register_operand")
+ (match_operand:VI1 2 "register_operand")
+ (match_operand:<ssedvecmode> 3 "register_operand")]
+ "TARGET_AVXVNNIINT8"
+{
+ operands[1] = lowpart_subreg (<ssedvecmode>mode,
+ force_reg (<MODE>mode, operands[1]),
+ <MODE>mode);
+ operands[2] = lowpart_subreg (<ssedvecmode>mode,
+ force_reg (<MODE>mode, operands[2]),
+ <MODE>mode);
+ emit_insn (gen_rtx_SET (operands[0], operands[3]));
+ emit_insn (gen_vpdpbuud_<ssedvecmodelower> (operands[0], operands[3],
+ operands[1], operands[2]));
+ DONE;
+})
+
(define_insn "vpdp<vpdotprodtype>_<mode>"
[(set (match_operand:VI4_AVX 0 "register_operand" "=x")
(unspec:VI4_AVX
diff --git a/gcc/testsuite/gcc.target/i386/vnniint8-auto-vectorize-1.c b/gcc/testsuite/gcc.target/i386/vnniint8-auto-vectorize-1.c
new file mode 100644
index 00000000000..9cadab6a845
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vnniint8-auto-vectorize-1.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-mavxvnniint8 -O2" } */
+/* { dg-final { scan-assembler "vpdpbssd\t" } } */
+/* { dg-final { scan-assembler "vpdpbuud\t" } } */
+
+int __attribute__((noinline, noclone, optimize("tree-vectorize")))
+sdot_prod_qi (char * restrict a, char * restrict b,
+ int c, int n)
+{
+ int i;
+ for (i = 0; i < n; i++)
+ {
+ c += ((int) a[i] * (int) b[i]);
+ }
+ return c;
+}
+
+int __attribute__((noinline, noclone, optimize("tree-vectorize")))
+udot_prod_qi (unsigned char * restrict a, unsigned char *restrict b,
+ int c, int n)
+{
+ int i;
+ for (i = 0; i < n; i++)
+ {
+ c += ((int) a[i] * (int) b[i]);
+ }
+ return c;
+}
diff --git a/gcc/testsuite/gcc.target/i386/vnniint8-auto-vectorize-2.c b/gcc/testsuite/gcc.target/i386/vnniint8-auto-vectorize-2.c
new file mode 100644
index 00000000000..99853e6c3b7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vnniint8-auto-vectorize-2.c
@@ -0,0 +1,75 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavxvnniint8" } */
+/* { dg-require-effective-target avxvnniint8 } */
+
+#define AVXVNNIINT8
+#ifndef CHECK
+#define CHECK "avx-check.h"
+#endif
+
+#ifndef TEST
+#define TEST avx_test
+#endif
+
+#include CHECK
+#include "vnniint8-auto-vectorize-1.c"
+
+#define N 256
+char a_i8[N], b_i8[N];
+unsigned char c_u8[N], d_u8[N];
+int i8_exp, i8_ref;
+
+int __attribute__((noipa, optimize("no-tree-vectorize")))
+sdot_prod_qi_scalar (char * restrict a, char * restrict b,
+ int c, int n)
+{
+ int i;
+ for (i = 0; i < n; i++)
+ {
+ c += ((int) a[i] * (int) b[i]);
+ }
+ return c;
+}
+
+int __attribute__((noipa, optimize("no-tree-vectorize")))
+udot_prod_qi_scalar (unsigned char * restrict a, unsigned char *restrict b,
+ int c, int n)
+{
+ int i;
+ for (i = 0; i < n; i++)
+ {
+ c += ((int) a[i] * (int) b[i]);
+ }
+ return c;
+}
+
+void init ()
+{
+ int i;
+
+ i8_exp = i8_ref = 127;
+
+ for (i = 0; i < N; i++)
+ {
+ a_i8[i] = (-i + 4) % 128;
+ b_i8[i] = (i + 1) % 128;
+ c_u8[i] = (i + 3) % 256;
+ d_u8[i] = (i + 5) % 256;
+ }
+}
+
+void
+TEST (void)
+{
+ init ();
+ i8_exp = sdot_prod_qi (a_i8, b_i8, i8_exp, N);
+ i8_ref = sdot_prod_qi_scalar (a_i8, b_i8, i8_ref, N);
+ if (i8_exp != i8_ref)
+ abort ();
+
+ init ();
+ i8_exp = udot_prod_qi (c_u8, d_u8, i8_exp, N);
+ i8_ref = udot_prod_qi_scalar (c_u8, d_u8, i8_ref, N);
+ if (i8_exp != i8_ref)
+ abort ();
+}
--
2.18.1
next prev parent reply other threads:[~2022-10-18 9:18 UTC|newest]
Thread overview: 45+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-10-14 7:54 [PATCH 0/6] Add Intel Sierra Forest Instructions Haochen Jiang
2022-10-14 7:54 ` [PATCH 1/6] Support Intel AVX-IFMA Haochen Jiang
2022-10-19 6:03 ` [PATCH] " Hongyu Wang
2022-10-21 0:52 ` Hongtao Liu
2022-10-14 7:54 ` [PATCH 2/6] Support Intel AVX-VNNI-INT8 Haochen Jiang
2022-10-17 4:05 ` Hongtao Liu
2022-10-17 6:27 ` Jiang, Haochen
2022-10-17 6:30 ` Hongtao Liu
2022-10-18 9:10 ` [PATCH v2] " Haochen Jiang
2022-10-19 1:41 ` Hongtao Liu
2022-10-21 0:53 ` Hongtao Liu
2022-10-14 7:54 ` [PATCH 3/6] i386: Add intrinsic for vector __bf16 Haochen Jiang
2022-10-14 7:54 ` [PATCH 4/6] Support Intel AVX-NE-CONVERT Haochen Jiang
2022-10-17 5:46 ` Hongtao Liu
2022-10-24 6:20 ` Kong, Lingling
2022-10-25 5:23 ` Hongtao Liu
2022-10-28 8:57 ` Kong, Lingling
2022-10-31 1:05 ` Liu, Hongtao
2022-10-14 7:54 ` [PATCH 5/6] Support Intel CMPccXADD Haochen Jiang
2022-10-24 9:01 ` [PATCH] " Haochen Jiang
2022-10-24 9:04 ` Jiang, Haochen
2022-10-24 9:44 ` Uros Bizjak
2022-11-03 6:26 ` Haochen Jiang
2022-11-03 7:47 ` Uros Bizjak
2022-10-14 7:54 ` [PATCH 6/6] Initial Sierra Forest Support Haochen Jiang
2022-11-03 6:31 ` Hongtao Liu
2022-10-14 8:14 ` [PATCH 0/6] Add Intel Sierra Forest Instructions Iain Sandoe
2022-10-14 8:20 ` Hongtao Liu
2022-10-14 8:24 ` Iain Sandoe
2022-10-14 8:30 ` Hongtao Liu
2022-10-14 8:36 ` Iain Sandoe
2022-10-17 1:02 ` Hongtao Liu
2022-10-17 1:30 ` Bernhard Reutner-Fischer
2022-10-17 1:56 ` Hongtao Liu
2022-10-19 11:09 ` Iain Sandoe
2022-10-20 1:11 ` Hongtao Liu
2022-10-20 9:09 ` Hongtao Liu
2022-10-20 9:17 ` Iain Sandoe
2022-10-20 9:20 ` Hongtao Liu
2022-10-20 15:23 ` Iain Sandoe
2022-10-21 0:06 ` Hongtao Liu
2022-10-14 8:35 ` Hongtao Liu
2022-10-18 9:17 ` Haochen Jiang [this message]
2022-10-19 1:43 ` [PATCH] i386: Auto vectorize sdot_prod, udot_prod with VNNIINT8 instruction Hongtao Liu
2022-10-21 0:54 ` Hongtao Liu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221018091727.82856-1-haochen.jiang@intel.com \
--to=haochen.jiang@intel.com \
--cc=gcc-patches@gcc.gnu.org \
--cc=hongtao.liu@intel.com \
--cc=ubizjak@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).