public inbox for libc-stable@sourceware.org
 help / color / mirror / Atom feed
From: Sunil Pandey <skpgkp2@gmail.com>
To: "H.J. Lu" <hjl.tools@gmail.com>,
	 Libc-stable Mailing List <libc-stable@sourceware.org>
Cc: Noah Goldstein <goldstein.w.n@gmail.com>,
	GNU C Library <libc-alpha@sourceware.org>
Subject: Re: [PATCH v4 21/23] x86: Add AVX2 optimized str{n}casecmp
Date: Thu, 12 May 2022 12:52:06 -0700	[thread overview]
Message-ID: <CAMAf5_eOfzQsDwEK3FajW2YqwYpXz1=c44sYQ=CSz0-bjHTD_g@mail.gmail.com> (raw)
In-Reply-To: <CAMe9rOqZ8h1dOUctD6jF3OmUv9TXjyA7_ser3p_6157myHq1Og@mail.gmail.com>

[-- Attachment #1: Type: text/plain, Size: 31021 bytes --]

On Fri, Mar 25, 2022 at 11:15 AM H.J. Lu via Libc-alpha
<libc-alpha@sourceware.org> wrote:
>
> On Thu, Mar 24, 2022 at 4:56 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
> >
> > geometric_mean(N=40) of all benchmarks AVX2 / SSE42: .702
> >
> > All string/memory tests pass.
> > Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
> > ---
> >  sysdeps/x86_64/multiarch/Makefile             |   4 +
> >  sysdeps/x86_64/multiarch/ifunc-impl-list.c    |  28 +++
> >  sysdeps/x86_64/multiarch/ifunc-strcasecmp.h   |  12 +
> >  .../x86_64/multiarch/strcasecmp_l-avx2-rtm.S  |  15 ++
> >  sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S  |  23 ++
> >  sysdeps/x86_64/multiarch/strcmp-avx2.S        | 237 +++++++++++++++---
> >  .../x86_64/multiarch/strncase_l-avx2-rtm.S    |  16 ++
> >  sysdeps/x86_64/multiarch/strncase_l-avx2.S    |  27 ++
> >  8 files changed, 331 insertions(+), 31 deletions(-)
> >  create mode 100644 sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S
> >  create mode 100644 sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S
> >  create mode 100644 sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S
> >  create mode 100644 sysdeps/x86_64/multiarch/strncase_l-avx2.S
> >
> > diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
> > index e7b413edad..06e1848823 100644
> > --- a/sysdeps/x86_64/multiarch/Makefile
> > +++ b/sysdeps/x86_64/multiarch/Makefile
> > @@ -55,6 +55,8 @@ sysdep_routines += \
> >    stpncpy-sse2-unaligned \
> >    stpncpy-ssse3 \
> >    strcasecmp_l-avx \
> > +  strcasecmp_l-avx2 \
> > +  strcasecmp_l-avx2-rtm \
> >    strcasecmp_l-sse2 \
> >    strcasecmp_l-sse4_2 \
> >    strcasecmp_l-ssse3 \
> > @@ -93,6 +95,8 @@ sysdep_routines += \
> >    strlen-evex \
> >    strlen-sse2 \
> >    strncase_l-avx \
> > +  strncase_l-avx2 \
> > +  strncase_l-avx2-rtm \
> >    strncase_l-sse2 \
> >    strncase_l-sse4_2 \
> >    strncase_l-ssse3 \
> > diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
> > index a594f4176e..3c556d07ac 100644
> > --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
> > +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
> > @@ -436,6 +436,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> >
> >    /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c.  */
> >    IFUNC_IMPL (i, name, strcasecmp,
> > +             IFUNC_IMPL_ADD (array, i, strcasecmp,
> > +                             CPU_FEATURE_USABLE (AVX2),
> > +                             __strcasecmp_avx2)
> > +             IFUNC_IMPL_ADD (array, i, strcasecmp,
> > +                             (CPU_FEATURE_USABLE (AVX2)
> > +                              && CPU_FEATURE_USABLE (RTM)),
> > +                             __strcasecmp_avx2_rtm)
> >               IFUNC_IMPL_ADD (array, i, strcasecmp,
> >                               CPU_FEATURE_USABLE (AVX),
> >                               __strcasecmp_avx)
> > @@ -449,6 +456,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> >
> >    /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c.  */
> >    IFUNC_IMPL (i, name, strcasecmp_l,
> > +             IFUNC_IMPL_ADD (array, i, strcasecmp,
> > +                             CPU_FEATURE_USABLE (AVX2),
> > +                             __strcasecmp_l_avx2)
> > +             IFUNC_IMPL_ADD (array, i, strcasecmp,
> > +                             (CPU_FEATURE_USABLE (AVX2)
> > +                              && CPU_FEATURE_USABLE (RTM)),
> > +                             __strcasecmp_l_avx2_rtm)
> >               IFUNC_IMPL_ADD (array, i, strcasecmp_l,
> >                               CPU_FEATURE_USABLE (AVX),
> >                               __strcasecmp_l_avx)
> > @@ -576,6 +590,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> >
> >    /* Support sysdeps/x86_64/multiarch/strncase_l.c.  */
> >    IFUNC_IMPL (i, name, strncasecmp,
> > +             IFUNC_IMPL_ADD (array, i, strncasecmp,
> > +                             CPU_FEATURE_USABLE (AVX2),
> > +                             __strncasecmp_avx2)
> > +             IFUNC_IMPL_ADD (array, i, strncasecmp,
> > +                             (CPU_FEATURE_USABLE (AVX2)
> > +                              && CPU_FEATURE_USABLE (RTM)),
> > +                             __strncasecmp_avx2_rtm)
> >               IFUNC_IMPL_ADD (array, i, strncasecmp,
> >                               CPU_FEATURE_USABLE (AVX),
> >                               __strncasecmp_avx)
> > @@ -590,6 +611,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> >
> >    /* Support sysdeps/x86_64/multiarch/strncase_l.c.  */
> >    IFUNC_IMPL (i, name, strncasecmp_l,
> > +             IFUNC_IMPL_ADD (array, i, strncasecmp,
> > +                             CPU_FEATURE_USABLE (AVX2),
> > +                             __strncasecmp_l_avx2)
> > +             IFUNC_IMPL_ADD (array, i, strncasecmp,
> > +                             (CPU_FEATURE_USABLE (AVX2)
> > +                              && CPU_FEATURE_USABLE (RTM)),
> > +                             __strncasecmp_l_avx2_rtm)
> >               IFUNC_IMPL_ADD (array, i, strncasecmp_l,
> >                               CPU_FEATURE_USABLE (AVX),
> >                               __strncasecmp_l_avx)
> > diff --git a/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
> > index 9e3cc61ac0..c4de111fd0 100644
> > --- a/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
> > +++ b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
> > @@ -23,12 +23,24 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
> >  extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
> >  extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden;
> >  extern __typeof (REDIRECT_NAME) OPTIMIZE (avx) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
> >
> >  static inline void *
> >  IFUNC_SELECTOR (void)
> >  {
> >    const struct cpu_features* cpu_features = __get_cpu_features ();
> >
> > +  if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
> > +      && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
> > +    {
> > +      if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
> > +        return OPTIMIZE (avx2_rtm);
> > +
> > +      if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
> > +        return OPTIMIZE (avx2);
> > +    }
> > +
> >    if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
> >      return OPTIMIZE (avx);
> >
> > diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S
> > new file mode 100644
> > index 0000000000..09957fc3c5
> > --- /dev/null
> > +++ b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S
> > @@ -0,0 +1,15 @@
> > +#ifndef STRCMP
> > +# define STRCMP        __strcasecmp_l_avx2_rtm
> > +#endif
> > +
> > +#define _GLABEL(x)     x ## _rtm
> > +#define GLABEL(x)      _GLABEL(x)
> > +
> > +#define ZERO_UPPER_VEC_REGISTERS_RETURN        \
> > +       ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
> > +
> > +#define VZEROUPPER_RETURN      jmp L(return_vzeroupper)
> > +
> > +#define SECTION(p)     p##.avx.rtm
> > +
> > +#include "strcasecmp_l-avx2.S"
> > diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S
> > new file mode 100644
> > index 0000000000..e2762f2a22
> > --- /dev/null
> > +++ b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S
> > @@ -0,0 +1,23 @@
> > +/* strcasecmp_l optimized with AVX2.
> > +   Copyright (C) 2017-2022 Free Software Foundation, Inc.
> > +   This file is part of the GNU C Library.
> > +
> > +   The GNU C Library is free software; you can redistribute it and/or
> > +   modify it under the terms of the GNU Lesser General Public
> > +   License as published by the Free Software Foundation; either
> > +   version 2.1 of the License, or (at your option) any later version.
> > +
> > +   The GNU C Library is distributed in the hope that it will be useful,
> > +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> > +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> > +   Lesser General Public License for more details.
> > +
> > +   You should have received a copy of the GNU Lesser General Public
> > +   License along with the GNU C Library; if not, see
> > +   <https://www.gnu.org/licenses/>.  */
> > +
> > +#ifndef STRCMP
> > +# define STRCMP        __strcasecmp_l_avx2
> > +#endif
> > +#define USE_AS_STRCASECMP_L
> > +#include "strcmp-avx2.S"
> > diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
> > index 86a86b68e3..8da09bd86d 100644
> > --- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
> > +++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
> > @@ -20,6 +20,10 @@
> >
> >  # include <sysdep.h>
> >
> > +# if defined USE_AS_STRCASECMP_L
> > +#  include "locale-defines.h"
> > +# endif
> > +
> >  # ifndef STRCMP
> >  #  define STRCMP       __strcmp_avx2
> >  # endif
> > @@ -74,13 +78,88 @@
> >  #  define VEC_OFFSET   (-VEC_SIZE)
> >  # endif
> >
> > +# ifdef USE_AS_STRCASECMP_L
> > +#  define BYTE_LOOP_REG        OFFSET_REG
> > +# else
> > +#  define BYTE_LOOP_REG        ecx
> > +# endif
> > +
> > +# ifdef USE_AS_STRCASECMP_L
> > +#  ifdef USE_AS_STRNCMP
> > +#   define STRCASECMP  __strncasecmp_avx2
> > +#   define LOCALE_REG  rcx
> > +#   define LOCALE_REG_LP       RCX_LP
> > +#   define STRCASECMP_NONASCII __strncasecmp_l_nonascii
> > +#  else
> > +#   define STRCASECMP  __strcasecmp_avx2
> > +#   define LOCALE_REG  rdx
> > +#   define LOCALE_REG_LP       RDX_LP
> > +#   define STRCASECMP_NONASCII __strcasecmp_l_nonascii
> > +#  endif
> > +# endif
> > +
> >  # define xmmZERO       xmm15
> >  # define ymmZERO       ymm15
> >
> > +# define LCASE_MIN_ymm %ymm10
> > +# define LCASE_MAX_ymm %ymm11
> > +# define CASE_ADD_ymm  %ymm12
> > +
> > +# define LCASE_MIN_xmm %xmm10
> > +# define LCASE_MAX_xmm %xmm11
> > +# define CASE_ADD_xmm  %xmm12
> > +
> > +       /* r11 is never use elsewhere so this is safe to maintain.  */
> > +# define TOLOWER_BASE  %r11
> > +
> >  # ifndef SECTION
> >  #  define SECTION(p)   p##.avx
> >  # endif
> >
> > +# ifdef USE_AS_STRCASECMP_L
> > +#  define REG(x, y) x ## y
> > +#  define TOLOWER(reg1_in, reg1_out, reg2_in, reg2_out, ext)                   \
> > +       vpaddb  REG(LCASE_MIN_, ext), reg1_in, REG(%ext, 8);                            \
> > +       vpaddb  REG(LCASE_MIN_, ext), reg2_in, REG(%ext, 9);                            \
> > +       vpcmpgtb REG(LCASE_MAX_, ext), REG(%ext, 8), REG(%ext, 8);                      \
> > +       vpcmpgtb REG(LCASE_MAX_, ext), REG(%ext, 9), REG(%ext, 9);                      \
> > +       vpandn  REG(CASE_ADD_, ext), REG(%ext, 8), REG(%ext, 8);                        \
> > +       vpandn  REG(CASE_ADD_, ext), REG(%ext, 9), REG(%ext, 9);                        \
> > +       vpaddb  REG(%ext, 8), reg1_in, reg1_out;                                                        \
> > +       vpaddb  REG(%ext, 9), reg2_in, reg2_out
> > +
> > +#  define TOLOWER_gpr(src, dst)        movl (TOLOWER_BASE, src, 4), dst
> > +#  define TOLOWER_ymm(...)     TOLOWER(__VA_ARGS__, ymm)
> > +#  define TOLOWER_xmm(...)     TOLOWER(__VA_ARGS__, xmm)
> > +
> > +#  define CMP_R1_R2(s1_reg, s2_reg, scratch_reg, reg_out, ext)                 \
> > +       TOLOWER (s1_reg, scratch_reg, s2_reg, s2_reg, ext);                                     \
> > +       VPCMPEQ scratch_reg, s2_reg, reg_out
> > +
> > +#  define CMP_R1_S2(s1_reg, s2_mem, scratch_reg, reg_out, ext)                 \
> > +       VMOVU   s2_mem, reg_out;                                                                                        \
> > +       CMP_R1_R2(s1_reg, reg_out, scratch_reg, reg_out, ext)
> > +
> > +#  define CMP_R1_R2_ymm(...) CMP_R1_R2(__VA_ARGS__, ymm)
> > +#  define CMP_R1_R2_xmm(...) CMP_R1_R2(__VA_ARGS__, xmm)
> > +
> > +#  define CMP_R1_S2_ymm(...) CMP_R1_S2(__VA_ARGS__, ymm)
> > +#  define CMP_R1_S2_xmm(...) CMP_R1_S2(__VA_ARGS__, xmm)
> > +
> > +# else
> > +#  define TOLOWER_gpr(...)
> > +#  define TOLOWER_ymm(...)
> > +#  define TOLOWER_xmm(...)
> > +
> > +#  define CMP_R1_R2_ymm(s1_reg, s2_reg, scratch_reg, reg_out)                  \
> > +       VPCMPEQ s2_reg, s1_reg, reg_out
> > +
> > +#  define CMP_R1_R2_xmm(...) CMP_R1_R2_ymm(__VA_ARGS__)
> > +
> > +#  define CMP_R1_S2_ymm(...) CMP_R1_R2_ymm(__VA_ARGS__)
> > +#  define CMP_R1_S2_xmm(...) CMP_R1_R2_xmm(__VA_ARGS__)
> > +# endif
> > +
> >  /* Warning!
> >             wcscmp/wcsncmp have to use SIGNED comparison for elements.
> >             strcmp/strncmp have to use UNSIGNED comparison for elements.
> > @@ -102,8 +181,49 @@
> >     returned.  */
> >
> >         .section SECTION(.text), "ax", @progbits
> > -ENTRY(STRCMP)
> > +       .align  16
> > +       .type   STRCMP, @function
> > +       .globl  STRCMP
> > +       .hidden STRCMP
> > +
> > +# ifndef GLABEL
> > +#  define GLABEL(...)  __VA_ARGS__
> > +# endif
> > +
> > +# ifdef USE_AS_STRCASECMP_L
> > +ENTRY (GLABEL(STRCASECMP))
> > +       movq    __libc_tsd_LOCALE@gottpoff(%rip), %rax
> > +       mov     %fs:(%rax), %LOCALE_REG_LP
> > +
> > +       /* Either 1 or 5 bytes (dependeing if CET is enabled).  */
> > +       .p2align 4
> > +END (GLABEL(STRCASECMP))
> > +       /* FALLTHROUGH to strcasecmp/strncasecmp_l.  */
> > +# endif
> > +
> > +       .p2align 4
> > +STRCMP:
> > +       cfi_startproc
> > +       _CET_ENDBR
> > +       CALL_MCOUNT
> > +
> > +# if defined USE_AS_STRCASECMP_L
> > +       /* We have to fall back on the C implementation for locales with
> > +          encodings not matching ASCII for single bytes.  */
> > +#  if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0
> > +       mov     LOCALE_T___LOCALES + LC_CTYPE * LP_SIZE(%LOCALE_REG), %RAX_LP
> > +#  else
> > +       mov     (%LOCALE_REG), %RAX_LP
> > +#  endif
> > +       testl   $1, LOCALE_DATA_VALUES + _NL_CTYPE_NONASCII_CASE * SIZEOF_VALUES(%rax)
> > +       jne     STRCASECMP_NONASCII
> > +       leaq    _nl_C_LC_CTYPE_tolower + 128 * 4(%rip), TOLOWER_BASE
> > +# endif
> > +
> >  # ifdef USE_AS_STRNCMP
> > +       /* Don't overwrite LOCALE_REG (rcx) until we have pass
> > +          L(one_or_less). Otherwise we might use the wrong locale in
> > +          the OVERFLOW_STRCMP (strcasecmp_l).  */
> >  #  ifdef __ILP32__
> >         /* Clear the upper 32 bits.  */
> >         movl    %edx, %edx
> > @@ -128,6 +248,30 @@ ENTRY(STRCMP)
> >  #  endif
> >  # endif
> >         vpxor   %xmmZERO, %xmmZERO, %xmmZERO
> > +# if defined USE_AS_STRCASECMP_L
> > +       .section .rodata.cst32, "aM", @progbits, 32
> > +       .align  32
> > +L(lcase_min):
> > +       .quad   0x3f3f3f3f3f3f3f3f
> > +       .quad   0x3f3f3f3f3f3f3f3f
> > +       .quad   0x3f3f3f3f3f3f3f3f
> > +       .quad   0x3f3f3f3f3f3f3f3f
> > +L(lcase_max):
> > +       .quad   0x9999999999999999
> > +       .quad   0x9999999999999999
> > +       .quad   0x9999999999999999
> > +       .quad   0x9999999999999999
> > +L(case_add):
> > +       .quad   0x2020202020202020
> > +       .quad   0x2020202020202020
> > +       .quad   0x2020202020202020
> > +       .quad   0x2020202020202020
> > +       .previous
> > +
> > +       vmovdqa L(lcase_min)(%rip), LCASE_MIN_ymm
> > +       vmovdqa L(lcase_max)(%rip), LCASE_MAX_ymm
> > +       vmovdqa L(case_add)(%rip), CASE_ADD_ymm
> > +# endif
> >         movl    %edi, %eax
> >         orl     %esi, %eax
> >         sall    $20, %eax
> > @@ -138,8 +282,10 @@ ENTRY(STRCMP)
> >  L(no_page_cross):
> >         /* Safe to compare 4x vectors.  */
> >         VMOVU   (%rdi), %ymm0
> > -       /* 1s where s1 and s2 equal.  */
> > -       VPCMPEQ (%rsi), %ymm0, %ymm1
> > +       /* 1s where s1 and s2 equal. Just VPCMPEQ if its not strcasecmp.
> > +          Otherwise converts ymm0 and load from rsi to lower. ymm2 is
> > +          scratch and ymm1 is the return.  */
> > +       CMP_R1_S2_ymm (%ymm0, (%rsi), %ymm2, %ymm1)
> >         /* 1s at null CHAR.  */
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         /* 1s where s1 and s2 equal AND not null CHAR.  */
> > @@ -172,6 +318,8 @@ L(return_vec_0):
> >  # else
> >         movzbl  (%rdi, %rcx), %eax
> >         movzbl  (%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >  # endif
> >  L(ret0):
> > @@ -192,6 +340,10 @@ L(ret_zero):
> >
> >         .p2align 4,, 5
> >  L(one_or_less):
> > +#  ifdef USE_AS_STRCASECMP_L
> > +       /* Set locale argument for strcasecmp.  */
> > +       movq    %LOCALE_REG, %rdx
> > +#  endif
> >         jb      L(ret_zero)
> >         /* 'nbe' covers the case where length is negative (large
> >            unsigned).  */
> > @@ -207,6 +359,8 @@ L(one_or_less):
> >  #  else
> >         movzbl  (%rdi), %eax
> >         movzbl  (%rsi), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >  #  endif
> >  L(ret1):
> > @@ -234,6 +388,8 @@ L(return_vec_1):
> >  # else
> >         movzbl  VEC_SIZE(%rdi, %rcx), %eax
> >         movzbl  VEC_SIZE(%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >  # endif
> >  L(ret2):
> > @@ -265,6 +421,8 @@ L(return_vec_2):
> >  # else
> >         movzbl  (VEC_SIZE * 2)(%rdi, %rcx), %eax
> >         movzbl  (VEC_SIZE * 2)(%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >  # endif
> >  L(ret3):
> > @@ -285,6 +443,8 @@ L(return_vec_3):
> >  #  else
> >         movzbl  (VEC_SIZE * 3)(%rdi, %rcx), %eax
> >         movzbl  (VEC_SIZE * 3)(%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >  #  endif
> >  L(ret4):
> > @@ -295,7 +455,7 @@ L(ret4):
> >  L(more_3x_vec):
> >         /* Safe to compare 4x vectors.  */
> >         VMOVU   VEC_SIZE(%rdi), %ymm0
> > -       VPCMPEQ VEC_SIZE(%rsi), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, VEC_SIZE(%rsi), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -308,7 +468,7 @@ L(more_3x_vec):
> >  # endif
> >
> >         VMOVU   (VEC_SIZE * 2)(%rdi), %ymm0
> > -       VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 2)(%rsi), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -316,7 +476,7 @@ L(more_3x_vec):
> >         jnz     L(return_vec_2)
> >
> >         VMOVU   (VEC_SIZE * 3)(%rdi), %ymm0
> > -       VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 3)(%rsi), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -391,12 +551,10 @@ L(loop_skip_page_cross_check):
> >         VMOVA   (VEC_SIZE * 3)(%rdi), %ymm6
> >
> >         /* ymm1 all 1s where s1 and s2 equal. All 0s otherwise.  */
> > -       VPCMPEQ (VEC_SIZE * 0)(%rsi), %ymm0, %ymm1
> > -
> > -       VPCMPEQ (VEC_SIZE * 1)(%rsi), %ymm2, %ymm3
> > -       VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm4, %ymm5
> > -       VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm6, %ymm7
> > -
> > +       CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 0)(%rsi), %ymm3, %ymm1)
> > +       CMP_R1_S2_ymm (%ymm2, (VEC_SIZE * 1)(%rsi), %ymm5, %ymm3)
> > +       CMP_R1_S2_ymm (%ymm4, (VEC_SIZE * 2)(%rsi), %ymm7, %ymm5)
> > +       CMP_R1_S2_ymm (%ymm6, (VEC_SIZE * 3)(%rsi), %ymm13, %ymm7)
> >
> >         /* If any mismatches or null CHAR then 0 CHAR, otherwise non-
> >            zero.  */
> > @@ -465,6 +623,8 @@ L(return_vec_2_3_end):
> >  # else
> >         movzbl  (VEC_SIZE * 2 - VEC_OFFSET)(%rdi, %LOOP_REG64), %eax
> >         movzbl  (VEC_SIZE * 2 - VEC_OFFSET)(%rsi, %LOOP_REG64), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >         xorl    %r8d, %eax
> >         subl    %r8d, %eax
> > @@ -508,6 +668,8 @@ L(return_vec_0_end):
> >  # else
> >         movzbl  (%rdi, %rcx), %eax
> >         movzbl  (%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >         xorl    %r8d, %eax
> >         subl    %r8d, %eax
> > @@ -530,6 +692,8 @@ L(return_vec_1_end):
> >  #  else
> >         movzbl  VEC_SIZE(%rdi, %rcx), %eax
> >         movzbl  VEC_SIZE(%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >         xorl    %r8d, %eax
> >         subl    %r8d, %eax
> > @@ -556,6 +720,8 @@ L(return_vec_2_end):
> >  # else
> >         movzbl  (VEC_SIZE * 2)(%rdi, %rcx), %eax
> >         movzbl  (VEC_SIZE * 2)(%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >         xorl    %r8d, %eax
> >         subl    %r8d, %eax
> > @@ -583,7 +749,7 @@ L(page_cross_during_loop):
> >         jle     L(less_1x_vec_till_page_cross)
> >
> >         VMOVA   (%rdi), %ymm0
> > -       VPCMPEQ (%rsi), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, (%rsi), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -605,7 +771,7 @@ L(less_1x_vec_till_page_cross):
> >            here, it means the previous page (rdi - VEC_SIZE) has already
> >            been loaded earlier so must be valid.  */
> >         VMOVU   -VEC_SIZE(%rdi, %rax), %ymm0
> > -       VPCMPEQ -VEC_SIZE(%rsi, %rax), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, -VEC_SIZE(%rsi, %rax), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -647,6 +813,8 @@ L(return_page_cross_cmp_mem):
> >  # else
> >         movzbl  VEC_OFFSET(%rdi, %rcx), %eax
> >         movzbl  VEC_OFFSET(%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >         xorl    %r8d, %eax
> >         subl    %r8d, %eax
> > @@ -673,7 +841,7 @@ L(more_2x_vec_till_page_cross):
> >            iteration here.  */
> >
> >         VMOVU   VEC_SIZE(%rdi), %ymm0
> > -       VPCMPEQ VEC_SIZE(%rsi), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, VEC_SIZE(%rsi), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -689,7 +857,7 @@ L(more_2x_vec_till_page_cross):
> >
> >         /* Safe to include comparisons from lower bytes.  */
> >         VMOVU   -(VEC_SIZE * 2)(%rdi, %rax), %ymm0
> > -       VPCMPEQ -(VEC_SIZE * 2)(%rsi, %rax), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, -(VEC_SIZE * 2)(%rsi, %rax), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -697,7 +865,7 @@ L(more_2x_vec_till_page_cross):
> >         jnz     L(return_vec_page_cross_0)
> >
> >         VMOVU   -(VEC_SIZE * 1)(%rdi, %rax), %ymm0
> > -       VPCMPEQ -(VEC_SIZE * 1)(%rsi, %rax), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, -(VEC_SIZE * 1)(%rsi, %rax), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -715,8 +883,8 @@ L(more_2x_vec_till_page_cross):
> >         VMOVA   (VEC_SIZE * 2)(%rdi), %ymm4
> >         VMOVA   (VEC_SIZE * 3)(%rdi), %ymm6
> >
> > -       VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm4, %ymm5
> > -       VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm6, %ymm7
> > +       CMP_R1_S2_ymm (%ymm4, (VEC_SIZE * 2)(%rsi), %ymm7, %ymm5)
> > +       CMP_R1_S2_ymm (%ymm6, (VEC_SIZE * 3)(%rsi), %ymm13, %ymm7)
> >         vpand   %ymm4, %ymm5, %ymm5
> >         vpand   %ymm6, %ymm7, %ymm7
> >         VPMINU  %ymm5, %ymm7, %ymm7
> > @@ -767,6 +935,8 @@ L(return_vec_page_cross_1):
> >  # else
> >         movzbl  VEC_OFFSET(%rdi, %rcx), %eax
> >         movzbl  VEC_OFFSET(%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >         xorl    %r8d, %eax
> >         subl    %r8d, %eax
> > @@ -822,7 +992,7 @@ L(page_cross):
> >  L(page_cross_loop):
> >
> >         VMOVU   (%rdi, %OFFSET_REG64), %ymm0
> > -       VPCMPEQ (%rsi, %OFFSET_REG64), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, (%rsi, %OFFSET_REG64), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -840,11 +1010,11 @@ L(page_cross_loop):
> >         subl    %eax, %OFFSET_REG
> >         /* OFFSET_REG has distance to page cross - VEC_SIZE. Guranteed
> >            to not cross page so is safe to load. Since we have already
> > -          loaded at least 1 VEC from rsi it is also guranteed to be safe.
> > -        */
> > +          loaded at least 1 VEC from rsi it is also guranteed to be
> > +          safe.  */
> >
> >         VMOVU   (%rdi, %OFFSET_REG64), %ymm0
> > -       VPCMPEQ (%rsi, %OFFSET_REG64), %ymm0, %ymm1
> > +       CMP_R1_S2_ymm (%ymm0, (%rsi, %OFFSET_REG64), %ymm2, %ymm1)
> >         VPCMPEQ %ymm0, %ymmZERO, %ymm2
> >         vpandn  %ymm1, %ymm2, %ymm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -877,6 +1047,8 @@ L(ret_vec_page_cross_cont):
> >  # else
> >         movzbl  (%rdi, %rcx), %eax
> >         movzbl  (%rsi, %rcx), %ecx
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %ecx)
> >         subl    %ecx, %eax
> >         xorl    %r8d, %eax
> >         subl    %r8d, %eax
> > @@ -930,7 +1102,7 @@ L(less_1x_vec_till_page):
> >         ja      L(less_16_till_page)
> >
> >         VMOVU   (%rdi), %xmm0
> > -       VPCMPEQ (%rsi), %xmm0, %xmm1
> > +       CMP_R1_S2_xmm (%xmm0, (%rsi), %xmm2, %xmm1)
> >         VPCMPEQ %xmm0, %xmmZERO, %xmm2
> >         vpandn  %xmm1, %xmm2, %xmm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -948,7 +1120,7 @@ L(less_1x_vec_till_page):
> >  # endif
> >
> >         VMOVU   (%rdi, %OFFSET_REG64), %xmm0
> > -       VPCMPEQ (%rsi, %OFFSET_REG64), %xmm0, %xmm1
> > +       CMP_R1_S2_xmm (%xmm0, (%rsi, %OFFSET_REG64), %xmm2, %xmm1)
> >         VPCMPEQ %xmm0, %xmmZERO, %xmm2
> >         vpandn  %xmm1, %xmm2, %xmm1
> >         vpmovmskb %ymm1, %ecx
> > @@ -986,7 +1158,7 @@ L(less_16_till_page):
> >         vmovq   (%rdi), %xmm0
> >         vmovq   (%rsi), %xmm1
> >         VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > -       VPCMPEQ %xmm1, %xmm0, %xmm1
> > +       CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
> >         vpandn  %xmm1, %xmm2, %xmm1
> >         vpmovmskb %ymm1, %ecx
> >         incb    %cl
> > @@ -1006,7 +1178,7 @@ L(less_16_till_page):
> >         vmovq   (%rdi, %OFFSET_REG64), %xmm0
> >         vmovq   (%rsi, %OFFSET_REG64), %xmm1
> >         VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > -       VPCMPEQ %xmm1, %xmm0, %xmm1
> > +       CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
> >         vpandn  %xmm1, %xmm2, %xmm1
> >         vpmovmskb %ymm1, %ecx
> >         incb    %cl
> > @@ -1062,7 +1234,7 @@ L(ret_less_8_wcs):
> >         vmovd   (%rdi), %xmm0
> >         vmovd   (%rsi), %xmm1
> >         VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > -       VPCMPEQ %xmm1, %xmm0, %xmm1
> > +       CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
> >         vpandn  %xmm1, %xmm2, %xmm1
> >         vpmovmskb %ymm1, %ecx
> >         subl    $0xf, %ecx
> > @@ -1081,7 +1253,7 @@ L(ret_less_8_wcs):
> >         vmovd   (%rdi, %OFFSET_REG64), %xmm0
> >         vmovd   (%rsi, %OFFSET_REG64), %xmm1
> >         VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > -       VPCMPEQ %xmm1, %xmm0, %xmm1
> > +       CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
> >         vpandn  %xmm1, %xmm2, %xmm1
> >         vpmovmskb %ymm1, %ecx
> >         subl    $0xf, %ecx
> > @@ -1115,7 +1287,9 @@ L(less_4_till_page):
> >  L(less_4_loop):
> >         movzbl  (%rdi), %eax
> >         movzbl  (%rsi, %rdi), %ecx
> > -       subl    %ecx, %eax
> > +       TOLOWER_gpr (%rax, %eax)
> > +       TOLOWER_gpr (%rcx, %BYTE_LOOP_REG)
> > +       subl    %BYTE_LOOP_REG, %eax
> >         jnz     L(ret_less_4_loop)
> >         testl   %ecx, %ecx
> >         jz      L(ret_zero_4_loop)
> > @@ -1142,5 +1316,6 @@ L(ret_less_4_loop):
> >         subl    %r8d, %eax
> >         ret
> >  # endif
> > -END(STRCMP)
> > +       cfi_endproc
> > +       .size   STRCMP, .-STRCMP
> >  #endif
> > diff --git a/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S
> > new file mode 100644
> > index 0000000000..58c05dcfb8
> > --- /dev/null
> > +++ b/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S
> > @@ -0,0 +1,16 @@
> > +#ifndef STRCMP
> > +# define STRCMP        __strncasecmp_l_avx2_rtm
> > +#endif
> > +
> > +#define _GLABEL(x)     x ## _rtm
> > +#define GLABEL(x)      _GLABEL(x)
> > +
> > +#define ZERO_UPPER_VEC_REGISTERS_RETURN        \
> > +       ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
> > +
> > +#define VZEROUPPER_RETURN      jmp L(return_vzeroupper)
> > +
> > +#define SECTION(p)     p##.avx.rtm
> > +#define OVERFLOW_STRCMP        __strcasecmp_l_avx2_rtm
> > +
> > +#include "strncase_l-avx2.S"
> > diff --git a/sysdeps/x86_64/multiarch/strncase_l-avx2.S b/sysdeps/x86_64/multiarch/strncase_l-avx2.S
> > new file mode 100644
> > index 0000000000..48c0aa21f8
> > --- /dev/null
> > +++ b/sysdeps/x86_64/multiarch/strncase_l-avx2.S
> > @@ -0,0 +1,27 @@
> > +/* strncasecmp_l optimized with AVX2.
> > +   Copyright (C) 2017-2022 Free Software Foundation, Inc.
> > +   This file is part of the GNU C Library.
> > +
> > +   The GNU C Library is free software; you can redistribute it and/or
> > +   modify it under the terms of the GNU Lesser General Public
> > +   License as published by the Free Software Foundation; either
> > +   version 2.1 of the License, or (at your option) any later version.
> > +
> > +   The GNU C Library is distributed in the hope that it will be useful,
> > +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> > +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> > +   Lesser General Public License for more details.
> > +
> > +   You should have received a copy of the GNU Lesser General Public
> > +   License along with the GNU C Library; if not, see
> > +   <https://www.gnu.org/licenses/>.  */
> > +
> > +#ifndef STRCMP
> > +# define STRCMP        __strncasecmp_l_avx2
> > +#endif
> > +#define USE_AS_STRCASECMP_L
> > +#define USE_AS_STRNCMP
> > +#ifndef OVERFLOW_STRCMP
> > +# define OVERFLOW_STRCMP       __strcasecmp_l_avx2
> > +#endif
> > +#include "strcmp-avx2.S"
> > --
> > 2.25.1
> >
>
> LGTM.
>
> Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
>
> Thanks.
>
> --
> H.J.

I would like to backport this patch to release branches.
Any comments or objections?

Conflict resolution patch attached.

--Sunil

[-- Attachment #2: 0015-x86-Add-AVX2-optimized-str-n-casecmp.patch --]
[-- Type: application/octet-stream, Size: 24673 bytes --]

From b382e4caf50dfee62e170f9b6617b470b1289dcb Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Thu, 24 Mar 2022 18:56:12 -0500
Subject: [PATCH 15/26] x86: Add AVX2 optimized str{n}casecmp

geometric_mean(N=40) of all benchmarks AVX2 / SSE42: .702

All string/memory tests pass.
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>

(cherry picked from commit bbf81222343fed5cd704001a2ae0d86c71544151)
---
 sysdeps/x86_64/multiarch/Makefile             |   4 +
 sysdeps/x86_64/multiarch/ifunc-impl-list.c    |  28 +++
 sysdeps/x86_64/multiarch/ifunc-strcasecmp.h   |  12 +
 .../x86_64/multiarch/strcasecmp_l-avx2-rtm.S  |  15 ++
 sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S  |  23 ++
 sysdeps/x86_64/multiarch/strcmp-avx2.S        | 237 +++++++++++++++---
 .../x86_64/multiarch/strncase_l-avx2-rtm.S    |  16 ++
 sysdeps/x86_64/multiarch/strncase_l-avx2.S    |  27 ++
 8 files changed, 331 insertions(+), 31 deletions(-)
 create mode 100644 sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S
 create mode 100644 sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S
 create mode 100644 sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S
 create mode 100644 sysdeps/x86_64/multiarch/strncase_l-avx2.S

diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index e7b413edad..06e1848823 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -55,6 +55,8 @@ sysdep_routines += \
   stpncpy-sse2-unaligned \
   stpncpy-ssse3 \
   strcasecmp_l-avx \
+  strcasecmp_l-avx2 \
+  strcasecmp_l-avx2-rtm \
   strcasecmp_l-sse2 \
   strcasecmp_l-sse4_2 \
   strcasecmp_l-ssse3 \
@@ -93,6 +95,8 @@ sysdep_routines += \
   strlen-evex \
   strlen-sse2 \
   strncase_l-avx \
+  strncase_l-avx2 \
+  strncase_l-avx2-rtm \
   strncase_l-sse2 \
   strncase_l-sse4_2 \
   strncase_l-ssse3 \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index a594f4176e..3c556d07ac 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -436,6 +436,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c.  */
   IFUNC_IMPL (i, name, strcasecmp,
+	      IFUNC_IMPL_ADD (array, i, strcasecmp,
+			      CPU_FEATURE_USABLE (AVX2),
+			      __strcasecmp_avx2)
+	      IFUNC_IMPL_ADD (array, i, strcasecmp,
+			      (CPU_FEATURE_USABLE (AVX2)
+			       && CPU_FEATURE_USABLE (RTM)),
+			      __strcasecmp_avx2_rtm)
 	      IFUNC_IMPL_ADD (array, i, strcasecmp,
 			      CPU_FEATURE_USABLE (AVX),
 			      __strcasecmp_avx)
@@ -449,6 +456,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c.  */
   IFUNC_IMPL (i, name, strcasecmp_l,
+	      IFUNC_IMPL_ADD (array, i, strcasecmp,
+			      CPU_FEATURE_USABLE (AVX2),
+			      __strcasecmp_l_avx2)
+	      IFUNC_IMPL_ADD (array, i, strcasecmp,
+			      (CPU_FEATURE_USABLE (AVX2)
+			       && CPU_FEATURE_USABLE (RTM)),
+			      __strcasecmp_l_avx2_rtm)
 	      IFUNC_IMPL_ADD (array, i, strcasecmp_l,
 			      CPU_FEATURE_USABLE (AVX),
 			      __strcasecmp_l_avx)
@@ -576,6 +590,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   /* Support sysdeps/x86_64/multiarch/strncase_l.c.  */
   IFUNC_IMPL (i, name, strncasecmp,
+	      IFUNC_IMPL_ADD (array, i, strncasecmp,
+			      CPU_FEATURE_USABLE (AVX2),
+			      __strncasecmp_avx2)
+	      IFUNC_IMPL_ADD (array, i, strncasecmp,
+			      (CPU_FEATURE_USABLE (AVX2)
+			       && CPU_FEATURE_USABLE (RTM)),
+			      __strncasecmp_avx2_rtm)
 	      IFUNC_IMPL_ADD (array, i, strncasecmp,
 			      CPU_FEATURE_USABLE (AVX),
 			      __strncasecmp_avx)
@@ -590,6 +611,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   /* Support sysdeps/x86_64/multiarch/strncase_l.c.  */
   IFUNC_IMPL (i, name, strncasecmp_l,
+	      IFUNC_IMPL_ADD (array, i, strncasecmp,
+			      CPU_FEATURE_USABLE (AVX2),
+			      __strncasecmp_l_avx2)
+	      IFUNC_IMPL_ADD (array, i, strncasecmp,
+			      (CPU_FEATURE_USABLE (AVX2)
+			       && CPU_FEATURE_USABLE (RTM)),
+			      __strncasecmp_l_avx2_rtm)
 	      IFUNC_IMPL_ADD (array, i, strncasecmp_l,
 			      CPU_FEATURE_USABLE (AVX),
 			      __strncasecmp_l_avx)
diff --git a/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
index 9e3cc61ac0..c4de111fd0 100644
--- a/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
+++ b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
@@ -23,12 +23,24 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
 extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
 extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden;
 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
 
 static inline void *
 IFUNC_SELECTOR (void)
 {
   const struct cpu_features* cpu_features = __get_cpu_features ();
 
+  if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+      && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+    {
+      if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+        return OPTIMIZE (avx2_rtm);
+
+      if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+        return OPTIMIZE (avx2);
+    }
+
   if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
     return OPTIMIZE (avx);
 
diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S
new file mode 100644
index 0000000000..09957fc3c5
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S
@@ -0,0 +1,15 @@
+#ifndef STRCMP
+# define STRCMP	__strcasecmp_l_avx2_rtm
+#endif
+
+#define _GLABEL(x)	x ## _rtm
+#define GLABEL(x)	_GLABEL(x)
+
+#define ZERO_UPPER_VEC_REGISTERS_RETURN	\
+	ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
+
+#define VZEROUPPER_RETURN	jmp L(return_vzeroupper)
+
+#define SECTION(p)	p##.avx.rtm
+
+#include "strcasecmp_l-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S
new file mode 100644
index 0000000000..e2762f2a22
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S
@@ -0,0 +1,23 @@
+/* strcasecmp_l optimized with AVX2.
+   Copyright (C) 2017-2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#ifndef STRCMP
+# define STRCMP	__strcasecmp_l_avx2
+#endif
+#define USE_AS_STRCASECMP_L
+#include "strcmp-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
index b94fc5c39a..3366d0b083 100644
--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
+++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
@@ -20,6 +20,10 @@
 
 # include <sysdep.h>
 
+# if defined USE_AS_STRCASECMP_L
+#  include "locale-defines.h"
+# endif
+
 # ifndef STRCMP
 #  define STRCMP	__strcmp_avx2
 # endif
@@ -74,13 +78,88 @@
 #  define VEC_OFFSET	(-VEC_SIZE)
 # endif
 
+# ifdef USE_AS_STRCASECMP_L
+#  define BYTE_LOOP_REG	OFFSET_REG
+# else
+#  define BYTE_LOOP_REG	ecx
+# endif
+
+# ifdef USE_AS_STRCASECMP_L
+#  ifdef USE_AS_STRNCMP
+#   define STRCASECMP	__strncasecmp_avx2
+#   define LOCALE_REG	rcx
+#   define LOCALE_REG_LP	RCX_LP
+#   define STRCASECMP_NONASCII	__strncasecmp_l_nonascii
+#  else
+#   define STRCASECMP	__strcasecmp_avx2
+#   define LOCALE_REG	rdx
+#   define LOCALE_REG_LP	RDX_LP
+#   define STRCASECMP_NONASCII	__strcasecmp_l_nonascii
+#  endif
+# endif
+
 # define xmmZERO	xmm15
 # define ymmZERO	ymm15
 
+# define LCASE_MIN_ymm	%ymm10
+# define LCASE_MAX_ymm	%ymm11
+# define CASE_ADD_ymm	%ymm12
+
+# define LCASE_MIN_xmm	%xmm10
+# define LCASE_MAX_xmm	%xmm11
+# define CASE_ADD_xmm	%xmm12
+
+	/* r11 is never use elsewhere so this is safe to maintain.  */
+# define TOLOWER_BASE	%r11
+
 # ifndef SECTION
 #  define SECTION(p)	p##.avx
 # endif
 
+# ifdef USE_AS_STRCASECMP_L
+#  define REG(x, y) x ## y
+#  define TOLOWER(reg1_in, reg1_out, reg2_in, reg2_out, ext)			\
+	vpaddb	REG(LCASE_MIN_, ext), reg1_in, REG(%ext, 8);				\
+	vpaddb	REG(LCASE_MIN_, ext), reg2_in, REG(%ext, 9);				\
+	vpcmpgtb REG(LCASE_MAX_, ext), REG(%ext, 8), REG(%ext, 8);			\
+	vpcmpgtb REG(LCASE_MAX_, ext), REG(%ext, 9), REG(%ext, 9);			\
+	vpandn	REG(CASE_ADD_, ext), REG(%ext, 8), REG(%ext, 8);			\
+	vpandn	REG(CASE_ADD_, ext), REG(%ext, 9), REG(%ext, 9);			\
+	vpaddb	REG(%ext, 8), reg1_in, reg1_out;							\
+	vpaddb	REG(%ext, 9), reg2_in, reg2_out
+
+#  define TOLOWER_gpr(src, dst)	movl (TOLOWER_BASE, src, 4), dst
+#  define TOLOWER_ymm(...)	TOLOWER(__VA_ARGS__, ymm)
+#  define TOLOWER_xmm(...)	TOLOWER(__VA_ARGS__, xmm)
+
+#  define CMP_R1_R2(s1_reg, s2_reg, scratch_reg, reg_out, ext)			\
+	TOLOWER	(s1_reg, scratch_reg, s2_reg, s2_reg, ext);					\
+	VPCMPEQ	scratch_reg, s2_reg, reg_out
+
+#  define CMP_R1_S2(s1_reg, s2_mem, scratch_reg, reg_out, ext)			\
+	VMOVU	s2_mem, reg_out;											\
+	CMP_R1_R2(s1_reg, reg_out, scratch_reg, reg_out, ext)
+
+#  define CMP_R1_R2_ymm(...) CMP_R1_R2(__VA_ARGS__, ymm)
+#  define CMP_R1_R2_xmm(...) CMP_R1_R2(__VA_ARGS__, xmm)
+
+#  define CMP_R1_S2_ymm(...) CMP_R1_S2(__VA_ARGS__, ymm)
+#  define CMP_R1_S2_xmm(...) CMP_R1_S2(__VA_ARGS__, xmm)
+
+# else
+#  define TOLOWER_gpr(...)
+#  define TOLOWER_ymm(...)
+#  define TOLOWER_xmm(...)
+
+#  define CMP_R1_R2_ymm(s1_reg, s2_reg, scratch_reg, reg_out)			\
+	VPCMPEQ	s2_reg, s1_reg, reg_out
+
+#  define CMP_R1_R2_xmm(...) CMP_R1_R2_ymm(__VA_ARGS__)
+
+#  define CMP_R1_S2_ymm(...) CMP_R1_R2_ymm(__VA_ARGS__)
+#  define CMP_R1_S2_xmm(...) CMP_R1_R2_xmm(__VA_ARGS__)
+# endif
+
 /* Warning!
            wcscmp/wcsncmp have to use SIGNED comparison for elements.
            strcmp/strncmp have to use UNSIGNED comparison for elements.
@@ -102,8 +181,49 @@
    returned.  */
 
 	.section SECTION(.text), "ax", @progbits
-ENTRY(STRCMP)
+	.align	16
+	.type	STRCMP, @function
+	.globl	STRCMP
+	.hidden	STRCMP
+
+# ifndef GLABEL
+#  define GLABEL(...)	__VA_ARGS__
+# endif
+
+# ifdef USE_AS_STRCASECMP_L
+ENTRY (GLABEL(STRCASECMP))
+	movq	__libc_tsd_LOCALE@gottpoff(%rip), %rax
+	mov	%fs:(%rax), %LOCALE_REG_LP
+
+	/* Either 1 or 5 bytes (dependeing if CET is enabled).  */
+	.p2align 4
+END (GLABEL(STRCASECMP))
+	/* FALLTHROUGH to strcasecmp/strncasecmp_l.  */
+# endif
+
+	.p2align 4
+STRCMP:
+	cfi_startproc
+	_CET_ENDBR
+	CALL_MCOUNT
+
+# if defined USE_AS_STRCASECMP_L
+	/* We have to fall back on the C implementation for locales with
+	   encodings not matching ASCII for single bytes.  */
+#  if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0
+	mov	LOCALE_T___LOCALES + LC_CTYPE * LP_SIZE(%LOCALE_REG), %RAX_LP
+#  else
+	mov	(%LOCALE_REG), %RAX_LP
+#  endif
+	testl	$1, LOCALE_DATA_VALUES + _NL_CTYPE_NONASCII_CASE * SIZEOF_VALUES(%rax)
+	jne	STRCASECMP_NONASCII
+	leaq	_nl_C_LC_CTYPE_tolower + 128 * 4(%rip), TOLOWER_BASE
+# endif
+
 # ifdef USE_AS_STRNCMP
+	/* Don't overwrite LOCALE_REG (rcx) until we have pass
+	   L(one_or_less). Otherwise we might use the wrong locale in
+	   the OVERFLOW_STRCMP (strcasecmp_l).  */
 #  ifdef __ILP32__
 	/* Clear the upper 32 bits.  */
 	movl	%edx, %edx
@@ -128,6 +248,30 @@ ENTRY(STRCMP)
 #  endif
 # endif
 	vpxor	%xmmZERO, %xmmZERO, %xmmZERO
+# if defined USE_AS_STRCASECMP_L
+	.section .rodata.cst32, "aM", @progbits, 32
+	.align	32
+L(lcase_min):
+	.quad	0x3f3f3f3f3f3f3f3f
+	.quad	0x3f3f3f3f3f3f3f3f
+	.quad	0x3f3f3f3f3f3f3f3f
+	.quad	0x3f3f3f3f3f3f3f3f
+L(lcase_max):
+	.quad	0x9999999999999999
+	.quad	0x9999999999999999
+	.quad	0x9999999999999999
+	.quad	0x9999999999999999
+L(case_add):
+	.quad	0x2020202020202020
+	.quad	0x2020202020202020
+	.quad	0x2020202020202020
+	.quad	0x2020202020202020
+	.previous
+
+	vmovdqa	L(lcase_min)(%rip), LCASE_MIN_ymm
+	vmovdqa	L(lcase_max)(%rip), LCASE_MAX_ymm
+	vmovdqa	L(case_add)(%rip), CASE_ADD_ymm
+# endif
 	movl	%edi, %eax
 	orl	%esi, %eax
 	sall	$20, %eax
@@ -138,8 +282,10 @@ ENTRY(STRCMP)
 L(no_page_cross):
 	/* Safe to compare 4x vectors.  */
 	VMOVU	(%rdi), %ymm0
-	/* 1s where s1 and s2 equal.  */
-	VPCMPEQ	(%rsi), %ymm0, %ymm1
+	/* 1s where s1 and s2 equal. Just VPCMPEQ if its not strcasecmp.
+	   Otherwise converts ymm0 and load from rsi to lower. ymm2 is
+	   scratch and ymm1 is the return.  */
+	CMP_R1_S2_ymm (%ymm0, (%rsi), %ymm2, %ymm1)
 	/* 1s at null CHAR.  */
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	/* 1s where s1 and s2 equal AND not null CHAR.  */
@@ -172,6 +318,8 @@ L(return_vec_0):
 # else
 	movzbl	(%rdi, %rcx), %eax
 	movzbl	(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 # endif
 L(ret0):
@@ -192,6 +340,10 @@ L(ret_zero):
 
 	.p2align 4,, 5
 L(one_or_less):
+#  ifdef USE_AS_STRCASECMP_L
+	/* Set locale argument for strcasecmp.  */
+	movq	%LOCALE_REG, %rdx
+#  endif
 	jb	L(ret_zero)
 #  ifdef USE_AS_WCSCMP
 	/* 'nbe' covers the case where length is negative (large
@@ -211,6 +363,8 @@ L(one_or_less):
 	jnbe	__strcmp_avx2
 	movzbl	(%rdi), %eax
 	movzbl	(%rsi), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 #  endif
 L(ret1):
@@ -238,6 +392,8 @@ L(return_vec_1):
 # else
 	movzbl	VEC_SIZE(%rdi, %rcx), %eax
 	movzbl	VEC_SIZE(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 # endif
 L(ret2):
@@ -269,6 +425,8 @@ L(return_vec_2):
 # else
 	movzbl	(VEC_SIZE * 2)(%rdi, %rcx), %eax
 	movzbl	(VEC_SIZE * 2)(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 # endif
 L(ret3):
@@ -289,6 +447,8 @@ L(return_vec_3):
 #  else
 	movzbl	(VEC_SIZE * 3)(%rdi, %rcx), %eax
 	movzbl	(VEC_SIZE * 3)(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 #  endif
 L(ret4):
@@ -299,7 +459,7 @@ L(ret4):
 L(more_3x_vec):
 	/* Safe to compare 4x vectors.  */
 	VMOVU	VEC_SIZE(%rdi), %ymm0
-	VPCMPEQ	VEC_SIZE(%rsi), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, VEC_SIZE(%rsi), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -312,7 +472,7 @@ L(more_3x_vec):
 # endif
 
 	VMOVU	(VEC_SIZE * 2)(%rdi), %ymm0
-	VPCMPEQ	(VEC_SIZE * 2)(%rsi), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 2)(%rsi), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -320,7 +480,7 @@ L(more_3x_vec):
 	jnz	L(return_vec_2)
 
 	VMOVU	(VEC_SIZE * 3)(%rdi), %ymm0
-	VPCMPEQ	(VEC_SIZE * 3)(%rsi), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 3)(%rsi), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -395,12 +555,10 @@ L(loop_skip_page_cross_check):
 	VMOVA	(VEC_SIZE * 3)(%rdi), %ymm6
 
 	/* ymm1 all 1s where s1 and s2 equal. All 0s otherwise.  */
-	VPCMPEQ	(VEC_SIZE * 0)(%rsi), %ymm0, %ymm1
-
-	VPCMPEQ	(VEC_SIZE * 1)(%rsi), %ymm2, %ymm3
-	VPCMPEQ	(VEC_SIZE * 2)(%rsi), %ymm4, %ymm5
-	VPCMPEQ	(VEC_SIZE * 3)(%rsi), %ymm6, %ymm7
-
+	CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 0)(%rsi), %ymm3, %ymm1)
+	CMP_R1_S2_ymm (%ymm2, (VEC_SIZE * 1)(%rsi), %ymm5, %ymm3)
+	CMP_R1_S2_ymm (%ymm4, (VEC_SIZE * 2)(%rsi), %ymm7, %ymm5)
+	CMP_R1_S2_ymm (%ymm6, (VEC_SIZE * 3)(%rsi), %ymm13, %ymm7)
 
 	/* If any mismatches or null CHAR then 0 CHAR, otherwise non-
 	   zero.  */
@@ -469,6 +627,8 @@ L(return_vec_2_3_end):
 # else
 	movzbl	(VEC_SIZE * 2 - VEC_OFFSET)(%rdi, %LOOP_REG64), %eax
 	movzbl	(VEC_SIZE * 2 - VEC_OFFSET)(%rsi, %LOOP_REG64), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 	xorl	%r8d, %eax
 	subl	%r8d, %eax
@@ -512,6 +672,8 @@ L(return_vec_0_end):
 # else
 	movzbl	(%rdi, %rcx), %eax
 	movzbl	(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 	xorl	%r8d, %eax
 	subl	%r8d, %eax
@@ -534,6 +696,8 @@ L(return_vec_1_end):
 #  else
 	movzbl	VEC_SIZE(%rdi, %rcx), %eax
 	movzbl	VEC_SIZE(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 	xorl	%r8d, %eax
 	subl	%r8d, %eax
@@ -560,6 +724,8 @@ L(return_vec_2_end):
 # else
 	movzbl	(VEC_SIZE * 2)(%rdi, %rcx), %eax
 	movzbl	(VEC_SIZE * 2)(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 	xorl	%r8d, %eax
 	subl	%r8d, %eax
@@ -587,7 +753,7 @@ L(page_cross_during_loop):
 	jle	L(less_1x_vec_till_page_cross)
 
 	VMOVA	(%rdi), %ymm0
-	VPCMPEQ	(%rsi), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, (%rsi), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -609,7 +775,7 @@ L(less_1x_vec_till_page_cross):
 	   here, it means the previous page (rdi - VEC_SIZE) has already
 	   been loaded earlier so must be valid.  */
 	VMOVU	-VEC_SIZE(%rdi, %rax), %ymm0
-	VPCMPEQ	-VEC_SIZE(%rsi, %rax), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, -VEC_SIZE(%rsi, %rax), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -651,6 +817,8 @@ L(return_page_cross_cmp_mem):
 # else
 	movzbl	VEC_OFFSET(%rdi, %rcx), %eax
 	movzbl	VEC_OFFSET(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 	xorl	%r8d, %eax
 	subl	%r8d, %eax
@@ -677,7 +845,7 @@ L(more_2x_vec_till_page_cross):
 	   iteration here.  */
 
 	VMOVU	VEC_SIZE(%rdi), %ymm0
-	VPCMPEQ	VEC_SIZE(%rsi), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, VEC_SIZE(%rsi), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -693,7 +861,7 @@ L(more_2x_vec_till_page_cross):
 
 	/* Safe to include comparisons from lower bytes.  */
 	VMOVU	-(VEC_SIZE * 2)(%rdi, %rax), %ymm0
-	VPCMPEQ	-(VEC_SIZE * 2)(%rsi, %rax), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, -(VEC_SIZE * 2)(%rsi, %rax), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -701,7 +869,7 @@ L(more_2x_vec_till_page_cross):
 	jnz	L(return_vec_page_cross_0)
 
 	VMOVU	-(VEC_SIZE * 1)(%rdi, %rax), %ymm0
-	VPCMPEQ	-(VEC_SIZE * 1)(%rsi, %rax), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, -(VEC_SIZE * 1)(%rsi, %rax), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -719,8 +887,8 @@ L(more_2x_vec_till_page_cross):
 	VMOVA	(VEC_SIZE * 2)(%rdi), %ymm4
 	VMOVA	(VEC_SIZE * 3)(%rdi), %ymm6
 
-	VPCMPEQ	(VEC_SIZE * 2)(%rsi), %ymm4, %ymm5
-	VPCMPEQ	(VEC_SIZE * 3)(%rsi), %ymm6, %ymm7
+	CMP_R1_S2_ymm (%ymm4, (VEC_SIZE * 2)(%rsi), %ymm7, %ymm5)
+	CMP_R1_S2_ymm (%ymm6, (VEC_SIZE * 3)(%rsi), %ymm13, %ymm7)
 	vpand	%ymm4, %ymm5, %ymm5
 	vpand	%ymm6, %ymm7, %ymm7
 	VPMINU	%ymm5, %ymm7, %ymm7
@@ -771,6 +939,8 @@ L(return_vec_page_cross_1):
 # else
 	movzbl	VEC_OFFSET(%rdi, %rcx), %eax
 	movzbl	VEC_OFFSET(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 	xorl	%r8d, %eax
 	subl	%r8d, %eax
@@ -826,7 +996,7 @@ L(page_cross):
 L(page_cross_loop):
 
 	VMOVU	(%rdi, %OFFSET_REG64), %ymm0
-	VPCMPEQ	(%rsi, %OFFSET_REG64), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, (%rsi, %OFFSET_REG64), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -844,11 +1014,11 @@ L(page_cross_loop):
 	subl	%eax, %OFFSET_REG
 	/* OFFSET_REG has distance to page cross - VEC_SIZE. Guranteed
 	   to not cross page so is safe to load. Since we have already
-	   loaded at least 1 VEC from rsi it is also guranteed to be safe.
-	 */
+	   loaded at least 1 VEC from rsi it is also guranteed to be
+	   safe.  */
 
 	VMOVU	(%rdi, %OFFSET_REG64), %ymm0
-	VPCMPEQ	(%rsi, %OFFSET_REG64), %ymm0, %ymm1
+	CMP_R1_S2_ymm (%ymm0, (%rsi, %OFFSET_REG64), %ymm2, %ymm1)
 	VPCMPEQ	%ymm0, %ymmZERO, %ymm2
 	vpandn	%ymm1, %ymm2, %ymm1
 	vpmovmskb %ymm1, %ecx
@@ -881,6 +1051,8 @@ L(ret_vec_page_cross_cont):
 # else
 	movzbl	(%rdi, %rcx), %eax
 	movzbl	(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
 	xorl	%r8d, %eax
 	subl	%r8d, %eax
@@ -934,7 +1106,7 @@ L(less_1x_vec_till_page):
 	ja	L(less_16_till_page)
 
 	VMOVU	(%rdi), %xmm0
-	VPCMPEQ	(%rsi), %xmm0, %xmm1
+	CMP_R1_S2_xmm (%xmm0, (%rsi), %xmm2, %xmm1)
 	VPCMPEQ	%xmm0, %xmmZERO, %xmm2
 	vpandn	%xmm1, %xmm2, %xmm1
 	vpmovmskb %ymm1, %ecx
@@ -952,7 +1124,7 @@ L(less_1x_vec_till_page):
 # endif
 
 	VMOVU	(%rdi, %OFFSET_REG64), %xmm0
-	VPCMPEQ	(%rsi, %OFFSET_REG64), %xmm0, %xmm1
+	CMP_R1_S2_xmm (%xmm0, (%rsi, %OFFSET_REG64), %xmm2, %xmm1)
 	VPCMPEQ	%xmm0, %xmmZERO, %xmm2
 	vpandn	%xmm1, %xmm2, %xmm1
 	vpmovmskb %ymm1, %ecx
@@ -990,7 +1162,7 @@ L(less_16_till_page):
 	vmovq	(%rdi), %xmm0
 	vmovq	(%rsi), %xmm1
 	VPCMPEQ	%xmm0, %xmmZERO, %xmm2
-	VPCMPEQ	%xmm1, %xmm0, %xmm1
+	CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
 	vpandn	%xmm1, %xmm2, %xmm1
 	vpmovmskb %ymm1, %ecx
 	incb	%cl
@@ -1010,7 +1182,7 @@ L(less_16_till_page):
 	vmovq	(%rdi, %OFFSET_REG64), %xmm0
 	vmovq	(%rsi, %OFFSET_REG64), %xmm1
 	VPCMPEQ	%xmm0, %xmmZERO, %xmm2
-	VPCMPEQ	%xmm1, %xmm0, %xmm1
+	CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
 	vpandn	%xmm1, %xmm2, %xmm1
 	vpmovmskb %ymm1, %ecx
 	incb	%cl
@@ -1066,7 +1238,7 @@ L(ret_less_8_wcs):
 	vmovd	(%rdi), %xmm0
 	vmovd	(%rsi), %xmm1
 	VPCMPEQ	%xmm0, %xmmZERO, %xmm2
-	VPCMPEQ	%xmm1, %xmm0, %xmm1
+	CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
 	vpandn	%xmm1, %xmm2, %xmm1
 	vpmovmskb %ymm1, %ecx
 	subl	$0xf, %ecx
@@ -1085,7 +1257,7 @@ L(ret_less_8_wcs):
 	vmovd	(%rdi, %OFFSET_REG64), %xmm0
 	vmovd	(%rsi, %OFFSET_REG64), %xmm1
 	VPCMPEQ	%xmm0, %xmmZERO, %xmm2
-	VPCMPEQ	%xmm1, %xmm0, %xmm1
+	CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
 	vpandn	%xmm1, %xmm2, %xmm1
 	vpmovmskb %ymm1, %ecx
 	subl	$0xf, %ecx
@@ -1119,7 +1291,9 @@ L(less_4_till_page):
 L(less_4_loop):
 	movzbl	(%rdi), %eax
 	movzbl	(%rsi, %rdi), %ecx
-	subl	%ecx, %eax
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %BYTE_LOOP_REG)
+	subl	%BYTE_LOOP_REG, %eax
 	jnz	L(ret_less_4_loop)
 	testl	%ecx, %ecx
 	jz	L(ret_zero_4_loop)
@@ -1146,5 +1320,6 @@ L(ret_less_4_loop):
 	subl	%r8d, %eax
 	ret
 # endif
-END(STRCMP)
+	cfi_endproc
+	.size	STRCMP, .-STRCMP
 #endif
diff --git a/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S
new file mode 100644
index 0000000000..58c05dcfb8
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S
@@ -0,0 +1,16 @@
+#ifndef STRCMP
+# define STRCMP	__strncasecmp_l_avx2_rtm
+#endif
+
+#define _GLABEL(x)	x ## _rtm
+#define GLABEL(x)	_GLABEL(x)
+
+#define ZERO_UPPER_VEC_REGISTERS_RETURN	\
+	ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
+
+#define VZEROUPPER_RETURN	jmp L(return_vzeroupper)
+
+#define SECTION(p)	p##.avx.rtm
+#define OVERFLOW_STRCMP	__strcasecmp_l_avx2_rtm
+
+#include "strncase_l-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/strncase_l-avx2.S b/sysdeps/x86_64/multiarch/strncase_l-avx2.S
new file mode 100644
index 0000000000..48c0aa21f8
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/strncase_l-avx2.S
@@ -0,0 +1,27 @@
+/* strncasecmp_l optimized with AVX2.
+   Copyright (C) 2017-2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#ifndef STRCMP
+# define STRCMP	__strncasecmp_l_avx2
+#endif
+#define USE_AS_STRCASECMP_L
+#define USE_AS_STRNCMP
+#ifndef OVERFLOW_STRCMP
+# define OVERFLOW_STRCMP	__strcasecmp_l_avx2
+#endif
+#include "strcmp-avx2.S"
-- 
2.35.1


      parent reply	other threads:[~2022-05-12 19:52 UTC|newest]

Thread overview: 3+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
     [not found] <20220323215734.3927131-21-goldstein.w.n@gmail.com>
     [not found] ` <20220324235613.2445581-1-goldstein.w.n@gmail.com>
     [not found]   ` <20220324235613.2445581-2-goldstein.w.n@gmail.com>
     [not found]     ` <CAMe9rOp3wFrvLmM9PhtHWJis-i85QCrL17Oua8+xj=x2eSVxkg@mail.gmail.com>
     [not found]       ` <CAFUsyf+FsB7x-O-FB87vsxNF5XaXJ8+CxOr-sgaO+5PbXcH6qQ@mail.gmail.com>
2022-05-12 19:47         ` [PATCH v4 22/23] x86: Add EVEX " Sunil Pandey
2022-05-12 19:52           ` Sunil Pandey
     [not found]   ` <CAMe9rOqZ8h1dOUctD6jF3OmUv9TXjyA7_ser3p_6157myHq1Og@mail.gmail.com>
2022-05-12 19:52     ` Sunil Pandey [this message]

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to='CAMAf5_eOfzQsDwEK3FajW2YqwYpXz1=c44sYQ=CSz0-bjHTD_g@mail.gmail.com' \
    --to=skpgkp2@gmail.com \
    --cc=goldstein.w.n@gmail.com \
    --cc=hjl.tools@gmail.com \
    --cc=libc-alpha@sourceware.org \
    --cc=libc-stable@sourceware.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).