Re: [PATCH v2 5/7] x86: Optimize strcmp-avx2.S

[Noah Goldstein <goldstein.w.n@gmail.com>]

On Sun, Jan 9, 2022 at 6:41 PM H.J. Lu <hjl.tools@gmail.com> wrote:
>
> On Sun, Jan 9, 2022 at 4:31 PM Noah Goldstein via Libc-alpha
> <libc-alpha@sourceware.org> wrote:
> >
> > Optimization are primarily to the loop logic and how the page cross
> > logic interacts with the loop.
> >
> > The page cross logic is at times more expensive for short strings near
> > the end of a page but not crossing the page. This is done to retest
> > the page cross conditions with a non-faulty check and to improve the
> > logic for entering the loop afterwards. This is only particular cases,
> > however, and is general made up for by more than 10x improvements on
> > the transition from the page cross -> loop case.
> >
> > The non-page cross cases are improved most for smaller sizes [0, 128]
> > and go about even for (128, 4096]. The loop page cross logic is
> > improved so some more significant speedup is seen there as well.
> >
> > test-strcmp, test-strncmp, test-wcscmp, and test-wcsncmp all pass.
> >
> > Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
> > ---
> >  sysdeps/x86_64/multiarch/strcmp-avx2.S | 1590 ++++++++++++++----------
> >  1 file changed, 939 insertions(+), 651 deletions(-)
> >
> > diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
> > index 9c73b5899d..28d6a0025a 100644
> > --- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
> > +++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
> > @@ -26,35 +26,57 @@
> >
> >  # define PAGE_SIZE     4096
> >
> > -/* VEC_SIZE = Number of bytes in a ymm register */
> > +       /* VEC_SIZE = Number of bytes in a ymm register.  */
> >  # define VEC_SIZE      32
> >
> > -/* Shift for dividing by (VEC_SIZE * 4).  */
> > -# define DIVIDE_BY_VEC_4_SHIFT 7
> > -# if (VEC_SIZE * 4) != (1 << DIVIDE_BY_VEC_4_SHIFT)
> > -#  error (VEC_SIZE * 4) != (1 << DIVIDE_BY_VEC_4_SHIFT)
> > -# endif
> > +# define VMOVU vmovdqu
> > +# define VMOVA vmovdqa
> >
> >  # ifdef USE_AS_WCSCMP
> > -/* Compare packed dwords.  */
> > +       /* Compare packed dwords.  */
> >  #  define VPCMPEQ      vpcmpeqd
> > -/* Compare packed dwords and store minimum.  */
> > +       /* Compare packed dwords and store minimum.  */
> >  #  define VPMINU       vpminud
> > -/* 1 dword char == 4 bytes.  */
> > +       /* 1 dword char == 4 bytes.  */
> >  #  define SIZE_OF_CHAR 4
> >  # else
> > -/* Compare packed bytes.  */
> > +       /* Compare packed bytes.  */
> >  #  define VPCMPEQ      vpcmpeqb
> > -/* Compare packed bytes and store minimum.  */
> > +       /* Compare packed bytes and store minimum.  */
> >  #  define VPMINU       vpminub
> > -/* 1 byte char == 1 byte.  */
> > +       /* 1 byte char == 1 byte.  */
> >  #  define SIZE_OF_CHAR 1
> >  # endif
> >
> > +# ifdef USE_AS_STRNCMP
> > +#  define LOOP_REG     r9d
> > +#  define LOOP_REG64   r9
> > +
> > +#  define OFFSET_REG8  r9b
> > +#  define OFFSET_REG   r9d
> > +#  define OFFSET_REG64 r9
> > +# else
> > +#  define LOOP_REG     edx
> > +#  define LOOP_REG64   rdx
> > +
> > +#  define OFFSET_REG8  dl
> > +#  define OFFSET_REG   edx
> > +#  define OFFSET_REG64 rdx
> > +# endif
> > +
> >  # ifndef VZEROUPPER
> >  #  define VZEROUPPER   vzeroupper
> >  # endif
> >
> > +# if defined USE_AS_STRNCMP
> > +#  define VEC_OFFSET   0
> > +# else
> > +#  define VEC_OFFSET   (-VEC_SIZE)
> > +# endif
> > +
> > +# define xmmZERO       xmm15
> > +# define ymmZERO       ymm15
> > +
> >  # ifndef SECTION
> >  #  define SECTION(p)   p##.avx
> >  # endif
> > @@ -79,783 +101,1049 @@
> >     the maximum offset is reached before a difference is found, zero is
> >     returned.  */
> >
> > -       .section SECTION(.text),"ax",@progbits
> > -ENTRY (STRCMP)
> > +       .section SECTION(.text), "ax", @progbits
> > +ENTRY(STRCMP)
> >  # ifdef USE_AS_STRNCMP
> > -       /* Check for simple cases (0 or 1) in offset.  */
> > +#  ifdef __ILP32__
> > +       /* Clear the upper 32 bits.  */
> > +       movl    %edx, %rdx
> > +#  endif
> >         cmp     $1, %RDX_LP
> > -       je      L(char0)
> > -       jb      L(zero)
> > +       /* Signed comparison intentional. We use this branch to also
> > +          test cases where length >= 2^63. These very large sizes can be
> > +          handled with strcmp as there is no way for that length to
> > +          actually bound the buffer.  */
> > +       jle     L(one_or_less)
> >  #  ifdef USE_AS_WCSCMP
> > -#  ifndef __ILP32__
> >         movq    %rdx, %rcx
> > -       /* Check if length could overflow when multiplied by
> > -          sizeof(wchar_t). Checking top 8 bits will cover all potential
> > -          overflow cases as well as redirect cases where its impossible to
> > -          length to bound a valid memory region. In these cases just use
> > -          'wcscmp'.  */
> > +
> > +       /* Multiplying length by sizeof(wchar_t) can result in overflow.
> > +          Check if that is possible. All cases where overflow are possible
> > +          are cases where length is large enough that it can never be a
> > +          bound on valid memory so just use wcscmp.  */
> >         shrq    $56, %rcx
> >         jnz     __wcscmp_avx2
> > +
> > +       leaq    (, %rdx, 4), %rdx
> >  #  endif
> > -       /* Convert units: from wide to byte char.  */
> > -       shl     $2, %RDX_LP
> > -#  endif
> > -       /* Register %r11 tracks the maximum offset.  */
> > -       mov     %RDX_LP, %R11_LP
> >  # endif
> > +       vpxor   %xmmZERO, %xmmZERO, %xmmZERO
> >         movl    %edi, %eax
> > -       xorl    %edx, %edx
> > -       /* Make %xmm7 (%ymm7) all zeros in this function.  */
> > -       vpxor   %xmm7, %xmm7, %xmm7
> >         orl     %esi, %eax
> > -       andl    $(PAGE_SIZE - 1), %eax
> > -       cmpl    $(PAGE_SIZE - (VEC_SIZE * 4)), %eax
> > -       jg      L(cross_page)
> > -       /* Start comparing 4 vectors.  */
> > -       vmovdqu (%rdi), %ymm1
> > -       VPCMPEQ (%rsi), %ymm1, %ymm0
> > -       VPMINU  %ymm1, %ymm0, %ymm0
> > -       VPCMPEQ %ymm7, %ymm0, %ymm0
> > -       vpmovmskb %ymm0, %ecx
> > -       testl   %ecx, %ecx
> > -       je      L(next_3_vectors)
> > -       tzcntl  %ecx, %edx
> > +       sall    $20, %eax
> > +       /* Check if s1 or s2 may cross a page  in next 4x VEC loads.  */
> > +       cmpl    $((PAGE_SIZE -(VEC_SIZE * 4)) << 20), %eax
> > +       ja      L(page_cross)
> > +
> > +L(no_page_cross):
> > +       /* Safe to compare 4x vectors.  */
> > +       VMOVU   (%rdi), %ymm0
> > +       /* 1s where s1 and s2 equal.  */
> > +       VPCMPEQ (%rsi), %ymm0, %ymm1
> > +       /* 1s at null CHAR.  */
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       /* 1s where s1 and s2 equal AND not null CHAR.  */
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +
> > +       /* All 1s -> keep going, any 0s -> return.  */
> > +       vpmovmskb %ymm1, %ecx
> >  # ifdef USE_AS_STRNCMP
> > -       /* Return 0 if the mismatched index (%rdx) is after the maximum
> > -          offset (%r11).   */
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > +       cmpq    $VEC_SIZE, %rdx
> > +       jbe     L(vec_0_test_len)
> >  # endif
> > +
> > +       /* All 1s represents all equals. incl will overflow to zero in
> > +          all equals case. Otherwise 1s will carry until position of first
> > +          mismatch.  */
> > +       incl    %ecx
> > +       jz      L(more_3x_vec)
> > +
> > +       .p2align 4,, 4
> > +L(return_vec_0):
> > +       tzcntl  %ecx, %ecx
> >  # ifdef USE_AS_WCSCMP
> > +       movl    (%rdi, %rcx), %edx
> >         xorl    %eax, %eax
> > -       movl    (%rdi, %rdx), %ecx
> > -       cmpl    (%rsi, %rdx), %ecx
> > -       je      L(return)
> > -L(wcscmp_return):
> > +       cmpl    (%rsi, %rcx), %edx
> > +       je      L(ret0)
> >         setl    %al
> >         negl    %eax
> >         orl     $1, %eax
> > -L(return):
> >  # else
> > -       movzbl  (%rdi, %rdx), %eax
> > -       movzbl  (%rsi, %rdx), %edx
> > -       subl    %edx, %eax
> > +       movzbl  (%rdi, %rcx), %eax
> > +       movzbl  (%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> >  # endif
> > +L(ret0):
> >  L(return_vzeroupper):
> >         ZERO_UPPER_VEC_REGISTERS_RETURN
> >
> > -       .p2align 4
> > -L(return_vec_size):
> > -       tzcntl  %ecx, %edx
> >  # ifdef USE_AS_STRNCMP
> > -       /* Return 0 if the mismatched index (%rdx + VEC_SIZE) is after
> > -          the maximum offset (%r11).  */
> > -       addq    $VEC_SIZE, %rdx
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > +       .p2align 4,, 8
> > +L(vec_0_test_len):
> > +       notl    %ecx
> > +       bzhil   %edx, %ecx, %eax
> > +       jnz     L(return_vec_0)
> > +       /* Align if will cross fetch block.  */
> > +       .p2align 4,, 2
> > +L(ret_zero):
> >         xorl    %eax, %eax
> > -       movl    (%rdi, %rdx), %ecx
> > -       cmpl    (%rsi, %rdx), %ecx
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rdi, %rdx), %eax
> > -       movzbl  (%rsi, %rdx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > -# else
> > +       VZEROUPPER_RETURN
> > +
> > +       .p2align 4,, 5
> > +L(one_or_less):
> > +       jb      L(ret_zero)
> >  #  ifdef USE_AS_WCSCMP
> > +       /* 'nbe' covers the case where length is negative (large
> > +          unsigned).  */
> > +       jnbe    __wcscmp_avx2
> > +       movl    (%rdi), %edx
> >         xorl    %eax, %eax
> > -       movl    VEC_SIZE(%rdi, %rdx), %ecx
> > -       cmpl    VEC_SIZE(%rsi, %rdx), %ecx
> > -       jne     L(wcscmp_return)
> > +       cmpl    (%rsi), %edx
> > +       je      L(ret1)
> > +       setl    %al
> > +       negl    %eax
> > +       orl     $1, %eax
> >  #  else
> > -       movzbl  VEC_SIZE(%rdi, %rdx), %eax
> > -       movzbl  VEC_SIZE(%rsi, %rdx), %edx
> > -       subl    %edx, %eax
> > +       /* 'nbe' covers the case where length is negative (large
> > +          unsigned).  */
> > +
> > +       jnbe    __strcmp_avx2
> > +       movzbl  (%rdi), %eax
> > +       movzbl  (%rsi), %ecx
> > +       subl    %ecx, %eax
> >  #  endif
> > +L(ret1):
> > +       ret
> >  # endif
> > -       VZEROUPPER_RETURN
> >
> > -       .p2align 4
> > -L(return_2_vec_size):
> > -       tzcntl  %ecx, %edx
> > +       .p2align 4,, 10
> > +L(return_vec_1):
> > +       tzcntl  %ecx, %ecx
> >  # ifdef USE_AS_STRNCMP
> > -       /* Return 0 if the mismatched index (%rdx + 2 * VEC_SIZE) is
> > -          after the maximum offset (%r11).  */
> > -       addq    $(VEC_SIZE * 2), %rdx
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > +       /* rdx must be > CHAR_PER_VEC so save to subtract w.o fear of
> > +          overflow.  */
> > +       addq    $-VEC_SIZE, %rdx
> > +       cmpq    %rcx, %rdx
> > +       jbe     L(ret_zero)
> > +# endif
> > +# ifdef USE_AS_WCSCMP
> > +       movl    VEC_SIZE(%rdi, %rcx), %edx
> >         xorl    %eax, %eax
> > -       movl    (%rdi, %rdx), %ecx
> > -       cmpl    (%rsi, %rdx), %ecx
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rdi, %rdx), %eax
> > -       movzbl  (%rsi, %rdx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       cmpl    VEC_SIZE(%rsi, %rcx), %edx
> > +       je      L(ret2)
> > +       setl    %al
> > +       negl    %eax
> > +       orl     $1, %eax
> >  # else
> > -#  ifdef USE_AS_WCSCMP
> > -       xorl    %eax, %eax
> > -       movl    (VEC_SIZE * 2)(%rdi, %rdx), %ecx
> > -       cmpl    (VEC_SIZE * 2)(%rsi, %rdx), %ecx
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (VEC_SIZE * 2)(%rdi, %rdx), %eax
> > -       movzbl  (VEC_SIZE * 2)(%rsi, %rdx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       movzbl  VEC_SIZE(%rdi, %rcx), %eax
> > +       movzbl  VEC_SIZE(%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> >  # endif
> > +L(ret2):
> >         VZEROUPPER_RETURN
> >
> > -       .p2align 4
> > -L(return_3_vec_size):
> > -       tzcntl  %ecx, %edx
> > +       .p2align 4,, 10
> >  # ifdef USE_AS_STRNCMP
> > -       /* Return 0 if the mismatched index (%rdx + 3 * VEC_SIZE) is
> > -          after the maximum offset (%r11).  */
> > -       addq    $(VEC_SIZE * 3), %rdx
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > +L(return_vec_3):
> > +       salq    $32, %rcx
> > +# endif
> > +
> > +L(return_vec_2):
> > +# ifndef USE_AS_STRNCMP
> > +       tzcntl  %ecx, %ecx
> > +# else
> > +       tzcntq  %rcx, %rcx
> > +       cmpq    %rcx, %rdx
> > +       jbe     L(ret_zero)
> > +# endif
> > +
> > +# ifdef USE_AS_WCSCMP
> > +       movl    (VEC_SIZE * 2)(%rdi, %rcx), %edx
> >         xorl    %eax, %eax
> > -       movl    (%rdi, %rdx), %ecx
> > -       cmpl    (%rsi, %rdx), %ecx
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rdi, %rdx), %eax
> > -       movzbl  (%rsi, %rdx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       cmpl    (VEC_SIZE * 2)(%rsi, %rcx), %edx
> > +       je      L(ret3)
> > +       setl    %al
> > +       negl    %eax
> > +       orl     $1, %eax
> >  # else
> > +       movzbl  (VEC_SIZE * 2)(%rdi, %rcx), %eax
> > +       movzbl  (VEC_SIZE * 2)(%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> > +# endif
> > +L(ret3):
> > +       VZEROUPPER_RETURN
> > +
> > +# ifndef USE_AS_STRNCMP
> > +       .p2align 4,, 10
> > +L(return_vec_3):
> > +       tzcntl  %ecx, %ecx
> >  #  ifdef USE_AS_WCSCMP
> > +       movl    (VEC_SIZE * 3)(%rdi, %rcx), %edx
> >         xorl    %eax, %eax
> > -       movl    (VEC_SIZE * 3)(%rdi, %rdx), %ecx
> > -       cmpl    (VEC_SIZE * 3)(%rsi, %rdx), %ecx
> > -       jne     L(wcscmp_return)
> > +       cmpl    (VEC_SIZE * 3)(%rsi, %rcx), %edx
> > +       je      L(ret4)
> > +       setl    %al
> > +       negl    %eax
> > +       orl     $1, %eax
> >  #  else
> > -       movzbl  (VEC_SIZE * 3)(%rdi, %rdx), %eax
> > -       movzbl  (VEC_SIZE * 3)(%rsi, %rdx), %edx
> > -       subl    %edx, %eax
> > +       movzbl  (VEC_SIZE * 3)(%rdi, %rcx), %eax
> > +       movzbl  (VEC_SIZE * 3)(%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> >  #  endif
> > -# endif
> > +L(ret4):
> >         VZEROUPPER_RETURN
> > +# endif
> > +
> > +       .p2align 4,, 10
> > +L(more_3x_vec):
> > +       /* Safe to compare 4x vectors.  */
> > +       VMOVU   VEC_SIZE(%rdi), %ymm0
> > +       VPCMPEQ VEC_SIZE(%rsi), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incl    %ecx
> > +       jnz     L(return_vec_1)
> > +
> > +# ifdef USE_AS_STRNCMP
> > +       subq    $(VEC_SIZE * 2), %rdx
> > +       jbe     L(ret_zero)
> > +# endif
> > +
> > +       VMOVU   (VEC_SIZE * 2)(%rdi), %ymm0
> > +       VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incl    %ecx
> > +       jnz     L(return_vec_2)
> > +
> > +       VMOVU   (VEC_SIZE * 3)(%rdi), %ymm0
> > +       VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incl    %ecx
> > +       jnz     L(return_vec_3)
> >
> > -       .p2align 4
> > -L(next_3_vectors):
> > -       vmovdqu VEC_SIZE(%rdi), %ymm6
> > -       VPCMPEQ VEC_SIZE(%rsi), %ymm6, %ymm3
> > -       VPMINU  %ymm6, %ymm3, %ymm3
> > -       VPCMPEQ %ymm7, %ymm3, %ymm3
> > -       vpmovmskb %ymm3, %ecx
> > -       testl   %ecx, %ecx
> > -       jne     L(return_vec_size)
> > -       vmovdqu (VEC_SIZE * 2)(%rdi), %ymm5
> > -       vmovdqu (VEC_SIZE * 3)(%rdi), %ymm4
> > -       vmovdqu (VEC_SIZE * 3)(%rsi), %ymm0
> > -       VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm5, %ymm2
> > -       VPMINU  %ymm5, %ymm2, %ymm2
> > -       VPCMPEQ %ymm4, %ymm0, %ymm0
> > -       VPCMPEQ %ymm7, %ymm2, %ymm2
> > -       vpmovmskb %ymm2, %ecx
> > -       testl   %ecx, %ecx
> > -       jne     L(return_2_vec_size)
> > -       VPMINU  %ymm4, %ymm0, %ymm0
> > -       VPCMPEQ %ymm7, %ymm0, %ymm0
> > -       vpmovmskb %ymm0, %ecx
> > -       testl   %ecx, %ecx
> > -       jne     L(return_3_vec_size)
> > -L(main_loop_header):
> > -       leaq    (VEC_SIZE * 4)(%rdi), %rdx
> > -       movl    $PAGE_SIZE, %ecx
> > -       /* Align load via RAX.  */
> > -       andq    $-(VEC_SIZE * 4), %rdx
> > -       subq    %rdi, %rdx
> > -       leaq    (%rdi, %rdx), %rax
> >  # ifdef USE_AS_STRNCMP
> > -       /* Starting from this point, the maximum offset, or simply the
> > -          'offset', DECREASES by the same amount when base pointers are
> > -          moved forward.  Return 0 when:
> > -            1) On match: offset <= the matched vector index.
> > -            2) On mistmach, offset is before the mistmatched index.
> > +       cmpq    $(VEC_SIZE * 2), %rdx
> > +       jbe     L(ret_zero)
> > +# endif
> > +
> > +# ifdef USE_AS_WCSCMP
> > +       /* any non-zero positive value that doesn't inference with 0x1.
> >          */
> > -       subq    %rdx, %r11
> > -       jbe     L(zero)
> > -# endif
> > -       addq    %rsi, %rdx
> > -       movq    %rdx, %rsi
> > -       andl    $(PAGE_SIZE - 1), %esi
> > -       /* Number of bytes before page crossing.  */
> > -       subq    %rsi, %rcx
> > -       /* Number of VEC_SIZE * 4 blocks before page crossing.  */
> > -       shrq    $DIVIDE_BY_VEC_4_SHIFT, %rcx
> > -       /* ESI: Number of VEC_SIZE * 4 blocks before page crossing.   */
> > -       movl    %ecx, %esi
> > -       jmp     L(loop_start)
> > +       movl    $2, %r8d
> >
> > +# else
> > +       xorl    %r8d, %r8d
> > +# endif
> > +
> > +       /* The prepare labels are various entry points from the page
> > +          cross logic.  */
> > +L(prepare_loop):
> > +
> > +# ifdef USE_AS_STRNCMP
> > +       /* Store N + (VEC_SIZE * 4) and place check at the begining of
> > +          the loop.  */
> > +       leaq    (VEC_SIZE * 2)(%rdi, %rdx), %rdx
> > +# endif
> > +L(prepare_loop_no_len):
> > +
> > +       /* Align s1 and adjust s2 accordingly.  */
> > +       subq    %rdi, %rsi
> > +       andq    $-(VEC_SIZE * 4), %rdi
> > +       addq    %rdi, %rsi
> > +
> > +# ifdef USE_AS_STRNCMP
> > +       subq    %rdi, %rdx
> > +# endif
> > +
> > +L(prepare_loop_aligned):
> > +       /* eax stores distance from rsi to next page cross. These cases
> > +          need to be handled specially as the 4x loop could potentially
> > +          read memory past the length of s1 or s2 and across a page
> > +          boundary.  */
> > +       movl    $-(VEC_SIZE * 4), %eax
> > +       subl    %esi, %eax
> > +       andl    $(PAGE_SIZE - 1), %eax
> > +
> > +       /* Loop 4x comparisons at a time.  */
> >         .p2align 4
> >  L(loop):
> > +
> > +       /* End condition for strncmp.  */
> >  # ifdef USE_AS_STRNCMP
> > -       /* Base pointers are moved forward by 4 * VEC_SIZE.  Decrease
> > -          the maximum offset (%r11) by the same amount.  */
> > -       subq    $(VEC_SIZE * 4), %r11
> > -       jbe     L(zero)
> > -# endif
> > -       addq    $(VEC_SIZE * 4), %rax
> > -       addq    $(VEC_SIZE * 4), %rdx
> > -L(loop_start):
> > -       testl   %esi, %esi
> > -       leal    -1(%esi), %esi
> > -       je      L(loop_cross_page)
> > -L(back_to_loop):
> > -       /* Main loop, comparing 4 vectors are a time.  */
> > -       vmovdqa (%rax), %ymm0
> > -       vmovdqa VEC_SIZE(%rax), %ymm3
> > -       VPCMPEQ (%rdx), %ymm0, %ymm4
> > -       VPCMPEQ VEC_SIZE(%rdx), %ymm3, %ymm1
> > -       VPMINU  %ymm0, %ymm4, %ymm4
> > -       VPMINU  %ymm3, %ymm1, %ymm1
> > -       vmovdqa (VEC_SIZE * 2)(%rax), %ymm2
> > -       VPMINU  %ymm1, %ymm4, %ymm0
> > -       vmovdqa (VEC_SIZE * 3)(%rax), %ymm3
> > -       VPCMPEQ (VEC_SIZE * 2)(%rdx), %ymm2, %ymm5
> > -       VPCMPEQ (VEC_SIZE * 3)(%rdx), %ymm3, %ymm6
> > -       VPMINU  %ymm2, %ymm5, %ymm5
> > -       VPMINU  %ymm3, %ymm6, %ymm6
> > -       VPMINU  %ymm5, %ymm0, %ymm0
> > -       VPMINU  %ymm6, %ymm0, %ymm0
> > -       VPCMPEQ %ymm7, %ymm0, %ymm0
> > -
> > -       /* Test each mask (32 bits) individually because for VEC_SIZE
> > -          == 32 is not possible to OR the four masks and keep all bits
> > -          in a 64-bit integer register, differing from SSE2 strcmp
> > -          where ORing is possible.  */
> > -       vpmovmskb %ymm0, %ecx
> > +       subq    $(VEC_SIZE * 4), %rdx
> > +       jbe     L(ret_zero)
> > +# endif
> > +
> > +       subq    $-(VEC_SIZE * 4), %rdi
> > +       subq    $-(VEC_SIZE * 4), %rsi
> > +
> > +       /* Check if rsi loads will cross a page boundary.  */
> > +       addl    $-(VEC_SIZE * 4), %eax
> > +       jnb     L(page_cross_during_loop)
> > +
> > +       /* Loop entry after handling page cross during loop.  */
> > +L(loop_skip_page_cross_check):
> > +       VMOVA   (VEC_SIZE * 0)(%rdi), %ymm0
> > +       VMOVA   (VEC_SIZE * 1)(%rdi), %ymm2
> > +       VMOVA   (VEC_SIZE * 2)(%rdi), %ymm4
> > +       VMOVA   (VEC_SIZE * 3)(%rdi), %ymm6
> > +
> > +       /* ymm1 all 1s where s1 and s2 equal. All 0s otherwise.  */
> > +       VPCMPEQ (VEC_SIZE * 0)(%rsi), %ymm0, %ymm1
> > +
> > +       VPCMPEQ (VEC_SIZE * 1)(%rsi), %ymm2, %ymm3
> > +       VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm4, %ymm5
> > +       VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm6, %ymm7
> > +
> > +
> > +       /* If any mismatches or null CHAR then 0 CHAR, otherwise non-
> > +          zero.  */
> > +       vpand   %ymm0, %ymm1, %ymm1
> > +
> > +
> > +       vpand   %ymm2, %ymm3, %ymm3
> > +       vpand   %ymm4, %ymm5, %ymm5
> > +       vpand   %ymm6, %ymm7, %ymm7
> > +
> > +       VPMINU  %ymm1, %ymm3, %ymm3
> > +       VPMINU  %ymm5, %ymm7, %ymm7
> > +
> > +       /* Reduce all 0 CHARs for the 4x VEC into ymm7.  */
> > +       VPMINU  %ymm3, %ymm7, %ymm7
> > +
> > +       /* If any 0 CHAR then done.  */
> > +       VPCMPEQ %ymm7, %ymmZERO, %ymm7
> > +       vpmovmskb %ymm7, %LOOP_REG
> > +       testl   %LOOP_REG, %LOOP_REG
> > +       jz      L(loop)
> > +
> > +       /* Find which VEC has the mismatch of end of string.  */
> > +       VPCMPEQ %ymm1, %ymmZERO, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> >         testl   %ecx, %ecx
> > -       je      L(loop)
> > -       VPCMPEQ %ymm7, %ymm4, %ymm0
> > -       vpmovmskb %ymm0, %edi
> > -       testl   %edi, %edi
> > -       je      L(test_vec)
> > -       tzcntl  %edi, %ecx
> > +       jnz     L(return_vec_0_end)
> > +
> > +
> > +       VPCMPEQ %ymm3, %ymmZERO, %ymm3
> > +       vpmovmskb %ymm3, %ecx
> > +       testl   %ecx, %ecx
> > +       jnz     L(return_vec_1_end)
> > +
> > +L(return_vec_2_3_end):
> >  # ifdef USE_AS_STRNCMP
> > -       cmpq    %rcx, %r11
> > -       jbe     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > +       subq    $(VEC_SIZE * 2), %rdx
> > +       jbe     L(ret_zero_end)
> > +# endif
> > +
> > +       VPCMPEQ %ymm5, %ymmZERO, %ymm5
> > +       vpmovmskb %ymm5, %ecx
> > +       testl   %ecx, %ecx
> > +       jnz     L(return_vec_2_end)
> > +
> > +       /* LOOP_REG contains matches for null/mismatch from the loop. If
> > +          VEC 0,1,and 2 all have no null and no mismatches then mismatch
> > +          must entirely be from VEC 3 which is fully represented by
> > +          LOOP_REG.  */
> > +       tzcntl  %LOOP_REG, %LOOP_REG
> > +
> > +# ifdef USE_AS_STRNCMP
> > +       subl    $-(VEC_SIZE), %LOOP_REG
> > +       cmpq    %LOOP_REG64, %rdx
> > +       jbe     L(ret_zero_end)
> > +# endif
> > +
> > +# ifdef USE_AS_WCSCMP
> > +       movl    (VEC_SIZE * 2 - VEC_OFFSET)(%rdi, %LOOP_REG64), %ecx
> >         xorl    %eax, %eax
> > -       movl    (%rsi, %rcx), %edi
> > -       cmpl    (%rdx, %rcx), %edi
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rax, %rcx), %eax
> > -       movzbl  (%rdx, %rcx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       cmpl    (VEC_SIZE * 2 - VEC_OFFSET)(%rsi, %LOOP_REG64), %ecx
> > +       je      L(ret5)
> > +       setl    %al
> > +       negl    %eax
> > +       xorl    %r8d, %eax
> >  # else
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > -       xorl    %eax, %eax
> > -       movl    (%rsi, %rcx), %edi
> > -       cmpl    (%rdx, %rcx), %edi
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rax, %rcx), %eax
> > -       movzbl  (%rdx, %rcx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       movzbl  (VEC_SIZE * 2 - VEC_OFFSET)(%rdi, %LOOP_REG64), %eax
> > +       movzbl  (VEC_SIZE * 2 - VEC_OFFSET)(%rsi, %LOOP_REG64), %ecx
> > +       subl    %ecx, %eax
> > +       xorl    %r8d, %eax
> > +       subl    %r8d, %eax
> >  # endif
> > +L(ret5):
> >         VZEROUPPER_RETURN
> >
> > -       .p2align 4
> > -L(test_vec):
> >  # ifdef USE_AS_STRNCMP
> > -       /* The first vector matched.  Return 0 if the maximum offset
> > -          (%r11) <= VEC_SIZE.  */
> > -       cmpq    $VEC_SIZE, %r11
> > -       jbe     L(zero)
> > +       .p2align 4,, 2
> > +L(ret_zero_end):
> > +       xorl    %eax, %eax
> > +       VZEROUPPER_RETURN
> >  # endif
> > -       VPCMPEQ %ymm7, %ymm1, %ymm1
> > -       vpmovmskb %ymm1, %ecx
> > -       testl   %ecx, %ecx
> > -       je      L(test_2_vec)
> > -       tzcntl  %ecx, %edi
> > +
> > +
> > +       /* The L(return_vec_N_end) differ from L(return_vec_N) in that
> > +          they use the value of `r8` to negate the return value. This is
> > +          because the page cross logic can swap `rdi` and `rsi`.  */
> > +       .p2align 4,, 10
> >  # ifdef USE_AS_STRNCMP
> > -       addq    $VEC_SIZE, %rdi
> > -       cmpq    %rdi, %r11
> > -       jbe     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > +L(return_vec_1_end):
> > +       salq    $32, %rcx
> > +# endif
> > +L(return_vec_0_end):
> > +# ifndef USE_AS_STRNCMP
> > +       tzcntl  %ecx, %ecx
> > +# else
> > +       tzcntq  %rcx, %rcx
> > +       cmpq    %rcx, %rdx
> > +       jbe     L(ret_zero_end)
> > +# endif
> > +
> > +# ifdef USE_AS_WCSCMP
> > +       movl    (%rdi, %rcx), %edx
> >         xorl    %eax, %eax
> > -       movl    (%rsi, %rdi), %ecx
> > -       cmpl    (%rdx, %rdi), %ecx
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rax, %rdi), %eax
> > -       movzbl  (%rdx, %rdi), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       cmpl    (%rsi, %rcx), %edx
> > +       je      L(ret6)
> > +       setl    %al
> > +       negl    %eax
> > +       xorl    %r8d, %eax
> >  # else
> > +       movzbl  (%rdi, %rcx), %eax
> > +       movzbl  (%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> > +       xorl    %r8d, %eax
> > +       subl    %r8d, %eax
> > +# endif
> > +L(ret6):
> > +       VZEROUPPER_RETURN
> > +
> > +# ifndef USE_AS_STRNCMP
> > +       .p2align 4,, 10
> > +L(return_vec_1_end):
> > +       tzcntl  %ecx, %ecx
> >  #  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > +       movl    VEC_SIZE(%rdi, %rcx), %edx
> >         xorl    %eax, %eax
> > -       movl    VEC_SIZE(%rsi, %rdi), %ecx
> > -       cmpl    VEC_SIZE(%rdx, %rdi), %ecx
> > -       jne     L(wcscmp_return)
> > +       cmpl    VEC_SIZE(%rsi, %rcx), %edx
> > +       je      L(ret7)
> > +       setl    %al
> > +       negl    %eax
> > +       xorl    %r8d, %eax
> >  #  else
> > -       movzbl  VEC_SIZE(%rax, %rdi), %eax
> > -       movzbl  VEC_SIZE(%rdx, %rdi), %edx
> > -       subl    %edx, %eax
> > +       movzbl  VEC_SIZE(%rdi, %rcx), %eax
> > +       movzbl  VEC_SIZE(%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> > +       xorl    %r8d, %eax
> > +       subl    %r8d, %eax
> >  #  endif
> > -# endif
> > +L(ret7):
> >         VZEROUPPER_RETURN
> > +# endif
> >
> > -       .p2align 4
> > -L(test_2_vec):
> > +       .p2align 4,, 10
> > +L(return_vec_2_end):
> > +       tzcntl  %ecx, %ecx
> >  # ifdef USE_AS_STRNCMP
> > -       /* The first 2 vectors matched.  Return 0 if the maximum offset
> > -          (%r11) <= 2 * VEC_SIZE.  */
> > -       cmpq    $(VEC_SIZE * 2), %r11
> > -       jbe     L(zero)
> > +       cmpq    %rcx, %rdx
> > +       jbe     L(ret_zero_page_cross)
> >  # endif
> > -       VPCMPEQ %ymm7, %ymm5, %ymm5
> > -       vpmovmskb %ymm5, %ecx
> > -       testl   %ecx, %ecx
> > -       je      L(test_3_vec)
> > -       tzcntl  %ecx, %edi
> > -# ifdef USE_AS_STRNCMP
> > -       addq    $(VEC_SIZE * 2), %rdi
> > -       cmpq    %rdi, %r11
> > -       jbe     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > +# ifdef USE_AS_WCSCMP
> > +       movl    (VEC_SIZE * 2)(%rdi, %rcx), %edx
> >         xorl    %eax, %eax
> > -       movl    (%rsi, %rdi), %ecx
> > -       cmpl    (%rdx, %rdi), %ecx
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rax, %rdi), %eax
> > -       movzbl  (%rdx, %rdi), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       cmpl    (VEC_SIZE * 2)(%rsi, %rcx), %edx
> > +       je      L(ret11)
> > +       setl    %al
> > +       negl    %eax
> > +       xorl    %r8d, %eax
> >  # else
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > -       xorl    %eax, %eax
> > -       movl    (VEC_SIZE * 2)(%rsi, %rdi), %ecx
> > -       cmpl    (VEC_SIZE * 2)(%rdx, %rdi), %ecx
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (VEC_SIZE * 2)(%rax, %rdi), %eax
> > -       movzbl  (VEC_SIZE * 2)(%rdx, %rdi), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       movzbl  (VEC_SIZE * 2)(%rdi, %rcx), %eax
> > +       movzbl  (VEC_SIZE * 2)(%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> > +       xorl    %r8d, %eax
> > +       subl    %r8d, %eax
> >  # endif
> > +L(ret11):
> >         VZEROUPPER_RETURN
> >
> > -       .p2align 4
> > -L(test_3_vec):
> > +
> > +       /* Page cross in rsi in next 4x VEC.  */
> > +
> > +       /* TODO: Improve logic here.  */
> > +       .p2align 4,, 10
> > +L(page_cross_during_loop):
> > +       /* eax contains [distance_from_page - (VEC_SIZE * 4)].  */
> > +
> > +       /* Optimistically rsi and rdi and both aligned inwhich case we
> > +          don't need any logic here.  */
> > +       cmpl    $-(VEC_SIZE * 4), %eax
> > +       /* Don't adjust eax before jumping back to loop and we will
> > +          never hit page cross case again.  */
> > +       je      L(loop_skip_page_cross_check)
> > +
> > +       /* Check if we can safely load a VEC.  */
> > +       cmpl    $-(VEC_SIZE * 3), %eax
> > +       jle     L(less_1x_vec_till_page_cross)
> > +
> > +       VMOVA   (%rdi), %ymm0
> > +       VPCMPEQ (%rsi), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incl    %ecx
> > +       jnz     L(return_vec_0_end)
> > +
> > +       /* if distance >= 2x VEC then eax > -(VEC_SIZE * 2).  */
> > +       cmpl    $-(VEC_SIZE * 2), %eax
> > +       jg      L(more_2x_vec_till_page_cross)
> > +
> > +       .p2align 4,, 4
> > +L(less_1x_vec_till_page_cross):
> > +       subl    $-(VEC_SIZE * 4), %eax
> > +       /* Guranteed safe to read from rdi - VEC_SIZE here. The only
> > +          concerning case is first iteration if incoming s1 was near start
> > +          of a page and s2 near end. If s1 was near the start of the page
> > +          we already aligned up to nearest VEC_SIZE * 4 so gurnateed safe
> > +          to read back -VEC_SIZE. If rdi is truly at the start of a page
> > +          here, it means the previous page (rdi - VEC_SIZE) has already
> > +          been loaded earlier so must be valid.  */
> > +       VMOVU   -VEC_SIZE(%rdi, %rax), %ymm0
> > +       VPCMPEQ -VEC_SIZE(%rsi, %rax), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +
> > +       /* Mask of potentially valid bits. The lower bits can be out of
> > +          range comparisons (but safe regarding page crosses).  */
> > +       movl    $-1, %r10d
> > +       shlxl   %esi, %r10d, %r10d
> > +       notl    %ecx
> > +
> >  # ifdef USE_AS_STRNCMP
> > -       /* The first 3 vectors matched.  Return 0 if the maximum offset
> > -          (%r11) <= 3 * VEC_SIZE.  */
> > -       cmpq    $(VEC_SIZE * 3), %r11
> > -       jbe     L(zero)
> > -# endif
> > -       VPCMPEQ %ymm7, %ymm6, %ymm6
> > -       vpmovmskb %ymm6, %esi
> > -       tzcntl  %esi, %ecx
> > +       cmpq    %rax, %rdx
> > +       jbe     L(return_page_cross_end_check)
> > +# endif
> > +       movl    %eax, %OFFSET_REG
> > +       addl    $(PAGE_SIZE - VEC_SIZE * 4), %eax
> > +
> > +       andl    %r10d, %ecx
> > +       jz      L(loop_skip_page_cross_check)
> > +
> > +       .p2align 4,, 3
> > +L(return_page_cross_end):
> > +       tzcntl  %ecx, %ecx
> > +
> >  # ifdef USE_AS_STRNCMP
> > -       addq    $(VEC_SIZE * 3), %rcx
> > -       cmpq    %rcx, %r11
> > -       jbe     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > -       xorl    %eax, %eax
> > -       movl    (%rsi, %rcx), %esi
> > -       cmpl    (%rdx, %rcx), %esi
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rax, %rcx), %eax
> > -       movzbl  (%rdx, %rcx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       leal    -VEC_SIZE(%OFFSET_REG64, %rcx), %ecx
> > +L(return_page_cross_cmp_mem):
> >  # else
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > +       addl    %OFFSET_REG, %ecx
> > +# endif
> > +# ifdef USE_AS_WCSCMP
> > +       movl    VEC_OFFSET(%rdi, %rcx), %edx
> >         xorl    %eax, %eax
> > -       movl    (VEC_SIZE * 3)(%rsi, %rcx), %esi
> > -       cmpl    (VEC_SIZE * 3)(%rdx, %rcx), %esi
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (VEC_SIZE * 3)(%rax, %rcx), %eax
> > -       movzbl  (VEC_SIZE * 3)(%rdx, %rcx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       cmpl    VEC_OFFSET(%rsi, %rcx), %edx
> > +       je      L(ret8)
> > +       setl    %al
> > +       negl    %eax
> > +       xorl    %r8d, %eax
> > +# else
> > +       movzbl  VEC_OFFSET(%rdi, %rcx), %eax
> > +       movzbl  VEC_OFFSET(%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> > +       xorl    %r8d, %eax
> > +       subl    %r8d, %eax
> >  # endif
> > +L(ret8):
> >         VZEROUPPER_RETURN
> >
> > -       .p2align 4
> > -L(loop_cross_page):
> > -       xorl    %r10d, %r10d
> > -       movq    %rdx, %rcx
> > -       /* Align load via RDX.  We load the extra ECX bytes which should
> > -          be ignored.  */
> > -       andl    $((VEC_SIZE * 4) - 1), %ecx
> > -       /* R10 is -RCX.  */
> > -       subq    %rcx, %r10
> > -
> > -       /* This works only if VEC_SIZE * 2 == 64. */
> > -# if (VEC_SIZE * 2) != 64
> > -#  error (VEC_SIZE * 2) != 64
> > -# endif
> > -
> > -       /* Check if the first VEC_SIZE * 2 bytes should be ignored.  */
> > -       cmpl    $(VEC_SIZE * 2), %ecx
> > -       jge     L(loop_cross_page_2_vec)
> > -
> > -       vmovdqu (%rax, %r10), %ymm2
> > -       vmovdqu VEC_SIZE(%rax, %r10), %ymm3
> > -       VPCMPEQ (%rdx, %r10), %ymm2, %ymm0
> > -       VPCMPEQ VEC_SIZE(%rdx, %r10), %ymm3, %ymm1
> > -       VPMINU  %ymm2, %ymm0, %ymm0
> > -       VPMINU  %ymm3, %ymm1, %ymm1
> > -       VPCMPEQ %ymm7, %ymm0, %ymm0
> > -       VPCMPEQ %ymm7, %ymm1, %ymm1
> > -
> > -       vpmovmskb %ymm0, %edi
> > -       vpmovmskb %ymm1, %esi
> > -
> > -       salq    $32, %rsi
> > -       xorq    %rsi, %rdi
> > -
> > -       /* Since ECX < VEC_SIZE * 2, simply skip the first ECX bytes.  */
> > -       shrq    %cl, %rdi
> > -
> > -       testq   %rdi, %rdi
> > -       je      L(loop_cross_page_2_vec)
> > -       tzcntq  %rdi, %rcx
> >  # ifdef USE_AS_STRNCMP
> > -       cmpq    %rcx, %r11
> > -       jbe     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > +       .p2align 4,, 10
> > +L(return_page_cross_end_check):
> > +       tzcntl  %ecx, %ecx
> > +       leal    -VEC_SIZE(%rax, %rcx), %ecx
> > +       cmpl    %ecx, %edx
> > +       ja      L(return_page_cross_cmp_mem)
> >         xorl    %eax, %eax
> > -       movl    (%rsi, %rcx), %edi
> > -       cmpl    (%rdx, %rcx), %edi
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rax, %rcx), %eax
> > -       movzbl  (%rdx, %rcx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > -# else
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > -       xorl    %eax, %eax
> > -       movl    (%rsi, %rcx), %edi
> > -       cmpl    (%rdx, %rcx), %edi
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rax, %rcx), %eax
> > -       movzbl  (%rdx, %rcx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > -# endif
> >         VZEROUPPER_RETURN
> > +# endif
> >
> > -       .p2align 4
> > -L(loop_cross_page_2_vec):
> > -       /* The first VEC_SIZE * 2 bytes match or are ignored.  */
> > -       vmovdqu (VEC_SIZE * 2)(%rax, %r10), %ymm2
> > -       vmovdqu (VEC_SIZE * 3)(%rax, %r10), %ymm3
> > -       VPCMPEQ (VEC_SIZE * 2)(%rdx, %r10), %ymm2, %ymm5
> > -       VPMINU  %ymm2, %ymm5, %ymm5
> > -       VPCMPEQ (VEC_SIZE * 3)(%rdx, %r10), %ymm3, %ymm6
> > -       VPCMPEQ %ymm7, %ymm5, %ymm5
> > -       VPMINU  %ymm3, %ymm6, %ymm6
> > -       VPCMPEQ %ymm7, %ymm6, %ymm6
> > -
> > -       vpmovmskb %ymm5, %edi
> > -       vpmovmskb %ymm6, %esi
> > -
> > -       salq    $32, %rsi
> > -       xorq    %rsi, %rdi
> >
> > -       xorl    %r8d, %r8d
> > -       /* If ECX > VEC_SIZE * 2, skip ECX - (VEC_SIZE * 2) bytes.  */
> > -       subl    $(VEC_SIZE * 2), %ecx
> > -       jle     1f
> > -       /* Skip ECX bytes.  */
> > -       shrq    %cl, %rdi
> > -       /* R8 has number of bytes skipped.  */
> > -       movl    %ecx, %r8d
> > -1:
> > -       /* Before jumping back to the loop, set ESI to the number of
> > -          VEC_SIZE * 4 blocks before page crossing.  */
> > -       movl    $(PAGE_SIZE / (VEC_SIZE * 4) - 1), %esi
> > -
> > -       testq   %rdi, %rdi
> > +       .p2align 4,, 10
> > +L(more_2x_vec_till_page_cross):
> > +       /* If more 2x vec till cross we will complete a full loop
> > +          iteration here.  */
> > +
> > +       VMOVU   VEC_SIZE(%rdi), %ymm0
> > +       VPCMPEQ VEC_SIZE(%rsi), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incl    %ecx
> > +       jnz     L(return_vec_1_end)
> > +
> >  # ifdef USE_AS_STRNCMP
> > -       /* At this point, if %rdi value is 0, it already tested
> > -          VEC_SIZE*4+%r10 byte starting from %rax. This label
> > -          checks whether strncmp maximum offset reached or not.  */
> > -       je      L(string_nbyte_offset_check)
> > -# else
> > -       je      L(back_to_loop)
> > +       cmpq    $(VEC_SIZE * 2), %rdx
> > +       jbe     L(ret_zero_in_loop_page_cross)
> >  # endif
> > -       tzcntq  %rdi, %rcx
> > -       addq    %r10, %rcx
> > -       /* Adjust for number of bytes skipped.  */
> > -       addq    %r8, %rcx
> > +
> > +       subl    $-(VEC_SIZE * 4), %eax
> > +
> > +       /* Safe to include comparisons from lower bytes.  */
> > +       VMOVU   -(VEC_SIZE * 2)(%rdi, %rax), %ymm0
> > +       VPCMPEQ -(VEC_SIZE * 2)(%rsi, %rax), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incl    %ecx
> > +       jnz     L(return_vec_page_cross_0)
> > +
> > +       VMOVU   -(VEC_SIZE * 1)(%rdi, %rax), %ymm0
> > +       VPCMPEQ -(VEC_SIZE * 1)(%rsi, %rax), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incl    %ecx
> > +       jnz     L(return_vec_page_cross_1)
> > +
> >  # ifdef USE_AS_STRNCMP
> > -       addq    $(VEC_SIZE * 2), %rcx
> > -       subq    %rcx, %r11
> > -       jbe     L(zero)
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > +       /* Must check length here as length might proclude reading next
> > +          page.  */
> > +       cmpq    %rax, %rdx
> > +       jbe     L(ret_zero_in_loop_page_cross)
> > +# endif
> > +
> > +       /* Finish the loop.  */
> > +       VMOVA   (VEC_SIZE * 2)(%rdi), %ymm4
> > +       VMOVA   (VEC_SIZE * 3)(%rdi), %ymm6
> > +
> > +       VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm4, %ymm5
> > +       VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm6, %ymm7
> > +       vpand   %ymm4, %ymm5, %ymm5
> > +       vpand   %ymm6, %ymm7, %ymm7
> > +       VPMINU  %ymm5, %ymm7, %ymm7
> > +       VPCMPEQ %ymm7, %ymmZERO, %ymm7
> > +       vpmovmskb %ymm7, %LOOP_REG
> > +       testl   %LOOP_REG, %LOOP_REG
> > +       jnz     L(return_vec_2_3_end)
> > +
> > +       /* Best for code size to include ucond-jmp here. Would be faster
> > +          if this case is hot to duplicate the L(return_vec_2_3_end) code
> > +          as fall-through and have jump back to loop on mismatch
> > +          comparison.  */
> > +       subq    $-(VEC_SIZE * 4), %rdi
> > +       subq    $-(VEC_SIZE * 4), %rsi
> > +       addl    $(PAGE_SIZE - VEC_SIZE * 8), %eax
> > +# ifdef USE_AS_STRNCMP
> > +       subq    $(VEC_SIZE * 4), %rdx
> > +       ja      L(loop_skip_page_cross_check)
> > +L(ret_zero_in_loop_page_cross):
> >         xorl    %eax, %eax
> > -       movl    (%rsi, %rcx), %edi
> > -       cmpl    (%rdx, %rcx), %edi
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rax, %rcx), %eax
> > -       movzbl  (%rdx, %rcx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       VZEROUPPER_RETURN
> >  # else
> > -#  ifdef USE_AS_WCSCMP
> > -       movq    %rax, %rsi
> > -       xorl    %eax, %eax
> > -       movl    (VEC_SIZE * 2)(%rsi, %rcx), %edi
> > -       cmpl    (VEC_SIZE * 2)(%rdx, %rcx), %edi
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (VEC_SIZE * 2)(%rax, %rcx), %eax
> > -       movzbl  (VEC_SIZE * 2)(%rdx, %rcx), %edx
> > -       subl    %edx, %eax
> > -#  endif
> > +       jmp     L(loop_skip_page_cross_check)
> >  # endif
> > -       VZEROUPPER_RETURN
> >
> > +
> > +       .p2align 4,, 10
> > +L(return_vec_page_cross_0):
> > +       addl    $-VEC_SIZE, %eax
> > +L(return_vec_page_cross_1):
> > +       tzcntl  %ecx, %ecx
> >  # ifdef USE_AS_STRNCMP
> > -L(string_nbyte_offset_check):
> > -       leaq    (VEC_SIZE * 4)(%r10), %r10
> > -       cmpq    %r10, %r11
> > -       jbe     L(zero)
> > -       jmp     L(back_to_loop)
> > +       leal    -VEC_SIZE(%rax, %rcx), %ecx
> > +       cmpq    %rcx, %rdx
> > +       jbe     L(ret_zero_in_loop_page_cross)
> > +# else
> > +       addl    %eax, %ecx
> >  # endif
> >
> > -       .p2align 4
> > -L(cross_page_loop):
> > -       /* Check one byte/dword at a time.  */
> >  # ifdef USE_AS_WCSCMP
> > -       cmpl    %ecx, %eax
> > +       movl    VEC_OFFSET(%rdi, %rcx), %edx
> > +       xorl    %eax, %eax
> > +       cmpl    VEC_OFFSET(%rsi, %rcx), %edx
> > +       je      L(ret9)
> > +       setl    %al
> > +       negl    %eax
> > +       xorl    %r8d, %eax
> >  # else
> > +       movzbl  VEC_OFFSET(%rdi, %rcx), %eax
> > +       movzbl  VEC_OFFSET(%rsi, %rcx), %ecx
> >         subl    %ecx, %eax
> > +       xorl    %r8d, %eax
> > +       subl    %r8d, %eax
> >  # endif
> > -       jne     L(different)
> > -       addl    $SIZE_OF_CHAR, %edx
> > -       cmpl    $(VEC_SIZE * 4), %edx
> > -       je      L(main_loop_header)
> > -# ifdef USE_AS_STRNCMP
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > +L(ret9):
> > +       VZEROUPPER_RETURN
> > +
> > +
> > +       .p2align 4,, 10
> > +L(page_cross):
> > +# ifndef USE_AS_STRNCMP
> > +       /* If both are VEC aligned we don't need any special logic here.
> > +          Only valid for strcmp where stop condition is guranteed to be
> > +          reachable by just reading memory.  */
> > +       testl   $((VEC_SIZE - 1) << 20), %eax
> > +       jz      L(no_page_cross)
> >  # endif
> > +
> > +       movl    %edi, %eax
> > +       movl    %esi, %ecx
> > +       andl    $(PAGE_SIZE - 1), %eax
> > +       andl    $(PAGE_SIZE - 1), %ecx
> > +
> > +       xorl    %OFFSET_REG, %OFFSET_REG
> > +
> > +       /* Check which is closer to page cross, s1 or s2.  */
> > +       cmpl    %eax, %ecx
> > +       jg      L(page_cross_s2)
> > +
> > +       /* The previous page cross check has false positives. Check for
> > +          true positive as page cross logic is very expensive.  */
> > +       subl    $(PAGE_SIZE - VEC_SIZE * 4), %eax
> > +       jbe     L(no_page_cross)
> > +
> > +       /* Set r8 to not interfere with normal return value (rdi and rsi
> > +          did not swap).  */
> >  # ifdef USE_AS_WCSCMP
> > -       movl    (%rdi, %rdx), %eax
> > -       movl    (%rsi, %rdx), %ecx
> > +       /* any non-zero positive value that doesn't inference with 0x1.
> > +        */
> > +       movl    $2, %r8d
> >  # else
> > -       movzbl  (%rdi, %rdx), %eax
> > -       movzbl  (%rsi, %rdx), %ecx
> > +       xorl    %r8d, %r8d
> >  # endif
> > -       /* Check null char.  */
> > -       testl   %eax, %eax
> > -       jne     L(cross_page_loop)
> > -       /* Since %eax == 0, subtract is OK for both SIGNED and UNSIGNED
> > -          comparisons.  */
> > -       subl    %ecx, %eax
> > -# ifndef USE_AS_WCSCMP
> > -L(different):
> > +
> > +       /* Check if less than 1x VEC till page cross.  */
> > +       subl    $(VEC_SIZE * 3), %eax
> > +       jg      L(less_1x_vec_till_page)
> > +
> > +       /* If more than 1x VEC till page cross, loop throuh safely
> > +          loadable memory until within 1x VEC of page cross.  */
> > +
> > +       .p2align 4,, 10
> > +L(page_cross_loop):
> > +
> > +       VMOVU   (%rdi, %OFFSET_REG64), %ymm0
> > +       VPCMPEQ (%rsi, %OFFSET_REG64), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incl    %ecx
> > +
> > +       jnz     L(check_ret_vec_page_cross)
> > +       addl    $VEC_SIZE, %OFFSET_REG
> > +# ifdef USE_AS_STRNCMP
> > +       cmpq    %OFFSET_REG64, %rdx
> > +       jbe     L(ret_zero_page_cross)
> >  # endif
> > -       VZEROUPPER_RETURN
> > +       addl    $VEC_SIZE, %eax
> > +       jl      L(page_cross_loop)
> > +
> > +       subl    %eax, %OFFSET_REG
> > +       /* OFFSET_REG has distance to page cross - VEC_SIZE. Guranteed
> > +          to not cross page so is safe to load. Since we have already
> > +          loaded at least 1 VEC from rsi it is also guranteed to be safe.
> > +        */
> > +
> > +       VMOVU   (%rdi, %OFFSET_REG64), %ymm0
> > +       VPCMPEQ (%rsi, %OFFSET_REG64), %ymm0, %ymm1
> > +       VPCMPEQ %ymm0, %ymmZERO, %ymm2
> > +       vpandn  %ymm1, %ymm2, %ymm1
> > +       vpmovmskb %ymm1, %ecx
> > +
> > +# ifdef USE_AS_STRNCMP
> > +       leal    VEC_SIZE(%OFFSET_REG64), %eax
> > +       cmpq    %rax, %rdx
> > +       jbe     L(check_ret_vec_page_cross2)
> > +       addq    %rdi, %rdx
> > +# endif
> > +       incl    %ecx
> > +       jz      L(prepare_loop_no_len)
> >
> > +       .p2align 4,, 4
> > +L(ret_vec_page_cross):
> > +# ifndef USE_AS_STRNCMP
> > +L(check_ret_vec_page_cross):
> > +# endif
> > +       tzcntl  %ecx, %ecx
> > +       addl    %OFFSET_REG, %ecx
> > +L(ret_vec_page_cross_cont):
> >  # ifdef USE_AS_WCSCMP
> > -       .p2align 4
> > -L(different):
> > -       /* Use movl to avoid modifying EFLAGS.  */
> > -       movl    $0, %eax
> > +       movl    (%rdi, %rcx), %edx
> > +       xorl    %eax, %eax
> > +       cmpl    (%rsi, %rcx), %edx
> > +       je      L(ret12)
> >         setl    %al
> >         negl    %eax
> > -       orl     $1, %eax
> > -       VZEROUPPER_RETURN
> > +       xorl    %r8d, %eax
> > +# else
> > +       movzbl  (%rdi, %rcx), %eax
> > +       movzbl  (%rsi, %rcx), %ecx
> > +       subl    %ecx, %eax
> > +       xorl    %r8d, %eax
> > +       subl    %r8d, %eax
> >  # endif
> > +L(ret12):
> > +       VZEROUPPER_RETURN
> >
> >  # ifdef USE_AS_STRNCMP
> > -       .p2align 4
> > -L(zero):
> > +       .p2align 4,, 10
> > +L(check_ret_vec_page_cross2):
> > +       incl    %ecx
> > +L(check_ret_vec_page_cross):
> > +       tzcntl  %ecx, %ecx
> > +       addl    %OFFSET_REG, %ecx
> > +       cmpq    %rcx, %rdx
> > +       ja      L(ret_vec_page_cross_cont)
> > +       .p2align 4,, 2
> > +L(ret_zero_page_cross):
> >         xorl    %eax, %eax
> >         VZEROUPPER_RETURN
> > +# endif
> >
> > -       .p2align 4
> > -L(char0):
> > -#  ifdef USE_AS_WCSCMP
> > -       xorl    %eax, %eax
> > -       movl    (%rdi), %ecx
> > -       cmpl    (%rsi), %ecx
> > -       jne     L(wcscmp_return)
> > -#  else
> > -       movzbl  (%rsi), %ecx
> > -       movzbl  (%rdi), %eax
> > -       subl    %ecx, %eax
> > -#  endif
> > -       VZEROUPPER_RETURN
> > +       .p2align 4,, 4
> > +L(page_cross_s2):
> > +       /* Ensure this is a true page cross.  */
> > +       subl    $(PAGE_SIZE - VEC_SIZE * 4), %ecx
> > +       jbe     L(no_page_cross)
> > +
> > +
> > +       movl    %ecx, %eax
> > +       movq    %rdi, %rcx
> > +       movq    %rsi, %rdi
> > +       movq    %rcx, %rsi
> > +
> > +       /* set r8 to negate return value as rdi and rsi swapped.  */
> > +# ifdef USE_AS_WCSCMP
> > +       movl    $-4, %r8d
> > +# else
> > +       movl    $-1, %r8d
> >  # endif
> > +       xorl    %OFFSET_REG, %OFFSET_REG
> >
> > -       .p2align 4
> > -L(last_vector):
> > -       addq    %rdx, %rdi
> > -       addq    %rdx, %rsi
> > +       /* Check if more than 1x VEC till page cross.  */
> > +       subl    $(VEC_SIZE * 3), %eax
> > +       jle     L(page_cross_loop)
> > +
> > +       .p2align 4,, 6
> > +L(less_1x_vec_till_page):
> > +       /* Find largest load size we can use.  */
> > +       cmpl    $16, %eax
> > +       ja      L(less_16_till_page)
> > +
> > +       VMOVU   (%rdi), %xmm0
> > +       VPCMPEQ (%rsi), %xmm0, %xmm1
> > +       VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > +       vpandn  %xmm1, %xmm2, %xmm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incw    %cx
> > +       jnz     L(check_ret_vec_page_cross)
> > +       movl    $16, %OFFSET_REG
> >  # ifdef USE_AS_STRNCMP
> > -       subq    %rdx, %r11
> > +       cmpq    %OFFSET_REG64, %rdx
> > +       jbe     L(ret_zero_page_cross_slow_case0)
> > +       subl    %eax, %OFFSET_REG
> > +# else
> > +       /* Explicit check for 16 byte alignment.  */
> > +       subl    %eax, %OFFSET_REG
> > +       jz      L(prepare_loop)
> >  # endif
> > -       tzcntl  %ecx, %edx
> > +
> > +       VMOVU   (%rdi, %OFFSET_REG64), %xmm0
> > +       VPCMPEQ (%rsi, %OFFSET_REG64), %xmm0, %xmm1
> > +       VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > +       vpandn  %xmm1, %xmm2, %xmm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incw    %cx
> > +       jnz     L(check_ret_vec_page_cross)
> > +
> >  # ifdef USE_AS_STRNCMP
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > +       addl    $16, %OFFSET_REG
> > +       subq    %OFFSET_REG64, %rdx
> > +       jbe     L(ret_zero_page_cross_slow_case0)
> > +       subq    $-(VEC_SIZE * 4), %rdx
> > +
> > +       leaq    -(VEC_SIZE * 4)(%rdi, %OFFSET_REG64), %rdi
> > +       leaq    -(VEC_SIZE * 4)(%rsi, %OFFSET_REG64), %rsi
> > +# else
> > +       leaq    (16 - VEC_SIZE * 4)(%rdi, %OFFSET_REG64), %rdi
> > +       leaq    (16 - VEC_SIZE * 4)(%rsi, %OFFSET_REG64), %rsi
> >  # endif
> > -# ifdef USE_AS_WCSCMP
> > +       jmp     L(prepare_loop_aligned)
> > +
> > +# ifdef USE_AS_STRNCMP
> > +       .p2align 4,, 2
> > +L(ret_zero_page_cross_slow_case0):
> >         xorl    %eax, %eax
> > -       movl    (%rdi, %rdx), %ecx
> > -       cmpl    (%rsi, %rdx), %ecx
> > -       jne     L(wcscmp_return)
> > -# else
> > -       movzbl  (%rdi, %rdx), %eax
> > -       movzbl  (%rsi, %rdx), %edx
> > -       subl    %edx, %eax
> > +       ret
> >  # endif
> > -       VZEROUPPER_RETURN
> >
> > -       /* Comparing on page boundary region requires special treatment:
> > -          It must done one vector at the time, starting with the wider
> > -          ymm vector if possible, if not, with xmm. If fetching 16 bytes
> > -          (xmm) still passes the boundary, byte comparison must be done.
> > -        */
> > -       .p2align 4
> > -L(cross_page):
> > -       /* Try one ymm vector at a time.  */
> > -       cmpl    $(PAGE_SIZE - VEC_SIZE), %eax
> > -       jg      L(cross_page_1_vector)
> > -L(loop_1_vector):
> > -       vmovdqu (%rdi, %rdx), %ymm1
> > -       VPCMPEQ (%rsi, %rdx), %ymm1, %ymm0
> > -       VPMINU  %ymm1, %ymm0, %ymm0
> > -       VPCMPEQ %ymm7, %ymm0, %ymm0
> > -       vpmovmskb %ymm0, %ecx
> > -       testl   %ecx, %ecx
> > -       jne     L(last_vector)
> >
> > -       addl    $VEC_SIZE, %edx
> > +       .p2align 4,, 10
> > +L(less_16_till_page):
> > +       /* Find largest load size we can use.  */
> > +       cmpl    $24, %eax
> > +       ja      L(less_8_till_page)
> >
> > -       addl    $VEC_SIZE, %eax
> > -# ifdef USE_AS_STRNCMP
> > -       /* Return 0 if the current offset (%rdx) >= the maximum offset
> > -          (%r11).  */
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > -# endif
> > -       cmpl    $(PAGE_SIZE - VEC_SIZE), %eax
> > -       jle     L(loop_1_vector)
> > -L(cross_page_1_vector):
> > -       /* Less than 32 bytes to check, try one xmm vector.  */
> > -       cmpl    $(PAGE_SIZE - 16), %eax
> > -       jg      L(cross_page_1_xmm)
> > -       vmovdqu (%rdi, %rdx), %xmm1
> > -       VPCMPEQ (%rsi, %rdx), %xmm1, %xmm0
> > -       VPMINU  %xmm1, %xmm0, %xmm0
> > -       VPCMPEQ %xmm7, %xmm0, %xmm0
> > -       vpmovmskb %xmm0, %ecx
> > -       testl   %ecx, %ecx
> > -       jne     L(last_vector)
> > +       vmovq   (%rdi), %xmm0
> > +       vmovq   (%rsi), %xmm1
> > +       VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > +       VPCMPEQ %xmm1, %xmm0, %xmm1
> > +       vpandn  %xmm1, %xmm2, %xmm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incb    %cl
> > +       jnz     L(check_ret_vec_page_cross)
> >
> > -       addl    $16, %edx
> > -# ifndef USE_AS_WCSCMP
> > -       addl    $16, %eax
> > +
> > +# ifdef USE_AS_STRNCMP
> > +       cmpq    $8, %rdx
> > +       jbe     L(ret_zero_page_cross_slow_case0)
> >  # endif
> > +       movl    $24, %OFFSET_REG
> > +       /* Explicit check for 16 byte alignment.  */
> > +       subl    %eax, %OFFSET_REG
> > +
> > +
> > +
> > +       vmovq   (%rdi, %OFFSET_REG64), %xmm0
> > +       vmovq   (%rsi, %OFFSET_REG64), %xmm1
> > +       VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > +       VPCMPEQ %xmm1, %xmm0, %xmm1
> > +       vpandn  %xmm1, %xmm2, %xmm1
> > +       vpmovmskb %ymm1, %ecx
> > +       incb    %cl
> > +       jnz     L(check_ret_vec_page_cross)
> > +
> >  # ifdef USE_AS_STRNCMP
> > -       /* Return 0 if the current offset (%rdx) >= the maximum offset
> > -          (%r11).  */
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > -# endif
> > -
> > -L(cross_page_1_xmm):
> > -# ifndef USE_AS_WCSCMP
> > -       /* Less than 16 bytes to check, try 8 byte vector.  NB: No need
> > -          for wcscmp nor wcsncmp since wide char is 4 bytes.   */
> > -       cmpl    $(PAGE_SIZE - 8), %eax
> > -       jg      L(cross_page_8bytes)
> > -       vmovq   (%rdi, %rdx), %xmm1
> > -       vmovq   (%rsi, %rdx), %xmm0
> > -       VPCMPEQ %xmm0, %xmm1, %xmm0
> > -       VPMINU  %xmm1, %xmm0, %xmm0
> > -       VPCMPEQ %xmm7, %xmm0, %xmm0
> > -       vpmovmskb %xmm0, %ecx
> > -       /* Only last 8 bits are valid.  */
> > -       andl    $0xff, %ecx
> > -       testl   %ecx, %ecx
> > -       jne     L(last_vector)
> > +       addl    $8, %OFFSET_REG
> > +       subq    %OFFSET_REG64, %rdx
> > +       jbe     L(ret_zero_page_cross_slow_case0)
> > +       subq    $-(VEC_SIZE * 4), %rdx
> >
> > -       addl    $8, %edx
> > -       addl    $8, %eax
> > +       leaq    -(VEC_SIZE * 4)(%rdi, %OFFSET_REG64), %rdi
> > +       leaq    -(VEC_SIZE * 4)(%rsi, %OFFSET_REG64), %rsi
> > +# else
> > +       leaq    (8 - VEC_SIZE * 4)(%rdi, %OFFSET_REG64), %rdi
> > +       leaq    (8 - VEC_SIZE * 4)(%rsi, %OFFSET_REG64), %rsi
> > +# endif
> > +       jmp     L(prepare_loop_aligned)
> > +
> > +
> > +       .p2align 4,, 10
> > +L(less_8_till_page):
> > +# ifdef USE_AS_WCSCMP
> > +       /* If using wchar then this is the only check before we reach
> > +          the page boundary.  */
> > +       movl    (%rdi), %eax
> > +       movl    (%rsi), %ecx
> > +       cmpl    %ecx, %eax
> > +       jnz     L(ret_less_8_wcs)
> >  #  ifdef USE_AS_STRNCMP
> > -       /* Return 0 if the current offset (%rdx) >= the maximum offset
> > -          (%r11).  */
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > +       addq    %rdi, %rdx
> > +       /* We already checked for len <= 1 so cannot hit that case here.
> > +        */
> >  #  endif
> > +       testl   %eax, %eax
> > +       jnz     L(prepare_loop_no_len)
> > +       ret
> >
> > -L(cross_page_8bytes):
> > -       /* Less than 8 bytes to check, try 4 byte vector.  */
> > -       cmpl    $(PAGE_SIZE - 4), %eax
> > -       jg      L(cross_page_4bytes)
> > -       vmovd   (%rdi, %rdx), %xmm1
> > -       vmovd   (%rsi, %rdx), %xmm0
> > -       VPCMPEQ %xmm0, %xmm1, %xmm0
> > -       VPMINU  %xmm1, %xmm0, %xmm0
> > -       VPCMPEQ %xmm7, %xmm0, %xmm0
> > -       vpmovmskb %xmm0, %ecx
> > -       /* Only last 4 bits are valid.  */
> > -       andl    $0xf, %ecx
> > -       testl   %ecx, %ecx
> > -       jne     L(last_vector)
> > +       .p2align 4,, 8
> > +L(ret_less_8_wcs):
> > +       setl    %OFFSET_REG8
> > +       negl    %OFFSET_REG
> > +       movl    %OFFSET_REG, %eax
> > +       xorl    %r8d, %eax
> > +       ret
> > +
> > +# else
> > +
> > +       /* Find largest load size we can use.  */
> > +       cmpl    $28, %eax
> > +       ja      L(less_4_till_page)
> > +
> > +       vmovd   (%rdi), %xmm0
> > +       vmovd   (%rsi), %xmm1
> > +       VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > +       VPCMPEQ %xmm1, %xmm0, %xmm1
> > +       vpandn  %xmm1, %xmm2, %xmm1
> > +       vpmovmskb %ymm1, %ecx
> > +       subl    $0xf, %ecx
> > +       jnz     L(check_ret_vec_page_cross)
> >
> > -       addl    $4, %edx
> >  #  ifdef USE_AS_STRNCMP
> > -       /* Return 0 if the current offset (%rdx) >= the maximum offset
> > -          (%r11).  */
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > +       cmpq    $4, %rdx
> > +       jbe     L(ret_zero_page_cross_slow_case1)
> >  #  endif
> > +       movl    $28, %OFFSET_REG
> > +       /* Explicit check for 16 byte alignment.  */
> > +       subl    %eax, %OFFSET_REG
> >
> > -L(cross_page_4bytes):
> > -# endif
> > -       /* Less than 4 bytes to check, try one byte/dword at a time.  */
> > -# ifdef USE_AS_STRNCMP
> > -       cmpq    %r11, %rdx
> > -       jae     L(zero)
> > -# endif
> > -# ifdef USE_AS_WCSCMP
> > -       movl    (%rdi, %rdx), %eax
> > -       movl    (%rsi, %rdx), %ecx
> > -# else
> > -       movzbl  (%rdi, %rdx), %eax
> > -       movzbl  (%rsi, %rdx), %ecx
> > -# endif
> > -       testl   %eax, %eax
> > -       jne     L(cross_page_loop)
> > +
> > +
> > +       vmovd   (%rdi, %OFFSET_REG64), %xmm0
> > +       vmovd   (%rsi, %OFFSET_REG64), %xmm1
> > +       VPCMPEQ %xmm0, %xmmZERO, %xmm2
> > +       VPCMPEQ %xmm1, %xmm0, %xmm1
> > +       vpandn  %xmm1, %xmm2, %xmm1
> > +       vpmovmskb %ymm1, %ecx
> > +       subl    $0xf, %ecx
> > +       jnz     L(check_ret_vec_page_cross)
> > +
> > +#  ifdef USE_AS_STRNCMP
> > +       addl    $4, %OFFSET_REG
> > +       subq    %OFFSET_REG64, %rdx
> > +       jbe     L(ret_zero_page_cross_slow_case1)
> > +       subq    $-(VEC_SIZE * 4), %rdx
> > +
> > +       leaq    -(VEC_SIZE * 4)(%rdi, %OFFSET_REG64), %rdi
> > +       leaq    -(VEC_SIZE * 4)(%rsi, %OFFSET_REG64), %rsi
> > +#  else
> > +       leaq    (4 - VEC_SIZE * 4)(%rdi, %OFFSET_REG64), %rdi
> > +       leaq    (4 - VEC_SIZE * 4)(%rsi, %OFFSET_REG64), %rsi
> > +#  endif
> > +       jmp     L(prepare_loop_aligned)
> > +
> > +#  ifdef USE_AS_STRNCMP
> > +       .p2align 4,, 2
> > +L(ret_zero_page_cross_slow_case1):
> > +       xorl    %eax, %eax
> > +       ret
> > +#  endif
> > +
> > +       .p2align 4,, 10
> > +L(less_4_till_page):
> > +       subq    %rdi, %rsi
> > +       /* Extremely slow byte comparison loop.  */
> > +L(less_4_loop):
> > +       movzbl  (%rdi), %eax
> > +       movzbl  (%rsi, %rdi), %ecx
> >         subl    %ecx, %eax
> > -       VZEROUPPER_RETURN
> > -END (STRCMP)
> > +       jnz     L(ret_less_4_loop)
> > +       testl   %ecx, %ecx
> > +       jz      L(ret_zero_4_loop)
> > +#  ifdef USE_AS_STRNCMP
> > +       decq    %rdx
> > +       jz      L(ret_zero_4_loop)
> > +#  endif
> > +       incq    %rdi
> > +       /* end condition is reach page boundary (rdi is aligned).  */
> > +       testl   $31, %edi
> > +       jnz     L(less_4_loop)
> > +       leaq    -(VEC_SIZE * 4)(%rdi, %rsi), %rsi
> > +       addq    $-(VEC_SIZE * 4), %rdi
> > +#  ifdef USE_AS_STRNCMP
> > +       subq    $-(VEC_SIZE * 4), %rdx
> > +#  endif
> > +       jmp     L(prepare_loop_aligned)
> > +
> > +L(ret_zero_4_loop):
> > +       xorl    %eax, %eax
> > +       ret
> > +L(ret_less_4_loop):
> > +       xorl    %r8d, %eax
> > +       subl    %r8d, %eax
> > +       ret
> > +# endif
> > +END(STRCMP)
> >  #endif
> > --
> > 2.25.1
> >
>
> LGTM.

Should I wait until 2.36 release to push the optimized versions?

There are alot of edge cases with these functions and last time we
tried to improve them in:

commit c46e9afb2df5fc9e39ff4d13777e4b4c26e04e55
Author: H.J. Lu <hjl.tools@gmail.com>
Date:   Fri Oct 29 12:40:20 2021 -0700

    x86-64: Improve EVEX strcmp with masked load


We missed a case:
https://bugzilla.redhat.com/show_bug.cgi?id=2026399#c19
>
> Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
>
> Thanks.
>
> --
> H.J.