From: Noah Goldstein <goldstein.w.n@gmail.com>
To: GNU C Library <libc-alpha@sourceware.org>
Subject: Re: [PATCH v3 3/6] x86: Reduce code size of Remove mem{move|pcpy|cpy}-ssse3
Date: Sat, 9 Apr 2022 19:48:41 -0500 [thread overview]
Message-ID: <CAFUsyf+oDr2_WdOWqrBG75_Tc6bzwGdm+oK0F7ZERzu3FJUWuA@mail.gmail.com> (raw)
In-Reply-To: <20220410004250.1448020-3-goldstein.w.n@gmail.com>
Disregard this patch. It's from the wrong patchset.
On Sat, Apr 9, 2022 at 7:46 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> The goal is to remove most SSSE3 function as SSE4, AVX2, and EVEX are
> generally preferable. memcpy/memmove is one exception where avoiding
> unaligned loads with `palignr` is important for some targets.
>
> This commit replaces memmove-ssse3 with a better optimized are lower
> code footprint verion. As well it aliases memcpy to memmove.
>
> Aside from this function all other SSSE3 functions should be safe to
> remove.
>
> The performance is not changed drastically although shows overall
> improvements without any major regressions or gains.
>
> bench-memcpy geometric_mean(N=50) New / Original: 0.962
>
> bench-memcpy-random geometric_mean(N=50) New / Original: 0.895
>
> bench-memcpy-large geometric_mean(N=50) New / Original: 0.894
>
> Benchmarks where run on Zhaoxin KX-6840@2000MHz See attached numbers
> for all results.
>
> More important this saves 7246 bytes of code size in memmove an
> additional 10741 bytes by reusing memmove code for memcpy (total 17987
> bytes saves). As well an additional 896 bytes of rodata for the jump
> table entries.
> ---
> sysdeps/x86_64/multiarch/Makefile | 1 -
> sysdeps/x86_64/multiarch/memcpy-ssse3.S | 3151 ----------------------
> sysdeps/x86_64/multiarch/memmove-ssse3.S | 386 ++-
> 3 files changed, 382 insertions(+), 3156 deletions(-)
> delete mode 100644 sysdeps/x86_64/multiarch/memcpy-ssse3.S
>
> diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
> index 303fb5d734..e7ea963fc0 100644
> --- a/sysdeps/x86_64/multiarch/Makefile
> +++ b/sysdeps/x86_64/multiarch/Makefile
> @@ -16,7 +16,6 @@ sysdep_routines += \
> memcmpeq-avx2-rtm \
> memcmpeq-evex \
> memcmpeq-sse2 \
> - memcpy-ssse3 \
> memmove-avx-unaligned-erms \
> memmove-avx-unaligned-erms-rtm \
> memmove-avx512-no-vzeroupper \
> diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3.S b/sysdeps/x86_64/multiarch/memcpy-ssse3.S
> deleted file mode 100644
> index 65644d3a09..0000000000
> --- a/sysdeps/x86_64/multiarch/memcpy-ssse3.S
> +++ /dev/null
> @@ -1,3151 +0,0 @@
> -/* memcpy with SSSE3
> - Copyright (C) 2010-2022 Free Software Foundation, Inc.
> - This file is part of the GNU C Library.
> -
> - The GNU C Library is free software; you can redistribute it and/or
> - modify it under the terms of the GNU Lesser General Public
> - License as published by the Free Software Foundation; either
> - version 2.1 of the License, or (at your option) any later version.
> -
> - The GNU C Library is distributed in the hope that it will be useful,
> - but WITHOUT ANY WARRANTY; without even the implied warranty of
> - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - Lesser General Public License for more details.
> -
> - You should have received a copy of the GNU Lesser General Public
> - License along with the GNU C Library; if not, see
> - <https://www.gnu.org/licenses/>. */
> -
> -#include <sysdep.h>
> -
> -#if IS_IN (libc)
> -
> -#include "asm-syntax.h"
> -
> -#ifndef MEMCPY
> -# define MEMCPY __memcpy_ssse3
> -# define MEMCPY_CHK __memcpy_chk_ssse3
> -# define MEMPCPY __mempcpy_ssse3
> -# define MEMPCPY_CHK __mempcpy_chk_ssse3
> -#endif
> -
> -#define JMPTBL(I, B) I - B
> -
> -/* Branch to an entry in a jump table. TABLE is a jump table with
> - relative offsets. INDEX is a register contains the index into the
> - jump table. SCALE is the scale of INDEX. */
> -#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \
> - lea TABLE(%rip), %r11; \
> - movslq (%r11, INDEX, SCALE), INDEX; \
> - lea (%r11, INDEX), INDEX; \
> - _CET_NOTRACK jmp *INDEX; \
> - ud2
> -
> - .section .text.ssse3,"ax",@progbits
> -#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE
> -ENTRY (MEMPCPY_CHK)
> - cmp %RDX_LP, %RCX_LP
> - jb HIDDEN_JUMPTARGET (__chk_fail)
> -END (MEMPCPY_CHK)
> -
> -ENTRY (MEMPCPY)
> - mov %RDI_LP, %RAX_LP
> - add %RDX_LP, %RAX_LP
> - jmp L(start)
> -END (MEMPCPY)
> -#endif
> -
> -#if !defined USE_AS_BCOPY
> -ENTRY (MEMCPY_CHK)
> - cmp %RDX_LP, %RCX_LP
> - jb HIDDEN_JUMPTARGET (__chk_fail)
> -END (MEMCPY_CHK)
> -#endif
> -
> -ENTRY (MEMCPY)
> - mov %RDI_LP, %RAX_LP
> -#ifdef USE_AS_MEMPCPY
> - add %RDX_LP, %RAX_LP
> -#endif
> -
> -#ifdef __ILP32__
> - /* Clear the upper 32 bits. */
> - mov %edx, %edx
> -#endif
> -
> -#ifdef USE_AS_MEMMOVE
> - cmp %rsi, %rdi
> - jb L(copy_forward)
> - je L(write_0bytes)
> - cmp $79, %rdx
> - jbe L(copy_forward)
> - jmp L(copy_backward)
> -L(copy_forward):
> -#endif
> -L(start):
> - cmp $79, %rdx
> - lea L(table_less_80bytes)(%rip), %r11
> - ja L(80bytesormore)
> - movslq (%r11, %rdx, 4), %r9
> - add %rdx, %rsi
> - add %rdx, %rdi
> - add %r11, %r9
> - _CET_NOTRACK jmp *%r9
> - ud2
> -
> - .p2align 4
> -L(80bytesormore):
> -#ifndef USE_AS_MEMMOVE
> - cmp %dil, %sil
> - jle L(copy_backward)
> -#endif
> -
> - movdqu (%rsi), %xmm0
> - mov %rdi, %rcx
> - and $-16, %rdi
> - add $16, %rdi
> - mov %rcx, %r8
> - sub %rdi, %rcx
> - add %rcx, %rdx
> - sub %rcx, %rsi
> -
> -#ifdef SHARED_CACHE_SIZE_HALF
> - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP
> -#else
> - mov __x86_shared_cache_size_half(%rip), %RCX_LP
> -#endif
> - cmp %rcx, %rdx
> - mov %rsi, %r9
> - ja L(large_page_fwd)
> - and $0xf, %r9
> - jz L(shl_0)
> -#ifdef DATA_CACHE_SIZE_HALF
> - mov $DATA_CACHE_SIZE_HALF, %RCX_LP
> -#else
> - mov __x86_data_cache_size_half(%rip), %RCX_LP
> -#endif
> - BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %r9, 4)
> -
> - .p2align 4
> -L(copy_backward):
> - movdqu -16(%rsi, %rdx), %xmm0
> - add %rdx, %rsi
> - lea -16(%rdi, %rdx), %r8
> - add %rdx, %rdi
> -
> - mov %rdi, %rcx
> - and $0xf, %rcx
> - xor %rcx, %rdi
> - sub %rcx, %rdx
> - sub %rcx, %rsi
> -
> -#ifdef SHARED_CACHE_SIZE_HALF
> - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP
> -#else
> - mov __x86_shared_cache_size_half(%rip), %RCX_LP
> -#endif
> -
> - cmp %rcx, %rdx
> - mov %rsi, %r9
> - ja L(large_page_bwd)
> - and $0xf, %r9
> - jz L(shl_0_bwd)
> -#ifdef DATA_CACHE_SIZE_HALF
> - mov $DATA_CACHE_SIZE_HALF, %RCX_LP
> -#else
> - mov __x86_data_cache_size_half(%rip), %RCX_LP
> -#endif
> - BRANCH_TO_JMPTBL_ENTRY (L(shl_table_bwd), %r9, 4)
> -
> - .p2align 4
> -L(shl_0):
> - sub $16, %rdx
> - movdqa (%rsi), %xmm1
> - add $16, %rsi
> - movdqa %xmm1, (%rdi)
> - add $16, %rdi
> - cmp $128, %rdx
> - movdqu %xmm0, (%r8)
> - ja L(shl_0_gobble)
> - cmp $64, %rdx
> - jb L(shl_0_less_64bytes)
> - movaps (%rsi), %xmm4
> - movaps 16(%rsi), %xmm1
> - movaps 32(%rsi), %xmm2
> - movaps 48(%rsi), %xmm3
> - movaps %xmm4, (%rdi)
> - movaps %xmm1, 16(%rdi)
> - movaps %xmm2, 32(%rdi)
> - movaps %xmm3, 48(%rdi)
> - sub $64, %rdx
> - add $64, %rsi
> - add $64, %rdi
> -L(shl_0_less_64bytes):
> - add %rdx, %rsi
> - add %rdx, %rdi
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_0_gobble):
> -#ifdef DATA_CACHE_SIZE_HALF
> - cmp $DATA_CACHE_SIZE_HALF, %RDX_LP
> -#else
> - cmp __x86_data_cache_size_half(%rip), %RDX_LP
> -#endif
> - lea -128(%rdx), %rdx
> - jae L(shl_0_gobble_mem_loop)
> -L(shl_0_gobble_cache_loop):
> - movdqa (%rsi), %xmm4
> - movaps 0x10(%rsi), %xmm1
> - movaps 0x20(%rsi), %xmm2
> - movaps 0x30(%rsi), %xmm3
> -
> - movdqa %xmm4, (%rdi)
> - movaps %xmm1, 0x10(%rdi)
> - movaps %xmm2, 0x20(%rdi)
> - movaps %xmm3, 0x30(%rdi)
> -
> - sub $128, %rdx
> - movaps 0x40(%rsi), %xmm4
> - movaps 0x50(%rsi), %xmm5
> - movaps 0x60(%rsi), %xmm6
> - movaps 0x70(%rsi), %xmm7
> - lea 0x80(%rsi), %rsi
> - movaps %xmm4, 0x40(%rdi)
> - movaps %xmm5, 0x50(%rdi)
> - movaps %xmm6, 0x60(%rdi)
> - movaps %xmm7, 0x70(%rdi)
> - lea 0x80(%rdi), %rdi
> -
> - jae L(shl_0_gobble_cache_loop)
> - cmp $-0x40, %rdx
> - lea 0x80(%rdx), %rdx
> - jl L(shl_0_cache_less_64bytes)
> -
> - movdqa (%rsi), %xmm4
> - sub $0x40, %rdx
> - movdqa 0x10(%rsi), %xmm1
> -
> - movdqa %xmm4, (%rdi)
> - movdqa %xmm1, 0x10(%rdi)
> -
> - movdqa 0x20(%rsi), %xmm4
> - movdqa 0x30(%rsi), %xmm1
> - add $0x40, %rsi
> -
> - movdqa %xmm4, 0x20(%rdi)
> - movdqa %xmm1, 0x30(%rdi)
> - add $0x40, %rdi
> -L(shl_0_cache_less_64bytes):
> - add %rdx, %rsi
> - add %rdx, %rdi
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_0_gobble_mem_loop):
> - prefetcht0 0x1c0(%rsi)
> - prefetcht0 0x280(%rsi)
> -
> - movdqa (%rsi), %xmm0
> - movdqa 0x10(%rsi), %xmm1
> - movdqa 0x20(%rsi), %xmm2
> - movdqa 0x30(%rsi), %xmm3
> - movdqa 0x40(%rsi), %xmm4
> - movdqa 0x50(%rsi), %xmm5
> - movdqa 0x60(%rsi), %xmm6
> - movdqa 0x70(%rsi), %xmm7
> - lea 0x80(%rsi), %rsi
> - sub $0x80, %rdx
> - movdqa %xmm0, (%rdi)
> - movdqa %xmm1, 0x10(%rdi)
> - movdqa %xmm2, 0x20(%rdi)
> - movdqa %xmm3, 0x30(%rdi)
> - movdqa %xmm4, 0x40(%rdi)
> - movdqa %xmm5, 0x50(%rdi)
> - movdqa %xmm6, 0x60(%rdi)
> - movdqa %xmm7, 0x70(%rdi)
> - lea 0x80(%rdi), %rdi
> -
> - jae L(shl_0_gobble_mem_loop)
> - cmp $-0x40, %rdx
> - lea 0x80(%rdx), %rdx
> - jl L(shl_0_mem_less_64bytes)
> -
> - movdqa (%rsi), %xmm0
> - sub $0x40, %rdx
> - movdqa 0x10(%rsi), %xmm1
> -
> - movdqa %xmm0, (%rdi)
> - movdqa %xmm1, 0x10(%rdi)
> -
> - movdqa 0x20(%rsi), %xmm0
> - movdqa 0x30(%rsi), %xmm1
> - add $0x40, %rsi
> -
> - movdqa %xmm0, 0x20(%rdi)
> - movdqa %xmm1, 0x30(%rdi)
> - add $0x40, %rdi
> -L(shl_0_mem_less_64bytes):
> - cmp $0x20, %rdx
> - jb L(shl_0_mem_less_32bytes)
> - movdqa (%rsi), %xmm0
> - sub $0x20, %rdx
> - movdqa 0x10(%rsi), %xmm1
> - add $0x20, %rsi
> - movdqa %xmm0, (%rdi)
> - movdqa %xmm1, 0x10(%rdi)
> - add $0x20, %rdi
> -L(shl_0_mem_less_32bytes):
> - add %rdx, %rdi
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_0_bwd):
> - sub $16, %rdx
> - movdqa -0x10(%rsi), %xmm1
> - sub $16, %rsi
> - movdqa %xmm1, -0x10(%rdi)
> - sub $16, %rdi
> - cmp $0x80, %rdx
> - movdqu %xmm0, (%r8)
> - ja L(shl_0_gobble_bwd)
> - cmp $64, %rdx
> - jb L(shl_0_less_64bytes_bwd)
> - movaps -0x10(%rsi), %xmm0
> - movaps -0x20(%rsi), %xmm1
> - movaps -0x30(%rsi), %xmm2
> - movaps -0x40(%rsi), %xmm3
> - movaps %xmm0, -0x10(%rdi)
> - movaps %xmm1, -0x20(%rdi)
> - movaps %xmm2, -0x30(%rdi)
> - movaps %xmm3, -0x40(%rdi)
> - sub $64, %rdx
> - sub $0x40, %rsi
> - sub $0x40, %rdi
> -L(shl_0_less_64bytes_bwd):
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_0_gobble_bwd):
> -#ifdef DATA_CACHE_SIZE_HALF
> - cmp $DATA_CACHE_SIZE_HALF, %RDX_LP
> -#else
> - cmp __x86_data_cache_size_half(%rip), %RDX_LP
> -#endif
> - lea -128(%rdx), %rdx
> - jae L(shl_0_gobble_mem_bwd_loop)
> -L(shl_0_gobble_bwd_loop):
> - movdqa -0x10(%rsi), %xmm0
> - movaps -0x20(%rsi), %xmm1
> - movaps -0x30(%rsi), %xmm2
> - movaps -0x40(%rsi), %xmm3
> -
> - movdqa %xmm0, -0x10(%rdi)
> - movaps %xmm1, -0x20(%rdi)
> - movaps %xmm2, -0x30(%rdi)
> - movaps %xmm3, -0x40(%rdi)
> -
> - sub $0x80, %rdx
> - movaps -0x50(%rsi), %xmm4
> - movaps -0x60(%rsi), %xmm5
> - movaps -0x70(%rsi), %xmm6
> - movaps -0x80(%rsi), %xmm7
> - lea -0x80(%rsi), %rsi
> - movaps %xmm4, -0x50(%rdi)
> - movaps %xmm5, -0x60(%rdi)
> - movaps %xmm6, -0x70(%rdi)
> - movaps %xmm7, -0x80(%rdi)
> - lea -0x80(%rdi), %rdi
> -
> - jae L(shl_0_gobble_bwd_loop)
> - cmp $-0x40, %rdx
> - lea 0x80(%rdx), %rdx
> - jl L(shl_0_gobble_bwd_less_64bytes)
> -
> - movdqa -0x10(%rsi), %xmm0
> - sub $0x40, %rdx
> - movdqa -0x20(%rsi), %xmm1
> -
> - movdqa %xmm0, -0x10(%rdi)
> - movdqa %xmm1, -0x20(%rdi)
> -
> - movdqa -0x30(%rsi), %xmm0
> - movdqa -0x40(%rsi), %xmm1
> - sub $0x40, %rsi
> -
> - movdqa %xmm0, -0x30(%rdi)
> - movdqa %xmm1, -0x40(%rdi)
> - sub $0x40, %rdi
> -L(shl_0_gobble_bwd_less_64bytes):
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_0_gobble_mem_bwd_loop):
> - prefetcht0 -0x1c0(%rsi)
> - prefetcht0 -0x280(%rsi)
> - movdqa -0x10(%rsi), %xmm0
> - movdqa -0x20(%rsi), %xmm1
> - movdqa -0x30(%rsi), %xmm2
> - movdqa -0x40(%rsi), %xmm3
> - movdqa -0x50(%rsi), %xmm4
> - movdqa -0x60(%rsi), %xmm5
> - movdqa -0x70(%rsi), %xmm6
> - movdqa -0x80(%rsi), %xmm7
> - lea -0x80(%rsi), %rsi
> - sub $0x80, %rdx
> - movdqa %xmm0, -0x10(%rdi)
> - movdqa %xmm1, -0x20(%rdi)
> - movdqa %xmm2, -0x30(%rdi)
> - movdqa %xmm3, -0x40(%rdi)
> - movdqa %xmm4, -0x50(%rdi)
> - movdqa %xmm5, -0x60(%rdi)
> - movdqa %xmm6, -0x70(%rdi)
> - movdqa %xmm7, -0x80(%rdi)
> - lea -0x80(%rdi), %rdi
> -
> - jae L(shl_0_gobble_mem_bwd_loop)
> - cmp $-0x40, %rdx
> - lea 0x80(%rdx), %rdx
> - jl L(shl_0_mem_bwd_less_64bytes)
> -
> - movdqa -0x10(%rsi), %xmm0
> - sub $0x40, %rdx
> - movdqa -0x20(%rsi), %xmm1
> -
> - movdqa %xmm0, -0x10(%rdi)
> - movdqa %xmm1, -0x20(%rdi)
> -
> - movdqa -0x30(%rsi), %xmm0
> - movdqa -0x40(%rsi), %xmm1
> - sub $0x40, %rsi
> -
> - movdqa %xmm0, -0x30(%rdi)
> - movdqa %xmm1, -0x40(%rdi)
> - sub $0x40, %rdi
> -L(shl_0_mem_bwd_less_64bytes):
> - cmp $0x20, %rdx
> - jb L(shl_0_mem_bwd_less_32bytes)
> - movdqa -0x10(%rsi), %xmm0
> - sub $0x20, %rdx
> - movdqa -0x20(%rsi), %xmm1
> - sub $0x20, %rsi
> - movdqa %xmm0, -0x10(%rdi)
> - movdqa %xmm1, -0x20(%rdi)
> - sub $0x20, %rdi
> -L(shl_0_mem_bwd_less_32bytes):
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_1):
> - lea (L(shl_1_loop_L1)-L(shl_1))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x01(%rsi), %xmm1
> - jb L(L1_fwd)
> - lea (L(shl_1_loop_L2)-L(shl_1_loop_L1))(%r9), %r9
> -L(L1_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_1_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_1_loop_L1):
> - sub $64, %rdx
> - movaps 0x0f(%rsi), %xmm2
> - movaps 0x1f(%rsi), %xmm3
> - movaps 0x2f(%rsi), %xmm4
> - movaps 0x3f(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $1, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $1, %xmm3, %xmm4
> - palignr $1, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $1, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_1_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_1_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_1_bwd):
> - lea (L(shl_1_bwd_loop_L1)-L(shl_1_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x01(%rsi), %xmm1
> - jb L(L1_bwd)
> - lea (L(shl_1_bwd_loop_L2)-L(shl_1_bwd_loop_L1))(%r9), %r9
> -L(L1_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_1_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_1_bwd_loop_L1):
> - movaps -0x11(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x21(%rsi), %xmm3
> - movaps -0x31(%rsi), %xmm4
> - movaps -0x41(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $1, %xmm2, %xmm1
> - palignr $1, %xmm3, %xmm2
> - palignr $1, %xmm4, %xmm3
> - palignr $1, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_1_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_1_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_2):
> - lea (L(shl_2_loop_L1)-L(shl_2))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x02(%rsi), %xmm1
> - jb L(L2_fwd)
> - lea (L(shl_2_loop_L2)-L(shl_2_loop_L1))(%r9), %r9
> -L(L2_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_2_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_2_loop_L1):
> - sub $64, %rdx
> - movaps 0x0e(%rsi), %xmm2
> - movaps 0x1e(%rsi), %xmm3
> - movaps 0x2e(%rsi), %xmm4
> - movaps 0x3e(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $2, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $2, %xmm3, %xmm4
> - palignr $2, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $2, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_2_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_2_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_2_bwd):
> - lea (L(shl_2_bwd_loop_L1)-L(shl_2_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x02(%rsi), %xmm1
> - jb L(L2_bwd)
> - lea (L(shl_2_bwd_loop_L2)-L(shl_2_bwd_loop_L1))(%r9), %r9
> -L(L2_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_2_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_2_bwd_loop_L1):
> - movaps -0x12(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x22(%rsi), %xmm3
> - movaps -0x32(%rsi), %xmm4
> - movaps -0x42(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $2, %xmm2, %xmm1
> - palignr $2, %xmm3, %xmm2
> - palignr $2, %xmm4, %xmm3
> - palignr $2, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_2_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_2_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_3):
> - lea (L(shl_3_loop_L1)-L(shl_3))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x03(%rsi), %xmm1
> - jb L(L3_fwd)
> - lea (L(shl_3_loop_L2)-L(shl_3_loop_L1))(%r9), %r9
> -L(L3_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_3_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_3_loop_L1):
> - sub $64, %rdx
> - movaps 0x0d(%rsi), %xmm2
> - movaps 0x1d(%rsi), %xmm3
> - movaps 0x2d(%rsi), %xmm4
> - movaps 0x3d(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $3, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $3, %xmm3, %xmm4
> - palignr $3, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $3, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_3_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_3_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_3_bwd):
> - lea (L(shl_3_bwd_loop_L1)-L(shl_3_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x03(%rsi), %xmm1
> - jb L(L3_bwd)
> - lea (L(shl_3_bwd_loop_L2)-L(shl_3_bwd_loop_L1))(%r9), %r9
> -L(L3_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_3_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_3_bwd_loop_L1):
> - movaps -0x13(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x23(%rsi), %xmm3
> - movaps -0x33(%rsi), %xmm4
> - movaps -0x43(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $3, %xmm2, %xmm1
> - palignr $3, %xmm3, %xmm2
> - palignr $3, %xmm4, %xmm3
> - palignr $3, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_3_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_3_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_4):
> - lea (L(shl_4_loop_L1)-L(shl_4))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x04(%rsi), %xmm1
> - jb L(L4_fwd)
> - lea (L(shl_4_loop_L2)-L(shl_4_loop_L1))(%r9), %r9
> -L(L4_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_4_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_4_loop_L1):
> - sub $64, %rdx
> - movaps 0x0c(%rsi), %xmm2
> - movaps 0x1c(%rsi), %xmm3
> - movaps 0x2c(%rsi), %xmm4
> - movaps 0x3c(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $4, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $4, %xmm3, %xmm4
> - palignr $4, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $4, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_4_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_4_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_4_bwd):
> - lea (L(shl_4_bwd_loop_L1)-L(shl_4_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x04(%rsi), %xmm1
> - jb L(L4_bwd)
> - lea (L(shl_4_bwd_loop_L2)-L(shl_4_bwd_loop_L1))(%r9), %r9
> -L(L4_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_4_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_4_bwd_loop_L1):
> - movaps -0x14(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x24(%rsi), %xmm3
> - movaps -0x34(%rsi), %xmm4
> - movaps -0x44(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $4, %xmm2, %xmm1
> - palignr $4, %xmm3, %xmm2
> - palignr $4, %xmm4, %xmm3
> - palignr $4, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_4_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_4_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_5):
> - lea (L(shl_5_loop_L1)-L(shl_5))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x05(%rsi), %xmm1
> - jb L(L5_fwd)
> - lea (L(shl_5_loop_L2)-L(shl_5_loop_L1))(%r9), %r9
> -L(L5_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_5_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_5_loop_L1):
> - sub $64, %rdx
> - movaps 0x0b(%rsi), %xmm2
> - movaps 0x1b(%rsi), %xmm3
> - movaps 0x2b(%rsi), %xmm4
> - movaps 0x3b(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $5, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $5, %xmm3, %xmm4
> - palignr $5, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $5, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_5_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_5_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_5_bwd):
> - lea (L(shl_5_bwd_loop_L1)-L(shl_5_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x05(%rsi), %xmm1
> - jb L(L5_bwd)
> - lea (L(shl_5_bwd_loop_L2)-L(shl_5_bwd_loop_L1))(%r9), %r9
> -L(L5_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_5_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_5_bwd_loop_L1):
> - movaps -0x15(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x25(%rsi), %xmm3
> - movaps -0x35(%rsi), %xmm4
> - movaps -0x45(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $5, %xmm2, %xmm1
> - palignr $5, %xmm3, %xmm2
> - palignr $5, %xmm4, %xmm3
> - palignr $5, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_5_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_5_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_6):
> - lea (L(shl_6_loop_L1)-L(shl_6))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x06(%rsi), %xmm1
> - jb L(L6_fwd)
> - lea (L(shl_6_loop_L2)-L(shl_6_loop_L1))(%r9), %r9
> -L(L6_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_6_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_6_loop_L1):
> - sub $64, %rdx
> - movaps 0x0a(%rsi), %xmm2
> - movaps 0x1a(%rsi), %xmm3
> - movaps 0x2a(%rsi), %xmm4
> - movaps 0x3a(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $6, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $6, %xmm3, %xmm4
> - palignr $6, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $6, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_6_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_6_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_6_bwd):
> - lea (L(shl_6_bwd_loop_L1)-L(shl_6_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x06(%rsi), %xmm1
> - jb L(L6_bwd)
> - lea (L(shl_6_bwd_loop_L2)-L(shl_6_bwd_loop_L1))(%r9), %r9
> -L(L6_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_6_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_6_bwd_loop_L1):
> - movaps -0x16(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x26(%rsi), %xmm3
> - movaps -0x36(%rsi), %xmm4
> - movaps -0x46(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $6, %xmm2, %xmm1
> - palignr $6, %xmm3, %xmm2
> - palignr $6, %xmm4, %xmm3
> - palignr $6, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_6_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_6_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_7):
> - lea (L(shl_7_loop_L1)-L(shl_7))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x07(%rsi), %xmm1
> - jb L(L7_fwd)
> - lea (L(shl_7_loop_L2)-L(shl_7_loop_L1))(%r9), %r9
> -L(L7_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_7_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_7_loop_L1):
> - sub $64, %rdx
> - movaps 0x09(%rsi), %xmm2
> - movaps 0x19(%rsi), %xmm3
> - movaps 0x29(%rsi), %xmm4
> - movaps 0x39(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $7, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $7, %xmm3, %xmm4
> - palignr $7, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $7, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_7_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_7_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_7_bwd):
> - lea (L(shl_7_bwd_loop_L1)-L(shl_7_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x07(%rsi), %xmm1
> - jb L(L7_bwd)
> - lea (L(shl_7_bwd_loop_L2)-L(shl_7_bwd_loop_L1))(%r9), %r9
> -L(L7_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_7_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_7_bwd_loop_L1):
> - movaps -0x17(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x27(%rsi), %xmm3
> - movaps -0x37(%rsi), %xmm4
> - movaps -0x47(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $7, %xmm2, %xmm1
> - palignr $7, %xmm3, %xmm2
> - palignr $7, %xmm4, %xmm3
> - palignr $7, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_7_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_7_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_8):
> - lea (L(shl_8_loop_L1)-L(shl_8))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x08(%rsi), %xmm1
> - jb L(L8_fwd)
> - lea (L(shl_8_loop_L2)-L(shl_8_loop_L1))(%r9), %r9
> -L(L8_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> -L(shl_8_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_8_loop_L1):
> - sub $64, %rdx
> - movaps 0x08(%rsi), %xmm2
> - movaps 0x18(%rsi), %xmm3
> - movaps 0x28(%rsi), %xmm4
> - movaps 0x38(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $8, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $8, %xmm3, %xmm4
> - palignr $8, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $8, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_8_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> - .p2align 4
> -L(shl_8_end):
> - lea 64(%rdx), %rdx
> - movaps %xmm4, -0x20(%rdi)
> - add %rdx, %rsi
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_8_bwd):
> - lea (L(shl_8_bwd_loop_L1)-L(shl_8_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x08(%rsi), %xmm1
> - jb L(L8_bwd)
> - lea (L(shl_8_bwd_loop_L2)-L(shl_8_bwd_loop_L1))(%r9), %r9
> -L(L8_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_8_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_8_bwd_loop_L1):
> - movaps -0x18(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x28(%rsi), %xmm3
> - movaps -0x38(%rsi), %xmm4
> - movaps -0x48(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $8, %xmm2, %xmm1
> - palignr $8, %xmm3, %xmm2
> - palignr $8, %xmm4, %xmm3
> - palignr $8, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_8_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_8_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_9):
> - lea (L(shl_9_loop_L1)-L(shl_9))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x09(%rsi), %xmm1
> - jb L(L9_fwd)
> - lea (L(shl_9_loop_L2)-L(shl_9_loop_L1))(%r9), %r9
> -L(L9_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_9_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_9_loop_L1):
> - sub $64, %rdx
> - movaps 0x07(%rsi), %xmm2
> - movaps 0x17(%rsi), %xmm3
> - movaps 0x27(%rsi), %xmm4
> - movaps 0x37(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $9, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $9, %xmm3, %xmm4
> - palignr $9, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $9, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_9_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_9_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_9_bwd):
> - lea (L(shl_9_bwd_loop_L1)-L(shl_9_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x09(%rsi), %xmm1
> - jb L(L9_bwd)
> - lea (L(shl_9_bwd_loop_L2)-L(shl_9_bwd_loop_L1))(%r9), %r9
> -L(L9_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_9_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_9_bwd_loop_L1):
> - movaps -0x19(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x29(%rsi), %xmm3
> - movaps -0x39(%rsi), %xmm4
> - movaps -0x49(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $9, %xmm2, %xmm1
> - palignr $9, %xmm3, %xmm2
> - palignr $9, %xmm4, %xmm3
> - palignr $9, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_9_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_9_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_10):
> - lea (L(shl_10_loop_L1)-L(shl_10))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0a(%rsi), %xmm1
> - jb L(L10_fwd)
> - lea (L(shl_10_loop_L2)-L(shl_10_loop_L1))(%r9), %r9
> -L(L10_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_10_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_10_loop_L1):
> - sub $64, %rdx
> - movaps 0x06(%rsi), %xmm2
> - movaps 0x16(%rsi), %xmm3
> - movaps 0x26(%rsi), %xmm4
> - movaps 0x36(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $10, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $10, %xmm3, %xmm4
> - palignr $10, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $10, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_10_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_10_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_10_bwd):
> - lea (L(shl_10_bwd_loop_L1)-L(shl_10_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0a(%rsi), %xmm1
> - jb L(L10_bwd)
> - lea (L(shl_10_bwd_loop_L2)-L(shl_10_bwd_loop_L1))(%r9), %r9
> -L(L10_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_10_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_10_bwd_loop_L1):
> - movaps -0x1a(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x2a(%rsi), %xmm3
> - movaps -0x3a(%rsi), %xmm4
> - movaps -0x4a(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $10, %xmm2, %xmm1
> - palignr $10, %xmm3, %xmm2
> - palignr $10, %xmm4, %xmm3
> - palignr $10, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_10_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_10_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_11):
> - lea (L(shl_11_loop_L1)-L(shl_11))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0b(%rsi), %xmm1
> - jb L(L11_fwd)
> - lea (L(shl_11_loop_L2)-L(shl_11_loop_L1))(%r9), %r9
> -L(L11_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_11_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_11_loop_L1):
> - sub $64, %rdx
> - movaps 0x05(%rsi), %xmm2
> - movaps 0x15(%rsi), %xmm3
> - movaps 0x25(%rsi), %xmm4
> - movaps 0x35(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $11, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $11, %xmm3, %xmm4
> - palignr $11, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $11, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_11_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_11_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_11_bwd):
> - lea (L(shl_11_bwd_loop_L1)-L(shl_11_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0b(%rsi), %xmm1
> - jb L(L11_bwd)
> - lea (L(shl_11_bwd_loop_L2)-L(shl_11_bwd_loop_L1))(%r9), %r9
> -L(L11_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_11_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_11_bwd_loop_L1):
> - movaps -0x1b(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x2b(%rsi), %xmm3
> - movaps -0x3b(%rsi), %xmm4
> - movaps -0x4b(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $11, %xmm2, %xmm1
> - palignr $11, %xmm3, %xmm2
> - palignr $11, %xmm4, %xmm3
> - palignr $11, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_11_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_11_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_12):
> - lea (L(shl_12_loop_L1)-L(shl_12))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0c(%rsi), %xmm1
> - jb L(L12_fwd)
> - lea (L(shl_12_loop_L2)-L(shl_12_loop_L1))(%r9), %r9
> -L(L12_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_12_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_12_loop_L1):
> - sub $64, %rdx
> - movaps 0x04(%rsi), %xmm2
> - movaps 0x14(%rsi), %xmm3
> - movaps 0x24(%rsi), %xmm4
> - movaps 0x34(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $12, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $12, %xmm3, %xmm4
> - palignr $12, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $12, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_12_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_12_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_12_bwd):
> - lea (L(shl_12_bwd_loop_L1)-L(shl_12_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0c(%rsi), %xmm1
> - jb L(L12_bwd)
> - lea (L(shl_12_bwd_loop_L2)-L(shl_12_bwd_loop_L1))(%r9), %r9
> -L(L12_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_12_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_12_bwd_loop_L1):
> - movaps -0x1c(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x2c(%rsi), %xmm3
> - movaps -0x3c(%rsi), %xmm4
> - movaps -0x4c(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $12, %xmm2, %xmm1
> - palignr $12, %xmm3, %xmm2
> - palignr $12, %xmm4, %xmm3
> - palignr $12, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_12_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_12_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_13):
> - lea (L(shl_13_loop_L1)-L(shl_13))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0d(%rsi), %xmm1
> - jb L(L13_fwd)
> - lea (L(shl_13_loop_L2)-L(shl_13_loop_L1))(%r9), %r9
> -L(L13_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_13_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_13_loop_L1):
> - sub $64, %rdx
> - movaps 0x03(%rsi), %xmm2
> - movaps 0x13(%rsi), %xmm3
> - movaps 0x23(%rsi), %xmm4
> - movaps 0x33(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $13, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $13, %xmm3, %xmm4
> - palignr $13, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $13, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_13_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_13_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_13_bwd):
> - lea (L(shl_13_bwd_loop_L1)-L(shl_13_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0d(%rsi), %xmm1
> - jb L(L13_bwd)
> - lea (L(shl_13_bwd_loop_L2)-L(shl_13_bwd_loop_L1))(%r9), %r9
> -L(L13_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_13_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_13_bwd_loop_L1):
> - movaps -0x1d(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x2d(%rsi), %xmm3
> - movaps -0x3d(%rsi), %xmm4
> - movaps -0x4d(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $13, %xmm2, %xmm1
> - palignr $13, %xmm3, %xmm2
> - palignr $13, %xmm4, %xmm3
> - palignr $13, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_13_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_13_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_14):
> - lea (L(shl_14_loop_L1)-L(shl_14))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0e(%rsi), %xmm1
> - jb L(L14_fwd)
> - lea (L(shl_14_loop_L2)-L(shl_14_loop_L1))(%r9), %r9
> -L(L14_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_14_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_14_loop_L1):
> - sub $64, %rdx
> - movaps 0x02(%rsi), %xmm2
> - movaps 0x12(%rsi), %xmm3
> - movaps 0x22(%rsi), %xmm4
> - movaps 0x32(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $14, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $14, %xmm3, %xmm4
> - palignr $14, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $14, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_14_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_14_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_14_bwd):
> - lea (L(shl_14_bwd_loop_L1)-L(shl_14_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0e(%rsi), %xmm1
> - jb L(L14_bwd)
> - lea (L(shl_14_bwd_loop_L2)-L(shl_14_bwd_loop_L1))(%r9), %r9
> -L(L14_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_14_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_14_bwd_loop_L1):
> - movaps -0x1e(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x2e(%rsi), %xmm3
> - movaps -0x3e(%rsi), %xmm4
> - movaps -0x4e(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $14, %xmm2, %xmm1
> - palignr $14, %xmm3, %xmm2
> - palignr $14, %xmm4, %xmm3
> - palignr $14, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_14_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_14_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_15):
> - lea (L(shl_15_loop_L1)-L(shl_15))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0f(%rsi), %xmm1
> - jb L(L15_fwd)
> - lea (L(shl_15_loop_L2)-L(shl_15_loop_L1))(%r9), %r9
> -L(L15_fwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_15_loop_L2):
> - prefetchnta 0x1c0(%rsi)
> -L(shl_15_loop_L1):
> - sub $64, %rdx
> - movaps 0x01(%rsi), %xmm2
> - movaps 0x11(%rsi), %xmm3
> - movaps 0x21(%rsi), %xmm4
> - movaps 0x31(%rsi), %xmm5
> - movdqa %xmm5, %xmm6
> - palignr $15, %xmm4, %xmm5
> - lea 64(%rsi), %rsi
> - palignr $15, %xmm3, %xmm4
> - palignr $15, %xmm2, %xmm3
> - lea 64(%rdi), %rdi
> - palignr $15, %xmm1, %xmm2
> - movdqa %xmm6, %xmm1
> - movdqa %xmm2, -0x40(%rdi)
> - movaps %xmm3, -0x30(%rdi)
> - jb L(shl_15_end)
> - movaps %xmm4, -0x20(%rdi)
> - movaps %xmm5, -0x10(%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_15_end):
> - movaps %xmm4, -0x20(%rdi)
> - lea 64(%rdx), %rdx
> - movaps %xmm5, -0x10(%rdi)
> - add %rdx, %rdi
> - movdqu %xmm0, (%r8)
> - add %rdx, %rsi
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(shl_15_bwd):
> - lea (L(shl_15_bwd_loop_L1)-L(shl_15_bwd))(%r9), %r9
> - cmp %rcx, %rdx
> - movaps -0x0f(%rsi), %xmm1
> - jb L(L15_bwd)
> - lea (L(shl_15_bwd_loop_L2)-L(shl_15_bwd_loop_L1))(%r9), %r9
> -L(L15_bwd):
> - lea -64(%rdx), %rdx
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_15_bwd_loop_L2):
> - prefetchnta -0x1c0(%rsi)
> -L(shl_15_bwd_loop_L1):
> - movaps -0x1f(%rsi), %xmm2
> - sub $0x40, %rdx
> - movaps -0x2f(%rsi), %xmm3
> - movaps -0x3f(%rsi), %xmm4
> - movaps -0x4f(%rsi), %xmm5
> - lea -0x40(%rsi), %rsi
> - palignr $15, %xmm2, %xmm1
> - palignr $15, %xmm3, %xmm2
> - palignr $15, %xmm4, %xmm3
> - palignr $15, %xmm5, %xmm4
> -
> - movaps %xmm1, -0x10(%rdi)
> - movaps %xmm5, %xmm1
> -
> - movaps %xmm2, -0x20(%rdi)
> - lea -0x40(%rdi), %rdi
> -
> - movaps %xmm3, 0x10(%rdi)
> - jb L(shl_15_bwd_end)
> - movaps %xmm4, (%rdi)
> - _CET_NOTRACK jmp *%r9
> - ud2
> -L(shl_15_bwd_end):
> - movaps %xmm4, (%rdi)
> - lea 64(%rdx), %rdx
> - movdqu %xmm0, (%r8)
> - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
> -
> - .p2align 4
> -L(write_72bytes):
> - movdqu -72(%rsi), %xmm0
> - movdqu -56(%rsi), %xmm1
> - mov -40(%rsi), %r8
> - mov -32(%rsi), %r9
> - mov -24(%rsi), %r10
> - mov -16(%rsi), %r11
> - mov -8(%rsi), %rcx
> - movdqu %xmm0, -72(%rdi)
> - movdqu %xmm1, -56(%rdi)
> - mov %r8, -40(%rdi)
> - mov %r9, -32(%rdi)
> - mov %r10, -24(%rdi)
> - mov %r11, -16(%rdi)
> - mov %rcx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_64bytes):
> - movdqu -64(%rsi), %xmm0
> - mov -48(%rsi), %rcx
> - mov -40(%rsi), %r8
> - mov -32(%rsi), %r9
> - mov -24(%rsi), %r10
> - mov -16(%rsi), %r11
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -64(%rdi)
> - mov %rcx, -48(%rdi)
> - mov %r8, -40(%rdi)
> - mov %r9, -32(%rdi)
> - mov %r10, -24(%rdi)
> - mov %r11, -16(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_56bytes):
> - movdqu -56(%rsi), %xmm0
> - mov -40(%rsi), %r8
> - mov -32(%rsi), %r9
> - mov -24(%rsi), %r10
> - mov -16(%rsi), %r11
> - mov -8(%rsi), %rcx
> - movdqu %xmm0, -56(%rdi)
> - mov %r8, -40(%rdi)
> - mov %r9, -32(%rdi)
> - mov %r10, -24(%rdi)
> - mov %r11, -16(%rdi)
> - mov %rcx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_48bytes):
> - mov -48(%rsi), %rcx
> - mov -40(%rsi), %r8
> - mov -32(%rsi), %r9
> - mov -24(%rsi), %r10
> - mov -16(%rsi), %r11
> - mov -8(%rsi), %rdx
> - mov %rcx, -48(%rdi)
> - mov %r8, -40(%rdi)
> - mov %r9, -32(%rdi)
> - mov %r10, -24(%rdi)
> - mov %r11, -16(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_40bytes):
> - mov -40(%rsi), %r8
> - mov -32(%rsi), %r9
> - mov -24(%rsi), %r10
> - mov -16(%rsi), %r11
> - mov -8(%rsi), %rdx
> - mov %r8, -40(%rdi)
> - mov %r9, -32(%rdi)
> - mov %r10, -24(%rdi)
> - mov %r11, -16(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_32bytes):
> - mov -32(%rsi), %r9
> - mov -24(%rsi), %r10
> - mov -16(%rsi), %r11
> - mov -8(%rsi), %rdx
> - mov %r9, -32(%rdi)
> - mov %r10, -24(%rdi)
> - mov %r11, -16(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_24bytes):
> - mov -24(%rsi), %r10
> - mov -16(%rsi), %r11
> - mov -8(%rsi), %rdx
> - mov %r10, -24(%rdi)
> - mov %r11, -16(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_16bytes):
> - mov -16(%rsi), %r11
> - mov -8(%rsi), %rdx
> - mov %r11, -16(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_8bytes):
> - mov -8(%rsi), %rdx
> - mov %rdx, -8(%rdi)
> -L(write_0bytes):
> - ret
> -
> - .p2align 4
> -L(write_73bytes):
> - movdqu -73(%rsi), %xmm0
> - movdqu -57(%rsi), %xmm1
> - mov -41(%rsi), %rcx
> - mov -33(%rsi), %r9
> - mov -25(%rsi), %r10
> - mov -17(%rsi), %r11
> - mov -9(%rsi), %r8
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -73(%rdi)
> - movdqu %xmm1, -57(%rdi)
> - mov %rcx, -41(%rdi)
> - mov %r9, -33(%rdi)
> - mov %r10, -25(%rdi)
> - mov %r11, -17(%rdi)
> - mov %r8, -9(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_65bytes):
> - movdqu -65(%rsi), %xmm0
> - movdqu -49(%rsi), %xmm1
> - mov -33(%rsi), %r9
> - mov -25(%rsi), %r10
> - mov -17(%rsi), %r11
> - mov -9(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -65(%rdi)
> - movdqu %xmm1, -49(%rdi)
> - mov %r9, -33(%rdi)
> - mov %r10, -25(%rdi)
> - mov %r11, -17(%rdi)
> - mov %rcx, -9(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_57bytes):
> - movdqu -57(%rsi), %xmm0
> - mov -41(%rsi), %r8
> - mov -33(%rsi), %r9
> - mov -25(%rsi), %r10
> - mov -17(%rsi), %r11
> - mov -9(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -57(%rdi)
> - mov %r8, -41(%rdi)
> - mov %r9, -33(%rdi)
> - mov %r10, -25(%rdi)
> - mov %r11, -17(%rdi)
> - mov %rcx, -9(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_49bytes):
> - movdqu -49(%rsi), %xmm0
> - mov -33(%rsi), %r9
> - mov -25(%rsi), %r10
> - mov -17(%rsi), %r11
> - mov -9(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -49(%rdi)
> - mov %r9, -33(%rdi)
> - mov %r10, -25(%rdi)
> - mov %r11, -17(%rdi)
> - mov %rcx, -9(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_41bytes):
> - mov -41(%rsi), %r8
> - mov -33(%rsi), %r9
> - mov -25(%rsi), %r10
> - mov -17(%rsi), %r11
> - mov -9(%rsi), %rcx
> - mov -1(%rsi), %dl
> - mov %r8, -41(%rdi)
> - mov %r9, -33(%rdi)
> - mov %r10, -25(%rdi)
> - mov %r11, -17(%rdi)
> - mov %rcx, -9(%rdi)
> - mov %dl, -1(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_33bytes):
> - mov -33(%rsi), %r9
> - mov -25(%rsi), %r10
> - mov -17(%rsi), %r11
> - mov -9(%rsi), %rcx
> - mov -1(%rsi), %dl
> - mov %r9, -33(%rdi)
> - mov %r10, -25(%rdi)
> - mov %r11, -17(%rdi)
> - mov %rcx, -9(%rdi)
> - mov %dl, -1(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_25bytes):
> - mov -25(%rsi), %r10
> - mov -17(%rsi), %r11
> - mov -9(%rsi), %rcx
> - mov -1(%rsi), %dl
> - mov %r10, -25(%rdi)
> - mov %r11, -17(%rdi)
> - mov %rcx, -9(%rdi)
> - mov %dl, -1(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_17bytes):
> - mov -17(%rsi), %r11
> - mov -9(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r11, -17(%rdi)
> - mov %rcx, -9(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_9bytes):
> - mov -9(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %rcx, -9(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_1bytes):
> - mov -1(%rsi), %dl
> - mov %dl, -1(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_74bytes):
> - movdqu -74(%rsi), %xmm0
> - movdqu -58(%rsi), %xmm1
> - mov -42(%rsi), %r8
> - mov -34(%rsi), %r9
> - mov -26(%rsi), %r10
> - mov -18(%rsi), %r11
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -74(%rdi)
> - movdqu %xmm1, -58(%rdi)
> - mov %r8, -42(%rdi)
> - mov %r9, -34(%rdi)
> - mov %r10, -26(%rdi)
> - mov %r11, -18(%rdi)
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_66bytes):
> - movdqu -66(%rsi), %xmm0
> - movdqu -50(%rsi), %xmm1
> - mov -42(%rsi), %r8
> - mov -34(%rsi), %r9
> - mov -26(%rsi), %r10
> - mov -18(%rsi), %r11
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -66(%rdi)
> - movdqu %xmm1, -50(%rdi)
> - mov %r8, -42(%rdi)
> - mov %r9, -34(%rdi)
> - mov %r10, -26(%rdi)
> - mov %r11, -18(%rdi)
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_58bytes):
> - movdqu -58(%rsi), %xmm1
> - mov -42(%rsi), %r8
> - mov -34(%rsi), %r9
> - mov -26(%rsi), %r10
> - mov -18(%rsi), %r11
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm1, -58(%rdi)
> - mov %r8, -42(%rdi)
> - mov %r9, -34(%rdi)
> - mov %r10, -26(%rdi)
> - mov %r11, -18(%rdi)
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_50bytes):
> - movdqu -50(%rsi), %xmm0
> - mov -34(%rsi), %r9
> - mov -26(%rsi), %r10
> - mov -18(%rsi), %r11
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -50(%rdi)
> - mov %r9, -34(%rdi)
> - mov %r10, -26(%rdi)
> - mov %r11, -18(%rdi)
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_42bytes):
> - mov -42(%rsi), %r8
> - mov -34(%rsi), %r9
> - mov -26(%rsi), %r10
> - mov -18(%rsi), %r11
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r8, -42(%rdi)
> - mov %r9, -34(%rdi)
> - mov %r10, -26(%rdi)
> - mov %r11, -18(%rdi)
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_34bytes):
> - mov -34(%rsi), %r9
> - mov -26(%rsi), %r10
> - mov -18(%rsi), %r11
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r9, -34(%rdi)
> - mov %r10, -26(%rdi)
> - mov %r11, -18(%rdi)
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_26bytes):
> - mov -26(%rsi), %r10
> - mov -18(%rsi), %r11
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r10, -26(%rdi)
> - mov %r11, -18(%rdi)
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_18bytes):
> - mov -18(%rsi), %r11
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r11, -18(%rdi)
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_10bytes):
> - mov -10(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %rcx, -10(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_2bytes):
> - mov -2(%rsi), %dx
> - mov %dx, -2(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_75bytes):
> - movdqu -75(%rsi), %xmm0
> - movdqu -59(%rsi), %xmm1
> - mov -43(%rsi), %r8
> - mov -35(%rsi), %r9
> - mov -27(%rsi), %r10
> - mov -19(%rsi), %r11
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -75(%rdi)
> - movdqu %xmm1, -59(%rdi)
> - mov %r8, -43(%rdi)
> - mov %r9, -35(%rdi)
> - mov %r10, -27(%rdi)
> - mov %r11, -19(%rdi)
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_67bytes):
> - movdqu -67(%rsi), %xmm0
> - movdqu -59(%rsi), %xmm1
> - mov -43(%rsi), %r8
> - mov -35(%rsi), %r9
> - mov -27(%rsi), %r10
> - mov -19(%rsi), %r11
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -67(%rdi)
> - movdqu %xmm1, -59(%rdi)
> - mov %r8, -43(%rdi)
> - mov %r9, -35(%rdi)
> - mov %r10, -27(%rdi)
> - mov %r11, -19(%rdi)
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_59bytes):
> - movdqu -59(%rsi), %xmm0
> - mov -43(%rsi), %r8
> - mov -35(%rsi), %r9
> - mov -27(%rsi), %r10
> - mov -19(%rsi), %r11
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -59(%rdi)
> - mov %r8, -43(%rdi)
> - mov %r9, -35(%rdi)
> - mov %r10, -27(%rdi)
> - mov %r11, -19(%rdi)
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_51bytes):
> - movdqu -51(%rsi), %xmm0
> - mov -35(%rsi), %r9
> - mov -27(%rsi), %r10
> - mov -19(%rsi), %r11
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -51(%rdi)
> - mov %r9, -35(%rdi)
> - mov %r10, -27(%rdi)
> - mov %r11, -19(%rdi)
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_43bytes):
> - mov -43(%rsi), %r8
> - mov -35(%rsi), %r9
> - mov -27(%rsi), %r10
> - mov -19(%rsi), %r11
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r8, -43(%rdi)
> - mov %r9, -35(%rdi)
> - mov %r10, -27(%rdi)
> - mov %r11, -19(%rdi)
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_35bytes):
> - mov -35(%rsi), %r9
> - mov -27(%rsi), %r10
> - mov -19(%rsi), %r11
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r9, -35(%rdi)
> - mov %r10, -27(%rdi)
> - mov %r11, -19(%rdi)
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_27bytes):
> - mov -27(%rsi), %r10
> - mov -19(%rsi), %r11
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r10, -27(%rdi)
> - mov %r11, -19(%rdi)
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_19bytes):
> - mov -19(%rsi), %r11
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r11, -19(%rdi)
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_11bytes):
> - mov -11(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %rcx, -11(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_3bytes):
> - mov -3(%rsi), %dx
> - mov -2(%rsi), %cx
> - mov %dx, -3(%rdi)
> - mov %cx, -2(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_76bytes):
> - movdqu -76(%rsi), %xmm0
> - movdqu -60(%rsi), %xmm1
> - mov -44(%rsi), %r8
> - mov -36(%rsi), %r9
> - mov -28(%rsi), %r10
> - mov -20(%rsi), %r11
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -76(%rdi)
> - movdqu %xmm1, -60(%rdi)
> - mov %r8, -44(%rdi)
> - mov %r9, -36(%rdi)
> - mov %r10, -28(%rdi)
> - mov %r11, -20(%rdi)
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_68bytes):
> - movdqu -68(%rsi), %xmm0
> - movdqu -52(%rsi), %xmm1
> - mov -36(%rsi), %r9
> - mov -28(%rsi), %r10
> - mov -20(%rsi), %r11
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -68(%rdi)
> - movdqu %xmm1, -52(%rdi)
> - mov %r9, -36(%rdi)
> - mov %r10, -28(%rdi)
> - mov %r11, -20(%rdi)
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_60bytes):
> - movdqu -60(%rsi), %xmm0
> - mov -44(%rsi), %r8
> - mov -36(%rsi), %r9
> - mov -28(%rsi), %r10
> - mov -20(%rsi), %r11
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -60(%rdi)
> - mov %r8, -44(%rdi)
> - mov %r9, -36(%rdi)
> - mov %r10, -28(%rdi)
> - mov %r11, -20(%rdi)
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_52bytes):
> - movdqu -52(%rsi), %xmm0
> - mov -36(%rsi), %r9
> - mov -28(%rsi), %r10
> - mov -20(%rsi), %r11
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - movdqu %xmm0, -52(%rdi)
> - mov %r9, -36(%rdi)
> - mov %r10, -28(%rdi)
> - mov %r11, -20(%rdi)
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_44bytes):
> - mov -44(%rsi), %r8
> - mov -36(%rsi), %r9
> - mov -28(%rsi), %r10
> - mov -20(%rsi), %r11
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r8, -44(%rdi)
> - mov %r9, -36(%rdi)
> - mov %r10, -28(%rdi)
> - mov %r11, -20(%rdi)
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_36bytes):
> - mov -36(%rsi), %r9
> - mov -28(%rsi), %r10
> - mov -20(%rsi), %r11
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r9, -36(%rdi)
> - mov %r10, -28(%rdi)
> - mov %r11, -20(%rdi)
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_28bytes):
> - mov -28(%rsi), %r10
> - mov -20(%rsi), %r11
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r10, -28(%rdi)
> - mov %r11, -20(%rdi)
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_20bytes):
> - mov -20(%rsi), %r11
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %r11, -20(%rdi)
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_12bytes):
> - mov -12(%rsi), %rcx
> - mov -4(%rsi), %edx
> - mov %rcx, -12(%rdi)
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_4bytes):
> - mov -4(%rsi), %edx
> - mov %edx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_77bytes):
> - movdqu -77(%rsi), %xmm0
> - movdqu -61(%rsi), %xmm1
> - mov -45(%rsi), %r8
> - mov -37(%rsi), %r9
> - mov -29(%rsi), %r10
> - mov -21(%rsi), %r11
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -77(%rdi)
> - movdqu %xmm1, -61(%rdi)
> - mov %r8, -45(%rdi)
> - mov %r9, -37(%rdi)
> - mov %r10, -29(%rdi)
> - mov %r11, -21(%rdi)
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_69bytes):
> - movdqu -69(%rsi), %xmm0
> - movdqu -53(%rsi), %xmm1
> - mov -37(%rsi), %r9
> - mov -29(%rsi), %r10
> - mov -21(%rsi), %r11
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -69(%rdi)
> - movdqu %xmm1, -53(%rdi)
> - mov %r9, -37(%rdi)
> - mov %r10, -29(%rdi)
> - mov %r11, -21(%rdi)
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_61bytes):
> - movdqu -61(%rsi), %xmm0
> - mov -45(%rsi), %r8
> - mov -37(%rsi), %r9
> - mov -29(%rsi), %r10
> - mov -21(%rsi), %r11
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -61(%rdi)
> - mov %r8, -45(%rdi)
> - mov %r9, -37(%rdi)
> - mov %r10, -29(%rdi)
> - mov %r11, -21(%rdi)
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_53bytes):
> - movdqu -53(%rsi), %xmm0
> - mov -45(%rsi), %r8
> - mov -37(%rsi), %r9
> - mov -29(%rsi), %r10
> - mov -21(%rsi), %r11
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -53(%rdi)
> - mov %r9, -37(%rdi)
> - mov %r10, -29(%rdi)
> - mov %r11, -21(%rdi)
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_45bytes):
> - mov -45(%rsi), %r8
> - mov -37(%rsi), %r9
> - mov -29(%rsi), %r10
> - mov -21(%rsi), %r11
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r8, -45(%rdi)
> - mov %r9, -37(%rdi)
> - mov %r10, -29(%rdi)
> - mov %r11, -21(%rdi)
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_37bytes):
> - mov -37(%rsi), %r9
> - mov -29(%rsi), %r10
> - mov -21(%rsi), %r11
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r9, -37(%rdi)
> - mov %r10, -29(%rdi)
> - mov %r11, -21(%rdi)
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_29bytes):
> - mov -29(%rsi), %r10
> - mov -21(%rsi), %r11
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r10, -29(%rdi)
> - mov %r11, -21(%rdi)
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_21bytes):
> - mov -21(%rsi), %r11
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r11, -21(%rdi)
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_13bytes):
> - mov -13(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %rcx, -13(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_5bytes):
> - mov -5(%rsi), %edx
> - mov -4(%rsi), %ecx
> - mov %edx, -5(%rdi)
> - mov %ecx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_78bytes):
> - movdqu -78(%rsi), %xmm0
> - movdqu -62(%rsi), %xmm1
> - mov -46(%rsi), %r8
> - mov -38(%rsi), %r9
> - mov -30(%rsi), %r10
> - mov -22(%rsi), %r11
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -78(%rdi)
> - movdqu %xmm1, -62(%rdi)
> - mov %r8, -46(%rdi)
> - mov %r9, -38(%rdi)
> - mov %r10, -30(%rdi)
> - mov %r11, -22(%rdi)
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_70bytes):
> - movdqu -70(%rsi), %xmm0
> - movdqu -54(%rsi), %xmm1
> - mov -38(%rsi), %r9
> - mov -30(%rsi), %r10
> - mov -22(%rsi), %r11
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -70(%rdi)
> - movdqu %xmm1, -54(%rdi)
> - mov %r9, -38(%rdi)
> - mov %r10, -30(%rdi)
> - mov %r11, -22(%rdi)
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_62bytes):
> - movdqu -62(%rsi), %xmm0
> - mov -46(%rsi), %r8
> - mov -38(%rsi), %r9
> - mov -30(%rsi), %r10
> - mov -22(%rsi), %r11
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -62(%rdi)
> - mov %r8, -46(%rdi)
> - mov %r9, -38(%rdi)
> - mov %r10, -30(%rdi)
> - mov %r11, -22(%rdi)
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_54bytes):
> - movdqu -54(%rsi), %xmm0
> - mov -38(%rsi), %r9
> - mov -30(%rsi), %r10
> - mov -22(%rsi), %r11
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -54(%rdi)
> - mov %r9, -38(%rdi)
> - mov %r10, -30(%rdi)
> - mov %r11, -22(%rdi)
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_46bytes):
> - mov -46(%rsi), %r8
> - mov -38(%rsi), %r9
> - mov -30(%rsi), %r10
> - mov -22(%rsi), %r11
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r8, -46(%rdi)
> - mov %r9, -38(%rdi)
> - mov %r10, -30(%rdi)
> - mov %r11, -22(%rdi)
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_38bytes):
> - mov -38(%rsi), %r9
> - mov -30(%rsi), %r10
> - mov -22(%rsi), %r11
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r9, -38(%rdi)
> - mov %r10, -30(%rdi)
> - mov %r11, -22(%rdi)
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_30bytes):
> - mov -30(%rsi), %r10
> - mov -22(%rsi), %r11
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r10, -30(%rdi)
> - mov %r11, -22(%rdi)
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_22bytes):
> - mov -22(%rsi), %r11
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r11, -22(%rdi)
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_14bytes):
> - mov -14(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %rcx, -14(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_6bytes):
> - mov -6(%rsi), %edx
> - mov -4(%rsi), %ecx
> - mov %edx, -6(%rdi)
> - mov %ecx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_79bytes):
> - movdqu -79(%rsi), %xmm0
> - movdqu -63(%rsi), %xmm1
> - mov -47(%rsi), %r8
> - mov -39(%rsi), %r9
> - mov -31(%rsi), %r10
> - mov -23(%rsi), %r11
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -79(%rdi)
> - movdqu %xmm1, -63(%rdi)
> - mov %r8, -47(%rdi)
> - mov %r9, -39(%rdi)
> - mov %r10, -31(%rdi)
> - mov %r11, -23(%rdi)
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_71bytes):
> - movdqu -71(%rsi), %xmm0
> - movdqu -55(%rsi), %xmm1
> - mov -39(%rsi), %r9
> - mov -31(%rsi), %r10
> - mov -23(%rsi), %r11
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -71(%rdi)
> - movdqu %xmm1, -55(%rdi)
> - mov %r9, -39(%rdi)
> - mov %r10, -31(%rdi)
> - mov %r11, -23(%rdi)
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_63bytes):
> - movdqu -63(%rsi), %xmm0
> - mov -47(%rsi), %r8
> - mov -39(%rsi), %r9
> - mov -31(%rsi), %r10
> - mov -23(%rsi), %r11
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -63(%rdi)
> - mov %r8, -47(%rdi)
> - mov %r9, -39(%rdi)
> - mov %r10, -31(%rdi)
> - mov %r11, -23(%rdi)
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_55bytes):
> - movdqu -55(%rsi), %xmm0
> - mov -39(%rsi), %r9
> - mov -31(%rsi), %r10
> - mov -23(%rsi), %r11
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - movdqu %xmm0, -55(%rdi)
> - mov %r9, -39(%rdi)
> - mov %r10, -31(%rdi)
> - mov %r11, -23(%rdi)
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_47bytes):
> - mov -47(%rsi), %r8
> - mov -39(%rsi), %r9
> - mov -31(%rsi), %r10
> - mov -23(%rsi), %r11
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r8, -47(%rdi)
> - mov %r9, -39(%rdi)
> - mov %r10, -31(%rdi)
> - mov %r11, -23(%rdi)
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_39bytes):
> - mov -39(%rsi), %r9
> - mov -31(%rsi), %r10
> - mov -23(%rsi), %r11
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r9, -39(%rdi)
> - mov %r10, -31(%rdi)
> - mov %r11, -23(%rdi)
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_31bytes):
> - mov -31(%rsi), %r10
> - mov -23(%rsi), %r11
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r10, -31(%rdi)
> - mov %r11, -23(%rdi)
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_23bytes):
> - mov -23(%rsi), %r11
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %r11, -23(%rdi)
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_15bytes):
> - mov -15(%rsi), %rcx
> - mov -8(%rsi), %rdx
> - mov %rcx, -15(%rdi)
> - mov %rdx, -8(%rdi)
> - ret
> -
> - .p2align 4
> -L(write_7bytes):
> - mov -7(%rsi), %edx
> - mov -4(%rsi), %ecx
> - mov %edx, -7(%rdi)
> - mov %ecx, -4(%rdi)
> - ret
> -
> - .p2align 4
> -L(large_page_fwd):
> - movdqu (%rsi), %xmm1
> - lea 16(%rsi), %rsi
> - movdqu %xmm0, (%r8)
> - movntdq %xmm1, (%rdi)
> - lea 16(%rdi), %rdi
> - lea -0x90(%rdx), %rdx
> -#ifdef USE_AS_MEMMOVE
> - mov %rsi, %r9
> - sub %rdi, %r9
> - cmp %rdx, %r9
> - jae L(memmove_is_memcpy_fwd)
> - shl $2, %rcx
> - cmp %rcx, %rdx
> - jb L(ll_cache_copy_fwd_start)
> -L(memmove_is_memcpy_fwd):
> -#endif
> -L(large_page_loop):
> - movdqu (%rsi), %xmm0
> - movdqu 0x10(%rsi), %xmm1
> - movdqu 0x20(%rsi), %xmm2
> - movdqu 0x30(%rsi), %xmm3
> - movdqu 0x40(%rsi), %xmm4
> - movdqu 0x50(%rsi), %xmm5
> - movdqu 0x60(%rsi), %xmm6
> - movdqu 0x70(%rsi), %xmm7
> - lea 0x80(%rsi), %rsi
> -
> - sub $0x80, %rdx
> - movntdq %xmm0, (%rdi)
> - movntdq %xmm1, 0x10(%rdi)
> - movntdq %xmm2, 0x20(%rdi)
> - movntdq %xmm3, 0x30(%rdi)
> - movntdq %xmm4, 0x40(%rdi)
> - movntdq %xmm5, 0x50(%rdi)
> - movntdq %xmm6, 0x60(%rdi)
> - movntdq %xmm7, 0x70(%rdi)
> - lea 0x80(%rdi), %rdi
> - jae L(large_page_loop)
> - cmp $-0x40, %rdx
> - lea 0x80(%rdx), %rdx
> - jl L(large_page_less_64bytes)
> -
> - movdqu (%rsi), %xmm0
> - movdqu 0x10(%rsi), %xmm1
> - movdqu 0x20(%rsi), %xmm2
> - movdqu 0x30(%rsi), %xmm3
> - lea 0x40(%rsi), %rsi
> -
> - movntdq %xmm0, (%rdi)
> - movntdq %xmm1, 0x10(%rdi)
> - movntdq %xmm2, 0x20(%rdi)
> - movntdq %xmm3, 0x30(%rdi)
> - lea 0x40(%rdi), %rdi
> - sub $0x40, %rdx
> -L(large_page_less_64bytes):
> - add %rdx, %rsi
> - add %rdx, %rdi
> - sfence
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> -#ifdef USE_AS_MEMMOVE
> - .p2align 4
> -L(ll_cache_copy_fwd_start):
> - prefetcht0 0x1c0(%rsi)
> - prefetcht0 0x200(%rsi)
> - movdqu (%rsi), %xmm0
> - movdqu 0x10(%rsi), %xmm1
> - movdqu 0x20(%rsi), %xmm2
> - movdqu 0x30(%rsi), %xmm3
> - movdqu 0x40(%rsi), %xmm4
> - movdqu 0x50(%rsi), %xmm5
> - movdqu 0x60(%rsi), %xmm6
> - movdqu 0x70(%rsi), %xmm7
> - lea 0x80(%rsi), %rsi
> -
> - sub $0x80, %rdx
> - movaps %xmm0, (%rdi)
> - movaps %xmm1, 0x10(%rdi)
> - movaps %xmm2, 0x20(%rdi)
> - movaps %xmm3, 0x30(%rdi)
> - movaps %xmm4, 0x40(%rdi)
> - movaps %xmm5, 0x50(%rdi)
> - movaps %xmm6, 0x60(%rdi)
> - movaps %xmm7, 0x70(%rdi)
> - lea 0x80(%rdi), %rdi
> - jae L(ll_cache_copy_fwd_start)
> - cmp $-0x40, %rdx
> - lea 0x80(%rdx), %rdx
> - jl L(large_page_ll_less_fwd_64bytes)
> -
> - movdqu (%rsi), %xmm0
> - movdqu 0x10(%rsi), %xmm1
> - movdqu 0x20(%rsi), %xmm2
> - movdqu 0x30(%rsi), %xmm3
> - lea 0x40(%rsi), %rsi
> -
> - movaps %xmm0, (%rdi)
> - movaps %xmm1, 0x10(%rdi)
> - movaps %xmm2, 0x20(%rdi)
> - movaps %xmm3, 0x30(%rdi)
> - lea 0x40(%rdi), %rdi
> - sub $0x40, %rdx
> -L(large_page_ll_less_fwd_64bytes):
> - add %rdx, %rsi
> - add %rdx, %rdi
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> -#endif
> - .p2align 4
> -L(large_page_bwd):
> - movdqu -0x10(%rsi), %xmm1
> - lea -16(%rsi), %rsi
> - movdqu %xmm0, (%r8)
> - movdqa %xmm1, -0x10(%rdi)
> - lea -16(%rdi), %rdi
> - lea -0x90(%rdx), %rdx
> -#ifdef USE_AS_MEMMOVE
> - mov %rdi, %r9
> - sub %rsi, %r9
> - cmp %rdx, %r9
> - jae L(memmove_is_memcpy_bwd)
> - cmp %rcx, %r9
> - jb L(ll_cache_copy_bwd_start)
> -L(memmove_is_memcpy_bwd):
> -#endif
> -L(large_page_bwd_loop):
> - movdqu -0x10(%rsi), %xmm0
> - movdqu -0x20(%rsi), %xmm1
> - movdqu -0x30(%rsi), %xmm2
> - movdqu -0x40(%rsi), %xmm3
> - movdqu -0x50(%rsi), %xmm4
> - movdqu -0x60(%rsi), %xmm5
> - movdqu -0x70(%rsi), %xmm6
> - movdqu -0x80(%rsi), %xmm7
> - lea -0x80(%rsi), %rsi
> -
> - sub $0x80, %rdx
> - movntdq %xmm0, -0x10(%rdi)
> - movntdq %xmm1, -0x20(%rdi)
> - movntdq %xmm2, -0x30(%rdi)
> - movntdq %xmm3, -0x40(%rdi)
> - movntdq %xmm4, -0x50(%rdi)
> - movntdq %xmm5, -0x60(%rdi)
> - movntdq %xmm6, -0x70(%rdi)
> - movntdq %xmm7, -0x80(%rdi)
> - lea -0x80(%rdi), %rdi
> - jae L(large_page_bwd_loop)
> - cmp $-0x40, %rdx
> - lea 0x80(%rdx), %rdx
> - jl L(large_page_less_bwd_64bytes)
> -
> - movdqu -0x10(%rsi), %xmm0
> - movdqu -0x20(%rsi), %xmm1
> - movdqu -0x30(%rsi), %xmm2
> - movdqu -0x40(%rsi), %xmm3
> - lea -0x40(%rsi), %rsi
> -
> - movntdq %xmm0, -0x10(%rdi)
> - movntdq %xmm1, -0x20(%rdi)
> - movntdq %xmm2, -0x30(%rdi)
> - movntdq %xmm3, -0x40(%rdi)
> - lea -0x40(%rdi), %rdi
> - sub $0x40, %rdx
> -L(large_page_less_bwd_64bytes):
> - sfence
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -
> -#ifdef USE_AS_MEMMOVE
> - .p2align 4
> -L(ll_cache_copy_bwd_start):
> - prefetcht0 -0x1c0(%rsi)
> - prefetcht0 -0x200(%rsi)
> - movdqu -0x10(%rsi), %xmm0
> - movdqu -0x20(%rsi), %xmm1
> - movdqu -0x30(%rsi), %xmm2
> - movdqu -0x40(%rsi), %xmm3
> - movdqu -0x50(%rsi), %xmm4
> - movdqu -0x60(%rsi), %xmm5
> - movdqu -0x70(%rsi), %xmm6
> - movdqu -0x80(%rsi), %xmm7
> - lea -0x80(%rsi), %rsi
> -
> - sub $0x80, %rdx
> - movaps %xmm0, -0x10(%rdi)
> - movaps %xmm1, -0x20(%rdi)
> - movaps %xmm2, -0x30(%rdi)
> - movaps %xmm3, -0x40(%rdi)
> - movaps %xmm4, -0x50(%rdi)
> - movaps %xmm5, -0x60(%rdi)
> - movaps %xmm6, -0x70(%rdi)
> - movaps %xmm7, -0x80(%rdi)
> - lea -0x80(%rdi), %rdi
> - jae L(ll_cache_copy_bwd_start)
> - cmp $-0x40, %rdx
> - lea 0x80(%rdx), %rdx
> - jl L(large_page_ll_less_bwd_64bytes)
> -
> - movdqu -0x10(%rsi), %xmm0
> - movdqu -0x20(%rsi), %xmm1
> - movdqu -0x30(%rsi), %xmm2
> - movdqu -0x40(%rsi), %xmm3
> - lea -0x40(%rsi), %rsi
> -
> - movaps %xmm0, -0x10(%rdi)
> - movaps %xmm1, -0x20(%rdi)
> - movaps %xmm2, -0x30(%rdi)
> - movaps %xmm3, -0x40(%rdi)
> - lea -0x40(%rdi), %rdi
> - sub $0x40, %rdx
> -L(large_page_ll_less_bwd_64bytes):
> - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
> -#endif
> -
> -END (MEMCPY)
> -
> - .section .rodata.ssse3,"a",@progbits
> - .p2align 3
> -L(table_less_80bytes):
> - .int JMPTBL (L(write_0bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_1bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_2bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_3bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_4bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_5bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_6bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_7bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_8bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_9bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_10bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_11bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_12bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_13bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_14bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_15bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_16bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_17bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_18bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_19bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_20bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_21bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_22bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_23bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_24bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_25bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_26bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_27bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_28bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_29bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_30bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_31bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_32bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_33bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_34bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_35bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_36bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_37bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_38bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_39bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_40bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_41bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_42bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_43bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_44bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_45bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_46bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_47bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_48bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_49bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_50bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_51bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_52bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_53bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_54bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_55bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_56bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_57bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_58bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_59bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_60bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_61bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_62bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_63bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_64bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_65bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_66bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_67bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_68bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_69bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_70bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_71bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_72bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_73bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_74bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_75bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_76bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_77bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_78bytes), L(table_less_80bytes))
> - .int JMPTBL (L(write_79bytes), L(table_less_80bytes))
> -
> - .p2align 3
> -L(shl_table):
> - .int JMPTBL (L(shl_0), L(shl_table))
> - .int JMPTBL (L(shl_1), L(shl_table))
> - .int JMPTBL (L(shl_2), L(shl_table))
> - .int JMPTBL (L(shl_3), L(shl_table))
> - .int JMPTBL (L(shl_4), L(shl_table))
> - .int JMPTBL (L(shl_5), L(shl_table))
> - .int JMPTBL (L(shl_6), L(shl_table))
> - .int JMPTBL (L(shl_7), L(shl_table))
> - .int JMPTBL (L(shl_8), L(shl_table))
> - .int JMPTBL (L(shl_9), L(shl_table))
> - .int JMPTBL (L(shl_10), L(shl_table))
> - .int JMPTBL (L(shl_11), L(shl_table))
> - .int JMPTBL (L(shl_12), L(shl_table))
> - .int JMPTBL (L(shl_13), L(shl_table))
> - .int JMPTBL (L(shl_14), L(shl_table))
> - .int JMPTBL (L(shl_15), L(shl_table))
> -
> - .p2align 3
> -L(shl_table_bwd):
> - .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd))
> - .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd))
> -
> -#endif
> diff --git a/sysdeps/x86_64/multiarch/memmove-ssse3.S b/sysdeps/x86_64/multiarch/memmove-ssse3.S
> index 295430b1ef..84e4e0f6cb 100644
> --- a/sysdeps/x86_64/multiarch/memmove-ssse3.S
> +++ b/sysdeps/x86_64/multiarch/memmove-ssse3.S
> @@ -1,4 +1,382 @@
> -#define USE_AS_MEMMOVE
> -#define MEMCPY __memmove_ssse3
> -#define MEMCPY_CHK __memmove_chk_ssse3
> -#include "memcpy-ssse3.S"
> +#include <sysdep.h>
> +
> +#ifndef MEMMOVE
> +# define MEMMOVE __memmove_ssse3
> +# define MEMMOVE_CHK __memmove_chk_ssse3
> +# define MEMCPY __memcpy_ssse3
> +# define MEMCPY_CHK __memcpy_chk_ssse3
> +# define MEMPCPY __mempcpy_ssse3
> +# define MEMPCPY_CHK __mempcpy_chk_ssse3
> +#endif
> +
> + .section .text.ssse3, "ax", @progbits
> +ENTRY(MEMPCPY_CHK)
> + cmp %RDX_LP, %RCX_LP
> + jb HIDDEN_JUMPTARGET(__chk_fail)
> +END(MEMPCPY_CHK)
> +
> +ENTRY(MEMPCPY)
> + mov %RDI_LP, %RAX_LP
> + add %RDX_LP, %RAX_LP
> + jmp L(start)
> +END(MEMPCPY)
> +
> +ENTRY(MEMMOVE_CHK)
> + cmp %RDX_LP, %RCX_LP
> + jb HIDDEN_JUMPTARGET(__chk_fail)
> +END(MEMMOVE_CHK)
> +
> +ENTRY_P2ALIGN(MEMMOVE, 6)
> + movq %rdi, %rax
> +L(start):
> + cmpq $16, %rdx
> + jb L(copy_0_15)
> +
> + /* These loads are always useful. */
> + movups 0(%rsi), %xmm0
> + movups -16(%rsi, %rdx), %xmm7
> + cmpq $32, %rdx
> + ja L(more_2x_vec)
> +
> + movups %xmm0, 0(%rdi)
> + movups %xmm7, -16(%rdi, %rdx)
> + ret
> +
> + .p2align 4,, 8
> +L(copy_4x_vec):
> + movups 16(%rsi), %xmm1
> + movups -32(%rsi, %rdx), %xmm2
> +
> + movups %xmm0, 0(%rdi)
> + movups %xmm1, 16(%rdi)
> + movups %xmm2, -32(%rdi, %rdx)
> + movups %xmm7, -16(%rdi, %rdx)
> + ret
> +
> + .p2align 4,, 8
> +L(copy_0_15):
> + cmpl $8, %edx
> + ja L(copy_9_15)
> +
> + cmpl $4, %edx
> + jb L(copy_0_3)
> +
> + movl 0(%rsi), %ecx
> + movl -4(%rsi, %rdx), %esi
> + movl %ecx, 0(%rdi)
> + movl %esi, -4(%rdi, %rdx)
> + ret
> +
> + .p2align 4,, 8
> +L(copy_9_15):
> + movq 0(%rsi), %rcx
> + movq -8(%rsi, %rdx), %rsi
> + movq %rcx, 0(%rdi)
> + movq %rsi, -8(%rdi, %rdx)
> + ret
> +
> + .p2align 4,, 4
> +L(copy_0_3):
> + cmpl $1, %edx
> + jl L(copy_0_0)
> + movzbl (%rsi), %ecx
> + je L(copy_0_1)
> +
> + movzwl -2(%rsi, %rdx), %esi
> + movw %si, -2(%rdi, %rdx)
> +L(copy_0_1):
> + movb %cl, (%rdi)
> +L(copy_0_0):
> +L(nop):
> + ret
> +
> + .p2align 4
> +L(more_2x_vec):
> + cmpq $64, %rdx
> + jbe L(copy_4x_vec)
> +
> + /* We use rcx later to get alignr value. */
> + movq %rdi, %rcx
> +
> + /* Backward copy for overlap + dst > src for memmove safety. */
> + subq %rsi, %rcx
> + cmpq %rdx, %rcx
> + jb L(copy_backward)
> +
> + /* Load tail. */
> +
> + /* -16(%rsi, %rdx) already loaded into xmm7. */
> + movups -32(%rsi, %rdx), %xmm8
> + movups -48(%rsi, %rdx), %xmm9
> +
> + /* Get misalignment. */
> + andl $0xf, %ecx
> +
> + movq %rsi, %r9
> + addq %rcx, %rsi
> + andq $-16, %rsi
> + /* Get first vec for `palignr`. */
> + movaps (%rsi), %xmm1
> +
> + /* We have loaded (%rsi) so safe to do this store before the
> + loop. */
> + movups %xmm0, (%rdi)
> +
> +#ifdef SHARED_CACHE_SIZE_HALF
> + cmp $SHARED_CACHE_SIZE_HALF, %RDX_LP
> +#else
> + cmp __x86_shared_cache_size_half(%rip), %rdx
> +#endif
> + ja L(large_memcpy)
> +
> + leaq -64(%rdi, %rdx), %r8
> + andq $-16, %rdi
> + movl $48, %edx
> +
> + leaq L(loop_fwd_start)(%rip), %r9
> + sall $6, %ecx
> + addq %r9, %rcx
> + jmp * %rcx
> +
> + .p2align 4,, 8
> +L(copy_backward):
> + testq %rcx, %rcx
> + jz L(nop)
> +
> + /* Preload tail. */
> +
> + /* (%rsi) already loaded into xmm0. */
> + movups 16(%rsi), %xmm4
> + movups 32(%rsi), %xmm5
> +
> + movq %rdi, %r8
> + subq %rdi, %rsi
> + leaq -49(%rdi, %rdx), %rdi
> + andq $-16, %rdi
> + addq %rdi, %rsi
> + andq $-16, %rsi
> +
> + movaps 48(%rsi), %xmm6
> +
> +
> + leaq L(loop_bkwd_start)(%rip), %r9
> + andl $0xf, %ecx
> + sall $6, %ecx
> + addq %r9, %rcx
> + jmp * %rcx
> +
> + .p2align 4,, 8
> +L(large_memcpy):
> + movups -64(%r9, %rdx), %xmm10
> + movups -80(%r9, %rdx), %xmm11
> +
> + sall $5, %ecx
> + leal (%rcx, %rcx, 2), %r8d
> + leaq -96(%rdi, %rdx), %rcx
> + andq $-16, %rdi
> + leaq L(large_loop_fwd_start)(%rip), %rdx
> + addq %r8, %rdx
> + jmp * %rdx
> +
> +
> + /* Instead of a typical jump table all 16 loops are exactly
> + 64-bytes in size. So, we can just jump to first loop + r8 *
> + 64. Before modifying any loop ensure all their sizes match!
> + */
> + .p2align 6
> +L(loop_fwd_start):
> +L(loop_fwd_0x0):
> + movaps 16(%rsi), %xmm1
> + movaps 32(%rsi), %xmm2
> + movaps 48(%rsi), %xmm3
> + movaps %xmm1, 16(%rdi)
> + movaps %xmm2, 32(%rdi)
> + movaps %xmm3, 48(%rdi)
> + addq %rdx, %rdi
> + addq %rdx, %rsi
> + cmpq %rdi, %r8
> + ja L(loop_fwd_0x0)
> +L(end_loop_fwd):
> + movups %xmm9, 16(%r8)
> + movups %xmm8, 32(%r8)
> + movups %xmm7, 48(%r8)
> + ret
> +
> + /* Extactly 64 bytes if `jmp L(end_loop_fwd)` is long encoding.
> + 60 bytes otherwise. */
> +#define ALIGNED_LOOP_FWD(align_by); \
> + .p2align 6; \
> +L(loop_fwd_ ## align_by): \
> + movaps 16(%rsi), %xmm0; \
> + movaps 32(%rsi), %xmm2; \
> + movaps 48(%rsi), %xmm3; \
> + movaps %xmm3, %xmm4; \
> + palignr $align_by, %xmm2, %xmm3; \
> + palignr $align_by, %xmm0, %xmm2; \
> + palignr $align_by, %xmm1, %xmm0; \
> + movaps %xmm4, %xmm1; \
> + movaps %xmm0, 16(%rdi); \
> + movaps %xmm2, 32(%rdi); \
> + movaps %xmm3, 48(%rdi); \
> + addq %rdx, %rdi; \
> + addq %rdx, %rsi; \
> + cmpq %rdi, %r8; \
> + ja L(loop_fwd_ ## align_by); \
> + jmp L(end_loop_fwd);
> +
> + /* Must be in descending order. */
> + ALIGNED_LOOP_FWD (0xf)
> + ALIGNED_LOOP_FWD (0xe)
> + ALIGNED_LOOP_FWD (0xd)
> + ALIGNED_LOOP_FWD (0xc)
> + ALIGNED_LOOP_FWD (0xb)
> + ALIGNED_LOOP_FWD (0xa)
> + ALIGNED_LOOP_FWD (0x9)
> + ALIGNED_LOOP_FWD (0x8)
> + ALIGNED_LOOP_FWD (0x7)
> + ALIGNED_LOOP_FWD (0x6)
> + ALIGNED_LOOP_FWD (0x5)
> + ALIGNED_LOOP_FWD (0x4)
> + ALIGNED_LOOP_FWD (0x3)
> + ALIGNED_LOOP_FWD (0x2)
> + ALIGNED_LOOP_FWD (0x1)
> +
> + .p2align 6
> +L(large_loop_fwd_start):
> +L(large_loop_fwd_0x0):
> + movaps 16(%rsi), %xmm1
> + movaps 32(%rsi), %xmm2
> + movaps 48(%rsi), %xmm3
> + movaps 64(%rsi), %xmm4
> + movaps 80(%rsi), %xmm5
> + movntps %xmm1, 16(%rdi)
> + movntps %xmm2, 32(%rdi)
> + movntps %xmm3, 48(%rdi)
> + movntps %xmm4, 64(%rdi)
> + movntps %xmm5, 80(%rdi)
> + addq $80, %rdi
> + addq $80, %rsi
> + cmpq %rdi, %rcx
> + ja L(large_loop_fwd_0x0)
> +
> + /* Ensure no icache line split on tail. */
> + .p2align 4
> +L(end_large_loop_fwd):
> + sfence
> + movups %xmm11, 16(%rcx)
> + movups %xmm10, 32(%rcx)
> + movups %xmm9, 48(%rcx)
> + movups %xmm8, 64(%rcx)
> + movups %xmm7, 80(%rcx)
> + ret
> +
> +
> + /* Size > 64 bytes and <= 96 bytes. 32-byte align between ensure
> + 96-byte spacing between each. */
> +#define ALIGNED_LARGE_LOOP_FWD(align_by); \
> + .p2align 5; \
> +L(large_loop_fwd_ ## align_by): \
> + movaps 16(%rsi), %xmm0; \
> + movaps 32(%rsi), %xmm2; \
> + movaps 48(%rsi), %xmm3; \
> + movaps 64(%rsi), %xmm4; \
> + movaps 80(%rsi), %xmm5; \
> + movaps %xmm5, %xmm6; \
> + palignr $align_by, %xmm4, %xmm5; \
> + palignr $align_by, %xmm3, %xmm4; \
> + palignr $align_by, %xmm2, %xmm3; \
> + palignr $align_by, %xmm0, %xmm2; \
> + palignr $align_by, %xmm1, %xmm0; \
> + movaps %xmm6, %xmm1; \
> + movntps %xmm0, 16(%rdi); \
> + movntps %xmm2, 32(%rdi); \
> + movntps %xmm3, 48(%rdi); \
> + movntps %xmm4, 64(%rdi); \
> + movntps %xmm5, 80(%rdi); \
> + addq $80, %rdi; \
> + addq $80, %rsi; \
> + cmpq %rdi, %rcx; \
> + ja L(large_loop_fwd_ ## align_by); \
> + jmp L(end_large_loop_fwd);
> +
> + /* Must be in descending order. */
> + ALIGNED_LARGE_LOOP_FWD (0xf)
> + ALIGNED_LARGE_LOOP_FWD (0xe)
> + ALIGNED_LARGE_LOOP_FWD (0xd)
> + ALIGNED_LARGE_LOOP_FWD (0xc)
> + ALIGNED_LARGE_LOOP_FWD (0xb)
> + ALIGNED_LARGE_LOOP_FWD (0xa)
> + ALIGNED_LARGE_LOOP_FWD (0x9)
> + ALIGNED_LARGE_LOOP_FWD (0x8)
> + ALIGNED_LARGE_LOOP_FWD (0x7)
> + ALIGNED_LARGE_LOOP_FWD (0x6)
> + ALIGNED_LARGE_LOOP_FWD (0x5)
> + ALIGNED_LARGE_LOOP_FWD (0x4)
> + ALIGNED_LARGE_LOOP_FWD (0x3)
> + ALIGNED_LARGE_LOOP_FWD (0x2)
> + ALIGNED_LARGE_LOOP_FWD (0x1)
> +
> +
> + .p2align 6
> +L(loop_bkwd_start):
> +L(loop_bkwd_0x0):
> + movaps 32(%rsi), %xmm1
> + movaps 16(%rsi), %xmm2
> + movaps 0(%rsi), %xmm3
> + movaps %xmm1, 32(%rdi)
> + movaps %xmm2, 16(%rdi)
> + movaps %xmm3, 0(%rdi)
> + subq $48, %rdi
> + subq $48, %rsi
> + cmpq %rdi, %r8
> + jb L(loop_bkwd_0x0)
> +L(end_loop_bkwd):
> + movups %xmm7, -16(%r8, %rdx)
> + movups %xmm0, 0(%r8)
> + movups %xmm4, 16(%r8)
> + movups %xmm5, 32(%r8)
> +
> + ret
> +
> +
> + /* Extactly 64 bytes if `jmp L(end_loop_bkwd)` is long encoding.
> + 60 bytes otherwise. */
> +#define ALIGNED_LOOP_BKWD(align_by); \
> + .p2align 6; \
> +L(loop_bkwd_ ## align_by): \
> + movaps 32(%rsi), %xmm1; \
> + movaps 16(%rsi), %xmm2; \
> + movaps 0(%rsi), %xmm3; \
> + palignr $align_by, %xmm1, %xmm6; \
> + palignr $align_by, %xmm2, %xmm1; \
> + palignr $align_by, %xmm3, %xmm2; \
> + movaps %xmm6, 32(%rdi); \
> + movaps %xmm1, 16(%rdi); \
> + movaps %xmm2, 0(%rdi); \
> + subq $48, %rdi; \
> + subq $48, %rsi; \
> + movaps %xmm3, %xmm6; \
> + cmpq %rdi, %r8; \
> + jb L(loop_bkwd_ ## align_by); \
> + jmp L(end_loop_bkwd);
> +
> + /* Must be in descending order. */
> + ALIGNED_LOOP_BKWD (0xf)
> + ALIGNED_LOOP_BKWD (0xe)
> + ALIGNED_LOOP_BKWD (0xd)
> + ALIGNED_LOOP_BKWD (0xc)
> + ALIGNED_LOOP_BKWD (0xb)
> + ALIGNED_LOOP_BKWD (0xa)
> + ALIGNED_LOOP_BKWD (0x9)
> + ALIGNED_LOOP_BKWD (0x8)
> + ALIGNED_LOOP_BKWD (0x7)
> + ALIGNED_LOOP_BKWD (0x6)
> + ALIGNED_LOOP_BKWD (0x5)
> + ALIGNED_LOOP_BKWD (0x4)
> + ALIGNED_LOOP_BKWD (0x3)
> + ALIGNED_LOOP_BKWD (0x2)
> + ALIGNED_LOOP_BKWD (0x1)
> +END(MEMMOVE)
> +
> +strong_alias (MEMMOVE, MEMCPY)
> +strong_alias (MEMMOVE_CHK, MEMCPY_CHK)
> --
> 2.25.1
>
next prev parent reply other threads:[~2022-04-10 0:48 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-03-25 18:36 [PATCH v1 1/6] x86: Remove {w}memcmp-ssse3 Noah Goldstein
2022-03-25 18:36 ` [PATCH v1 2/6] x86: Remove str{n}{case}cmp-ssse3 Noah Goldstein
2022-03-25 19:55 ` H.J. Lu
2022-03-25 20:44 ` [PATCH v2 1/6] x86: Remove {w}memcmp-ssse3 Noah Goldstein
2022-03-25 20:44 ` [PATCH v2 2/6] x86: Remove str{n}{case}cmp-ssse3 Noah Goldstein
2022-03-25 20:44 ` [PATCH v2 3/6] x86: Remove mem{move|cpy}-ssse3 Noah Goldstein
2022-04-10 0:57 ` [PATCH v4 6/6] x86: Reduce code size of mem{move|pcpy|cpy}-ssse3 Noah Goldstein
2022-03-25 20:44 ` [PATCH v2 4/6] x86: Remove mem{move|cpy}-ssse3-back Noah Goldstein
2022-03-25 20:44 ` [PATCH v2 5/6] x86: Remove str{n}cat-ssse3 Noah Goldstein
2022-03-25 20:44 ` [PATCH v2 6/6] x86: Remove str{p}{n}cpy-ssse3 Noah Goldstein
2022-04-10 0:42 ` [PATCH v3 1/6] " Noah Goldstein
2022-04-10 0:48 ` Noah Goldstein
2022-04-10 0:42 ` [PATCH v3 2/6] x86: Remove mem{move|cpy}-ssse3-back Noah Goldstein
2022-04-10 0:48 ` Noah Goldstein
2022-04-10 0:42 ` [PATCH v3 3/6] x86: Reduce code size of Remove mem{move|pcpy|cpy}-ssse3 Noah Goldstein
2022-04-10 0:48 ` Noah Goldstein [this message]
2022-04-10 0:42 ` [PATCH v3 4/6] x86: Optimize memcmp SSE2 in memcmp.S Noah Goldstein
2022-04-10 0:48 ` Noah Goldstein
2022-04-10 0:42 ` [PATCH v3 5/6] x86: Remove memcmp-sse4.S Noah Goldstein
2022-04-10 0:48 ` Noah Goldstein
2022-04-10 0:42 ` [PATCH v3 6/6] x86: Cleanup page cross code in memcmp-avx2-movbe.S Noah Goldstein
2022-04-10 0:48 ` Noah Goldstein
2022-04-10 0:54 ` [PATCH v4 1/6] x86: Remove {w}memcmp-ssse3 Noah Goldstein
2022-04-10 0:54 ` [PATCH v4 2/6] x86: Remove str{n}{case}cmp-ssse3 Noah Goldstein
2022-04-10 0:54 ` [PATCH v4 3/6] x86: Remove str{n}cat-ssse3 Noah Goldstein
2022-04-10 0:54 ` [PATCH v4 4/6] x86: Remove str{p}{n}cpy-ssse3 Noah Goldstein
2022-04-10 0:54 ` [PATCH v4 5/6] x86: Remove mem{move|cpy}-ssse3-back Noah Goldstein
2022-04-14 16:47 ` [PATCH v5 1/6] x86: Remove {w}memcmp-ssse3 Noah Goldstein
2022-04-14 16:47 ` [PATCH v5 2/6] x86: Remove str{n}{case}cmp-ssse3 Noah Goldstein
2022-04-14 18:05 ` H.J. Lu
2022-04-14 16:47 ` [PATCH v5 3/6] x86: Remove str{n}cat-ssse3 Noah Goldstein
2022-04-14 18:06 ` H.J. Lu
2022-04-14 16:47 ` [PATCH v5 4/6] x86: Remove str{p}{n}cpy-ssse3 Noah Goldstein
2022-04-14 18:10 ` H.J. Lu
2022-04-14 16:47 ` [PATCH v5 5/6] x86: Remove mem{move|cpy}-ssse3-back Noah Goldstein
2022-04-14 18:13 ` H.J. Lu
2022-04-14 16:47 ` [PATCH v5 6/6] x86: Reduce code size of mem{move|pcpy|cpy}-ssse3 Noah Goldstein
2022-04-14 18:04 ` [PATCH v5 1/6] x86: Remove {w}memcmp-ssse3 H.J. Lu
2022-03-25 18:36 ` [PATCH v1 3/6] x86: Remove mem{move|cpy}-ssse3 Noah Goldstein
2022-03-25 19:56 ` H.J. Lu
2022-03-25 18:36 ` [PATCH v1 4/6] x86: Remove mem{move|cpy}-ssse3-back Noah Goldstein
2022-03-25 19:56 ` H.J. Lu
2022-03-25 18:36 ` [PATCH v1 5/6] x86: Remove str{n}cat-ssse3 Noah Goldstein
2022-03-25 19:57 ` H.J. Lu
2022-03-25 18:36 ` [PATCH v1 6/6] x86: Remove str{p}{n}cpy-ssse3 Noah Goldstein
2022-03-25 19:57 ` H.J. Lu
2022-03-25 19:54 ` [PATCH v1 1/6] x86: Remove {w}memcmp-ssse3 H.J. Lu
2022-03-25 20:34 ` Andreas Schwab
2022-03-25 20:40 ` Noah Goldstein
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=CAFUsyf+oDr2_WdOWqrBG75_Tc6bzwGdm+oK0F7ZERzu3FJUWuA@mail.gmail.com \
--to=goldstein.w.n@gmail.com \
--cc=libc-alpha@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).