From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-pf1-x42e.google.com (mail-pf1-x42e.google.com [IPv6:2607:f8b0:4864:20::42e]) by sourceware.org (Postfix) with ESMTPS id 3D6173857343 for ; Thu, 9 Jun 2022 19:33:58 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org 3D6173857343 Received: by mail-pf1-x42e.google.com with SMTP id x4so13360214pfj.10 for ; Thu, 09 Jun 2022 12:33:58 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:mime-version:references:in-reply-to:from:date :message-id:subject:to:cc; bh=QNXZgo+Vu/9n2Z8crqgatKV1DyTegL00T4fWube64QQ=; b=klMfsprV9lwXdaA3iGA/a1vNabGkmR/yzNLLx4HMD+fjz4p77k0GAxuY75w34p0z9M c01MR7T4Y1iJWHtz2NoUphsP4sLNSapHxkpzMS5yzrlnSuW5h3PUDNwcHVc7YKGLFbJL zZICBezHEMAy7W5PJRvd4fZx7TgIW+ow3wNgG3iKE5+XcRS9Odx7lCiGCqSQof0YBj2U il1UavQZV+JA7XBERH2MEJqy1MIKcKgy3lOuuceAI0kLKeTmLSOJm0RhyT/T4lGzeTA0 EL8dSEzPAt4DqfiqT6SnQbsSn7Qe19BfKTGCMnE78kmcgWwVfcYYyu9kpEcJsYUImfSA RPcg== X-Gm-Message-State: AOAM530aii4nEiJ+OW8Pr6oMzWMXUTc0d10iOi80n2EMQ0uFf0xjU+Vi 3XXSBUuI1Eq/u4VK9BGZG2QJt3+zB+XPF4ojQ+aIpcZ3 X-Google-Smtp-Source: ABdhPJyzaU9wBZIm0/fiLt3Wr14CL4MED6bHPErnp7XVDr4Y/DAVnvXMWeZChP57GhxxkDMt4VDDvP64k2rJrKJBo1U= X-Received: by 2002:a63:90c8:0:b0:3fc:ad6f:6e96 with SMTP id a191-20020a6390c8000000b003fcad6f6e96mr36428567pge.256.1654803236993; Thu, 09 Jun 2022 12:33:56 -0700 (PDT) MIME-Version: 1.0 References: <20220607200650.3179339-1-goldstein.w.n@gmail.com> <20220609181636.2530997-1-goldstein.w.n@gmail.com> In-Reply-To: <20220609181636.2530997-1-goldstein.w.n@gmail.com> From: "H.J. Lu" Date: Thu, 9 Jun 2022 12:33:21 -0700 Message-ID: Subject: Re: [PATCH v4 1/7] x86: Improve svml_s_atanhf16_core_avx512.S To: Noah Goldstein Cc: GNU C Library , "Carlos O'Donell" Content-Type: text/plain; charset="UTF-8" X-Spam-Status: No, score=-3025.1 required=5.0 tests=BAYES_00, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, DKIM_VALID_EF, FREEMAIL_FROM, GIT_PATCH_0, RCVD_IN_DNSWL_NONE, SPF_HELO_NONE, SPF_PASS, TXREP, T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org X-BeenThere: libc-alpha@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Libc-alpha mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 09 Jun 2022 19:34:01 -0000 On Thu, Jun 9, 2022 at 11:16 AM Noah Goldstein wrote: > > Improvements are: > 1. Reduce code size (-64 bytes). > 2. Remove redundant move instructions. > 3. Slightly improve instruction selection/scheduling where > possible. > 4. Reduce rodata size ([-128, -188] bytes). > > The throughput improvement is not significant as the port 0 bottleneck > is unavoidable. > > Function, New Time, Old Time, New / Old > _ZGVeN16v_atanhf, 1.39, 1.408, 0.987 > --- > .../multiarch/svml_s_atanhf16_core_avx512.S | 474 +++++++++--------- > 1 file changed, 244 insertions(+), 230 deletions(-) > > diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf16_core_avx512.S > index a1cd920a0f..f42462c581 100644 > --- a/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf16_core_avx512.S > +++ b/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf16_core_avx512.S > @@ -31,53 +31,50 @@ > * > */ > > -/* Offsets for data table __svml_satanh_data_internal_avx512 > - */ > -#define Log_tbl_H 0 > -#define Log_tbl_L 128 > -#define One 256 > -#define AbsMask 320 > -#define AddB5 384 > -#define RcpBitMask 448 > -#define poly_coeff3 512 > -#define poly_coeff2 576 > -#define poly_coeff1 640 > -#define poly_coeff0 704 > -#define Half 768 > -#define L2H 832 > -#define L2L 896 > +/* Offsets for data table __svml_satanh_data_internal_avx512 and > + __svml_satanh_data_internal_avx512_al64. Ordered by use in the > + function. On cold-starts this might help the prefetcher. Possibly > + a better idea is to interleave start/end so that the prefetcher is > + less likely to detect a stream and pull irrelivant lines into > + cache. */ > + > +/* Offset into __svml_satanh_data_internal_avx512. 4-byte aligned as > + the memory is broadcast to {1to16}. */ > +#define AbsMask 0 > + > +/* Offset into __svml_satanh_data_internal_avx512_al64. The full value > + is used here. */ > +#define One 0 > +#define AddB5 64 > +#define RcpBitMask 128 > +#define Log_tbl_L_lo 192 > +#define Log_tbl_L_hi 256 > +#define Log_tbl_H_lo 320 > +#define Log_tbl_H_hi 384 > +#define L2H 448 > +#define L2L 512 > +#define poly_coeff3 576 > +#define poly_coeff2 640 > +#define poly_coeff1 704 > > #include > > +#define ATANHF_DATA(x) ((x)+__svml_satanh_data_internal_avx512_al64) > + > .section .text.exex512, "ax", @progbits > ENTRY(_ZGVeN16v_atanhf_skx) > - pushq %rbp > - cfi_def_cfa_offset(16) > - movq %rsp, %rbp > - cfi_def_cfa(6, 16) > - cfi_offset(6, -16) > - andq $-64, %rsp > - subq $192, %rsp > - vmovups One+__svml_satanh_data_internal_avx512(%rip), %zmm4 > - > - /* round reciprocals to 1+5b mantissas */ > - vmovups AddB5+__svml_satanh_data_internal_avx512(%rip), %zmm14 > - vmovups RcpBitMask+__svml_satanh_data_internal_avx512(%rip), %zmm1 > - vmovaps %zmm0, %zmm11 > - vandps AbsMask+__svml_satanh_data_internal_avx512(%rip), %zmm11, %zmm6 > + vandps AbsMask+__svml_satanh_data_internal_avx512(%rip){1to16}, %zmm0, %zmm6 > + vmovups ATANHF_DATA(One)(%rip), %zmm4 > > /* 1+y */ > vaddps {rn-sae}, %zmm4, %zmm6, %zmm9 > > /* 1-y */ > vsubps {rn-sae}, %zmm6, %zmm4, %zmm8 > - vxorps %zmm6, %zmm11, %zmm10 > - > - /* Yp_high */ > - vsubps {rn-sae}, %zmm4, %zmm9, %zmm2 > > - /* -Ym_high */ > - vsubps {rn-sae}, %zmm4, %zmm8, %zmm5 > + /* round reciprocals to 1+5b mantissas */ > + vmovups ATANHF_DATA(AddB5)(%rip), %zmm14 > + vmovups ATANHF_DATA(RcpBitMask)(%rip), %zmm1 > > /* RcpP ~ 1/Yp */ > vrcp14ps %zmm9, %zmm12 > @@ -85,15 +82,21 @@ ENTRY(_ZGVeN16v_atanhf_skx) > /* RcpM ~ 1/Ym */ > vrcp14ps %zmm8, %zmm13 > > + /* Yp_high */ > + vsubps {rn-sae}, %zmm4, %zmm9, %zmm2 > + > + /* -Ym_high */ > + vsubps {rn-sae}, %zmm4, %zmm8, %zmm5 > + > + > /* input outside (-1, 1) ? */ > - vcmpps $21, {sae}, %zmm4, %zmm6, %k0 > vpaddd %zmm14, %zmm12, %zmm15 > - vpaddd %zmm14, %zmm13, %zmm0 > + vpaddd %zmm14, %zmm13, %zmm12 > > /* Yp_low */ > vsubps {rn-sae}, %zmm2, %zmm6, %zmm3 > vandps %zmm1, %zmm15, %zmm7 > - vandps %zmm1, %zmm0, %zmm12 > + vandps %zmm1, %zmm12, %zmm12 > > /* Ym_low */ > vaddps {rn-sae}, %zmm5, %zmm6, %zmm5 > @@ -102,225 +105,199 @@ ENTRY(_ZGVeN16v_atanhf_skx) > vfmsub213ps {rn-sae}, %zmm4, %zmm7, %zmm9 > > /* Reduced argument: Rm = (RcpM*Ym - 1)+RcpM*Ym_low */ > - vfmsub231ps {rn-sae}, %zmm12, %zmm8, %zmm4 > - vmovups Log_tbl_L+__svml_satanh_data_internal_avx512(%rip), %zmm8 > - vmovups Log_tbl_L+64+__svml_satanh_data_internal_avx512(%rip), %zmm13 > + vfmsub213ps {rn-sae}, %zmm4, %zmm12, %zmm8 > + > + vmovups ATANHF_DATA(Log_tbl_L_lo)(%rip), %zmm10 > + vmovups ATANHF_DATA(Log_tbl_L_hi)(%rip), %zmm13 > > /* exponents */ > - vgetexpps {sae}, %zmm7, %zmm15 > vfmadd231ps {rn-sae}, %zmm7, %zmm3, %zmm9 > + vgetexpps {sae}, %zmm7, %zmm15 > + > > /* Table lookups */ > - vmovups __svml_satanh_data_internal_avx512(%rip), %zmm6 > + vfnmadd231ps {rn-sae}, %zmm12, %zmm5, %zmm8 > vgetexpps {sae}, %zmm12, %zmm14 > - vfnmadd231ps {rn-sae}, %zmm12, %zmm5, %zmm4 > + > > /* Prepare table index */ > vpsrld $18, %zmm7, %zmm3 > vpsrld $18, %zmm12, %zmm2 > - vmovups Log_tbl_H+64+__svml_satanh_data_internal_avx512(%rip), %zmm7 > - vmovups poly_coeff1+__svml_satanh_data_internal_avx512(%rip), %zmm12 > - > + vmovups ATANHF_DATA(Log_tbl_H_lo)(%rip), %zmm11 > + vmovups ATANHF_DATA(Log_tbl_H_hi)(%rip), %zmm7 > /* Km-Kp */ > + > + vmovaps %zmm3, %zmm5 > + vpermi2ps %zmm13, %zmm10, %zmm3 > + vpermt2ps %zmm13, %zmm2, %zmm10 > + vpermi2ps %zmm7, %zmm11, %zmm5 > + vpermt2ps %zmm7, %zmm2, %zmm11 > vsubps {rn-sae}, %zmm15, %zmm14, %zmm1 > - kmovw %k0, %edx > - vmovaps %zmm3, %zmm0 > - vpermi2ps %zmm13, %zmm8, %zmm3 > - vpermt2ps %zmm13, %zmm2, %zmm8 > - vpermi2ps %zmm7, %zmm6, %zmm0 > - vpermt2ps %zmm7, %zmm2, %zmm6 > - vsubps {rn-sae}, %zmm3, %zmm8, %zmm5 > + vsubps {rn-sae}, %zmm3, %zmm10, %zmm7 > > /* K*L2H + Th */ > - vmovups L2H+__svml_satanh_data_internal_avx512(%rip), %zmm2 > + vmovups ATANHF_DATA(L2H)(%rip), %zmm2 > > /* K*L2L + Tl */ > - vmovups L2L+__svml_satanh_data_internal_avx512(%rip), %zmm3 > - > - /* polynomials */ > - vmovups poly_coeff3+__svml_satanh_data_internal_avx512(%rip), %zmm7 > - vmovups poly_coeff0+__svml_satanh_data_internal_avx512(%rip), %zmm13 > + vmovups ATANHF_DATA(L2L)(%rip), %zmm3 > > /* table values */ > - vsubps {rn-sae}, %zmm0, %zmm6, %zmm0 > - vfmadd231ps {rn-sae}, %zmm1, %zmm2, %zmm0 > - vfmadd213ps {rn-sae}, %zmm5, %zmm3, %zmm1 > - vmovups poly_coeff2+__svml_satanh_data_internal_avx512(%rip), %zmm3 > - vmovaps %zmm3, %zmm2 > - vfmadd231ps {rn-sae}, %zmm9, %zmm7, %zmm2 > - vfmadd231ps {rn-sae}, %zmm4, %zmm7, %zmm3 > - vfmadd213ps {rn-sae}, %zmm12, %zmm9, %zmm2 > - vfmadd213ps {rn-sae}, %zmm12, %zmm4, %zmm3 > - vfmadd213ps {rn-sae}, %zmm13, %zmm9, %zmm2 > - vfmadd213ps {rn-sae}, %zmm13, %zmm4, %zmm3 > + vsubps {rn-sae}, %zmm5, %zmm11, %zmm5 > + vfmadd231ps {rn-sae}, %zmm1, %zmm2, %zmm5 > + vfmadd213ps {rn-sae}, %zmm7, %zmm3, %zmm1 > + /* polynomials */ > + vmovups ATANHF_DATA(poly_coeff3)(%rip), %zmm7 > + vmovups ATANHF_DATA(poly_coeff2)(%rip), %zmm10 > + vmovaps %zmm10, %zmm14 > + vfmadd231ps {rn-sae}, %zmm9, %zmm7, %zmm10 > + vfmadd231ps {rn-sae}, %zmm8, %zmm7, %zmm14 > + vmovups ATANHF_DATA(poly_coeff1)(%rip), %zmm12 > + vfmadd213ps {rn-sae}, %zmm12, %zmm9, %zmm10 > + vfmadd213ps {rn-sae}, %zmm12, %zmm8, %zmm14 > + vfmadd213ps {rn-sae}, %zmm4, %zmm9, %zmm10 > + vfmadd213ps {rn-sae}, %zmm4, %zmm8, %zmm14 > > /* (K*L2L + Tl) + Rp*PolyP */ > - vfmadd213ps {rn-sae}, %zmm1, %zmm9, %zmm2 > - vorps Half+__svml_satanh_data_internal_avx512(%rip), %zmm10, %zmm9 > + vfmadd213ps {rn-sae}, %zmm1, %zmm9, %zmm10 > + > + /* zmm12 = zmm12 & (zmm4 | zmm0). */ > + vpternlogq $0xe0, %zmm0, %zmm4, %zmm12 > > /* (K*L2L + Tl) + Rp*PolyP -Rm*PolyM */ > - vfnmadd213ps {rn-sae}, %zmm2, %zmm4, %zmm3 > - vaddps {rn-sae}, %zmm3, %zmm0, %zmm4 > - vmulps {rn-sae}, %zmm9, %zmm4, %zmm0 > + vfnmadd213ps {rn-sae}, %zmm5, %zmm8, %zmm14 > + vaddps {rn-sae}, %zmm14, %zmm10, %zmm8 > + > + vcmpps $21, {sae}, %zmm4, %zmm6, %k0 > + kmovw %k0, %edx > testl %edx, %edx > > /* Go to special inputs processing branch */ > jne L(SPECIAL_VALUES_BRANCH) > - # LOE rbx r12 r13 r14 r15 edx zmm0 zmm11 > + # LOE rbx r12 r13 r14 r15 zmm0 zmm8 zmm12 > + vmulps {rn-sae}, %zmm12, %zmm8, %zmm0 > > - /* Restore registers > - * and exit the function > - */ > - > -L(EXIT): > - movq %rbp, %rsp > - popq %rbp > - cfi_def_cfa(7, 8) > - cfi_restore(6) > + /* No register to restore on fast path. */ > ret > - cfi_def_cfa(6, 16) > - cfi_offset(6, -16) > - > - /* Branch to process > - * special inputs > - */ > > + /* Cold case. edx has 1s where there was a special value that > + needs to be handled by a atanhf call. Optimize for code size > + more so than speed here. */ > L(SPECIAL_VALUES_BRANCH): > - vmovups %zmm11, 64(%rsp) > - vmovups %zmm0, 128(%rsp) > - # LOE rbx r12 r13 r14 r15 edx zmm0 > - > - xorl %eax, %eax > - # LOE rbx r12 r13 r14 r15 eax edx > - > - vzeroupper > - movq %r12, 16(%rsp) > - /* DW_CFA_expression: r12 (r12) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -64; DW_OP_and; DW_OP_const4s: -176; DW_OP_plus) */ > - .cfi_escape 0x10, 0x0c, 0x0e, 0x38, 0x1c, 0x0d, 0xc0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0x50, 0xff, 0xff, 0xff, 0x22 > - movl %eax, %r12d > - movq %r13, 8(%rsp) > - /* DW_CFA_expression: r13 (r13) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -64; DW_OP_and; DW_OP_const4s: -184; DW_OP_plus) */ > - .cfi_escape 0x10, 0x0d, 0x0e, 0x38, 0x1c, 0x0d, 0xc0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0x48, 0xff, 0xff, 0xff, 0x22 > - movl %edx, %r13d > - movq %r14, (%rsp) > - /* DW_CFA_expression: r14 (r14) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -64; DW_OP_and; DW_OP_const4s: -192; DW_OP_plus) */ > - .cfi_escape 0x10, 0x0e, 0x0e, 0x38, 0x1c, 0x0d, 0xc0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0x40, 0xff, 0xff, 0xff, 0x22 > - # LOE rbx r15 r12d r13d > - > - /* Range mask > - * bits check > + # LOE rbx rdx r12 r13 r14 r15 zmm0 zmm8 zmm12 > + /* Use r13 to save/restore the stack. This allows us to use rbp as > + callee save register saving code size. */ > + pushq %r13 > + cfi_adjust_cfa_offset(8) > + cfi_offset(r13, -16) > + /* Need to callee save registers to preserve state across tanhf calls. > */ > + pushq %rbx > + cfi_adjust_cfa_offset(8) > + cfi_offset(rbx, -24) > + pushq %rbp > + cfi_adjust_cfa_offset(8) > + cfi_offset(rbp, -32) > + movq %rsp, %r13 > + cfi_def_cfa_register(r13) > > -L(RANGEMASK_CHECK): > - btl %r12d, %r13d > - > - /* Call scalar math function */ > - jc L(SCALAR_MATH_CALL) > - # LOE rbx r15 r12d r13d > - > - /* Special inputs > - * processing loop > - */ > + /* Align stack and make room for 2x zmm vectors. */ > + andq $-64, %rsp > + addq $-128, %rsp > + vmulps {rn-sae}, %zmm12, %zmm8, %zmm1 > + vmovaps %zmm1, (%rsp) > + vmovaps %zmm0, 64(%rsp) > + vzeroupper > > + /* edx has 1s where there was a special value that needs to be handled > + by a atanhf call. */ > + movl %edx, %ebx > L(SPECIAL_VALUES_LOOP): > - incl %r12d > - cmpl $16, %r12d > - > - /* Check bits in range mask */ > - jl L(RANGEMASK_CHECK) > - # LOE rbx r15 r12d r13d > - > - movq 16(%rsp), %r12 > - cfi_restore(12) > - movq 8(%rsp), %r13 > - cfi_restore(13) > - movq (%rsp), %r14 > - cfi_restore(14) > - vmovups 128(%rsp), %zmm0 > - > - /* Go to exit */ > - jmp L(EXIT) > - /* DW_CFA_expression: r12 (r12) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -64; DW_OP_and; DW_OP_const4s: -176; DW_OP_plus) */ > - .cfi_escape 0x10, 0x0c, 0x0e, 0x38, 0x1c, 0x0d, 0xc0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0x50, 0xff, 0xff, 0xff, 0x22 > - /* DW_CFA_expression: r13 (r13) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -64; DW_OP_and; DW_OP_const4s: -184; DW_OP_plus) */ > - .cfi_escape 0x10, 0x0d, 0x0e, 0x38, 0x1c, 0x0d, 0xc0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0x48, 0xff, 0xff, 0xff, 0x22 > - /* DW_CFA_expression: r14 (r14) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -64; DW_OP_and; DW_OP_const4s: -192; DW_OP_plus) */ > - .cfi_escape 0x10, 0x0e, 0x0e, 0x38, 0x1c, 0x0d, 0xc0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0x40, 0xff, 0xff, 0xff, 0x22 > - # LOE rbx r12 r13 r14 r15 zmm0 > - > - /* Scalar math fucntion call > - * to process special input > - */ > - > -L(SCALAR_MATH_CALL): > - movl %r12d, %r14d > - movss 64(%rsp, %r14, 4), %xmm0 > + # LOE rbx rbp r12 r13 r14 r15 > + /* use rbp as index for special value that is saved across calls to > + atanhf. We technically don't need a callee save register here as offset > + to rsp is always [0, 56] so we can restore rsp by realigning to 64. > + Essentially the tradeoff is 1 extra save/restore vs 2 extra instructions > + in the loop. Realigning also costs more code size. */ > + xorl %ebp, %ebp > + tzcntl %ebx, %ebp > + > + /* Scalar math fucntion call to process special input. */ > + movss 64(%rsp, %rbp, 4), %xmm0 > call atanhf@PLT > - # LOE rbx r14 r15 r12d r13d xmm0 > - > - movss %xmm0, 128(%rsp, %r14, 4) > > - /* Process special inputs in loop */ > - jmp L(SPECIAL_VALUES_LOOP) > - # LOE rbx r15 r12d r13d > + /* No good way to avoid the store-forwarding fault this will cause on > + return. `lfence` avoids the SF fault but at greater cost as it > + serialized stack/callee save restoration. */ > + movss %xmm0, (%rsp, %rbp, 4) > + > + blsrl %ebx, %ebx > + jnz L(SPECIAL_VALUES_LOOP) > + # LOE r12 r13 r14 r15 > + > + /* All results have been written to (%rsp). */ > + vmovaps (%rsp), %zmm0 > + /* Restore rsp. */ > + movq %r13, %rsp > + cfi_def_cfa_register(rsp) > + /* Restore callee save registers. */ > + popq %rbp > + cfi_adjust_cfa_offset(-8) > + cfi_restore(rbp) > + popq %rbx > + cfi_adjust_cfa_offset(-8) > + cfi_restore(rbp) > + popq %r13 > + cfi_adjust_cfa_offset(-8) > + cfi_restore(r13) > + ret > END(_ZGVeN16v_atanhf_skx) > > .section .rodata, "a" > - .align 64 > - > + .align 4 > #ifdef __svml_satanh_data_internal_avx512_typedef > typedef unsigned int VUINT32; > -typedef struct { > - __declspec(align(64)) VUINT32 Log_tbl_H[32][1]; > - __declspec(align(64)) VUINT32 Log_tbl_L[32][1]; > +typedef struct{ > + __declspec(align(4)) VUINT32 AbsMask[1][1]; > __declspec(align(64)) VUINT32 One[16][1]; > - __declspec(align(64)) VUINT32 AbsMask[16][1]; > __declspec(align(64)) VUINT32 AddB5[16][1]; > __declspec(align(64)) VUINT32 RcpBitMask[16][1]; > + __declspec(align(64)) VUINT32 Log_tbl_L_lo[16][1]; > + __declspec(align(64)) VUINT32 Log_tbl_L_hi[16][1]; > + __declspec(align(64)) VUINT32 Log_tbl_H_lo[16][1]; > + __declspec(align(64)) VUINT32 Log_tbl_H_hi[16][1]; > + __declspec(align(64)) VUINT32 L2H[16][1]; > + __declspec(align(64)) VUINT32 L2L[16][1]; > __declspec(align(64)) VUINT32 poly_coeff3[16][1]; > __declspec(align(64)) VUINT32 poly_coeff2[16][1]; > __declspec(align(64)) VUINT32 poly_coeff1[16][1]; > - __declspec(align(64)) VUINT32 poly_coeff0[16][1]; > - __declspec(align(64)) VUINT32 Half[16][1]; > - __declspec(align(64)) VUINT32 L2H[16][1]; > - __declspec(align(64)) VUINT32 L2L[16][1]; > } __svml_satanh_data_internal_avx512; > #endif > __svml_satanh_data_internal_avx512: > - /* Log_tbl_H */ > - .long 0x00000000 > - .long 0x3cfc0000 > - .long 0x3d780000 > - .long 0x3db78000 > - .long 0x3df10000 > - .long 0x3e14c000 > - .long 0x3e300000 > - .long 0x3e4a8000 > - .long 0x3e648000 > - .long 0x3e7dc000 > - .long 0x3e8b4000 > - .long 0x3e974000 > - .long 0x3ea30000 > - .long 0x3eae8000 > - .long 0x3eb9c000 > - .long 0x3ec4e000 > - .long 0x3ecfa000 > - .long 0x3eda2000 > - .long 0x3ee48000 > - .long 0x3eeea000 > - .long 0x3ef8a000 > - .long 0x3f013000 > - .long 0x3f05f000 > - .long 0x3f0aa000 > - .long 0x3f0f4000 > - .long 0x3f13d000 > - .long 0x3f184000 > - .long 0x3f1ca000 > - .long 0x3f20f000 > - .long 0x3f252000 > - .long 0x3f295000 > - .long 0x3f2d7000 > - /* Log_tbl_L */ > + /* Leave this at front so we can potentially save space due to > + smaller alignment constraint. */ > + .align 4 > + /* AbsMask */ > + .long 0x7fffffff > + .align 64 > +__svml_satanh_data_internal_avx512_al64: > + /* One */ > + .long 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 > + .long 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 > + .long 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 > + .long 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 > + /* AddB5 */ > + .align 64 > + .long 0x00020000, 0x00020000, 0x00020000, 0x00020000 > + .long 0x00020000, 0x00020000, 0x00020000, 0x00020000 > + .long 0x00020000, 0x00020000, 0x00020000, 0x00020000 > + .long 0x00020000, 0x00020000, 0x00020000, 0x00020000 > + /* RcpBitMask */ > + .align 64 > + .long 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000 > + .long 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000 > + .long 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000 > + .long 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000 > + /* Log_tbl_L_lo */ > .align 64 > .long 0x00000000 > .long 0x3726c39e > @@ -338,6 +315,8 @@ __svml_satanh_data_internal_avx512: > .long 0x38dedfac > .long 0x38ebfb5e > .long 0xb8e63c9f > + /* Log_tbl_L_hi */ > + .align 64 > .long 0xb85c1340 > .long 0x38777bcd > .long 0xb6038656 > @@ -354,39 +333,74 @@ __svml_satanh_data_internal_avx512: > .long 0x38f85db0 > .long 0x37b4996f > .long 0xb8bfb3ca > - /* One */ > + /* Log_tbl_H_lo */ > .align 64 > - .long 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 > - /* AbsMask */ > + .long 0x00000000 > + .long 0x3cfc0000 > + .long 0x3d780000 > + .long 0x3db78000 > + .long 0x3df10000 > + .long 0x3e14c000 > + .long 0x3e300000 > + .long 0x3e4a8000 > + .long 0x3e648000 > + .long 0x3e7dc000 > + .long 0x3e8b4000 > + .long 0x3e974000 > + .long 0x3ea30000 > + .long 0x3eae8000 > + .long 0x3eb9c000 > + .long 0x3ec4e000 > + /* Log_tbl_H_hi */ > .align 64 > - .long 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff > - /* AddB5 */ > + .long 0x3ecfa000 > + .long 0x3eda2000 > + .long 0x3ee48000 > + .long 0x3eeea000 > + .long 0x3ef8a000 > + .long 0x3f013000 > + .long 0x3f05f000 > + .long 0x3f0aa000 > + .long 0x3f0f4000 > + .long 0x3f13d000 > + .long 0x3f184000 > + .long 0x3f1ca000 > + .long 0x3f20f000 > + .long 0x3f252000 > + .long 0x3f295000 > + .long 0x3f2d7000 > + /* L2H = log(2)_high */ > .align 64 > - .long 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000, 0x00020000 > - /* RcpBitMask */ > + .long 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000 > + .long 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000 > + .long 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000 > + .long 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000 > + /* L2L = log(2)_low */ > .align 64 > - .long 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000, 0xfffc0000 > + .long 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4 > + .long 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4 > + .long 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4 > + .long 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4 > /* poly_coeff3 */ > .align 64 > - .long 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810 > + .long 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810 > + .long 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810 > + .long 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810 > + .long 0xbe800810, 0xbe800810, 0xbe800810, 0xbe800810 > /* poly_coeff2 */ > .align 64 > - .long 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e > + .long 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e > + .long 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e > + .long 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e > + .long 0x3eaab11e, 0x3eaab11e, 0x3eaab11e, 0x3eaab11e > /* poly_coeff1 */ > .align 64 > - .long 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 > - /* poly_coeff0 */ > - .align 64 > - .long 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 > - /* Half */ > - .align 64 > - .long 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000, 0x3f000000 > - /* L2H = log(2)_high */ > - .align 64 > - .long 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000, 0x3f317000 > - /* L2L = log(2)_low */ > - .align 64 > - .long 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4, 0x3805fdf4 > + .long 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 > + .long 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 > + .long 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 > + .long 0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 > .align 64 > + .type __svml_satanh_data_internal_avx512_al64, @object > + .size __svml_satanh_data_internal_avx512_al64, .-__svml_satanh_data_internal_avx512_al64 > .type __svml_satanh_data_internal_avx512, @object > .size __svml_satanh_data_internal_avx512, .-__svml_satanh_data_internal_avx512 > -- > 2.34.1 > LGTM. Thanks. -- H.J.