* [PATCH] x86_64: Add expm1 with FMA
@ 2023-08-11 15:04 H.J. Lu
2023-08-14 4:50 ` Noah Goldstein
2023-08-17 6:46 ` Paul Zimmermann
0 siblings, 2 replies; 3+ messages in thread
From: H.J. Lu @ 2023-08-11 15:04 UTC (permalink / raw)
To: libc-alpha
On Skylake, it improves expm1 bench performance by:
Before After Improvement
max 70.204 68.054 3%
min 20.709 16.2 22%
mean 22.1221 16.7367 24%
NB: Add
extern long double __expm1l (long double);
extern long double __expm1f128 (long double);
for __typeof (__expm1l) and __typeof (__expm1f128) when __expm1 is
defined since __expm1 may be expanded in their declarations which
causes the build failure.
---
sysdeps/ieee754/dbl-64/s_expm1.c | 7 +++++
sysdeps/x86_64/fpu/multiarch/Makefile | 2 ++
sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c | 10 ++++++
sysdeps/x86_64/fpu/multiarch/s_expm1.c | 36 ++++++++++++++++++++++
4 files changed, 55 insertions(+)
create mode 100644 sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c
create mode 100644 sysdeps/x86_64/fpu/multiarch/s_expm1.c
diff --git a/sysdeps/ieee754/dbl-64/s_expm1.c b/sysdeps/ieee754/dbl-64/s_expm1.c
index 8f1c95bd04..1cafeca9c0 100644
--- a/sysdeps/ieee754/dbl-64/s_expm1.c
+++ b/sysdeps/ieee754/dbl-64/s_expm1.c
@@ -130,6 +130,11 @@ static const double
4.00821782732936239552e-06, /* 3ED0CFCA 86E65239 */
-2.01099218183624371326e-07 }; /* BE8AFDB7 6E09C32D */
+#ifndef SECTION
+# define SECTION
+#endif
+
+SECTION
double
__expm1 (double x)
{
@@ -258,4 +263,6 @@ __expm1 (double x)
}
return y;
}
+#ifndef __expm1
libm_alias_double (__expm1, expm1)
+#endif
diff --git a/sysdeps/x86_64/fpu/multiarch/Makefile b/sysdeps/x86_64/fpu/multiarch/Makefile
index f773255721..add339a876 100644
--- a/sysdeps/x86_64/fpu/multiarch/Makefile
+++ b/sysdeps/x86_64/fpu/multiarch/Makefile
@@ -37,6 +37,7 @@ libm-sysdep_routines += \
e_log2-fma \
e_pow-fma \
s_atan-fma \
+ s_expm1-fma \
s_sin-fma \
s_sincos-fma \
s_tan-fma \
@@ -49,6 +50,7 @@ CFLAGS-e_log-fma.c = -mfma -mavx2
CFLAGS-e_log2-fma.c = -mfma -mavx2
CFLAGS-e_pow-fma.c = -mfma -mavx2
CFLAGS-s_atan-fma.c = -mfma -mavx2
+CFLAGS-s_expm1-fma.c = -mfma -mavx2
CFLAGS-s_sin-fma.c = -mfma -mavx2
CFLAGS-s_tan-fma.c = -mfma -mavx2
CFLAGS-s_sincos-fma.c = -mfma -mavx2
diff --git a/sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c b/sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c
new file mode 100644
index 0000000000..3ee2bd804e
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c
@@ -0,0 +1,10 @@
+#define __expm1 __expm1_fma
+
+/* NB: __expm1 may be expanded to __expm1_fma in the following
+ prototypes. */
+extern long double __expm1l (long double);
+extern long double __expm1f128 (long double);
+
+#define SECTION __attribute__ ((section (".text.fma")))
+
+#include <sysdeps/ieee754/dbl-64/s_expm1.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_expm1.c b/sysdeps/x86_64/fpu/multiarch/s_expm1.c
new file mode 100644
index 0000000000..2cae83fb7f
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_expm1.c
@@ -0,0 +1,36 @@
+/* Multiple versions of expm1.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <libm-alias-double.h>
+
+extern double __redirect_expm1 (double);
+
+#define SYMBOL_NAME expm1
+#include "ifunc-fma.h"
+
+libc_ifunc_redirected (__redirect_expm1, __expm1, IFUNC_SELECTOR ());
+libm_alias_double (__expm1, expm1)
+
+#define __expm1 __expm1_sse2
+
+/* NB: __expm1 may be expanded to __expm1_sse2 in the following
+ prototypes. */
+extern long double __expm1l (long double);
+extern long double __expm1f128 (long double);
+
+#include <sysdeps/ieee754/dbl-64/s_expm1.c>
--
2.41.0
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH] x86_64: Add expm1 with FMA
2023-08-11 15:04 [PATCH] x86_64: Add expm1 with FMA H.J. Lu
@ 2023-08-14 4:50 ` Noah Goldstein
2023-08-17 6:46 ` Paul Zimmermann
1 sibling, 0 replies; 3+ messages in thread
From: Noah Goldstein @ 2023-08-14 4:50 UTC (permalink / raw)
To: H.J. Lu; +Cc: libc-alpha
On Fri, Aug 11, 2023 at 8:04 AM H.J. Lu via Libc-alpha
<libc-alpha@sourceware.org> wrote:
>
> On Skylake, it improves expm1 bench performance by:
>
> Before After Improvement
> max 70.204 68.054 3%
> min 20.709 16.2 22%
> mean 22.1221 16.7367 24%
>
> NB: Add
>
> extern long double __expm1l (long double);
> extern long double __expm1f128 (long double);
>
> for __typeof (__expm1l) and __typeof (__expm1f128) when __expm1 is
> defined since __expm1 may be expanded in their declarations which
> causes the build failure.
> ---
> sysdeps/ieee754/dbl-64/s_expm1.c | 7 +++++
> sysdeps/x86_64/fpu/multiarch/Makefile | 2 ++
> sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c | 10 ++++++
> sysdeps/x86_64/fpu/multiarch/s_expm1.c | 36 ++++++++++++++++++++++
> 4 files changed, 55 insertions(+)
> create mode 100644 sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c
> create mode 100644 sysdeps/x86_64/fpu/multiarch/s_expm1.c
>
> diff --git a/sysdeps/ieee754/dbl-64/s_expm1.c b/sysdeps/ieee754/dbl-64/s_expm1.c
> index 8f1c95bd04..1cafeca9c0 100644
> --- a/sysdeps/ieee754/dbl-64/s_expm1.c
> +++ b/sysdeps/ieee754/dbl-64/s_expm1.c
> @@ -130,6 +130,11 @@ static const double
> 4.00821782732936239552e-06, /* 3ED0CFCA 86E65239 */
> -2.01099218183624371326e-07 }; /* BE8AFDB7 6E09C32D */
>
> +#ifndef SECTION
> +# define SECTION
> +#endif
> +
> +SECTION
> double
> __expm1 (double x)
> {
> @@ -258,4 +263,6 @@ __expm1 (double x)
> }
> return y;
> }
> +#ifndef __expm1
> libm_alias_double (__expm1, expm1)
> +#endif
> diff --git a/sysdeps/x86_64/fpu/multiarch/Makefile b/sysdeps/x86_64/fpu/multiarch/Makefile
> index f773255721..add339a876 100644
> --- a/sysdeps/x86_64/fpu/multiarch/Makefile
> +++ b/sysdeps/x86_64/fpu/multiarch/Makefile
> @@ -37,6 +37,7 @@ libm-sysdep_routines += \
> e_log2-fma \
> e_pow-fma \
> s_atan-fma \
> + s_expm1-fma \
> s_sin-fma \
> s_sincos-fma \
> s_tan-fma \
> @@ -49,6 +50,7 @@ CFLAGS-e_log-fma.c = -mfma -mavx2
> CFLAGS-e_log2-fma.c = -mfma -mavx2
> CFLAGS-e_pow-fma.c = -mfma -mavx2
> CFLAGS-s_atan-fma.c = -mfma -mavx2
> +CFLAGS-s_expm1-fma.c = -mfma -mavx2
> CFLAGS-s_sin-fma.c = -mfma -mavx2
> CFLAGS-s_tan-fma.c = -mfma -mavx2
> CFLAGS-s_sincos-fma.c = -mfma -mavx2
> diff --git a/sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c b/sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c
> new file mode 100644
> index 0000000000..3ee2bd804e
> --- /dev/null
> +++ b/sysdeps/x86_64/fpu/multiarch/s_expm1-fma.c
> @@ -0,0 +1,10 @@
> +#define __expm1 __expm1_fma
> +
> +/* NB: __expm1 may be expanded to __expm1_fma in the following
> + prototypes. */
> +extern long double __expm1l (long double);
> +extern long double __expm1f128 (long double);
> +
> +#define SECTION __attribute__ ((section (".text.fma")))
> +
> +#include <sysdeps/ieee754/dbl-64/s_expm1.c>
> diff --git a/sysdeps/x86_64/fpu/multiarch/s_expm1.c b/sysdeps/x86_64/fpu/multiarch/s_expm1.c
> new file mode 100644
> index 0000000000..2cae83fb7f
> --- /dev/null
> +++ b/sysdeps/x86_64/fpu/multiarch/s_expm1.c
> @@ -0,0 +1,36 @@
> +/* Multiple versions of expm1.
> + Copyright (C) 2023 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +#include <libm-alias-double.h>
> +
> +extern double __redirect_expm1 (double);
> +
> +#define SYMBOL_NAME expm1
> +#include "ifunc-fma.h"
> +
> +libc_ifunc_redirected (__redirect_expm1, __expm1, IFUNC_SELECTOR ());
> +libm_alias_double (__expm1, expm1)
> +
> +#define __expm1 __expm1_sse2
> +
> +/* NB: __expm1 may be expanded to __expm1_sse2 in the following
> + prototypes. */
> +extern long double __expm1l (long double);
> +extern long double __expm1f128 (long double);
> +
> +#include <sysdeps/ieee754/dbl-64/s_expm1.c>
> --
> 2.41.0
>
LGTM.
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH] x86_64: Add expm1 with FMA
2023-08-11 15:04 [PATCH] x86_64: Add expm1 with FMA H.J. Lu
2023-08-14 4:50 ` Noah Goldstein
@ 2023-08-17 6:46 ` Paul Zimmermann
1 sibling, 0 replies; 3+ messages in thread
From: Paul Zimmermann @ 2023-08-17 6:46 UTC (permalink / raw)
To: H.J. Lu; +Cc: libc-alpha
Hi H.J.,
> On Skylake, it improves expm1 bench performance by:
>
> Before After Improvement
> max 70.204 68.054 3%
> min 20.709 16.2 22%
> mean 22.1221 16.7367 24%
and it improves slightly the accuracy. The previous largest known error was
0.914 ulp (x=0x1.62f69d171fa65p-2), it is now 0.911:
expm1 0 -1 0x1.63be411e096ep-2 [1] [0.911] 0.910655 0.9106541427171914
LGTM
Paul
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2023-08-17 6:46 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-08-11 15:04 [PATCH] x86_64: Add expm1 with FMA H.J. Lu
2023-08-14 4:50 ` Noah Goldstein
2023-08-17 6:46 ` Paul Zimmermann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).