From: Florian Weimer <fweimer@redhat.com>
To: libc-alpha@sourceware.org
Subject: [PATCH v4 03/37] nptl: Move legacy unwinding implementation into libc
Date: Fri, 16 Apr 2021 11:20:27 +0200 [thread overview]
Message-ID: <96089e4a5014fe1c0ddcbf929118772f03618675.1618564630.git.fweimer@redhat.com> (raw)
In-Reply-To: <cover.1618564630.git.fweimer@redhat.com>
It is still used internally. Since unwinding is now available
unconditionally, avoid indirect calls through function pointers loaded
from the stack by inlining the non-cancellation cleanup code. This
avoids a regression in security hardening.
The out-of-line __libc_cleanup_routine implementation is no longer
needed because the inline definition is now static __always_inline.
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
---
nptl/Versions | 2 +
nptl/cleanup_defer_compat.c | 56 ++--------------------------
nptl/libc-cleanup.c | 64 ++++++++++++++++++++++++++++++--
nptl/nptl-init.c | 2 -
sysdeps/nptl/libc-lock.h | 59 ++++++++++++++---------------
sysdeps/nptl/libc-lockP.h | 26 +------------
sysdeps/nptl/pthread-functions.h | 4 --
7 files changed, 97 insertions(+), 116 deletions(-)
diff --git a/nptl/Versions b/nptl/Versions
index 60202b4969..2e5a964b11 100644
--- a/nptl/Versions
+++ b/nptl/Versions
@@ -87,6 +87,8 @@ libc {
__futex_abstimed_wait64;
__futex_abstimed_wait_cancelable64;
__libc_alloca_cutoff;
+ __libc_cleanup_pop_restore;
+ __libc_cleanup_push_defer;
__libc_dl_error_tsd;
__libc_pthread_init;
__lll_clocklock_elision;
diff --git a/nptl/cleanup_defer_compat.c b/nptl/cleanup_defer_compat.c
index 49ef53ea60..1957318208 100644
--- a/nptl/cleanup_defer_compat.c
+++ b/nptl/cleanup_defer_compat.c
@@ -17,41 +17,15 @@
<https://www.gnu.org/licenses/>. */
#include "pthreadP.h"
-
+#include <libc-lock.h>
void
_pthread_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer,
void (*routine) (void *), void *arg)
{
- struct pthread *self = THREAD_SELF;
-
buffer->__routine = routine;
buffer->__arg = arg;
- buffer->__prev = THREAD_GETMEM (self, cleanup);
-
- int cancelhandling = THREAD_GETMEM (self, cancelhandling);
-
- /* Disable asynchronous cancellation for now. */
- if (__glibc_unlikely (cancelhandling & CANCELTYPE_BITMASK))
- while (1)
- {
- int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
- cancelhandling
- & ~CANCELTYPE_BITMASK,
- cancelhandling);
- if (__glibc_likely (curval == cancelhandling))
- /* Successfully replaced the value. */
- break;
-
- /* Prepare for the next round. */
- cancelhandling = curval;
- }
-
- buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
- ? PTHREAD_CANCEL_ASYNCHRONOUS
- : PTHREAD_CANCEL_DEFERRED);
-
- THREAD_SETMEM (self, cleanup, buffer);
+ __libc_cleanup_push_defer (buffer);
}
strong_alias (_pthread_cleanup_push_defer, __pthread_cleanup_push_defer)
@@ -60,31 +34,7 @@ void
_pthread_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer,
int execute)
{
- struct pthread *self = THREAD_SELF;
-
- THREAD_SETMEM (self, cleanup, buffer->__prev);
-
- int cancelhandling;
- if (__builtin_expect (buffer->__canceltype != PTHREAD_CANCEL_DEFERRED, 0)
- && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
- & CANCELTYPE_BITMASK) == 0)
- {
- while (1)
- {
- int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
- cancelhandling
- | CANCELTYPE_BITMASK,
- cancelhandling);
- if (__glibc_likely (curval == cancelhandling))
- /* Successfully replaced the value. */
- break;
-
- /* Prepare for the next round. */
- cancelhandling = curval;
- }
-
- CANCELLATION_P (self);
- }
+ __libc_cleanup_pop_restore (buffer);
/* If necessary call the cleanup routine after we removed the
current cleanup block from the list. */
diff --git a/nptl/libc-cleanup.c b/nptl/libc-cleanup.c
index 61f97eceda..14ccfe9285 100644
--- a/nptl/libc-cleanup.c
+++ b/nptl/libc-cleanup.c
@@ -17,11 +17,69 @@
<https://www.gnu.org/licenses/>. */
#include "pthreadP.h"
+#include <tls.h>
+#include <libc-lock.h>
+void
+__libc_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer)
+{
+ struct pthread *self = THREAD_SELF;
+
+ buffer->__prev = THREAD_GETMEM (self, cleanup);
+
+ int cancelhandling = THREAD_GETMEM (self, cancelhandling);
+
+ /* Disable asynchronous cancellation for now. */
+ if (__glibc_unlikely (cancelhandling & CANCELTYPE_BITMASK))
+ while (1)
+ {
+ int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+ cancelhandling
+ & ~CANCELTYPE_BITMASK,
+ cancelhandling);
+ if (__glibc_likely (curval == cancelhandling))
+ /* Successfully replaced the value. */
+ break;
+
+ /* Prepare for the next round. */
+ cancelhandling = curval;
+ }
+
+ buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
+ ? PTHREAD_CANCEL_ASYNCHRONOUS
+ : PTHREAD_CANCEL_DEFERRED);
+
+ THREAD_SETMEM (self, cleanup, buffer);
+}
+libc_hidden_def (__libc_cleanup_push_defer)
void
-__libc_cleanup_routine (struct __pthread_cleanup_frame *f)
+__libc_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer)
{
- if (f->__do_it)
- f->__cancel_routine (f->__cancel_arg);
+ struct pthread *self = THREAD_SELF;
+
+ THREAD_SETMEM (self, cleanup, buffer->__prev);
+
+ int cancelhandling;
+ if (__builtin_expect (buffer->__canceltype != PTHREAD_CANCEL_DEFERRED, 0)
+ && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
+ & CANCELTYPE_BITMASK) == 0)
+ {
+ while (1)
+ {
+ int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+ cancelhandling
+ | CANCELTYPE_BITMASK,
+ cancelhandling);
+ if (__glibc_likely (curval == cancelhandling))
+ /* Successfully replaced the value. */
+ break;
+
+ /* Prepare for the next round. */
+ cancelhandling = curval;
+ }
+
+ CANCELLATION_P (self);
+ }
}
+libc_hidden_def (__libc_cleanup_pop_restore)
diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c
index 2c7e2222d4..43e2564e59 100644
--- a/nptl/nptl-init.c
+++ b/nptl/nptl-init.c
@@ -95,8 +95,6 @@ static const struct pthread_functions pthread_functions =
.ptr___pthread_key_create = __pthread_key_create,
.ptr___pthread_getspecific = __pthread_getspecific,
.ptr___pthread_setspecific = __pthread_setspecific,
- .ptr__pthread_cleanup_push_defer = __pthread_cleanup_push_defer,
- .ptr__pthread_cleanup_pop_restore = __pthread_cleanup_pop_restore,
.ptr_nthreads = &__nptl_nthreads,
.ptr___pthread_unwind = &__pthread_unwind,
.ptr__nptl_deallocate_tsd = __nptl_deallocate_tsd,
diff --git a/sysdeps/nptl/libc-lock.h b/sysdeps/nptl/libc-lock.h
index dea96121b3..e8a5e68a12 100644
--- a/sysdeps/nptl/libc-lock.h
+++ b/sysdeps/nptl/libc-lock.h
@@ -143,39 +143,40 @@ typedef struct __libc_lock_recursive_opaque__ __libc_lock_recursive_t;
__libc_maybe_call (__pthread_mutex_unlock, (&(NAME).mutex), 0)
#endif
-/* Note that for I/O cleanup handling we are using the old-style
- cancel handling. It does not have to be integrated with C++ since
- no C++ code is called in the middle. The old-style handling is
- faster and the support is not going away. */
-extern void _pthread_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer,
- void (*routine) (void *), void *arg);
-extern void _pthread_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer,
- int execute);
+/* Put the unwind buffer BUFFER on the per-thread callback stack. The
+ caller must fill BUFFER->__routine and BUFFER->__arg before calling
+ this function. */
+void __libc_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer);
+libc_hidden_proto (__libc_cleanup_push_defer)
+/* Remove BUFFER from the unwind callback stack. The caller must invoke
+ the callback if desired. */
+void __libc_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer);
+libc_hidden_proto (__libc_cleanup_pop_restore)
/* Start critical region with cleanup. */
-#define __libc_cleanup_region_start(DOIT, FCT, ARG) \
- { struct _pthread_cleanup_buffer _buffer; \
- int _avail; \
- if (DOIT) { \
- _avail = PTFAVAIL (_pthread_cleanup_push_defer); \
- if (_avail) { \
- __libc_ptf_call_always (_pthread_cleanup_push_defer, (&_buffer, FCT, \
- ARG)); \
- } else { \
- _buffer.__routine = (FCT); \
- _buffer.__arg = (ARG); \
- } \
- } else { \
- _avail = 0; \
- }
+#define __libc_cleanup_region_start(DOIT, FCT, ARG) \
+ { bool _cleanup_start_doit; \
+ struct _pthread_cleanup_buffer _buffer; \
+ /* Non-addressable copy of FCT, so that we avoid indirect calls on \
+ the non-unwinding path. */ \
+ void (*_cleanup_routine) (void *) = (FCT); \
+ _buffer.__arg = (ARG); \
+ if (DOIT) \
+ { \
+ _cleanup_start_doit = true; \
+ _buffer.__routine = _cleanup_routine; \
+ __libc_cleanup_push_defer (&_buffer); \
+ } \
+ else \
+ _cleanup_start_doit = false;
/* End critical region with cleanup. */
-#define __libc_cleanup_region_end(DOIT) \
- if (_avail) { \
- __libc_ptf_call_always (_pthread_cleanup_pop_restore, (&_buffer, DOIT));\
- } else if (DOIT) \
- _buffer.__routine (_buffer.__arg); \
- }
+#define __libc_cleanup_region_end(DOIT) \
+ if (_cleanup_start_doit) \
+ __libc_cleanup_pop_restore (&_buffer); \
+ if (DOIT) \
+ _cleanup_routine (_buffer.__arg); \
+ } /* matches __libc_cleanup_region_start */
/* Hide the definitions which are only supposed to be used inside libc in
diff --git a/sysdeps/nptl/libc-lockP.h b/sysdeps/nptl/libc-lockP.h
index 63b605dee2..2c928b7a76 100644
--- a/sysdeps/nptl/libc-lockP.h
+++ b/sysdeps/nptl/libc-lockP.h
@@ -251,32 +251,12 @@ _Static_assert (LLL_LOCK_INITIALIZER == 0, "LLL_LOCK_INITIALIZER != 0");
/* Get once control variable. */
#define __libc_once_get(ONCE_CONTROL) ((ONCE_CONTROL) != PTHREAD_ONCE_INIT)
-/* Note that for I/O cleanup handling we are using the old-style
- cancel handling. It does not have to be integrated with C++ snce
- no C++ code is called in the middle. The old-style handling is
- faster and the support is not going away. */
-extern void _pthread_cleanup_push (struct _pthread_cleanup_buffer *buffer,
- void (*routine) (void *), void *arg);
-extern void _pthread_cleanup_pop (struct _pthread_cleanup_buffer *buffer,
- int execute);
-extern void _pthread_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer,
- void (*routine) (void *), void *arg);
-extern void _pthread_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer,
- int execute);
-
-/* Sometimes we have to exit the block in the middle. */
-#define __libc_cleanup_end(DOIT) \
- if (_avail) { \
- __libc_ptf_call_always (_pthread_cleanup_pop_restore, (&_buffer, DOIT));\
- } else if (DOIT) \
- _buffer.__routine (_buffer.__arg)
-
/* __libc_cleanup_push and __libc_cleanup_pop depend on exception
handling and stack unwinding. */
#ifdef __EXCEPTIONS
/* Normal cleanup handling, based on C cleanup attribute. */
-__extern_inline void
+static __always_inline void
__libc_cleanup_routine (struct __pthread_cleanup_frame *f)
{
if (f->__do_it)
@@ -388,8 +368,6 @@ weak_extern (__pthread_once)
weak_extern (__pthread_initialize)
weak_extern (__pthread_atfork)
weak_extern (__pthread_setcancelstate)
-weak_extern (_pthread_cleanup_push_defer)
-weak_extern (_pthread_cleanup_pop_restore)
# else
# pragma weak __pthread_mutex_init
# pragma weak __pthread_mutex_destroy
@@ -412,8 +390,6 @@ weak_extern (_pthread_cleanup_pop_restore)
# pragma weak __pthread_initialize
# pragma weak __pthread_atfork
# pragma weak __pthread_setcancelstate
-# pragma weak _pthread_cleanup_push_defer
-# pragma weak _pthread_cleanup_pop_restore
# endif
#endif
diff --git a/sysdeps/nptl/pthread-functions.h b/sysdeps/nptl/pthread-functions.h
index 97a5c48939..4268084b66 100644
--- a/sysdeps/nptl/pthread-functions.h
+++ b/sysdeps/nptl/pthread-functions.h
@@ -57,10 +57,6 @@ struct pthread_functions
int (*ptr___pthread_key_create) (pthread_key_t *, void (*) (void *));
void *(*ptr___pthread_getspecific) (pthread_key_t);
int (*ptr___pthread_setspecific) (pthread_key_t, const void *);
- void (*ptr__pthread_cleanup_push_defer) (struct _pthread_cleanup_buffer *,
- void (*) (void *), void *);
- void (*ptr__pthread_cleanup_pop_restore) (struct _pthread_cleanup_buffer *,
- int);
#define HAVE_PTR_NTHREADS
unsigned int *ptr_nthreads;
void (*ptr___pthread_unwind) (__pthread_unwind_buf_t *)
--
2.30.2
next prev parent reply other threads:[~2021-04-16 9:20 UTC|newest]
Thread overview: 43+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-16 9:20 [PATCH v4 00/37] libpthread: Remove NPTL forwarders Florian Weimer
2021-04-16 9:20 ` [PATCH v4 01/37] nptl: Move pthread_mutex_consistent into libc Florian Weimer
2021-04-16 9:20 ` [PATCH v4 02/37] nptl: Move __pthread_cleanup_routine " Florian Weimer
2021-04-16 9:20 ` Florian Weimer [this message]
2021-04-16 9:20 ` [PATCH v4 04/37] nptl: Move legacy cancelation handling into libc as compat symbols Florian Weimer
2021-04-16 9:20 ` [PATCH v4 05/37] nptl: Remove longjmp, siglongjmp from libpthread Florian Weimer
2021-04-16 9:20 ` [PATCH v4 06/37] x86: Restore compile-time check for shadow stack pointer in longjmp Florian Weimer
2021-04-16 9:20 ` [PATCH v4 07/37] nptl: Move __pthread_cleanup_upto into libc Florian Weimer
2021-04-16 9:20 ` [PATCH v4 08/37] nptl: Move pthread_once and __pthread_once " Florian Weimer
2021-04-16 9:21 ` [PATCH v4 09/37] nptl: Move __pthread_unwind_next " Florian Weimer
2021-04-16 9:21 ` [PATCH v4 10/37] csu: Move calling main out of __libc_start_main_impl Florian Weimer
2021-04-16 9:21 ` [PATCH v4 11/37] nptl: Move internal __nptl_nthreads variable into libc Florian Weimer
2021-04-16 9:21 ` [PATCH v4 12/37] nptl_db: Introduce DB_MAIN_ARRAY_VARIABLE Florian Weimer
2021-04-16 9:21 ` [PATCH v4 13/37] nptl: Move __pthread_keys global variable into libc Florian Weimer
2021-04-16 9:21 ` [PATCH v4 14/37] nptl: Move __nptl_deallocate_tsd " Florian Weimer
2021-04-16 9:21 ` [PATCH v4 15/37] nptl: Move pthread_exit " Florian Weimer
2021-04-16 9:22 ` [PATCH v4 16/37] nptl: Move pthread_setcancelstate " Florian Weimer
2021-04-16 9:22 ` [PATCH v4 17/37] nptl: Move pthread_setcanceltype " Florian Weimer
2021-04-16 9:22 ` [PATCH v4 18/37] nptl: Invoke the set_robust_list system call directly in fork Florian Weimer
2021-04-16 9:22 ` [PATCH v4 19/37] dlfcn: Failures after dlmopen should not terminate process [BZ #24772] Florian Weimer
2021-04-16 9:22 ` [PATCH v4 20/37] dlfcn: dlerror needs to call free from the base namespace [BZ #24773] Florian Weimer
2021-04-16 9:22 ` [PATCH v4 21/37] Remove pthread_key_create-related internals from libc-lock.h Florian Weimer
2021-04-16 9:22 ` [PATCH v4 22/37] elf: Introduce __tls_init_tp for second-phase TCB initialization Florian Weimer
2021-04-16 9:22 ` [PATCH v4 23/37] nptl: Move part of TCB initialization from libpthread to __tls_init_tp Florian Weimer
2021-04-16 9:22 ` [PATCH v4 24/37] nptl: Move pthread_key_create, __pthread_key_create into libc Florian Weimer
2021-04-16 9:23 ` [PATCH v4 25/37] nptl: Move pthread_getspecific, __pthread_getspecific " Florian Weimer
2021-04-16 9:23 ` [PATCH v4 26/37] nptl: Move pthread_setspecific, __pthread_setspecific " Florian Weimer
2021-04-16 9:23 ` [PATCH v4 27/37] nptl: Move pthread_key_delete " Florian Weimer
2021-04-16 9:23 ` [PATCH v4 28/37] nptl: Move rwlock functions with forwarders " Florian Weimer
2021-04-16 9:23 ` [PATCH v4 29/37] nptl: Move the internal thread priority protection symbols " Florian Weimer
2021-04-16 9:23 ` [PATCH v4 30/37] pthread: Introduce __pthread_early_init Florian Weimer
2021-04-16 9:23 ` [PATCH v4 31/37] nptl: Move internal symbol __mutex_aconf into libc Florian Weimer
2021-04-16 9:24 ` [PATCH v4 32/37] nptl: pthread_mutex_lock, pthread_mutex_unock single-threaded optimization Florian Weimer
2021-04-16 9:24 ` [PATCH v4 33/37] x86: Remove low-level lock optimization Florian Weimer
2021-04-16 9:24 ` [PATCH v4 34/37] nptl: Move core mutex functions into libc Florian Weimer
2021-04-16 9:24 ` [PATCH v4 35/37] nptl: Move core condition variable " Florian Weimer
2021-04-16 9:24 ` [PATCH v4 36/37] nptl: Move setxid broadcast implementation " Florian Weimer
2021-04-16 9:24 ` [PATCH v4 37/37] nptl: Remove remnants of the libc/libpthread forwarder interface Florian Weimer
2021-04-21 11:47 ` [PATCH v4 00/37] libpthread: Remove NPTL forwarders Florian Weimer
2021-04-21 13:42 ` H.J. Lu
2021-04-22 7:35 ` Szabolcs Nagy
2021-04-22 7:58 ` Florian Weimer
2021-04-22 8:56 ` Szabolcs Nagy
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=96089e4a5014fe1c0ddcbf929118772f03618675.1618564630.git.fweimer@redhat.com \
--to=fweimer@redhat.com \
--cc=libc-alpha@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).