From: Florian Weimer <fweimer@redhat.com>
To: libc-alpha@sourceware.org
Subject: [PATCH 13/13] Linux: Move __reclaim_stacks into the fork implementation in libc
Date: Thu, 06 May 2021 20:11:22 +0200 [thread overview]
Message-ID: <55e7367c90178cd8fe4cddb84ace7e31b6648f52.1620323953.git.fweimer@redhat.com> (raw)
In-Reply-To: <cover.1620323953.git.fweimer@redhat.com>
As a result, __libc_pthread_init is no longer needed.
---
nptl/Versions | 1 -
nptl/allocatestack.c | 108 ------------------------------------------
nptl/nptl-init.c | 3 --
nptl/pthreadP.h | 7 ---
sysdeps/nptl/fork.c | 110 +++++++++++++++++++++++++++++++++++++++++++
5 files changed, 110 insertions(+), 119 deletions(-)
diff --git a/nptl/Versions b/nptl/Versions
index d439a023b7..4c1c4ee0a7 100644
--- a/nptl/Versions
+++ b/nptl/Versions
@@ -308,7 +308,6 @@ libc {
__libc_cleanup_push_defer;
__libc_dl_error_tsd;
__libc_multiple_threads;
- __libc_pthread_init;
__lll_clocklock_elision;
__lll_lock_elision;
__lll_lock_wait;
diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c
index 076cffd35b..8672e89e75 100644
--- a/nptl/allocatestack.c
+++ b/nptl/allocatestack.c
@@ -754,111 +754,3 @@ __deallocate_stack (struct pthread *pd)
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
}
-
-/* In case of a fork() call the memory allocation in the child will be
- the same but only one thread is running. All stacks except that of
- the one running thread are not used anymore. We have to recycle
- them. */
-void
-__reclaim_stacks (void)
-{
- struct pthread *self = (struct pthread *) THREAD_SELF;
-
- /* No locking necessary. The caller is the only stack in use. But
- we have to be aware that we might have interrupted a list
- operation. */
-
- if (GL (dl_in_flight_stack) != 0)
- {
- bool add_p = GL (dl_in_flight_stack) & 1;
- list_t *elem = (list_t *) (GL (dl_in_flight_stack) & ~(uintptr_t) 1);
-
- if (add_p)
- {
- /* We always add at the beginning of the list. So in this case we
- only need to check the beginning of these lists to see if the
- pointers at the head of the list are inconsistent. */
- list_t *l = NULL;
-
- if (GL (dl_stack_used).next->prev != &GL (dl_stack_used))
- l = &GL (dl_stack_used);
- else if (GL (dl_stack_cache).next->prev != &GL (dl_stack_cache))
- l = &GL (dl_stack_cache);
-
- if (l != NULL)
- {
- assert (l->next->prev == elem);
- elem->next = l->next;
- elem->prev = l;
- l->next = elem;
- }
- }
- else
- {
- /* We can simply always replay the delete operation. */
- elem->next->prev = elem->prev;
- elem->prev->next = elem->next;
- }
-
- GL (dl_in_flight_stack) = 0;
- }
-
- /* Mark all stacks except the still running one as free. */
- list_t *runp;
- list_for_each (runp, &GL (dl_stack_used))
- {
- struct pthread *curp = list_entry (runp, struct pthread, list);
- if (curp != self)
- {
- /* This marks the stack as free. */
- curp->tid = 0;
-
- /* Account for the size of the stack. */
- GL (dl_stack_cache_actsize) += curp->stackblock_size;
-
- if (curp->specific_used)
- {
- /* Clear the thread-specific data. */
- memset (curp->specific_1stblock, '\0',
- sizeof (curp->specific_1stblock));
-
- curp->specific_used = false;
-
- for (size_t cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
- if (curp->specific[cnt] != NULL)
- {
- memset (curp->specific[cnt], '\0',
- sizeof (curp->specific_1stblock));
-
- /* We have allocated the block which we do not
- free here so re-set the bit. */
- curp->specific_used = true;
- }
- }
- }
- }
-
- /* Add the stack of all running threads to the cache. */
- list_splice (&GL (dl_stack_used), &GL (dl_stack_cache));
-
- /* Remove the entry for the current thread to from the cache list
- and add it to the list of running threads. Which of the two
- lists is decided by the user_stack flag. */
- list_del (&self->list);
-
- /* Re-initialize the lists for all the threads. */
- INIT_LIST_HEAD (&GL (dl_stack_used));
- INIT_LIST_HEAD (&GL (dl_stack_user));
-
- if (__glibc_unlikely (THREAD_GETMEM (self, user_stack)))
- list_add (&self->list, &GL (dl_stack_user));
- else
- list_add (&self->list, &GL (dl_stack_used));
-
- /* There is one thread running. */
- __nptl_nthreads = 1;
-
- /* Initialize locks. */
- GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER;
- __default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
-}
diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c
index 4c89e7a792..16fb66bdf5 100644
--- a/nptl/nptl-init.c
+++ b/nptl/nptl-init.c
@@ -172,9 +172,6 @@ __pthread_initialize_minimal_internal (void)
__default_pthread_attr.internal.stacksize = limit.rlim_cur;
__default_pthread_attr.internal.guardsize = GLRO (dl_pagesize);
lll_unlock (__default_pthread_attr_lock, LLL_PRIVATE);
-
- /* Register the fork generation counter with the libc. */
- __libc_pthread_init (__reclaim_stacks);
}
strong_alias (__pthread_initialize_minimal_internal,
__pthread_initialize_minimal)
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 6b912f053b..d9b97c814a 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -333,10 +333,6 @@ extern void __free_tcb (struct pthread *pd) attribute_hidden;
/* Free allocated stack. */
extern void __deallocate_stack (struct pthread *pd) attribute_hidden;
-/* Mark all the stacks except for the current one as available. This
- function also re-initializes the lock for the stack cache. */
-extern void __reclaim_stacks (void) attribute_hidden;
-
/* Change the permissions of a thread stack. Called from
_dl_make_stacks_executable and pthread_create. */
int
@@ -372,9 +368,6 @@ extern unsigned long int __fork_generation attribute_hidden;
/* Pointer to the fork generation counter in the thread library. */
extern unsigned long int *__fork_generation_pointer attribute_hidden;
-/* Register the generation counter in the libpthread with the libc. */
-extern void __libc_pthread_init (void (*reclaim) (void));
-
extern size_t __pthread_get_minstack (const pthread_attr_t *attr);
/* Namespace save aliases. */
diff --git a/sysdeps/nptl/fork.c b/sysdeps/nptl/fork.c
index f41c40fca0..062b01265a 100644
--- a/sysdeps/nptl/fork.c
+++ b/sysdeps/nptl/fork.c
@@ -35,6 +35,7 @@
#include <nss/nss_database.h>
#include <unwind-link.h>
#include <sys/single_threaded.h>
+#include <list.h>
static void
fresetlockfiles (void)
@@ -46,6 +47,106 @@ fresetlockfiles (void)
_IO_lock_init (*((_IO_lock_t *) _IO_iter_file(i)->_lock));
}
+/* In case of a fork() call the memory allocation in the child will be
+ the same but only one thread is running. All stacks except that of
+ the one running thread are not used anymore. We have to recycle
+ them. */
+static void
+reclaim_stacks (void)
+{
+ struct pthread *self = (struct pthread *) THREAD_SELF;
+
+ /* No locking necessary. The caller is the only stack in use. But
+ we have to be aware that we might have interrupted a list
+ operation. */
+
+ if (GL (dl_in_flight_stack) != 0)
+ {
+ bool add_p = GL (dl_in_flight_stack) & 1;
+ list_t *elem = (list_t *) (GL (dl_in_flight_stack) & ~(uintptr_t) 1);
+
+ if (add_p)
+ {
+ /* We always add at the beginning of the list. So in this case we
+ only need to check the beginning of these lists to see if the
+ pointers at the head of the list are inconsistent. */
+ list_t *l = NULL;
+
+ if (GL (dl_stack_used).next->prev != &GL (dl_stack_used))
+ l = &GL (dl_stack_used);
+ else if (GL (dl_stack_cache).next->prev != &GL (dl_stack_cache))
+ l = &GL (dl_stack_cache);
+
+ if (l != NULL)
+ {
+ assert (l->next->prev == elem);
+ elem->next = l->next;
+ elem->prev = l;
+ l->next = elem;
+ }
+ }
+ else
+ {
+ /* We can simply always replay the delete operation. */
+ elem->next->prev = elem->prev;
+ elem->prev->next = elem->next;
+ }
+
+ GL (dl_in_flight_stack) = 0;
+ }
+
+ /* Mark all stacks except the still running one as free. */
+ list_t *runp;
+ list_for_each (runp, &GL (dl_stack_used))
+ {
+ struct pthread *curp = list_entry (runp, struct pthread, list);
+ if (curp != self)
+ {
+ /* This marks the stack as free. */
+ curp->tid = 0;
+
+ /* Account for the size of the stack. */
+ GL (dl_stack_cache_actsize) += curp->stackblock_size;
+
+ if (curp->specific_used)
+ {
+ /* Clear the thread-specific data. */
+ memset (curp->specific_1stblock, '\0',
+ sizeof (curp->specific_1stblock));
+
+ curp->specific_used = false;
+
+ for (size_t cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
+ if (curp->specific[cnt] != NULL)
+ {
+ memset (curp->specific[cnt], '\0',
+ sizeof (curp->specific_1stblock));
+
+ /* We have allocated the block which we do not
+ free here so re-set the bit. */
+ curp->specific_used = true;
+ }
+ }
+ }
+ }
+
+ /* Add the stack of all running threads to the cache. */
+ list_splice (&GL (dl_stack_used), &GL (dl_stack_cache));
+
+ /* Remove the entry for the current thread to from the cache list
+ and add it to the list of running threads. Which of the two
+ lists is decided by the user_stack flag. */
+ list_del (&self->list);
+
+ /* Re-initialize the lists for all the threads. */
+ INIT_LIST_HEAD (&GL (dl_stack_used));
+ INIT_LIST_HEAD (&GL (dl_stack_user));
+
+ if (__glibc_unlikely (THREAD_GETMEM (self, user_stack)))
+ list_add (&self->list, &GL (dl_stack_user));
+ else
+ list_add (&self->list, &GL (dl_stack_used));
+}
pid_t
__libc_fork (void)
@@ -112,6 +213,13 @@ __libc_fork (void)
{
__libc_unwind_link_after_fork ();
+ /* There is one thread running. */
+ __nptl_nthreads = 1;
+
+ /* Initialize thread library locks. */
+ GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER;
+ __default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
+
/* Release malloc locks. */
call_function_static_weak (__malloc_fork_unlock_child);
@@ -128,6 +236,8 @@ __libc_fork (void)
/* Reset the lock the dynamic loader uses to protect its data. */
__rtld_lock_initialize (GL(dl_load_lock));
+ reclaim_stacks ();
+
/* Run the handlers registered for the child. */
__run_fork_handlers (atfork_run_child, multiple_threads);
}
--
2.30.2
next prev parent reply other threads:[~2021-05-06 18:11 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-06 18:08 [PATCH 00/13] Linux: Move most stack management out of libpthread Florian Weimer
2021-05-06 18:08 ` [PATCH 01/13] scripts/versions.awk: Add strings and hashes to <first-versions.h> Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:09 ` [PATCH v2 02/13] elf, nptl: Resolve recursive lock implementation early Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-10 5:54 ` Florian Weimer
2021-05-06 18:10 ` [PATCH 03/13] nptl: Export __libc_multiple_threads from libc as an internal symbol Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:10 ` [PATCH 04/13] Linux: Explicitly disable cancellation checking in the dynamic loader Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:10 ` [PATCH 05/13] Linux: Simplify and fix the definition of SINGLE_THREAD_P Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:10 ` [PATCH 06/13] nptl: Eliminate __pthread_multiple_threads Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:10 ` [PATCH 07/13] elf: Introduce __tls_pre_init_tp Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:10 ` [PATCH 08/13] nptl: Move more stack management variables into _rtld_global Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:11 ` [PATCH 09/13] nptl: Simplify the change_stack_perm calling convention Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:11 ` [PATCH 10/13] nptl: Move changing of stack permissions into ld.so Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:11 ` [PATCH 11/13] nptl: Simplify resetting the in-flight stack in __reclaim_stacks Florian Weimer
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:11 ` [PATCH 12/13] nptl: Move __default_pthread_attr, __default_pthread_attr_lock into libc Florian Weimer
2021-05-09 21:41 ` Carlos O'Donell
2021-05-09 21:42 ` Carlos O'Donell
2021-05-06 18:11 ` Florian Weimer [this message]
2021-05-09 21:41 ` [PATCH 13/13] Linux: Move __reclaim_stacks into the fork implementation in libc Carlos O'Donell
2021-05-09 21:42 ` [PATCH 00/13] Linux: Move most stack management out of libpthread Carlos O'Donell
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=55e7367c90178cd8fe4cddb84ace7e31b6648f52.1620323953.git.fweimer@redhat.com \
--to=fweimer@redhat.com \
--cc=libc-alpha@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).