From d717623535955e121873b7c39db3c36cd63d7d5f Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 30 Mar 2022 14:01:41 -0500 Subject: [PATCH v1 1/4] clang-format pthread_rwlock_common.c --- nptl/pthread_rwlock_common.c | 966 ++++++++++++++++++----------------- 1 file changed, 487 insertions(+), 479 deletions(-) diff --git a/nptl/pthread_rwlock_common.c b/nptl/pthread_rwlock_common.c index bdc55d9592..6474d9ec65 100644 --- a/nptl/pthread_rwlock_common.c +++ b/nptl/pthread_rwlock_common.c @@ -26,7 +26,6 @@ #include #include - /* A reader--writer lock that fulfills the POSIX requirements (but operations on this lock are not necessarily full barriers, as one may interpret the POSIX requirement about "synchronizing memory"). All critical sections are @@ -71,18 +70,18 @@ #1 0 0 0 0 Lock is idle (and in a read phase). #2 0 0 >0 0 Readers have acquired the lock. #3 0 1 0 0 Lock is not acquired; a writer will try to start a - write phase. + write phase. #4 0 1 >0 0 Readers have acquired the lock; a writer is waiting - and explicit hand-over to the writer is required. + and explicit hand-over to the writer is required. #4a 0 1 >0 1 Same as #4 except that there are further readers - waiting because the writer is to be preferred. + waiting because the writer is to be preferred. #5 1 0 0 0 Lock is idle (and in a write phase). #6 1 0 >0 0 Write phase; readers will try to start a read phase - (requires explicit hand-over to all readers that - do not start the read phase). + (requires explicit hand-over to all readers that + do not start the read phase). #7 1 1 0 0 Lock is acquired by a writer. #8 1 1 >0 0 Lock acquired by a writer and readers are waiting; - explicit hand-over to the readers is required. + explicit hand-over to the readers is required. WP (PTHREAD_RWLOCK_WRPHASE) is true if the lock is in a write phase, so potentially acquired by a primary writer. @@ -214,7 +213,6 @@ deciding when to use elision so that enabling it would lead to consistently better performance. */ - static int __pthread_rwlock_get_private (pthread_rwlock_t *rwlock) { @@ -235,48 +233,48 @@ __pthread_rwlock_rdunlock (pthread_rwlock_t *rwlock) { rnew = r - (1 << PTHREAD_RWLOCK_READER_SHIFT); /* If we are the last reader, we also need to unblock any readers - that are waiting for a writer to go first (PTHREAD_RWLOCK_RWAITING) - so that they can register while the writer is active. */ + that are waiting for a writer to go first (PTHREAD_RWLOCK_RWAITING) + so that they can register while the writer is active. */ if ((rnew >> PTHREAD_RWLOCK_READER_SHIFT) == 0) - { - if ((rnew & PTHREAD_RWLOCK_WRLOCKED) != 0) - rnew |= PTHREAD_RWLOCK_WRPHASE; - rnew &= ~(unsigned int) PTHREAD_RWLOCK_RWAITING; - } + { + if ((rnew & PTHREAD_RWLOCK_WRLOCKED) != 0) + rnew |= PTHREAD_RWLOCK_WRPHASE; + rnew &= ~(unsigned int) PTHREAD_RWLOCK_RWAITING; + } /* We need release MO here for three reasons. First, so that we - synchronize with subsequent writers. Second, we might have been the - first reader and set __wrphase_futex to 0, so we need to synchronize - with the last reader that will set it to 1 (note that we will always - change __readers before the last reader, or we are the last reader). - Third, a writer that takes part in explicit hand-over needs to see - the first reader's store to __wrphase_futex (or a later value) if - the writer observes that a write phase has been started. */ - if (atomic_compare_exchange_weak_release (&rwlock->__data.__readers, - &r, rnew)) - break; + synchronize with subsequent writers. Second, we might have been the + first reader and set __wrphase_futex to 0, so we need to synchronize + with the last reader that will set it to 1 (note that we will always + change __readers before the last reader, or we are the last reader). + Third, a writer that takes part in explicit hand-over needs to see + the first reader's store to __wrphase_futex (or a later value) if + the writer observes that a write phase has been started. */ + if (atomic_compare_exchange_weak_release (&rwlock->__data.__readers, &r, + rnew)) + break; /* TODO Back-off. */ } if ((rnew & PTHREAD_RWLOCK_WRPHASE) != 0) { /* We need to do explicit hand-over. We need the acquire MO fence so - that our modification of _wrphase_futex happens after a store by - another reader that started a read phase. Relaxed MO is sufficient - for the modification of __wrphase_futex because it is just used - to delay acquisition by a writer until all threads are unblocked - irrespective of whether they are looking at __readers or - __wrphase_futex; any other synchronizes-with relations that are - necessary are established through __readers. */ + that our modification of _wrphase_futex happens after a store by + another reader that started a read phase. Relaxed MO is sufficient + for the modification of __wrphase_futex because it is just used + to delay acquisition by a writer until all threads are unblocked + irrespective of whether they are looking at __readers or + __wrphase_futex; any other synchronizes-with relations that are + necessary are established through __readers. */ atomic_thread_fence_acquire (); if ((atomic_exchange_relaxed (&rwlock->__data.__wrphase_futex, 1) - & PTHREAD_RWLOCK_FUTEX_USED) != 0) - futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); + & PTHREAD_RWLOCK_FUTEX_USED) + != 0) + futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); } /* Also wake up waiting readers if we did reset the RWAITING flag. */ if ((r & PTHREAD_RWLOCK_RWAITING) != (rnew & PTHREAD_RWLOCK_RWAITING)) futex_wake (&rwlock->__data.__readers, INT_MAX, private); } - static __always_inline int __pthread_rwlock_rdlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, const struct __timespec64 *abstime) @@ -289,14 +287,15 @@ __pthread_rwlock_rdlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, requires that "the validity of the abstime parameter need not be checked if the lock can be immediately acquired" (i.e., we need not but may check it). */ - if (abstime && __glibc_unlikely (!futex_abstimed_supported_clockid (clockid) - || ! valid_nanoseconds (abstime->tv_nsec))) + if (abstime + && __glibc_unlikely (!futex_abstimed_supported_clockid (clockid) + || !valid_nanoseconds (abstime->tv_nsec))) return EINVAL; /* Make sure we are not holding the rwlock as a writer. This is a deadlock situation we recognize and report. */ if (__glibc_unlikely (atomic_load_relaxed (&rwlock->__data.__cur_writer) - == THREAD_GETMEM (THREAD_SELF, tid))) + == THREAD_GETMEM (THREAD_SELF, tid))) return EDEADLK; /* If we prefer writers, recursive rdlock is disallowed, we are in a read @@ -311,47 +310,47 @@ __pthread_rwlock_rdlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, { r = atomic_load_relaxed (&rwlock->__data.__readers); while ((r & PTHREAD_RWLOCK_WRPHASE) == 0 - && (r & PTHREAD_RWLOCK_WRLOCKED) != 0 - && (r >> PTHREAD_RWLOCK_READER_SHIFT) > 0) - { - /* TODO Spin first. */ - /* Try setting the flag signaling that we are waiting without having - incremented the number of readers. Relaxed MO is fine because - this is just about waiting for a state change in __readers. */ - if (atomic_compare_exchange_weak_relaxed - (&rwlock->__data.__readers, &r, r | PTHREAD_RWLOCK_RWAITING)) - { - /* Wait for as long as the flag is set. An ABA situation is - harmless because the flag is just about the state of - __readers, and all threads set the flag under the same - conditions. */ - while (((r = atomic_load_relaxed (&rwlock->__data.__readers)) - & PTHREAD_RWLOCK_RWAITING) != 0) - { - int private = __pthread_rwlock_get_private (rwlock); - int err = __futex_abstimed_wait64 (&rwlock->__data.__readers, - r, clockid, abstime, - private); - /* We ignore EAGAIN and EINTR. On time-outs, we can just - return because we don't need to clean up anything. */ - if (err == ETIMEDOUT || err == EOVERFLOW) - return err; - } - /* It makes sense to not break out of the outer loop here - because we might be in the same situation again. */ - } - else - { - /* TODO Back-off. */ - } - } + && (r & PTHREAD_RWLOCK_WRLOCKED) != 0 + && (r >> PTHREAD_RWLOCK_READER_SHIFT) > 0) + { + /* TODO Spin first. */ + /* Try setting the flag signaling that we are waiting without having + incremented the number of readers. Relaxed MO is fine because + this is just about waiting for a state change in __readers. */ + if (atomic_compare_exchange_weak_relaxed ( + &rwlock->__data.__readers, &r, r | PTHREAD_RWLOCK_RWAITING)) + { + /* Wait for as long as the flag is set. An ABA situation is + harmless because the flag is just about the state of + __readers, and all threads set the flag under the same + conditions. */ + while (((r = atomic_load_relaxed (&rwlock->__data.__readers)) + & PTHREAD_RWLOCK_RWAITING) + != 0) + { + int private = __pthread_rwlock_get_private (rwlock); + int err = __futex_abstimed_wait64 ( + &rwlock->__data.__readers, r, clockid, abstime, private); + /* We ignore EAGAIN and EINTR. On time-outs, we can just + return because we don't need to clean up anything. */ + if (err == ETIMEDOUT || err == EOVERFLOW) + return err; + } + /* It makes sense to not break out of the outer loop here + because we might be in the same situation again. */ + } + else + { + /* TODO Back-off. */ + } + } } /* Register as a reader, using an add-and-fetch so that R can be used as expected value for future operations. Acquire MO so we synchronize with prior writers as well as the last reader of the previous read phase (see below). */ r = (atomic_fetch_add_acquire (&rwlock->__data.__readers, - (1 << PTHREAD_RWLOCK_READER_SHIFT)) + (1 << PTHREAD_RWLOCK_READER_SHIFT)) + (1 << PTHREAD_RWLOCK_READER_SHIFT)); /* Check whether there is an overflow in the number of readers. We assume @@ -370,12 +369,12 @@ __pthread_rwlock_rdlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, while (__glibc_unlikely (r >= PTHREAD_RWLOCK_READER_OVERFLOW)) { /* Relaxed MO is okay because we just want to undo our registration and - cannot have changed the rwlock state substantially if the CAS - succeeds. */ - if (atomic_compare_exchange_weak_relaxed - (&rwlock->__data.__readers, - &r, r - (1 << PTHREAD_RWLOCK_READER_SHIFT))) - return EAGAIN; + cannot have changed the rwlock state substantially if the CAS + succeeds. */ + if (atomic_compare_exchange_weak_relaxed ( + &rwlock->__data.__readers, &r, + r - (1 << PTHREAD_RWLOCK_READER_SHIFT))) + return EAGAIN; } /* We have registered as a reader, so if we are in a read phase, we have @@ -393,35 +392,36 @@ __pthread_rwlock_rdlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, for explicit hand-over of the read phase; the only exception is if we can start a read phase if there is no primary writer currently. */ while ((r & PTHREAD_RWLOCK_WRPHASE) != 0 - && (r & PTHREAD_RWLOCK_WRLOCKED) == 0) + && (r & PTHREAD_RWLOCK_WRLOCKED) == 0) { /* Try to enter a read phase: If the CAS below succeeds, we have - ownership; if it fails, we will simply retry and reassess the - situation. - Acquire MO so we synchronize with prior writers. */ + ownership; if it fails, we will simply retry and reassess the + situation. + Acquire MO so we synchronize with prior writers. */ if (atomic_compare_exchange_weak_acquire (&rwlock->__data.__readers, &r, - r ^ PTHREAD_RWLOCK_WRPHASE)) - { - /* We started the read phase, so we are also responsible for - updating the write-phase futex. Relaxed MO is sufficient. - We have to do the same steps as a writer would when handing - over the read phase to us because other readers cannot - distinguish between us and the writer; this includes - explicit hand-over and potentially having to wake other readers - (but we can pretend to do the setting and unsetting of WRLOCKED - atomically, and thus can skip this step). */ - if ((atomic_exchange_relaxed (&rwlock->__data.__wrphase_futex, 0) - & PTHREAD_RWLOCK_FUTEX_USED) != 0) - { - int private = __pthread_rwlock_get_private (rwlock); - futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); - } - return 0; - } + r ^ PTHREAD_RWLOCK_WRPHASE)) + { + /* We started the read phase, so we are also responsible for + updating the write-phase futex. Relaxed MO is sufficient. + We have to do the same steps as a writer would when handing + over the read phase to us because other readers cannot + distinguish between us and the writer; this includes + explicit hand-over and potentially having to wake other readers + (but we can pretend to do the setting and unsetting of WRLOCKED + atomically, and thus can skip this step). */ + if ((atomic_exchange_relaxed (&rwlock->__data.__wrphase_futex, 0) + & PTHREAD_RWLOCK_FUTEX_USED) + != 0) + { + int private = __pthread_rwlock_get_private (rwlock); + futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); + } + return 0; + } else - { - /* TODO Back off before retrying. Also see above. */ - } + { + /* TODO Back off before retrying. Also see above. */ + } } /* We were in a write phase but did not install the read phase. We cannot @@ -449,80 +449,81 @@ __pthread_rwlock_rdlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, for (;;) { while (((wpf = atomic_load_relaxed (&rwlock->__data.__wrphase_futex)) - | PTHREAD_RWLOCK_FUTEX_USED) == (1 | PTHREAD_RWLOCK_FUTEX_USED)) - { - int private = __pthread_rwlock_get_private (rwlock); - if (((wpf & PTHREAD_RWLOCK_FUTEX_USED) == 0) - && (!atomic_compare_exchange_weak_relaxed - (&rwlock->__data.__wrphase_futex, - &wpf, wpf | PTHREAD_RWLOCK_FUTEX_USED))) - continue; - int err = __futex_abstimed_wait64 (&rwlock->__data.__wrphase_futex, - 1 | PTHREAD_RWLOCK_FUTEX_USED, - clockid, abstime, private); - if (err == ETIMEDOUT || err == EOVERFLOW) - { - /* If we timed out, we need to unregister. If no read phase - has been installed while we waited, we can just decrement - the number of readers. Otherwise, we just acquire the - lock, which is allowed because we give no precise timing - guarantees, and because the timeout is only required to - be in effect if we would have had to wait for other - threads (e.g., if futex_wait would time-out immediately - because the given absolute time is in the past). */ - r = atomic_load_relaxed (&rwlock->__data.__readers); - while ((r & PTHREAD_RWLOCK_WRPHASE) != 0) - { - /* We don't need to make anything else visible to - others besides unregistering, so relaxed MO is - sufficient. */ - if (atomic_compare_exchange_weak_relaxed - (&rwlock->__data.__readers, &r, - r - (1 << PTHREAD_RWLOCK_READER_SHIFT))) - return err; - /* TODO Back-off. */ - } - /* Use the acquire MO fence to mirror the steps taken in the - non-timeout case. Note that the read can happen both - in the atomic_load above as well as in the failure case - of the CAS operation. */ - atomic_thread_fence_acquire (); - /* We still need to wait for explicit hand-over, but we must - not use futex_wait anymore because we would just time out - in this case and thus make the spin-waiting we need - unnecessarily expensive. */ - while ((atomic_load_relaxed (&rwlock->__data.__wrphase_futex) - | PTHREAD_RWLOCK_FUTEX_USED) - == (1 | PTHREAD_RWLOCK_FUTEX_USED)) - { - /* TODO Back-off? */ - } - ready = true; - break; - } - /* If we got interrupted (EINTR) or the futex word does not have the - expected value (EAGAIN), retry. */ - } + | PTHREAD_RWLOCK_FUTEX_USED) + == (1 | PTHREAD_RWLOCK_FUTEX_USED)) + { + int private = __pthread_rwlock_get_private (rwlock); + if (((wpf & PTHREAD_RWLOCK_FUTEX_USED) == 0) + && (!atomic_compare_exchange_weak_relaxed ( + &rwlock->__data.__wrphase_futex, &wpf, + wpf | PTHREAD_RWLOCK_FUTEX_USED))) + continue; + int err = __futex_abstimed_wait64 (&rwlock->__data.__wrphase_futex, + 1 | PTHREAD_RWLOCK_FUTEX_USED, + clockid, abstime, private); + if (err == ETIMEDOUT || err == EOVERFLOW) + { + /* If we timed out, we need to unregister. If no read phase + has been installed while we waited, we can just decrement + the number of readers. Otherwise, we just acquire the + lock, which is allowed because we give no precise timing + guarantees, and because the timeout is only required to + be in effect if we would have had to wait for other + threads (e.g., if futex_wait would time-out immediately + because the given absolute time is in the past). */ + r = atomic_load_relaxed (&rwlock->__data.__readers); + while ((r & PTHREAD_RWLOCK_WRPHASE) != 0) + { + /* We don't need to make anything else visible to + others besides unregistering, so relaxed MO is + sufficient. */ + if (atomic_compare_exchange_weak_relaxed ( + &rwlock->__data.__readers, &r, + r - (1 << PTHREAD_RWLOCK_READER_SHIFT))) + return err; + /* TODO Back-off. */ + } + /* Use the acquire MO fence to mirror the steps taken in the + non-timeout case. Note that the read can happen both + in the atomic_load above as well as in the failure case + of the CAS operation. */ + atomic_thread_fence_acquire (); + /* We still need to wait for explicit hand-over, but we must + not use futex_wait anymore because we would just time out + in this case and thus make the spin-waiting we need + unnecessarily expensive. */ + while ((atomic_load_relaxed (&rwlock->__data.__wrphase_futex) + | PTHREAD_RWLOCK_FUTEX_USED) + == (1 | PTHREAD_RWLOCK_FUTEX_USED)) + { + /* TODO Back-off? */ + } + ready = true; + break; + } + /* If we got interrupted (EINTR) or the futex word does not have the + expected value (EAGAIN), retry. */ + } if (ready) - /* See below. */ - break; + /* See below. */ + break; /* We need acquire MO here so that we synchronize with the lock - release of the writer, and so that we observe a recent value of - __wrphase_futex (see below). */ + release of the writer, and so that we observe a recent value of + __wrphase_futex (see below). */ if ((atomic_load_acquire (&rwlock->__data.__readers) - & PTHREAD_RWLOCK_WRPHASE) == 0) - /* We are in a read phase now, so the least recent modification of - __wrphase_futex we can read from is the store by the writer - with value 1. Thus, only now we can assume that if we observe - a value of 0, explicit hand-over is finished. Retry the loop - above one more time. */ - ready = true; + & PTHREAD_RWLOCK_WRPHASE) + == 0) + /* We are in a read phase now, so the least recent modification of + __wrphase_futex we can read from is the store by the writer + with value 1. Thus, only now we can assume that if we observe + a value of 0, explicit hand-over is finished. Retry the loop + above one more time. */ + ready = true; } return 0; } - static __always_inline void __pthread_rwlock_wrunlock (pthread_rwlock_t *rwlock) { @@ -532,25 +533,27 @@ __pthread_rwlock_wrunlock (pthread_rwlock_t *rwlock) /* Disable waiting by writers. We will wake up after we decided how to proceed. */ bool wake_writers - = ((atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0) - & PTHREAD_RWLOCK_FUTEX_USED) != 0); + = ((atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0) + & PTHREAD_RWLOCK_FUTEX_USED) + != 0); if (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP) { /* First, try to hand over to another writer. */ unsigned int w = atomic_load_relaxed (&rwlock->__data.__writers); while (w != 0) - { - /* Release MO so that another writer that gets WRLOCKED from us will - synchronize with us and thus can take over our view of - __readers (including, for example, whether we are in a write - phase or not). */ - if (atomic_compare_exchange_weak_release - (&rwlock->__data.__writers, &w, w | PTHREAD_RWLOCK_WRHANDOVER)) - /* Another writer will take over. */ - goto done; - /* TODO Back-off. */ - } + { + /* Release MO so that another writer that gets WRLOCKED from us will + synchronize with us and thus can take over our view of + __readers (including, for example, whether we are in a write + phase or not). */ + if (atomic_compare_exchange_weak_release ( + &rwlock->__data.__writers, &w, + w | PTHREAD_RWLOCK_WRHANDOVER)) + /* Another writer will take over. */ + goto done; + /* TODO Back-off. */ + } } /* We have done everything we needed to do to prefer writers, so now we @@ -558,32 +561,32 @@ __pthread_rwlock_wrunlock (pthread_rwlock_t *rwlock) stay in a write phase. See pthread_rwlock_rdunlock for more details. */ unsigned int r = atomic_load_relaxed (&rwlock->__data.__readers); /* Release MO so that subsequent readers or writers synchronize with us. */ - while (!atomic_compare_exchange_weak_release - (&rwlock->__data.__readers, &r, - ((r ^ PTHREAD_RWLOCK_WRLOCKED) - ^ ((r >> PTHREAD_RWLOCK_READER_SHIFT) == 0 ? 0 - : PTHREAD_RWLOCK_WRPHASE)))) + while (!atomic_compare_exchange_weak_release ( + &rwlock->__data.__readers, &r, + ((r ^ PTHREAD_RWLOCK_WRLOCKED) + ^ ((r >> PTHREAD_RWLOCK_READER_SHIFT) == 0 ? 0 + : PTHREAD_RWLOCK_WRPHASE)))) { /* TODO Back-off. */ } if ((r >> PTHREAD_RWLOCK_READER_SHIFT) != 0) { /* We must hand over explicitly through __wrphase_futex. Relaxed MO is - sufficient because it is just used to delay acquisition by a writer; - any other synchronizes-with relations that are necessary are - established through __readers. */ + sufficient because it is just used to delay acquisition by a writer; + any other synchronizes-with relations that are necessary are + established through __readers. */ if ((atomic_exchange_relaxed (&rwlock->__data.__wrphase_futex, 0) - & PTHREAD_RWLOCK_FUTEX_USED) != 0) - futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); + & PTHREAD_RWLOCK_FUTEX_USED) + != 0) + futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); } - done: +done: /* We released WRLOCKED in some way, so wake a writer. */ if (wake_writers) futex_wake (&rwlock->__data.__writers_futex, 1, private); } - static __always_inline int __pthread_rwlock_wrlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, const struct __timespec64 *abstime) @@ -594,14 +597,15 @@ __pthread_rwlock_wrlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, requires that "the validity of the abstime parameter need not be checked if the lock can be immediately acquired" (i.e., we need not but may check it). */ - if (abstime && __glibc_unlikely (!futex_abstimed_supported_clockid (clockid) - || ! valid_nanoseconds (abstime->tv_nsec))) + if (abstime + && __glibc_unlikely (!futex_abstimed_supported_clockid (clockid) + || !valid_nanoseconds (abstime->tv_nsec))) return EINVAL; /* Make sure we are not holding the rwlock as a writer. This is a deadlock situation we recognize and report. */ if (__glibc_unlikely (atomic_load_relaxed (&rwlock->__data.__cur_writer) - == THREAD_GETMEM (THREAD_SELF, tid))) + == THREAD_GETMEM (THREAD_SELF, tid))) return EDEADLK; /* First we try to acquire the role of primary writer by setting WRLOCKED; @@ -620,154 +624,155 @@ __pthread_rwlock_wrlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, this could be less scalable if readers arrive and leave frequently. */ bool may_share_futex_used_flag = false; unsigned int r = atomic_fetch_or_acquire (&rwlock->__data.__readers, - PTHREAD_RWLOCK_WRLOCKED); + PTHREAD_RWLOCK_WRLOCKED); if (__glibc_unlikely ((r & PTHREAD_RWLOCK_WRLOCKED) != 0)) { /* There is another primary writer. */ bool prefer_writer - = (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP); + = (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP); if (prefer_writer) - { - /* We register as a waiting writer, so that we can make use of - writer--writer hand-over. Relaxed MO is fine because we just - want to register. We assume that the maximum number of threads - is less than the capacity in __writers. */ - atomic_fetch_add_relaxed (&rwlock->__data.__writers, 1); - } + { + /* We register as a waiting writer, so that we can make use of + writer--writer hand-over. Relaxed MO is fine because we just + want to register. We assume that the maximum number of threads + is less than the capacity in __writers. */ + atomic_fetch_add_relaxed (&rwlock->__data.__writers, 1); + } for (;;) - { - /* TODO Spin until WRLOCKED is 0 before trying the CAS below. - But pay attention to not delay trying writer--writer hand-over - for too long (which we must try eventually anyway). */ - if ((r & PTHREAD_RWLOCK_WRLOCKED) == 0) - { - /* Try to become the primary writer or retry. Acquire MO as in - the fetch_or above. */ - if (atomic_compare_exchange_weak_acquire - (&rwlock->__data.__readers, &r, r | PTHREAD_RWLOCK_WRLOCKED)) - { - if (prefer_writer) - { - /* Unregister as a waiting writer. Note that because we - acquired WRLOCKED, WRHANDOVER will not be set. - Acquire MO on the CAS above ensures that - unregistering happens after the previous writer; - this sorts the accesses to __writers by all - primary writers in a useful way (e.g., any other - primary writer acquiring after us or getting it from - us through WRHANDOVER will see both our changes to - __writers). - ??? Perhaps this is not strictly necessary for - reasons we do not yet know of. */ - atomic_fetch_add_relaxed (&rwlock->__data.__writers, -1); - } - break; - } - /* Retry if the CAS fails (r will have been updated). */ - continue; - } - /* If writer--writer hand-over is available, try to become the - primary writer this way by grabbing the WRHANDOVER token. If we - succeed, we own WRLOCKED. */ - if (prefer_writer) - { - unsigned int w = atomic_load_relaxed (&rwlock->__data.__writers); - if ((w & PTHREAD_RWLOCK_WRHANDOVER) != 0) - { - /* Acquire MO is required here so that we synchronize with - the writer that handed over WRLOCKED. We also need this - for the reload of __readers below because our view of - __readers must be at least as recent as the view of the - writer that handed over WRLOCKED; we must avoid an ABA - through WRHANDOVER, which could, for example, lead to us - assuming we are still in a write phase when in fact we - are not. */ - if (atomic_compare_exchange_weak_acquire - (&rwlock->__data.__writers, - &w, (w - PTHREAD_RWLOCK_WRHANDOVER - 1))) - { - /* Reload so our view is consistent with the view of - the previous owner of WRLOCKED. See above. */ - r = atomic_load_relaxed (&rwlock->__data.__readers); - break; - } - /* We do not need to reload __readers here. We should try - to perform writer--writer hand-over if possible; if it - is not possible anymore, we will reload __readers - elsewhere in this loop. */ - continue; - } - } - /* We did not acquire WRLOCKED nor were able to use writer--writer - hand-over, so we block on __writers_futex. */ - int private = __pthread_rwlock_get_private (rwlock); - unsigned int wf - = atomic_load_relaxed (&rwlock->__data.__writers_futex); - if (((wf & ~(unsigned int) PTHREAD_RWLOCK_FUTEX_USED) != 1) - || ((wf != (1 | PTHREAD_RWLOCK_FUTEX_USED)) - && (!atomic_compare_exchange_weak_relaxed - (&rwlock->__data.__writers_futex, &wf, - 1 | PTHREAD_RWLOCK_FUTEX_USED)))) - { - /* If we cannot block on __writers_futex because there is no - primary writer, or we cannot set PTHREAD_RWLOCK_FUTEX_USED, - we retry. We must reload __readers here in case we cannot - block on __writers_futex so that we can become the primary - writer and are not stuck in a loop that just continuously - fails to block on __writers_futex. */ - r = atomic_load_relaxed (&rwlock->__data.__readers); - continue; - } - /* We set the flag that signals that the futex is used, or we could - have set it if we had been faster than other waiters. As a - result, we may share the flag with an unknown number of other - writers. Therefore, we must keep this flag set when we acquire - the lock. We do not need to do this when we do not reach this - point here because then we are not part of the group that may - share the flag, and another writer will wake one of the writers - in this group. */ - may_share_futex_used_flag = true; - int err = __futex_abstimed_wait64 (&rwlock->__data.__writers_futex, - 1 | PTHREAD_RWLOCK_FUTEX_USED, - clockid, abstime, private); - if (err == ETIMEDOUT || err == EOVERFLOW) - { - if (prefer_writer) - { - /* We need to unregister as a waiting writer. If we are the - last writer and writer--writer hand-over is available, - we must make use of it because nobody else will reset - WRLOCKED otherwise. (If we use it, we simply pretend - that this happened before the timeout; see - pthread_rwlock_rdlock_full for the full reasoning.) - Also see the similar code above. */ - unsigned int w - = atomic_load_relaxed (&rwlock->__data.__writers); - while (!atomic_compare_exchange_weak_acquire - (&rwlock->__data.__writers, &w, - (w == PTHREAD_RWLOCK_WRHANDOVER + 1 ? 0 : w - 1))) - { - /* TODO Back-off. */ - } - if (w == PTHREAD_RWLOCK_WRHANDOVER + 1) - { - /* We must continue as primary writer. See above. */ - r = atomic_load_relaxed (&rwlock->__data.__readers); - break; - } - } - /* We cleaned up and cannot have stolen another waiting writer's - futex wake-up, so just return. */ - return err; - } - /* If we got interrupted (EINTR) or the futex word does not have the - expected value (EAGAIN), retry after reloading __readers. */ - r = atomic_load_relaxed (&rwlock->__data.__readers); - } + { + /* TODO Spin until WRLOCKED is 0 before trying the CAS below. + But pay attention to not delay trying writer--writer hand-over + for too long (which we must try eventually anyway). */ + if ((r & PTHREAD_RWLOCK_WRLOCKED) == 0) + { + /* Try to become the primary writer or retry. Acquire MO as in + the fetch_or above. */ + if (atomic_compare_exchange_weak_acquire ( + &rwlock->__data.__readers, &r, + r | PTHREAD_RWLOCK_WRLOCKED)) + { + if (prefer_writer) + { + /* Unregister as a waiting writer. Note that because we + acquired WRLOCKED, WRHANDOVER will not be set. + Acquire MO on the CAS above ensures that + unregistering happens after the previous writer; + this sorts the accesses to __writers by all + primary writers in a useful way (e.g., any other + primary writer acquiring after us or getting it from + us through WRHANDOVER will see both our changes to + __writers). + ??? Perhaps this is not strictly necessary for + reasons we do not yet know of. */ + atomic_fetch_add_relaxed (&rwlock->__data.__writers, -1); + } + break; + } + /* Retry if the CAS fails (r will have been updated). */ + continue; + } + /* If writer--writer hand-over is available, try to become the + primary writer this way by grabbing the WRHANDOVER token. If we + succeed, we own WRLOCKED. */ + if (prefer_writer) + { + unsigned int w = atomic_load_relaxed (&rwlock->__data.__writers); + if ((w & PTHREAD_RWLOCK_WRHANDOVER) != 0) + { + /* Acquire MO is required here so that we synchronize with + the writer that handed over WRLOCKED. We also need this + for the reload of __readers below because our view of + __readers must be at least as recent as the view of the + writer that handed over WRLOCKED; we must avoid an ABA + through WRHANDOVER, which could, for example, lead to us + assuming we are still in a write phase when in fact we + are not. */ + if (atomic_compare_exchange_weak_acquire ( + &rwlock->__data.__writers, &w, + (w - PTHREAD_RWLOCK_WRHANDOVER - 1))) + { + /* Reload so our view is consistent with the view of + the previous owner of WRLOCKED. See above. */ + r = atomic_load_relaxed (&rwlock->__data.__readers); + break; + } + /* We do not need to reload __readers here. We should try + to perform writer--writer hand-over if possible; if it + is not possible anymore, we will reload __readers + elsewhere in this loop. */ + continue; + } + } + /* We did not acquire WRLOCKED nor were able to use writer--writer + hand-over, so we block on __writers_futex. */ + int private = __pthread_rwlock_get_private (rwlock); + unsigned int wf + = atomic_load_relaxed (&rwlock->__data.__writers_futex); + if (((wf & ~(unsigned int) PTHREAD_RWLOCK_FUTEX_USED) != 1) + || ((wf != (1 | PTHREAD_RWLOCK_FUTEX_USED)) + && (!atomic_compare_exchange_weak_relaxed ( + &rwlock->__data.__writers_futex, &wf, + 1 | PTHREAD_RWLOCK_FUTEX_USED)))) + { + /* If we cannot block on __writers_futex because there is no + primary writer, or we cannot set PTHREAD_RWLOCK_FUTEX_USED, + we retry. We must reload __readers here in case we cannot + block on __writers_futex so that we can become the primary + writer and are not stuck in a loop that just continuously + fails to block on __writers_futex. */ + r = atomic_load_relaxed (&rwlock->__data.__readers); + continue; + } + /* We set the flag that signals that the futex is used, or we could + have set it if we had been faster than other waiters. As a + result, we may share the flag with an unknown number of other + writers. Therefore, we must keep this flag set when we acquire + the lock. We do not need to do this when we do not reach this + point here because then we are not part of the group that may + share the flag, and another writer will wake one of the writers + in this group. */ + may_share_futex_used_flag = true; + int err = __futex_abstimed_wait64 (&rwlock->__data.__writers_futex, + 1 | PTHREAD_RWLOCK_FUTEX_USED, + clockid, abstime, private); + if (err == ETIMEDOUT || err == EOVERFLOW) + { + if (prefer_writer) + { + /* We need to unregister as a waiting writer. If we are the + last writer and writer--writer hand-over is available, + we must make use of it because nobody else will reset + WRLOCKED otherwise. (If we use it, we simply pretend + that this happened before the timeout; see + pthread_rwlock_rdlock_full for the full reasoning.) + Also see the similar code above. */ + unsigned int w + = atomic_load_relaxed (&rwlock->__data.__writers); + while (!atomic_compare_exchange_weak_acquire ( + &rwlock->__data.__writers, &w, + (w == PTHREAD_RWLOCK_WRHANDOVER + 1 ? 0 : w - 1))) + { + /* TODO Back-off. */ + } + if (w == PTHREAD_RWLOCK_WRHANDOVER + 1) + { + /* We must continue as primary writer. See above. */ + r = atomic_load_relaxed (&rwlock->__data.__readers); + break; + } + } + /* We cleaned up and cannot have stolen another waiting writer's + futex wake-up, so just return. */ + return err; + } + /* If we got interrupted (EINTR) or the futex word does not have the + expected value (EAGAIN), retry after reloading __readers. */ + r = atomic_load_relaxed (&rwlock->__data.__readers); + } /* Our snapshot of __readers is up-to-date at this point because we - either set WRLOCKED using a CAS (and update r accordingly below, - which was used as expected value for the CAS) or got WRLOCKED from - another writer whose snapshot of __readers we inherit. */ + either set WRLOCKED using a CAS (and update r accordingly below, + which was used as expected value for the CAS) or got WRLOCKED from + another writer whose snapshot of __readers we inherit. */ r |= PTHREAD_RWLOCK_WRLOCKED; } @@ -775,9 +780,9 @@ __pthread_rwlock_wrlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, MO is sufficient for futex words; acquire MO on the previous modifications of __readers ensures that this store happens after the store of value 0 by the previous primary writer. */ - atomic_store_relaxed (&rwlock->__data.__writers_futex, - 1 | (may_share_futex_used_flag - ? PTHREAD_RWLOCK_FUTEX_USED : 0)); + atomic_store_relaxed ( + &rwlock->__data.__writers_futex, + 1 | (may_share_futex_used_flag ? PTHREAD_RWLOCK_FUTEX_USED : 0)); /* If we are in a write phase, we have acquired the lock. */ if ((r & PTHREAD_RWLOCK_WRPHASE) != 0) @@ -786,23 +791,23 @@ __pthread_rwlock_wrlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, /* If we are in a read phase and there are no readers, try to start a write phase. */ while ((r & PTHREAD_RWLOCK_WRPHASE) == 0 - && (r >> PTHREAD_RWLOCK_READER_SHIFT) == 0) + && (r >> PTHREAD_RWLOCK_READER_SHIFT) == 0) { /* Acquire MO so that we synchronize with prior writers and do - not interfere with their updates to __writers_futex, as well - as regarding prior readers and their updates to __wrphase_futex, - respectively. */ - if (atomic_compare_exchange_weak_acquire (&rwlock->__data.__readers, - &r, r | PTHREAD_RWLOCK_WRPHASE)) - { - /* We have started a write phase, so need to enable readers to wait. - See the similar case in __pthread_rwlock_rdlock_full. Unlike in - that similar case, we are the (only) primary writer and so do - not need to wake another writer. */ - atomic_store_relaxed (&rwlock->__data.__wrphase_futex, 1); - - goto done; - } + not interfere with their updates to __writers_futex, as well + as regarding prior readers and their updates to __wrphase_futex, + respectively. */ + if (atomic_compare_exchange_weak_acquire (&rwlock->__data.__readers, &r, + r | PTHREAD_RWLOCK_WRPHASE)) + { + /* We have started a write phase, so need to enable readers to wait. + See the similar case in __pthread_rwlock_rdlock_full. Unlike in + that similar case, we are the (only) primary writer and so do + not need to wake another writer. */ + atomic_store_relaxed (&rwlock->__data.__wrphase_futex, 1); + + goto done; + } /* TODO Back-off. */ } @@ -818,133 +823,136 @@ __pthread_rwlock_wrlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, for (;;) { while (((wpf = atomic_load_relaxed (&rwlock->__data.__wrphase_futex)) - | PTHREAD_RWLOCK_FUTEX_USED) == PTHREAD_RWLOCK_FUTEX_USED) - { - int private = __pthread_rwlock_get_private (rwlock); - if ((wpf & PTHREAD_RWLOCK_FUTEX_USED) == 0 - && (!atomic_compare_exchange_weak_relaxed - (&rwlock->__data.__wrphase_futex, &wpf, - PTHREAD_RWLOCK_FUTEX_USED))) - continue; - int err = __futex_abstimed_wait64 (&rwlock->__data.__wrphase_futex, - PTHREAD_RWLOCK_FUTEX_USED, - clockid, abstime, private); - if (err == ETIMEDOUT || err == EOVERFLOW) - { - if (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP) - { - /* We try writer--writer hand-over. */ - unsigned int w - = atomic_load_relaxed (&rwlock->__data.__writers); - if (w != 0) - { - /* We are about to hand over WRLOCKED, so we must - release __writers_futex too; otherwise, we'd have - a pending store, which could at least prevent - other threads from waiting using the futex - because it could interleave with the stores - by subsequent writers. In turn, this means that - we have to clean up when we do not hand over - WRLOCKED. - Release MO so that another writer that gets - WRLOCKED from us can take over our view of - __readers. */ - unsigned int wf - = atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0); - while (w != 0) - { - if (atomic_compare_exchange_weak_release - (&rwlock->__data.__writers, &w, - w | PTHREAD_RWLOCK_WRHANDOVER)) - { - /* Wake other writers. */ - if ((wf & PTHREAD_RWLOCK_FUTEX_USED) != 0) - futex_wake (&rwlock->__data.__writers_futex, - 1, private); - return err; - } - /* TODO Back-off. */ - } - /* We still own WRLOCKED and someone else might set - a write phase concurrently, so enable waiting - again. Make sure we don't loose the flag that - signals whether there are threads waiting on - this futex. */ - atomic_store_relaxed (&rwlock->__data.__writers_futex, wf); - } - } - /* If we timed out and we are not in a write phase, we can - just stop being a primary writer. Otherwise, we just - acquire the lock. */ - r = atomic_load_relaxed (&rwlock->__data.__readers); - if ((r & PTHREAD_RWLOCK_WRPHASE) == 0) - { - /* We are about to release WRLOCKED, so we must release - __writers_futex too; see the handling of - writer--writer hand-over above. */ - unsigned int wf - = atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0); - while ((r & PTHREAD_RWLOCK_WRPHASE) == 0) - { - /* While we don't need to make anything from a - caller's critical section visible to other - threads, we need to ensure that our changes to - __writers_futex are properly ordered. - Therefore, use release MO to synchronize with - subsequent primary writers. Also wake up any - waiting readers as they are waiting because of - us. */ - if (atomic_compare_exchange_weak_release - (&rwlock->__data.__readers, &r, - (r ^ PTHREAD_RWLOCK_WRLOCKED) - & ~(unsigned int) PTHREAD_RWLOCK_RWAITING)) - { - /* Wake other writers. */ - if ((wf & PTHREAD_RWLOCK_FUTEX_USED) != 0) - futex_wake (&rwlock->__data.__writers_futex, - 1, private); - /* Wake waiting readers. */ - if ((r & PTHREAD_RWLOCK_RWAITING) != 0) - futex_wake (&rwlock->__data.__readers, - INT_MAX, private); - return ETIMEDOUT; - } - } - /* We still own WRLOCKED and someone else might set a - write phase concurrently, so enable waiting again. - Make sure we don't loose the flag that signals - whether there are threads waiting on this futex. */ - atomic_store_relaxed (&rwlock->__data.__writers_futex, wf); - } - /* Use the acquire MO fence to mirror the steps taken in the - non-timeout case. Note that the read can happen both - in the atomic_load above as well as in the failure case - of the CAS operation. */ - atomic_thread_fence_acquire (); - /* We still need to wait for explicit hand-over, but we must - not use futex_wait anymore. */ - while ((atomic_load_relaxed (&rwlock->__data.__wrphase_futex) - | PTHREAD_RWLOCK_FUTEX_USED) - == PTHREAD_RWLOCK_FUTEX_USED) - { - /* TODO Back-off. */ - } - ready = true; - break; - } - /* If we got interrupted (EINTR) or the futex word does not have - the expected value (EAGAIN), retry. */ - } + | PTHREAD_RWLOCK_FUTEX_USED) + == PTHREAD_RWLOCK_FUTEX_USED) + { + int private = __pthread_rwlock_get_private (rwlock); + if ((wpf & PTHREAD_RWLOCK_FUTEX_USED) == 0 + && (!atomic_compare_exchange_weak_relaxed ( + &rwlock->__data.__wrphase_futex, &wpf, + PTHREAD_RWLOCK_FUTEX_USED))) + continue; + int err = __futex_abstimed_wait64 (&rwlock->__data.__wrphase_futex, + PTHREAD_RWLOCK_FUTEX_USED, + clockid, abstime, private); + if (err == ETIMEDOUT || err == EOVERFLOW) + { + if (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP) + { + /* We try writer--writer hand-over. */ + unsigned int w + = atomic_load_relaxed (&rwlock->__data.__writers); + if (w != 0) + { + /* We are about to hand over WRLOCKED, so we must + release __writers_futex too; otherwise, we'd have + a pending store, which could at least prevent + other threads from waiting using the futex + because it could interleave with the stores + by subsequent writers. In turn, this means that + we have to clean up when we do not hand over + WRLOCKED. + Release MO so that another writer that gets + WRLOCKED from us can take over our view of + __readers. */ + unsigned int wf = atomic_exchange_relaxed ( + &rwlock->__data.__writers_futex, 0); + while (w != 0) + { + if (atomic_compare_exchange_weak_release ( + &rwlock->__data.__writers, &w, + w | PTHREAD_RWLOCK_WRHANDOVER)) + { + /* Wake other writers. */ + if ((wf & PTHREAD_RWLOCK_FUTEX_USED) != 0) + futex_wake (&rwlock->__data.__writers_futex, 1, + private); + return err; + } + /* TODO Back-off. */ + } + /* We still own WRLOCKED and someone else might set + a write phase concurrently, so enable waiting + again. Make sure we don't loose the flag that + signals whether there are threads waiting on + this futex. */ + atomic_store_relaxed (&rwlock->__data.__writers_futex, + wf); + } + } + /* If we timed out and we are not in a write phase, we can + just stop being a primary writer. Otherwise, we just + acquire the lock. */ + r = atomic_load_relaxed (&rwlock->__data.__readers); + if ((r & PTHREAD_RWLOCK_WRPHASE) == 0) + { + /* We are about to release WRLOCKED, so we must release + __writers_futex too; see the handling of + writer--writer hand-over above. */ + unsigned int wf = atomic_exchange_relaxed ( + &rwlock->__data.__writers_futex, 0); + while ((r & PTHREAD_RWLOCK_WRPHASE) == 0) + { + /* While we don't need to make anything from a + caller's critical section visible to other + threads, we need to ensure that our changes to + __writers_futex are properly ordered. + Therefore, use release MO to synchronize with + subsequent primary writers. Also wake up any + waiting readers as they are waiting because of + us. */ + if (atomic_compare_exchange_weak_release ( + &rwlock->__data.__readers, &r, + (r ^ PTHREAD_RWLOCK_WRLOCKED) + & ~(unsigned int) PTHREAD_RWLOCK_RWAITING)) + { + /* Wake other writers. */ + if ((wf & PTHREAD_RWLOCK_FUTEX_USED) != 0) + futex_wake (&rwlock->__data.__writers_futex, 1, + private); + /* Wake waiting readers. */ + if ((r & PTHREAD_RWLOCK_RWAITING) != 0) + futex_wake (&rwlock->__data.__readers, INT_MAX, + private); + return ETIMEDOUT; + } + } + /* We still own WRLOCKED and someone else might set a + write phase concurrently, so enable waiting again. + Make sure we don't loose the flag that signals + whether there are threads waiting on this futex. */ + atomic_store_relaxed (&rwlock->__data.__writers_futex, wf); + } + /* Use the acquire MO fence to mirror the steps taken in the + non-timeout case. Note that the read can happen both + in the atomic_load above as well as in the failure case + of the CAS operation. */ + atomic_thread_fence_acquire (); + /* We still need to wait for explicit hand-over, but we must + not use futex_wait anymore. */ + while ((atomic_load_relaxed (&rwlock->__data.__wrphase_futex) + | PTHREAD_RWLOCK_FUTEX_USED) + == PTHREAD_RWLOCK_FUTEX_USED) + { + /* TODO Back-off. */ + } + ready = true; + break; + } + /* If we got interrupted (EINTR) or the futex word does not have + the expected value (EAGAIN), retry. */ + } /* See pthread_rwlock_rdlock_full. */ if (ready) - break; + break; if ((atomic_load_acquire (&rwlock->__data.__readers) - & PTHREAD_RWLOCK_WRPHASE) != 0) - ready = true; + & PTHREAD_RWLOCK_WRPHASE) + != 0) + ready = true; } - done: +done: atomic_store_relaxed (&rwlock->__data.__cur_writer, - THREAD_GETMEM (THREAD_SELF, tid)); + THREAD_GETMEM (THREAD_SELF, tid)); return 0; } -- 2.25.1