diff --git a/libstdc++-v3/include/bits/c++config b/libstdc++-v3/include/bits/c++config index 69ace386dd7..543fac268b4 100644 --- a/libstdc++-v3/include/bits/c++config +++ b/libstdc++-v3/include/bits/c++config @@ -126,20 +126,29 @@ # define _GLIBCXX_ABI_TAG_CXX11 __attribute ((__abi_tag__ ("cxx11"))) #endif // Macro to warn about unused results. #if __cplusplus >= 201703L # define _GLIBCXX_NODISCARD [[__nodiscard__]] #else # define _GLIBCXX_NODISCARD #endif +// Macro for TSAN. +#if __SANITIZE_THREAD__ +# define _GLIBCXX_TSAN 1 +#elif defined __has_feature +# if __has_feature(thread_sanitizer) +# define _GLIBCXX_TSAN 1 +# endif +#endif + #if __cplusplus // Macro for constexpr, to support in mixed 03/0x mode. #ifndef _GLIBCXX_CONSTEXPR # if __cplusplus >= 201103L # define _GLIBCXX_CONSTEXPR constexpr # define _GLIBCXX_USE_CONSTEXPR constexpr # else diff --git a/libstdc++-v3/include/bits/shared_ptr_base.h b/libstdc++-v3/include/bits/shared_ptr_base.h index 5be935d174d..834ca13bdc6 100644 --- a/libstdc++-v3/include/bits/shared_ptr_base.h +++ b/libstdc++-v3/include/bits/shared_ptr_base.h @@ -153,20 +153,93 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION if (!_M_add_ref_lock_nothrow()) __throw_bad_weak_ptr(); } bool _M_add_ref_lock_nothrow() noexcept; void _M_release() noexcept { +#if _GLIBCXX_TSAN + _M_release_orig(); + return; +#endif + constexpr bool __lock_free + = __atomic_always_lock_free(sizeof(long long), 0) + && __atomic_always_lock_free(sizeof(_Atomic_word), 0); + constexpr bool __double_word + = sizeof(long long) == 2 * sizeof(_Atomic_word); + // The ref-count members follow the vptr, so are aligned to + // alignof(void*). + constexpr bool __aligned = alignof(long long) <= alignof(void*); + if _GLIBCXX17_CONSTEXPR + (__lock_free && __double_word && __aligned) + { + _M_release_double_width_cas(); + return; + } + else + { + _M_release_orig(); + } + } + + void + _M_release_double_width_cas() noexcept + { + _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(&_M_use_count); + _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(&_M_weak_count); + if (__atomic_load_n((long long*)(&_M_use_count), __ATOMIC_ACQUIRE) + == (1LL + (1LL << (__CHAR_BIT__ * sizeof(_Atomic_word))))) + { + // Both counts are 1, so there are no weak references and + // we are releasing the last strong reference. No other + // threads can observe the effects of this _M_release() + // call (e.g. calling use_count()) without a data race. + *(long long*)(&_M_use_count) = 0; + _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(&_M_use_count); + _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(&_M_weak_count); + _M_dispose(); + _M_destroy(); + } + else + { + if ((__gnu_cxx::__exchange_and_add(&_M_use_count, -1) == 1)) + { + _M_release_last_use(); + } + } + } + + void + __attribute__ ((noinline)) + _M_release_last_use() noexcept + { + _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(&_M_use_count); + _M_dispose(); + if (_Mutex_base<_Lp>::_S_need_barriers) + { + __atomic_thread_fence (__ATOMIC_ACQ_REL); + } + _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(&_M_weak_count); + if (__gnu_cxx::__exchange_and_add_dispatch(&_M_weak_count, + -1) == 1) + { + _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(&_M_weak_count); + _M_destroy(); + } + } + + void + _M_release_orig() noexcept + { // Be race-detector-friendly. For more info see bits/c++config. _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(&_M_use_count); if (__gnu_cxx::__exchange_and_add_dispatch(&_M_use_count, -1) == 1) { _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(&_M_use_count); _M_dispose(); // There must be a memory barrier between dispose() and destroy() // to ensure that the effects of dispose() are observed in the // thread that runs destroy(). // See http://gcc.gnu.org/ml/libstdc++/2005-11/msg00136.html @@ -279,20 +352,27 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION _Sp_counted_base<_S_single>::_M_release() noexcept { if (--_M_use_count == 0) { _M_dispose(); if (--_M_weak_count == 0) _M_destroy(); } } + template<> + inline void + _Sp_counted_base<_S_mutex>::_M_release() noexcept + { + _M_release_orig(); + } + template<> inline void _Sp_counted_base<_S_single>::_M_weak_add_ref() noexcept { ++_M_weak_count; } template<> inline void _Sp_counted_base<_S_single>::_M_weak_release() noexcept { if (--_M_weak_count == 0)