public inbox for gcc-cvs@sourceware.org
help / color / mirror / Atom feed
* [gcc r12-2423] libsanitizer: Merge with upstream
@ 2021-07-20 21:22 H.J. Lu
0 siblings, 0 replies; only message in thread
From: H.J. Lu @ 2021-07-20 21:22 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:90e46074e6b3561ae7d8ebd205127f286cc0c6b6
commit r12-2423-g90e46074e6b3561ae7d8ebd205127f286cc0c6b6
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Tue Jul 20 10:44:37 2021 -0700
libsanitizer: Merge with upstream
Merged revision: 7704fedfff6ef5676adb6415f3be0ac927d1a746
Diff:
---
libsanitizer/MERGE | 2 +-
libsanitizer/asan/Makefile.am | 1 -
libsanitizer/asan/Makefile.in | 8 +-
libsanitizer/asan/asan_allocator.cpp | 10 +-
libsanitizer/asan/asan_errors.cpp | 1 -
libsanitizer/asan/asan_fake_stack.cpp | 50 +++-
libsanitizer/asan/asan_flags.cpp | 4 -
libsanitizer/asan/asan_flags.inc | 3 +-
libsanitizer/asan/asan_globals.cpp | 19 ++
libsanitizer/asan/asan_interceptors.cpp | 28 +-
libsanitizer/asan/asan_interceptors.h | 11 +-
.../asan/asan_interceptors_memintrinsics.cpp | 6 +-
libsanitizer/asan/asan_interface.inc | 11 +
libsanitizer/asan/asan_internal.h | 15 +-
libsanitizer/asan/asan_malloc_linux.cpp | 36 +--
libsanitizer/asan/asan_malloc_local.h | 52 ----
libsanitizer/asan/asan_mapping.h | 31 +--
libsanitizer/asan/asan_mapping_myriad.h | 85 -------
libsanitizer/asan/asan_new_delete.cpp | 20 +-
libsanitizer/asan/asan_poisoning.cpp | 19 +-
libsanitizer/asan/asan_poisoning.h | 3 -
libsanitizer/asan/asan_rtems.cpp | 266 -------------------
libsanitizer/asan/asan_rtl.cpp | 47 ++--
libsanitizer/asan/asan_shadow_setup.cpp | 11 +-
libsanitizer/asan/asan_stack.cpp | 3 +-
libsanitizer/asan/asan_thread.cpp | 45 ++--
libsanitizer/asan/asan_thread.h | 17 +-
libsanitizer/hwasan/Makefile.am | 2 +
libsanitizer/hwasan/Makefile.in | 9 +-
libsanitizer/hwasan/hwasan.cpp | 77 +++++-
libsanitizer/hwasan/hwasan.h | 42 ++-
.../hwasan/hwasan_allocation_functions.cpp | 172 +++++++++++++
libsanitizer/hwasan/hwasan_allocator.cpp | 35 ++-
libsanitizer/hwasan/hwasan_allocator.h | 7 +-
libsanitizer/hwasan/hwasan_dynamic_shadow.cpp | 4 +-
libsanitizer/hwasan/hwasan_fuchsia.cpp | 192 ++++++++++++++
libsanitizer/hwasan/hwasan_interceptors.cpp | 182 +------------
libsanitizer/hwasan/hwasan_linux.cpp | 166 +++++-------
libsanitizer/hwasan/hwasan_mapping.h | 17 +-
libsanitizer/hwasan/hwasan_new_delete.cpp | 39 ++-
libsanitizer/hwasan/hwasan_poisoning.cpp | 24 --
libsanitizer/hwasan/hwasan_report.cpp | 206 +++++++++------
libsanitizer/hwasan/hwasan_thread.cpp | 21 +-
libsanitizer/hwasan/hwasan_thread.h | 11 +-
libsanitizer/hwasan/hwasan_thread_list.cpp | 2 +-
libsanitizer/hwasan/hwasan_thread_list.h | 8 +-
libsanitizer/include/sanitizer/dfsan_interface.h | 95 ++++---
libsanitizer/interception/interception.h | 33 +--
libsanitizer/lsan/lsan.cpp | 14 +-
libsanitizer/lsan/lsan_common.h | 4 +-
libsanitizer/lsan/lsan_thread.cpp | 7 +-
libsanitizer/sanitizer_common/Makefile.am | 2 +-
libsanitizer/sanitizer_common/Makefile.in | 19 +-
.../sanitizer_common/sanitizer_addrhashmap.h | 106 ++++----
.../sanitizer_common/sanitizer_allocator.cpp | 38 +--
.../sanitizer_allocator_combined.h | 4 +-
.../sanitizer_allocator_local_cache.h | 19 +-
.../sanitizer_allocator_primary32.h | 4 +-
.../sanitizer_allocator_primary64.h | 170 ++++++-------
.../sanitizer_allocator_secondary.h | 8 +-
libsanitizer/sanitizer_common/sanitizer_common.cpp | 15 +-
libsanitizer/sanitizer_common/sanitizer_common.h | 23 +-
.../sanitizer_common_interceptors.inc | 84 ++++--
.../sanitizer_common/sanitizer_common_libcdep.cpp | 4 +-
.../sanitizer_common/sanitizer_common_nolibc.cpp | 1 -
.../sanitizer_deadlock_detector1.cpp | 2 +-
.../sanitizer_deadlock_detector2.cpp | 32 ++-
libsanitizer/sanitizer_common/sanitizer_errno.h | 3 +-
.../sanitizer_common/sanitizer_fuchsia.cpp | 33 ++-
libsanitizer/sanitizer_common/sanitizer_fuchsia.h | 2 +
libsanitizer/sanitizer_common/sanitizer_libc.h | 3 +-
.../sanitizer_common/sanitizer_libignore.cpp | 25 ++
.../sanitizer_common/sanitizer_libignore.h | 35 +--
libsanitizer/sanitizer_common/sanitizer_linux.cpp | 46 ++--
.../sanitizer_common/sanitizer_linux_libcdep.cpp | 41 ++-
libsanitizer/sanitizer_common/sanitizer_mac.cpp | 25 +-
libsanitizer/sanitizer_common/sanitizer_mac.h | 20 --
libsanitizer/sanitizer_common/sanitizer_mutex.cpp | 39 +++
libsanitizer/sanitizer_common/sanitizer_mutex.h | 272 +++++++++++++++++---
libsanitizer/sanitizer_common/sanitizer_netbsd.cpp | 11 +-
libsanitizer/sanitizer_common/sanitizer_platform.h | 25 +-
.../sanitizer_platform_interceptors.h | 12 +-
.../sanitizer_platform_limits_linux.cpp | 7 +-
.../sanitizer_platform_limits_posix.h | 12 +-
.../sanitizer_common/sanitizer_posix_libcdep.cpp | 15 +-
libsanitizer/sanitizer_common/sanitizer_printf.cpp | 27 +-
.../sanitizer_common/sanitizer_quarantine.h | 3 +-
libsanitizer/sanitizer_common/sanitizer_rtems.cpp | 281 ---------------------
libsanitizer/sanitizer_common/sanitizer_rtems.h | 20 --
.../sanitizer_common/sanitizer_solaris.cpp | 18 +-
.../sanitizer_common/sanitizer_stacktrace.cpp | 17 +-
.../sanitizer_common/sanitizer_stacktrace.h | 36 ++-
.../sanitizer_stacktrace_libcdep.cpp | 185 +++++++++-----
.../sanitizer_symbolizer_markup.cpp | 15 +-
.../sanitizer_symbolizer_report.cpp | 22 +-
.../sanitizer_common/sanitizer_symbolizer_rtems.h | 40 ---
.../sanitizer_common/sanitizer_thread_registry.cpp | 29 +--
.../sanitizer_common/sanitizer_thread_registry.h | 18 +-
.../sanitizer_common/sanitizer_thread_safety.h | 42 +++
libsanitizer/sanitizer_common/sanitizer_win.cpp | 26 +-
libsanitizer/tsan/Makefile.am | 3 +-
libsanitizer/tsan/Makefile.in | 8 +-
libsanitizer/tsan/tsan_clock.cpp | 29 +--
libsanitizer/tsan/tsan_defs.h | 16 --
libsanitizer/tsan/tsan_interceptors.h | 16 +-
libsanitizer/tsan/tsan_interceptors_mac.cpp | 9 +-
libsanitizer/tsan/tsan_interceptors_posix.cpp | 41 ++-
libsanitizer/tsan/tsan_interface.h | 3 +-
libsanitizer/tsan/tsan_interface_ann.cpp | 6 +-
libsanitizer/tsan/tsan_interface_atomic.cpp | 54 ++--
libsanitizer/tsan/tsan_mman.cpp | 5 +-
libsanitizer/tsan/tsan_mutex.cpp | 11 +-
libsanitizer/tsan/tsan_mutex.h | 5 +-
libsanitizer/tsan/tsan_platform.h | 54 ++++
libsanitizer/tsan/tsan_platform_linux.cpp | 6 +
libsanitizer/tsan/tsan_platform_posix.cpp | 20 +-
libsanitizer/tsan/tsan_rtl.cpp | 51 +---
libsanitizer/tsan/tsan_rtl.h | 21 --
libsanitizer/tsan/tsan_rtl_mutex.cpp | 17 --
libsanitizer/tsan/tsan_rtl_ppc64.S | 1 -
libsanitizer/tsan/tsan_rtl_s390x.S | 47 ++++
libsanitizer/tsan/tsan_rtl_thread.cpp | 12 -
libsanitizer/tsan/tsan_stack_trace.cpp | 8 +-
libsanitizer/tsan/tsan_stat.cpp | 186 --------------
libsanitizer/tsan/tsan_stat.h | 191 --------------
libsanitizer/tsan/tsan_sync.cpp | 5 +-
libsanitizer/tsan/tsan_trace.h | 4 +-
libsanitizer/tsan/tsan_update_shadow_word_inl.h | 12 +-
libsanitizer/ubsan/ubsan_diag_standalone.cpp | 8 +-
libsanitizer/ubsan/ubsan_flags.cpp | 1 -
libsanitizer/ubsan/ubsan_handlers.cpp | 15 --
libsanitizer/ubsan/ubsan_handlers.h | 8 -
libsanitizer/ubsan/ubsan_platform.h | 10 +-
133 files changed, 2284 insertions(+), 2708 deletions(-)
diff --git a/libsanitizer/MERGE b/libsanitizer/MERGE
index c4731d0866c..81d00f27de4 100644
--- a/libsanitizer/MERGE
+++ b/libsanitizer/MERGE
@@ -1,4 +1,4 @@
-f58e0513dd95944b81ce7a6e7b49ba656de7d75f
+7704fedfff6ef5676adb6415f3be0ac927d1a746
The first line of this file holds the git revision number of the
last merge done from the master library sources.
diff --git a/libsanitizer/asan/Makefile.am b/libsanitizer/asan/Makefile.am
index 2e6385509b4..74658ca7b9c 100644
--- a/libsanitizer/asan/Makefile.am
+++ b/libsanitizer/asan/Makefile.am
@@ -38,7 +38,6 @@ asan_files = \
asan_posix.cpp \
asan_premap_shadow.cpp \
asan_report.cpp \
- asan_rtems.cpp \
asan_rtl.cpp \
asan_shadow_setup.cpp \
asan_stack.cpp \
diff --git a/libsanitizer/asan/Makefile.in b/libsanitizer/asan/Makefile.in
index 25c7fd7b759..53efe526f9c 100644
--- a/libsanitizer/asan/Makefile.in
+++ b/libsanitizer/asan/Makefile.in
@@ -156,9 +156,9 @@ am__objects_1 = asan_activation.lo asan_allocator.lo asan_debugging.lo \
asan_interceptors_memintrinsics.lo asan_linux.lo asan_mac.lo \
asan_malloc_linux.lo asan_malloc_mac.lo asan_malloc_win.lo \
asan_memory_profile.lo asan_new_delete.lo asan_poisoning.lo \
- asan_posix.lo asan_premap_shadow.lo asan_report.lo \
- asan_rtems.lo asan_rtl.lo asan_shadow_setup.lo asan_stack.lo \
- asan_stats.lo asan_suppressions.lo asan_thread.lo asan_win.lo \
+ asan_posix.lo asan_premap_shadow.lo asan_report.lo asan_rtl.lo \
+ asan_shadow_setup.lo asan_stack.lo asan_stats.lo \
+ asan_suppressions.lo asan_thread.lo asan_win.lo \
asan_win_dll_thunk.lo asan_win_dynamic_runtime_thunk.lo \
asan_interceptors_vfork.lo
am_libasan_la_OBJECTS = $(am__objects_1)
@@ -446,7 +446,6 @@ asan_files = \
asan_posix.cpp \
asan_premap_shadow.cpp \
asan_report.cpp \
- asan_rtems.cpp \
asan_rtl.cpp \
asan_shadow_setup.cpp \
asan_stack.cpp \
@@ -604,7 +603,6 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_posix.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_premap_shadow.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_report.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_rtems.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_rtl.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_shadow_setup.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_stack.Plo@am__quote@
diff --git a/libsanitizer/asan/asan_allocator.cpp b/libsanitizer/asan/asan_allocator.cpp
index 7c8bb504332..414fba3b427 100644
--- a/libsanitizer/asan/asan_allocator.cpp
+++ b/libsanitizer/asan/asan_allocator.cpp
@@ -852,12 +852,12 @@ struct Allocator {
quarantine.PrintStats();
}
- void ForceLock() {
+ void ForceLock() ACQUIRE(fallback_mutex) {
allocator.ForceLock();
fallback_mutex.Lock();
}
- void ForceUnlock() {
+ void ForceUnlock() RELEASE(fallback_mutex) {
fallback_mutex.Unlock();
allocator.ForceUnlock();
}
@@ -1081,11 +1081,9 @@ uptr asan_mz_size(const void *ptr) {
return instance.AllocationSize(reinterpret_cast<uptr>(ptr));
}
-void asan_mz_force_lock() {
- instance.ForceLock();
-}
+void asan_mz_force_lock() NO_THREAD_SAFETY_ANALYSIS { instance.ForceLock(); }
-void asan_mz_force_unlock() {
+void asan_mz_force_unlock() NO_THREAD_SAFETY_ANALYSIS {
instance.ForceUnlock();
}
diff --git a/libsanitizer/asan/asan_errors.cpp b/libsanitizer/asan/asan_errors.cpp
index e68e6971f96..45166c06487 100644
--- a/libsanitizer/asan/asan_errors.cpp
+++ b/libsanitizer/asan/asan_errors.cpp
@@ -533,7 +533,6 @@ static void PrintLegend(InternalScopedString *str) {
PrintShadowByte(str, " ASan internal: ", kAsanInternalHeapMagic);
PrintShadowByte(str, " Left alloca redzone: ", kAsanAllocaLeftMagic);
PrintShadowByte(str, " Right alloca redzone: ", kAsanAllocaRightMagic);
- PrintShadowByte(str, " Shadow gap: ", kAsanShadowGap);
}
static void PrintShadowBytes(InternalScopedString *str, const char *before,
diff --git a/libsanitizer/asan/asan_fake_stack.cpp b/libsanitizer/asan/asan_fake_stack.cpp
index 1f873fec7d7..bf5c342ee59 100644
--- a/libsanitizer/asan/asan_fake_stack.cpp
+++ b/libsanitizer/asan/asan_fake_stack.cpp
@@ -187,7 +187,7 @@ void SetTLSFakeStack(FakeStack *fs) { }
static FakeStack *GetFakeStack() {
AsanThread *t = GetCurrentThread();
if (!t) return nullptr;
- return t->fake_stack();
+ return t->get_or_create_fake_stack();
}
static FakeStack *GetFakeStackFast() {
@@ -198,7 +198,13 @@ static FakeStack *GetFakeStackFast() {
return GetFakeStack();
}
-ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) {
+static FakeStack *GetFakeStackFastAlways() {
+ if (FakeStack *fs = GetTLSFakeStack())
+ return fs;
+ return GetFakeStack();
+}
+
+static ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) {
FakeStack *fs = GetFakeStackFast();
if (!fs) return 0;
uptr local_stack;
@@ -210,7 +216,21 @@ ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) {
return ptr;
}
-ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) {
+static ALWAYS_INLINE uptr OnMallocAlways(uptr class_id, uptr size) {
+ FakeStack *fs = GetFakeStackFastAlways();
+ if (!fs)
+ return 0;
+ uptr local_stack;
+ uptr real_stack = reinterpret_cast<uptr>(&local_stack);
+ FakeFrame *ff = fs->Allocate(fs->stack_size_log(), class_id, real_stack);
+ if (!ff)
+ return 0; // Out of fake stack.
+ uptr ptr = reinterpret_cast<uptr>(ff);
+ SetShadow(ptr, size, class_id, 0);
+ return ptr;
+}
+
+static ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) {
FakeStack::Deallocate(ptr, class_id);
SetShadow(ptr, size, class_id, kMagic8);
}
@@ -219,14 +239,18 @@ ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) {
// ---------------------- Interface ---------------- {{{1
using namespace __asan;
-#define DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(class_id) \
- extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \
- __asan_stack_malloc_##class_id(uptr size) { \
- return OnMalloc(class_id, size); \
- } \
- extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __asan_stack_free_##class_id( \
- uptr ptr, uptr size) { \
- OnFree(ptr, class_id, size); \
+#define DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(class_id) \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \
+ __asan_stack_malloc_##class_id(uptr size) { \
+ return OnMalloc(class_id, size); \
+ } \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \
+ __asan_stack_malloc_always_##class_id(uptr size) { \
+ return OnMallocAlways(class_id, size); \
+ } \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __asan_stack_free_##class_id( \
+ uptr ptr, uptr size) { \
+ OnFree(ptr, class_id, size); \
}
DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(0)
@@ -240,7 +264,11 @@ DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(7)
DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(8)
DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(9)
DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(10)
+
extern "C" {
+// TODO: remove this method and fix tests that use it by setting
+// -asan-use-after-return=never, after modal UAR flag lands
+// (https://github.com/google/sanitizers/issues/1394)
SANITIZER_INTERFACE_ATTRIBUTE
void *__asan_get_current_fake_stack() { return GetFakeStackFast(); }
diff --git a/libsanitizer/asan/asan_flags.cpp b/libsanitizer/asan/asan_flags.cpp
index cb6a89fe32c..c64e4647028 100644
--- a/libsanitizer/asan/asan_flags.cpp
+++ b/libsanitizer/asan/asan_flags.cpp
@@ -155,10 +155,6 @@ void InitializeFlags() {
CHECK_LE(f->max_redzone, 2048);
CHECK(IsPowerOfTwo(f->redzone));
CHECK(IsPowerOfTwo(f->max_redzone));
- if (SANITIZER_RTEMS) {
- CHECK(!f->unmap_shadow_on_exit);
- CHECK(!f->protect_shadow_gap);
- }
// quarantine_size is deprecated but we still honor it.
// quarantine_size can not be used together with quarantine_size_mb.
diff --git a/libsanitizer/asan/asan_flags.inc b/libsanitizer/asan/asan_flags.inc
index 43c70dbca56..514b225c407 100644
--- a/libsanitizer/asan/asan_flags.inc
+++ b/libsanitizer/asan/asan_flags.inc
@@ -87,8 +87,7 @@ ASAN_FLAG(bool, check_malloc_usable_size, true,
"295.*.")
ASAN_FLAG(bool, unmap_shadow_on_exit, false,
"If set, explicitly unmaps the (huge) shadow at exit.")
-ASAN_FLAG(bool, protect_shadow_gap, !SANITIZER_RTEMS,
- "If set, mprotect the shadow gap")
+ASAN_FLAG(bool, protect_shadow_gap, true, "If set, mprotect the shadow gap")
ASAN_FLAG(bool, print_stats, false,
"Print various statistics after printing an error message or if "
"atexit=1.")
diff --git a/libsanitizer/asan/asan_globals.cpp b/libsanitizer/asan/asan_globals.cpp
index e045c31cd1c..9d7dbc6f264 100644
--- a/libsanitizer/asan/asan_globals.cpp
+++ b/libsanitizer/asan/asan_globals.cpp
@@ -154,6 +154,23 @@ static void CheckODRViolationViaIndicator(const Global *g) {
}
}
+// Check ODR violation for given global G by checking if it's already poisoned.
+// We use this method in case compiler doesn't use private aliases for global
+// variables.
+static void CheckODRViolationViaPoisoning(const Global *g) {
+ if (__asan_region_is_poisoned(g->beg, g->size_with_redzone)) {
+ // This check may not be enough: if the first global is much larger
+ // the entire redzone of the second global may be within the first global.
+ for (ListOfGlobals *l = list_of_all_globals; l; l = l->next) {
+ if (g->beg == l->g->beg &&
+ (flags()->detect_odr_violation >= 2 || g->size != l->g->size) &&
+ !IsODRViolationSuppressed(g->name))
+ ReportODRViolation(g, FindRegistrationSite(g),
+ l->g, FindRegistrationSite(l->g));
+ }
+ }
+}
+
// Clang provides two different ways for global variables protection:
// it can poison the global itself or its private alias. In former
// case we may poison same symbol multiple times, that can help us to
@@ -199,6 +216,8 @@ static void RegisterGlobal(const Global *g) {
// where two globals with the same name are defined in different modules.
if (UseODRIndicator(g))
CheckODRViolationViaIndicator(g);
+ else
+ CheckODRViolationViaPoisoning(g);
}
if (CanPoisonMemory())
PoisonRedZones(*g);
diff --git a/libsanitizer/asan/asan_interceptors.cpp b/libsanitizer/asan/asan_interceptors.cpp
index 9db7db89fa1..d0a6dd48a74 100644
--- a/libsanitizer/asan/asan_interceptors.cpp
+++ b/libsanitizer/asan/asan_interceptors.cpp
@@ -23,25 +23,25 @@
#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_libc.h"
-// There is no general interception at all on Fuchsia and RTEMS.
+// There is no general interception at all on Fuchsia.
// Only the functions in asan_interceptors_memintrinsics.cpp are
// really defined to replace libc functions.
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#if !SANITIZER_FUCHSIA
-#if SANITIZER_POSIX
-#include "sanitizer_common/sanitizer_posix.h"
-#endif
+# if SANITIZER_POSIX
+# include "sanitizer_common/sanitizer_posix.h"
+# endif
-#if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION || \
- ASAN_INTERCEPT__SJLJ_UNWIND_RAISEEXCEPTION
-#include <unwind.h>
-#endif
+# if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION || \
+ ASAN_INTERCEPT__SJLJ_UNWIND_RAISEEXCEPTION
+# include <unwind.h>
+# endif
-#if defined(__i386) && SANITIZER_LINUX
-#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.1"
-#elif defined(__mips__) && SANITIZER_LINUX
-#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.2"
-#endif
+# if defined(__i386) && SANITIZER_LINUX
+# define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.1"
+# elif defined(__mips__) && SANITIZER_LINUX
+# define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.2"
+# endif
namespace __asan {
diff --git a/libsanitizer/asan/asan_interceptors.h b/libsanitizer/asan/asan_interceptors.h
index 5bd5be6bac2..a9249dea45b 100644
--- a/libsanitizer/asan/asan_interceptors.h
+++ b/libsanitizer/asan/asan_interceptors.h
@@ -34,10 +34,10 @@ void InitializePlatformInterceptors();
} // namespace __asan
-// There is no general interception at all on Fuchsia and RTEMS.
+// There is no general interception at all on Fuchsia.
// Only the functions in asan_interceptors_memintrinsics.h are
// really defined to replace libc functions.
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#if !SANITIZER_FUCHSIA
// Use macro to describe if specific function should be
// intercepted on a given platform.
@@ -81,12 +81,7 @@ void InitializePlatformInterceptors();
#if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && !SANITIZER_SOLARIS && \
!SANITIZER_NETBSD
# define ASAN_INTERCEPT___CXA_THROW 1
-# if ! defined(ASAN_HAS_CXA_RETHROW_PRIMARY_EXCEPTION) \
- || ASAN_HAS_CXA_RETHROW_PRIMARY_EXCEPTION
-# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1
-# else
-# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 0
-# endif
+# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1
# if defined(_GLIBCXX_SJLJ_EXCEPTIONS) || (SANITIZER_IOS && defined(__arm__))
# define ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION 1
# else
diff --git a/libsanitizer/asan/asan_interceptors_memintrinsics.cpp b/libsanitizer/asan/asan_interceptors_memintrinsics.cpp
index ccdd5159042..9c316bb9574 100644
--- a/libsanitizer/asan/asan_interceptors_memintrinsics.cpp
+++ b/libsanitizer/asan/asan_interceptors_memintrinsics.cpp
@@ -30,9 +30,9 @@ void *__asan_memmove(void *to, const void *from, uptr size) {
ASAN_MEMMOVE_IMPL(nullptr, to, from, size);
}
-#if SANITIZER_FUCHSIA || SANITIZER_RTEMS
+#if SANITIZER_FUCHSIA
-// Fuchsia and RTEMS don't use sanitizer_common_interceptors.inc, but
+// Fuchsia doesn't use sanitizer_common_interceptors.inc, but
// the only things there it wants are these three. Just define them
// as aliases here rather than repeating the contents.
@@ -40,4 +40,4 @@ extern "C" decltype(__asan_memcpy) memcpy[[gnu::alias("__asan_memcpy")]];
extern "C" decltype(__asan_memmove) memmove[[gnu::alias("__asan_memmove")]];
extern "C" decltype(__asan_memset) memset[[gnu::alias("__asan_memset")]];
-#endif // SANITIZER_FUCHSIA || SANITIZER_RTEMS
+#endif // SANITIZER_FUCHSIA
diff --git a/libsanitizer/asan/asan_interface.inc b/libsanitizer/asan/asan_interface.inc
index 94801043982..ea28fc8ae87 100644
--- a/libsanitizer/asan/asan_interface.inc
+++ b/libsanitizer/asan/asan_interface.inc
@@ -134,6 +134,17 @@ INTERFACE_FUNCTION(__asan_stack_malloc_7)
INTERFACE_FUNCTION(__asan_stack_malloc_8)
INTERFACE_FUNCTION(__asan_stack_malloc_9)
INTERFACE_FUNCTION(__asan_stack_malloc_10)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_0)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_1)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_2)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_3)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_4)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_5)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_6)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_7)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_8)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_9)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_10)
INTERFACE_FUNCTION(__asan_store1)
INTERFACE_FUNCTION(__asan_store2)
INTERFACE_FUNCTION(__asan_store4)
diff --git a/libsanitizer/asan/asan_internal.h b/libsanitizer/asan/asan_internal.h
index cfb54927c6c..ad3320304d0 100644
--- a/libsanitizer/asan/asan_internal.h
+++ b/libsanitizer/asan/asan_internal.h
@@ -35,11 +35,11 @@
// If set, values like allocator chunk size, as well as defaults for some flags
// will be changed towards less memory overhead.
#ifndef ASAN_LOW_MEMORY
-# if SANITIZER_IOS || SANITIZER_ANDROID || SANITIZER_RTEMS
-# define ASAN_LOW_MEMORY 1
-# else
-# define ASAN_LOW_MEMORY 0
-# endif
+# if SANITIZER_IOS || SANITIZER_ANDROID
+# define ASAN_LOW_MEMORY 1
+# else
+# define ASAN_LOW_MEMORY 0
+# endif
#endif
#ifndef ASAN_DYNAMIC
@@ -77,7 +77,7 @@ void InitializeShadowMemory();
// asan_malloc_linux.cpp / asan_malloc_mac.cpp
void ReplaceSystemMalloc();
-// asan_linux.cpp / asan_mac.cpp / asan_rtems.cpp / asan_win.cpp
+// asan_linux.cpp / asan_mac.cpp / asan_win.cpp
uptr FindDynamicShadowStart();
void *AsanDoesNotSupportStaticLinkage();
void AsanCheckDynamicRTPrereqs();
@@ -159,9 +159,6 @@ const int kAsanArrayCookieMagic = 0xac;
const int kAsanIntraObjectRedzone = 0xbb;
const int kAsanAllocaLeftMagic = 0xca;
const int kAsanAllocaRightMagic = 0xcb;
-// Used to populate the shadow gap for systems without memory
-// protection there (i.e. Myriad).
-const int kAsanShadowGap = 0xcc;
static const uptr kCurrentStackFrameMagic = 0x41B58AB3;
static const uptr kRetiredStackFrameMagic = 0x45E0360E;
diff --git a/libsanitizer/asan/asan_malloc_linux.cpp b/libsanitizer/asan/asan_malloc_linux.cpp
index 9c3f0a5338e..c6bec8551bc 100644
--- a/libsanitizer/asan/asan_malloc_linux.cpp
+++ b/libsanitizer/asan/asan_malloc_linux.cpp
@@ -15,23 +15,22 @@
#include "sanitizer_common/sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \
- SANITIZER_NETBSD || SANITIZER_RTEMS || SANITIZER_SOLARIS
+ SANITIZER_NETBSD || SANITIZER_SOLARIS
-#include "sanitizer_common/sanitizer_allocator_checks.h"
-#include "sanitizer_common/sanitizer_errno.h"
-#include "sanitizer_common/sanitizer_tls_get_addr.h"
-#include "asan_allocator.h"
-#include "asan_interceptors.h"
-#include "asan_internal.h"
-#include "asan_malloc_local.h"
-#include "asan_stack.h"
+# include "asan_allocator.h"
+# include "asan_interceptors.h"
+# include "asan_internal.h"
+# include "asan_stack.h"
+# include "sanitizer_common/sanitizer_allocator_checks.h"
+# include "sanitizer_common/sanitizer_errno.h"
+# include "sanitizer_common/sanitizer_tls_get_addr.h"
// ---------------------- Replacement functions ---------------- {{{1
using namespace __asan;
static uptr allocated_for_dlsym;
static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = SANITIZER_RTEMS ? 4096 : 1024;
+static const uptr kDlsymAllocPoolSize = 1024;
static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
static inline bool IsInDlsymAllocPool(const void *ptr) {
@@ -82,27 +81,12 @@ static int PosixMemalignFromLocalPool(void **memptr, uptr alignment,
return 0;
}
-#if SANITIZER_RTEMS
-void* MemalignFromLocalPool(uptr alignment, uptr size) {
- void *ptr = nullptr;
- alignment = Max(alignment, kWordSize);
- PosixMemalignFromLocalPool(&ptr, alignment, size);
- return ptr;
-}
-
-bool IsFromLocalPool(const void *ptr) {
- return IsInDlsymAllocPool(ptr);
-}
-#endif
-
static inline bool MaybeInDlsym() {
// Fuchsia doesn't use dlsym-based interceptors.
return !SANITIZER_FUCHSIA && asan_init_is_running;
}
-static inline bool UseLocalPool() {
- return EarlyMalloc() || MaybeInDlsym();
-}
+static inline bool UseLocalPool() { return MaybeInDlsym(); }
static void *ReallocFromLocalPool(void *ptr, uptr size) {
const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
diff --git a/libsanitizer/asan/asan_malloc_local.h b/libsanitizer/asan/asan_malloc_local.h
deleted file mode 100644
index e2c9be0379f..00000000000
--- a/libsanitizer/asan/asan_malloc_local.h
+++ /dev/null
@@ -1,52 +0,0 @@
-//===-- asan_malloc_local.h -------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of AddressSanitizer, an address sanity checker.
-//
-// Provide interfaces to check for and handle local pool memory allocation.
-//===----------------------------------------------------------------------===//
-
-#ifndef ASAN_MALLOC_LOCAL_H
-#define ASAN_MALLOC_LOCAL_H
-
-#include "sanitizer_common/sanitizer_platform.h"
-#include "asan_internal.h"
-
-static inline bool EarlyMalloc() {
- return SANITIZER_RTEMS &&
- (!__asan::asan_inited || __asan::asan_init_is_running);
-}
-
-#if SANITIZER_RTEMS
-
-bool IsFromLocalPool(const void *ptr);
-void *MemalignFromLocalPool(uptr alignment, uptr size);
-
-// On RTEMS, we use the local pool to handle memory allocation when the ASan
-// run-time is not up. This macro is expanded in the context of the operator new
-// implementation.
-#define MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow) \
- do { \
- if (UNLIKELY(EarlyMalloc())) { \
- void *res = MemalignFromLocalPool(SHADOW_GRANULARITY, size); \
- if (!nothrow) \
- CHECK(res); \
- return res; \
- } \
- } while (0)
-
-#define IS_FROM_LOCAL_POOL(ptr) UNLIKELY(IsFromLocalPool(ptr))
-
-#else // SANITIZER_RTEMS
-
-#define MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow)
-#define IS_FROM_LOCAL_POOL(ptr) 0
-
-#endif // SANITIZER_RTEMS
-
-#endif // ASAN_MALLOC_LOCAL_H
diff --git a/libsanitizer/asan/asan_mapping.h b/libsanitizer/asan/asan_mapping.h
index 2511e7043df..e5a7f2007ae 100644
--- a/libsanitizer/asan/asan_mapping.h
+++ b/libsanitizer/asan/asan_mapping.h
@@ -150,17 +150,11 @@
// || `[0x36000000, 0x39ffffff]` || ShadowGap ||
// || `[0x30000000, 0x35ffffff]` || LowShadow ||
// || `[0x00000000, 0x2fffffff]` || LowMem ||
-//
-// Shadow mapping on Myriad2 (for shadow scale 5):
-// || `[0x9ff80000, 0x9fffffff]` || ShadowGap ||
-// || `[0x9f000000, 0x9ff7ffff]` || LowShadow ||
-// || `[0x80000000, 0x9effffff]` || LowMem ||
-// || `[0x00000000, 0x7fffffff]` || Ignored ||
#if defined(ASAN_SHADOW_SCALE)
static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE;
#else
-static const u64 kDefaultShadowScale = SANITIZER_MYRIAD2 ? 5 : 3;
+static const u64 kDefaultShadowScale = 3;
#endif
static const u64 kDefaultShadowSentinel = ~(uptr)0;
static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000
@@ -171,7 +165,7 @@ static const u64 kAArch64_ShadowOffset64 = 1ULL << 36;
static const u64 kRiscv64_ShadowOffset64 = 0xd55550000;
static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000;
static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37;
-static const u64 kPPC64_ShadowOffset64 = 1ULL << 41;
+static const u64 kPPC64_ShadowOffset64 = 1ULL << 44;
static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52;
static const u64 kSPARC64_ShadowOffset64 = 1ULL << 43; // 0x80000000000
static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000
@@ -180,15 +174,6 @@ static const u64 kNetBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000
static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000
static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
-static const u64 kMyriadMemoryOffset32 = 0x80000000ULL;
-static const u64 kMyriadMemorySize32 = 0x20000000ULL;
-static const u64 kMyriadMemoryEnd32 =
- kMyriadMemoryOffset32 + kMyriadMemorySize32 - 1;
-static const u64 kMyriadShadowOffset32 =
- (kMyriadMemoryOffset32 + kMyriadMemorySize32 -
- (kMyriadMemorySize32 >> kDefaultShadowScale));
-static const u64 kMyriadCacheBitMask32 = 0x40000000ULL;
-
#define SHADOW_SCALE kDefaultShadowScale
#if SANITIZER_FUCHSIA
@@ -206,8 +191,6 @@ static const u64 kMyriadCacheBitMask32 = 0x40000000ULL;
# define SHADOW_OFFSET kWindowsShadowOffset32
# elif SANITIZER_IOS
# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
-# elif SANITIZER_MYRIAD2
-# define SHADOW_OFFSET kMyriadShadowOffset32
# else
# define SHADOW_OFFSET kDefaultShadowOffset32
# endif
@@ -278,10 +261,8 @@ extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; // Initialized in __asan_init.
} // namespace __asan
-#if SANITIZER_MYRIAD2
-#include "asan_mapping_myriad.h"
-#elif defined(__sparc__) && SANITIZER_WORDSIZE == 64
-#include "asan_mapping_sparc64.h"
+#if defined(__sparc__) && SANITIZER_WORDSIZE == 64
+# include "asan_mapping_sparc64.h"
#else
#define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET))
@@ -363,7 +344,7 @@ static inline bool AddrIsInShadowGap(uptr a) {
} // namespace __asan
-#endif // SANITIZER_MYRIAD2
+#endif
namespace __asan {
@@ -393,8 +374,6 @@ static inline bool AddrIsAlignedByGranularity(uptr a) {
static inline bool AddressIsPoisoned(uptr a) {
PROFILE_ASAN_MAPPING();
- if (SANITIZER_MYRIAD2 && !AddrIsInMem(a) && !AddrIsInShadow(a))
- return false;
const uptr kAccessSize = 1;
u8 *shadow_address = (u8*)MEM_TO_SHADOW(a);
s8 shadow_value = *shadow_address;
diff --git a/libsanitizer/asan/asan_mapping_myriad.h b/libsanitizer/asan/asan_mapping_myriad.h
deleted file mode 100644
index 6969e3a4931..00000000000
--- a/libsanitizer/asan/asan_mapping_myriad.h
+++ /dev/null
@@ -1,85 +0,0 @@
-//===-- asan_mapping_myriad.h -----------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of AddressSanitizer, an address sanity checker.
-//
-// Myriad-specific definitions for ASan memory mapping.
-//===----------------------------------------------------------------------===//
-#ifndef ASAN_MAPPING_MYRIAD_H
-#define ASAN_MAPPING_MYRIAD_H
-
-#define RAW_ADDR(mem) ((mem) & ~kMyriadCacheBitMask32)
-#define MEM_TO_SHADOW(mem) \
- (((RAW_ADDR(mem) - kLowMemBeg) >> SHADOW_SCALE) + (SHADOW_OFFSET))
-
-#define kLowMemBeg kMyriadMemoryOffset32
-#define kLowMemEnd (SHADOW_OFFSET - 1)
-
-#define kLowShadowBeg SHADOW_OFFSET
-#define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd)
-
-#define kHighMemBeg 0
-
-#define kHighShadowBeg 0
-#define kHighShadowEnd 0
-
-#define kMidShadowBeg 0
-#define kMidShadowEnd 0
-
-#define kShadowGapBeg (kLowShadowEnd + 1)
-#define kShadowGapEnd kMyriadMemoryEnd32
-
-#define kShadowGap2Beg 0
-#define kShadowGap2End 0
-
-#define kShadowGap3Beg 0
-#define kShadowGap3End 0
-
-namespace __asan {
-
-static inline bool AddrIsInLowMem(uptr a) {
- PROFILE_ASAN_MAPPING();
- a = RAW_ADDR(a);
- return a >= kLowMemBeg && a <= kLowMemEnd;
-}
-
-static inline bool AddrIsInLowShadow(uptr a) {
- PROFILE_ASAN_MAPPING();
- a = RAW_ADDR(a);
- return a >= kLowShadowBeg && a <= kLowShadowEnd;
-}
-
-static inline bool AddrIsInMidMem(uptr a) {
- PROFILE_ASAN_MAPPING();
- return false;
-}
-
-static inline bool AddrIsInMidShadow(uptr a) {
- PROFILE_ASAN_MAPPING();
- return false;
-}
-
-static inline bool AddrIsInHighMem(uptr a) {
- PROFILE_ASAN_MAPPING();
- return false;
-}
-
-static inline bool AddrIsInHighShadow(uptr a) {
- PROFILE_ASAN_MAPPING();
- return false;
-}
-
-static inline bool AddrIsInShadowGap(uptr a) {
- PROFILE_ASAN_MAPPING();
- a = RAW_ADDR(a);
- return a >= kShadowGapBeg && a <= kShadowGapEnd;
-}
-
-} // namespace __asan
-
-#endif // ASAN_MAPPING_MYRIAD_H
diff --git a/libsanitizer/asan/asan_new_delete.cpp b/libsanitizer/asan/asan_new_delete.cpp
index 92a8648452b..da446072de1 100644
--- a/libsanitizer/asan/asan_new_delete.cpp
+++ b/libsanitizer/asan/asan_new_delete.cpp
@@ -11,16 +11,14 @@
// Interceptors for operators new and delete.
//===----------------------------------------------------------------------===//
+#include <stddef.h>
+
#include "asan_allocator.h"
#include "asan_internal.h"
-#include "asan_malloc_local.h"
#include "asan_report.h"
#include "asan_stack.h"
-
#include "interception/interception.h"
-#include <stddef.h>
-
// C++ operators can't have dllexport attributes on Windows. We export them
// anyway by passing extra -export flags to the linker, which is exactly that
// dllexport would normally do. We need to export them in order to make the
@@ -72,14 +70,12 @@ enum class align_val_t: size_t {};
// For local pool allocation, align to SHADOW_GRANULARITY to match asan
// allocator behavior.
#define OPERATOR_NEW_BODY(type, nothrow) \
- MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow); \
GET_STACK_TRACE_MALLOC; \
void *res = asan_memalign(0, size, &stack, type); \
if (!nothrow && UNLIKELY(!res)) \
ReportOutOfMemory(size, &stack); \
return res;
#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \
- MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow); \
GET_STACK_TRACE_MALLOC; \
void *res = asan_memalign((uptr)align, size, &stack, type); \
if (!nothrow && UNLIKELY(!res)) \
@@ -135,23 +131,19 @@ INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
#endif // !SANITIZER_MAC
#define OPERATOR_DELETE_BODY(type) \
- if (IS_FROM_LOCAL_POOL(ptr)) return;\
- GET_STACK_TRACE_FREE;\
+ GET_STACK_TRACE_FREE; \
asan_delete(ptr, 0, 0, &stack, type);
#define OPERATOR_DELETE_BODY_SIZE(type) \
- if (IS_FROM_LOCAL_POOL(ptr)) return;\
- GET_STACK_TRACE_FREE;\
+ GET_STACK_TRACE_FREE; \
asan_delete(ptr, size, 0, &stack, type);
#define OPERATOR_DELETE_BODY_ALIGN(type) \
- if (IS_FROM_LOCAL_POOL(ptr)) return;\
- GET_STACK_TRACE_FREE;\
+ GET_STACK_TRACE_FREE; \
asan_delete(ptr, 0, static_cast<uptr>(align), &stack, type);
#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \
- if (IS_FROM_LOCAL_POOL(ptr)) return;\
- GET_STACK_TRACE_FREE;\
+ GET_STACK_TRACE_FREE; \
asan_delete(ptr, size, static_cast<uptr>(align), &stack, type);
#if !SANITIZER_MAC
diff --git a/libsanitizer/asan/asan_poisoning.cpp b/libsanitizer/asan/asan_poisoning.cpp
index fa149ecfde6..5f215fe0f9b 100644
--- a/libsanitizer/asan/asan_poisoning.cpp
+++ b/libsanitizer/asan/asan_poisoning.cpp
@@ -173,17 +173,13 @@ int __asan_address_is_poisoned(void const volatile *addr) {
}
uptr __asan_region_is_poisoned(uptr beg, uptr size) {
- if (!size) return 0;
+ if (!size)
+ return 0;
uptr end = beg + size;
- if (SANITIZER_MYRIAD2) {
- // On Myriad, address not in DRAM range need to be treated as
- // unpoisoned.
- if (!AddrIsInMem(beg) && !AddrIsInShadow(beg)) return 0;
- if (!AddrIsInMem(end) && !AddrIsInShadow(end)) return 0;
- } else {
- if (!AddrIsInMem(beg)) return beg;
- if (!AddrIsInMem(end)) return end;
- }
+ if (!AddrIsInMem(beg))
+ return beg;
+ if (!AddrIsInMem(end))
+ return end;
CHECK_LT(beg, end);
uptr aligned_b = RoundUpTo(beg, SHADOW_GRANULARITY);
uptr aligned_e = RoundDownTo(end, SHADOW_GRANULARITY);
@@ -192,8 +188,7 @@ uptr __asan_region_is_poisoned(uptr beg, uptr size) {
// First check the first and the last application bytes,
// then check the SHADOW_GRANULARITY-aligned region by calling
// mem_is_zero on the corresponding shadow.
- if (!__asan::AddressIsPoisoned(beg) &&
- !__asan::AddressIsPoisoned(end - 1) &&
+ if (!__asan::AddressIsPoisoned(beg) && !__asan::AddressIsPoisoned(end - 1) &&
(shadow_end <= shadow_beg ||
__sanitizer::mem_is_zero((const char *)shadow_beg,
shadow_end - shadow_beg)))
diff --git a/libsanitizer/asan/asan_poisoning.h b/libsanitizer/asan/asan_poisoning.h
index 62dd9bd0edd..3d536f2d309 100644
--- a/libsanitizer/asan/asan_poisoning.h
+++ b/libsanitizer/asan/asan_poisoning.h
@@ -51,9 +51,6 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size,
// probably provide higher-level interface for these operations.
// For now, just memset on Windows.
if (value || SANITIZER_WINDOWS == 1 ||
- // RTEMS doesn't have have pages, let alone a fast way to zero
- // them, so default to memset.
- SANITIZER_RTEMS == 1 ||
shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg);
} else {
diff --git a/libsanitizer/asan/asan_rtems.cpp b/libsanitizer/asan/asan_rtems.cpp
deleted file mode 100644
index ea0b4ad9db6..00000000000
--- a/libsanitizer/asan/asan_rtems.cpp
+++ /dev/null
@@ -1,266 +0,0 @@
-//===-- asan_rtems.cpp ----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of AddressSanitizer, an address sanity checker.
-//
-// RTEMS-specific details.
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_common/sanitizer_rtems.h"
-#if SANITIZER_RTEMS
-
-#include "asan_internal.h"
-#include "asan_interceptors.h"
-#include "asan_mapping.h"
-#include "asan_poisoning.h"
-#include "asan_report.h"
-#include "asan_stack.h"
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_libc.h"
-
-#include <pthread.h>
-#include <stdlib.h>
-
-namespace __asan {
-
-static void ResetShadowMemory() {
- uptr shadow_start = SHADOW_OFFSET;
- uptr shadow_end = MEM_TO_SHADOW(kMyriadMemoryEnd32);
- uptr gap_start = MEM_TO_SHADOW(shadow_start);
- uptr gap_end = MEM_TO_SHADOW(shadow_end);
-
- REAL(memset)((void *)shadow_start, 0, shadow_end - shadow_start);
- REAL(memset)((void *)gap_start, kAsanShadowGap, gap_end - gap_start);
-}
-
-void InitializeShadowMemory() {
- kHighMemEnd = 0;
- kMidMemBeg = 0;
- kMidMemEnd = 0;
-
- ResetShadowMemory();
-}
-
-void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
- UNIMPLEMENTED();
-}
-
-void FlushUnneededASanShadowMemory(uptr p, uptr size) {
- // Since asan's mapping is compacting, the shadow chunk may be
- // not page-aligned, so we only flush the page-aligned portion.
- ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
-}
-
-void AsanCheckDynamicRTPrereqs() {}
-void AsanCheckIncompatibleRT() {}
-void InitializeAsanInterceptors() {}
-void InitializePlatformInterceptors() {}
-void InitializePlatformExceptionHandlers() {}
-
-// RTEMS only support static linking; it sufficies to return with no
-// error.
-void *AsanDoesNotSupportStaticLinkage() { return nullptr; }
-
-void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
- UNIMPLEMENTED();
-}
-
-bool PlatformUnpoisonStacks() { return false; }
-
-void EarlyInit() {
- // Provide early initialization of shadow memory so that
- // instrumented code running before full initialzation will not
- // report spurious errors.
- ResetShadowMemory();
-}
-
-// We can use a plain thread_local variable for TSD.
-static thread_local void *per_thread;
-
-void *AsanTSDGet() { return per_thread; }
-
-void AsanTSDSet(void *tsd) { per_thread = tsd; }
-
-// There's no initialization needed, and the passed-in destructor
-// will never be called. Instead, our own thread destruction hook
-// (below) will call AsanThread::TSDDtor directly.
-void AsanTSDInit(void (*destructor)(void *tsd)) {
- DCHECK(destructor == &PlatformTSDDtor);
-}
-
-void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); }
-
-//
-// Thread registration. We provide an API similar to the Fushia port.
-//
-
-struct AsanThread::InitOptions {
- uptr stack_bottom, stack_size, tls_bottom, tls_size;
-};
-
-// Shared setup between thread creation and startup for the initial thread.
-static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
- uptr user_id, bool detached,
- uptr stack_bottom, uptr stack_size,
- uptr tls_bottom, uptr tls_size) {
- // In lieu of AsanThread::Create.
- AsanThread *thread = (AsanThread *)MmapOrDie(sizeof(AsanThread), __func__);
- AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
- asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args);
-
- // On other systems, AsanThread::Init() is called from the new
- // thread itself. But on RTEMS we already know the stack address
- // range beforehand, so we can do most of the setup right now.
- const AsanThread::InitOptions options = {stack_bottom, stack_size,
- tls_bottom, tls_size};
- thread->Init(&options);
- return thread;
-}
-
-// This gets the same arguments passed to Init by CreateAsanThread, above.
-// We're in the creator thread before the new thread is actually started, but
-// its stack and tls address range are already known.
-void AsanThread::SetThreadStackAndTls(const AsanThread::InitOptions *options) {
- DCHECK_NE(GetCurrentThread(), this);
- DCHECK_NE(GetCurrentThread(), nullptr);
- CHECK_NE(options->stack_bottom, 0);
- CHECK_NE(options->stack_size, 0);
- stack_bottom_ = options->stack_bottom;
- stack_top_ = options->stack_bottom + options->stack_size;
- tls_begin_ = options->tls_bottom;
- tls_end_ = options->tls_bottom + options->tls_size;
-}
-
-// Called by __asan::AsanInitInternal (asan_rtl.c). Unlike other ports, the
-// main thread on RTEMS does not require special treatment; its AsanThread is
-// already created by the provided hooks. This function simply looks up and
-// returns the created thread.
-AsanThread *CreateMainThread() {
- return GetThreadContextByTidLocked(0)->thread;
-}
-
-// This is called before each thread creation is attempted. So, in
-// its first call, the calling thread is the initial and sole thread.
-static void *BeforeThreadCreateHook(uptr user_id, bool detached,
- uptr stack_bottom, uptr stack_size,
- uptr tls_bottom, uptr tls_size) {
- EnsureMainThreadIDIsCorrect();
- // Strict init-order checking is thread-hostile.
- if (flags()->strict_init_order) StopInitOrderChecking();
-
- GET_STACK_TRACE_THREAD;
- u32 parent_tid = GetCurrentTidOrInvalid();
-
- return CreateAsanThread(&stack, parent_tid, user_id, detached,
- stack_bottom, stack_size, tls_bottom, tls_size);
-}
-
-// This is called after creating a new thread (in the creating thread),
-// with the pointer returned by BeforeThreadCreateHook (above).
-static void ThreadCreateHook(void *hook, bool aborted) {
- AsanThread *thread = static_cast<AsanThread *>(hook);
- if (!aborted) {
- // The thread was created successfully.
- // ThreadStartHook is already running in the new thread.
- } else {
- // The thread wasn't created after all.
- // Clean up everything we set up in BeforeThreadCreateHook.
- asanThreadRegistry().FinishThread(thread->tid());
- UnmapOrDie(thread, sizeof(AsanThread));
- }
-}
-
-// This is called (1) in the newly-created thread before it runs anything else,
-// with the pointer returned by BeforeThreadCreateHook (above). (2) before a
-// thread restart.
-static void ThreadStartHook(void *hook, uptr os_id) {
- if (!hook)
- return;
-
- AsanThread *thread = static_cast<AsanThread *>(hook);
- SetCurrentThread(thread);
-
- ThreadStatus status =
- asanThreadRegistry().GetThreadLocked(thread->tid())->status;
- DCHECK(status == ThreadStatusCreated || status == ThreadStatusRunning);
- // Determine whether we are starting or restarting the thread.
- if (status == ThreadStatusCreated) {
- // In lieu of AsanThread::ThreadStart.
- asanThreadRegistry().StartThread(thread->tid(), os_id, ThreadType::Regular,
- nullptr);
- } else {
- // In a thread restart, a thread may resume execution at an
- // arbitrary function entry point, with its stack and TLS state
- // reset. We unpoison the stack in that case.
- PoisonShadow(thread->stack_bottom(), thread->stack_size(), 0);
- }
-}
-
-// Each thread runs this just before it exits,
-// with the pointer returned by BeforeThreadCreateHook (above).
-// All per-thread destructors have already been called.
-static void ThreadExitHook(void *hook, uptr os_id) {
- AsanThread *thread = static_cast<AsanThread *>(hook);
- if (thread)
- AsanThread::TSDDtor(thread->context());
-}
-
-static void HandleExit() {
- // Disable ASan by setting it to uninitialized. Also reset the
- // shadow memory to avoid reporting errors after the run-time has
- // been desroyed.
- if (asan_inited) {
- asan_inited = false;
- ResetShadowMemory();
- }
-}
-
-bool HandleDlopenInit() {
- // Not supported on this platform.
- static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN,
- "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false");
- return false;
-}
-} // namespace __asan
-
-// These are declared (in extern "C") by <some_path/sanitizer.h>.
-// The system runtime will call our definitions directly.
-
-extern "C" {
-void __sanitizer_early_init() {
- __asan::EarlyInit();
-}
-
-void *__sanitizer_before_thread_create_hook(uptr thread, bool detached,
- const char *name,
- void *stack_base, size_t stack_size,
- void *tls_base, size_t tls_size) {
- return __asan::BeforeThreadCreateHook(
- thread, detached,
- reinterpret_cast<uptr>(stack_base), stack_size,
- reinterpret_cast<uptr>(tls_base), tls_size);
-}
-
-void __sanitizer_thread_create_hook(void *handle, uptr thread, int status) {
- __asan::ThreadCreateHook(handle, status != 0);
-}
-
-void __sanitizer_thread_start_hook(void *handle, uptr self) {
- __asan::ThreadStartHook(handle, self);
-}
-
-void __sanitizer_thread_exit_hook(void *handle, uptr self) {
- __asan::ThreadExitHook(handle, self);
-}
-
-void __sanitizer_exit() {
- __asan::HandleExit();
-}
-} // "C"
-
-#endif // SANITIZER_RTEMS
diff --git a/libsanitizer/asan/asan_rtl.cpp b/libsanitizer/asan/asan_rtl.cpp
index e715d774228..e06a1113f4e 100644
--- a/libsanitizer/asan/asan_rtl.cpp
+++ b/libsanitizer/asan/asan_rtl.cpp
@@ -13,6 +13,7 @@
#include "asan_activation.h"
#include "asan_allocator.h"
+#include "asan_fake_stack.h"
#include "asan_interceptors.h"
#include "asan_interface_internal.h"
#include "asan_internal.h"
@@ -23,11 +24,11 @@
#include "asan_stats.h"
#include "asan_suppressions.h"
#include "asan_thread.h"
+#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
-#include "lsan/lsan_common.h"
#include "ubsan/ubsan_init.h"
#include "ubsan/ubsan_platform.h"
@@ -137,24 +138,21 @@ ASAN_REPORT_ERROR_N(load, false)
ASAN_REPORT_ERROR_N(store, true)
#define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg, fatal) \
- if (SANITIZER_MYRIAD2 && !AddrIsInMem(addr) && !AddrIsInShadow(addr)) \
- return; \
- uptr sp = MEM_TO_SHADOW(addr); \
- uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \
- : *reinterpret_cast<u16 *>(sp); \
- if (UNLIKELY(s)) { \
- if (UNLIKELY(size >= SHADOW_GRANULARITY || \
- ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >= \
- (s8)s)) { \
- if (__asan_test_only_reported_buggy_pointer) { \
- *__asan_test_only_reported_buggy_pointer = addr; \
- } else { \
- GET_CALLER_PC_BP_SP; \
- ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, \
- fatal); \
- } \
+ uptr sp = MEM_TO_SHADOW(addr); \
+ uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \
+ : *reinterpret_cast<u16 *>(sp); \
+ if (UNLIKELY(s)) { \
+ if (UNLIKELY(size >= SHADOW_GRANULARITY || \
+ ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >= \
+ (s8)s)) { \
+ if (__asan_test_only_reported_buggy_pointer) { \
+ *__asan_test_only_reported_buggy_pointer = addr; \
+ } else { \
+ GET_CALLER_PC_BP_SP; \
+ ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, fatal); \
} \
- }
+ } \
+ }
#define ASAN_MEMORY_ACCESS_CALLBACK(type, is_write, size) \
extern "C" NOINLINE INTERFACE_ATTRIBUTE \
@@ -305,7 +303,6 @@ static void asan_atexit() {
}
static void InitializeHighMemEnd() {
-#if !SANITIZER_MYRIAD2
#if !ASAN_FIXED_MAPPING
kHighMemEnd = GetMaxUserVirtualAddress();
// Increase kHighMemEnd to make sure it's properly
@@ -313,7 +310,6 @@ static void InitializeHighMemEnd() {
kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
#endif // !ASAN_FIXED_MAPPING
CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0);
-#endif // !SANITIZER_MYRIAD2
}
void PrintAddressSpaceLayout() {
@@ -569,9 +565,6 @@ static void UnpoisonDefaultStack() {
const uptr page_size = GetPageSizeCached();
top = curr_thread->stack_top();
bottom = ((uptr)&local_stack - page_size) & ~(page_size - 1);
- } else if (SANITIZER_RTEMS) {
- // Give up On RTEMS.
- return;
} else {
CHECK(!SANITIZER_FUCHSIA);
// If we haven't seen this thread, try asking the OS for stack bounds.
@@ -586,8 +579,12 @@ static void UnpoisonDefaultStack() {
static void UnpoisonFakeStack() {
AsanThread *curr_thread = GetCurrentThread();
- if (curr_thread && curr_thread->has_fake_stack())
- curr_thread->fake_stack()->HandleNoReturn();
+ if (!curr_thread)
+ return;
+ FakeStack *stack = curr_thread->get_fake_stack();
+ if (!stack)
+ return;
+ stack->HandleNoReturn();
}
} // namespace __asan
diff --git a/libsanitizer/asan/asan_shadow_setup.cpp b/libsanitizer/asan/asan_shadow_setup.cpp
index 2ead4425add..6e6260d3413 100644
--- a/libsanitizer/asan/asan_shadow_setup.cpp
+++ b/libsanitizer/asan/asan_shadow_setup.cpp
@@ -13,12 +13,11 @@
#include "sanitizer_common/sanitizer_platform.h"
-// asan_fuchsia.cpp and asan_rtems.cpp have their own
-// InitializeShadowMemory implementation.
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+// asan_fuchsia.cpp has their own InitializeShadowMemory implementation.
+#if !SANITIZER_FUCHSIA
-#include "asan_internal.h"
-#include "asan_mapping.h"
+# include "asan_internal.h"
+# include "asan_mapping.h"
namespace __asan {
@@ -123,4 +122,4 @@ void InitializeShadowMemory() {
} // namespace __asan
-#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#endif // !SANITIZER_FUCHSIA
diff --git a/libsanitizer/asan/asan_stack.cpp b/libsanitizer/asan/asan_stack.cpp
index b7f4e6aeeab..048295d6928 100644
--- a/libsanitizer/asan/asan_stack.cpp
+++ b/libsanitizer/asan/asan_stack.cpp
@@ -74,7 +74,8 @@ void __sanitizer::BufferedStackTrace::UnwindImpl(
if (SANITIZER_MIPS && t &&
!IsValidFrame(bp, t->stack_top(), t->stack_bottom()))
return;
- Unwind(max_depth, pc, bp, context, 0, 0, false);
+ Unwind(max_depth, pc, bp, context, t ? t->stack_top() : 0,
+ t ? t->stack_bottom() : 0, false);
}
// ------------------ Interface -------------- {{{1
diff --git a/libsanitizer/asan/asan_thread.cpp b/libsanitizer/asan/asan_thread.cpp
index 9c3c86f5735..35d4467e7b5 100644
--- a/libsanitizer/asan/asan_thread.cpp
+++ b/libsanitizer/asan/asan_thread.cpp
@@ -60,8 +60,8 @@ ThreadRegistry &asanThreadRegistry() {
// in TSD and can't reliably tell when no more TSD destructors will
// be called. It would be wrong to reuse AsanThreadContext for another
// thread before all TSD destructors will be called for it.
- asan_thread_registry = new(thread_registry_placeholder) ThreadRegistry(
- GetAsanThreadContext, kMaxNumberOfThreads, kMaxNumberOfThreads);
+ asan_thread_registry =
+ new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext);
initialized = true;
}
return *asan_thread_registry;
@@ -257,10 +257,9 @@ void AsanThread::Init(const InitOptions *options) {
&local);
}
-// Fuchsia and RTEMS don't use ThreadStart.
-// asan_fuchsia.c/asan_rtems.c define CreateMainThread and
-// SetThreadStackAndTls.
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+// Fuchsia doesn't use ThreadStart.
+// asan_fuchsia.c definies CreateMainThread and SetThreadStackAndTls.
+#if !SANITIZER_FUCHSIA
thread_return_t AsanThread::ThreadStart(tid_t os_id) {
Init();
@@ -317,7 +316,7 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) {
}
}
-#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#endif // !SANITIZER_FUCHSIA
void AsanThread::ClearShadowForThreadStackAndTLS() {
if (stack_top_ != stack_bottom_)
@@ -339,8 +338,8 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr,
uptr bottom = 0;
if (AddrIsInStack(addr)) {
bottom = stack_bottom();
- } else if (has_fake_stack()) {
- bottom = fake_stack()->AddrIsInFakeStack(addr);
+ } else if (FakeStack *fake_stack = get_fake_stack()) {
+ bottom = fake_stack->AddrIsInFakeStack(addr);
CHECK(bottom);
access->offset = addr - bottom;
access->frame_pc = ((uptr*)bottom)[2];
@@ -380,8 +379,8 @@ uptr AsanThread::GetStackVariableShadowStart(uptr addr) {
uptr bottom = 0;
if (AddrIsInStack(addr)) {
bottom = stack_bottom();
- } else if (has_fake_stack()) {
- bottom = fake_stack()->AddrIsInFakeStack(addr);
+ } else if (FakeStack *fake_stack = get_fake_stack()) {
+ bottom = fake_stack->AddrIsInFakeStack(addr);
if (bottom == 0) {
return 0;
}
@@ -409,19 +408,19 @@ bool AsanThread::AddrIsInStack(uptr addr) {
static bool ThreadStackContainsAddress(ThreadContextBase *tctx_base,
void *addr) {
- AsanThreadContext *tctx = static_cast<AsanThreadContext*>(tctx_base);
+ AsanThreadContext *tctx = static_cast<AsanThreadContext *>(tctx_base);
AsanThread *t = tctx->thread;
- if (!t) return false;
- if (t->AddrIsInStack((uptr)addr)) return true;
- if (t->has_fake_stack() && t->fake_stack()->AddrIsInFakeStack((uptr)addr))
+ if (!t)
+ return false;
+ if (t->AddrIsInStack((uptr)addr))
return true;
- return false;
+ FakeStack *fake_stack = t->get_fake_stack();
+ if (!fake_stack)
+ return false;
+ return fake_stack->AddrIsInFakeStack((uptr)addr);
}
AsanThread *GetCurrentThread() {
- if (SANITIZER_RTEMS && !asan_inited)
- return nullptr;
-
AsanThreadContext *context =
reinterpret_cast<AsanThreadContext *>(AsanTSDGet());
if (!context) {
@@ -503,8 +502,12 @@ void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches) {}
void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
void *arg) {
__asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id);
- if (t && t->has_fake_stack())
- t->fake_stack()->ForEachFakeFrame(callback, arg);
+ if (!t)
+ return;
+ __asan::FakeStack *fake_stack = t->get_fake_stack();
+ if (!fake_stack)
+ return;
+ fake_stack->ForEachFakeFrame(callback, arg);
}
void LockThreadRegistry() {
diff --git a/libsanitizer/asan/asan_thread.h b/libsanitizer/asan/asan_thread.h
index 200069ce0dd..801a3960ec6 100644
--- a/libsanitizer/asan/asan_thread.h
+++ b/libsanitizer/asan/asan_thread.h
@@ -28,8 +28,6 @@ struct DTLS;
namespace __asan {
-const u32 kMaxNumberOfThreads = (1 << 22); // 4M
-
class AsanThread;
// These objects are created for every thread and are never deleted,
@@ -104,17 +102,18 @@ class AsanThread {
void FinishSwitchFiber(FakeStack *fake_stack_save, uptr *bottom_old,
uptr *size_old);
- bool has_fake_stack() {
- return !atomic_load(&stack_switching_, memory_order_relaxed) &&
- (reinterpret_cast<uptr>(fake_stack_) > 1);
+ FakeStack *get_fake_stack() {
+ if (atomic_load(&stack_switching_, memory_order_relaxed))
+ return nullptr;
+ if (reinterpret_cast<uptr>(fake_stack_) <= 1)
+ return nullptr;
+ return fake_stack_;
}
- FakeStack *fake_stack() {
- if (!__asan_option_detect_stack_use_after_return)
- return nullptr;
+ FakeStack *get_or_create_fake_stack() {
if (atomic_load(&stack_switching_, memory_order_relaxed))
return nullptr;
- if (!has_fake_stack())
+ if (reinterpret_cast<uptr>(fake_stack_) <= 1)
return AsyncSignalSafeLazyInitFakeStack();
return fake_stack_;
}
diff --git a/libsanitizer/hwasan/Makefile.am b/libsanitizer/hwasan/Makefile.am
index 2226f68ab29..5e3a0f1b0a1 100644
--- a/libsanitizer/hwasan/Makefile.am
+++ b/libsanitizer/hwasan/Makefile.am
@@ -13,11 +13,13 @@ ACLOCAL_AMFLAGS = -I $(top_srcdir) -I $(top_srcdir)/config
toolexeclib_LTLIBRARIES = libhwasan.la
hwasan_files = \
+ hwasan_allocation_functions.cpp \
hwasan_allocator.cpp \
hwasan.cpp \
hwasan_dynamic_shadow.cpp \
hwasan_exceptions.cpp \
hwasan_flags.inc \
+ hwasan_fuchsia.cpp \
hwasan_globals.cpp \
hwasan_interceptors.cpp \
hwasan_interceptors_vfork.S \
diff --git a/libsanitizer/hwasan/Makefile.in b/libsanitizer/hwasan/Makefile.in
index 542af8f193e..22c5266a120 100644
--- a/libsanitizer/hwasan/Makefile.in
+++ b/libsanitizer/hwasan/Makefile.in
@@ -146,8 +146,9 @@ am__DEPENDENCIES_1 =
libhwasan_la_DEPENDENCIES = \
$(top_builddir)/sanitizer_common/libsanitizer_common.la \
$(am__append_1) $(am__append_2) $(am__DEPENDENCIES_1)
-am__objects_1 = hwasan_allocator.lo hwasan.lo hwasan_dynamic_shadow.lo \
- hwasan_exceptions.lo hwasan_globals.lo hwasan_interceptors.lo \
+am__objects_1 = hwasan_allocation_functions.lo hwasan_allocator.lo \
+ hwasan.lo hwasan_dynamic_shadow.lo hwasan_exceptions.lo \
+ hwasan_fuchsia.lo hwasan_globals.lo hwasan_interceptors.lo \
hwasan_interceptors_vfork.lo hwasan_linux.lo \
hwasan_memintrinsics.lo hwasan_new_delete.lo \
hwasan_poisoning.lo hwasan_report.lo hwasan_setjmp.lo \
@@ -411,11 +412,13 @@ AM_CXXFLAGS = -Wall -W -Wno-unused-parameter -Wwrite-strings -pedantic \
ACLOCAL_AMFLAGS = -I $(top_srcdir) -I $(top_srcdir)/config
toolexeclib_LTLIBRARIES = libhwasan.la
hwasan_files = \
+ hwasan_allocation_functions.cpp \
hwasan_allocator.cpp \
hwasan.cpp \
hwasan_dynamic_shadow.cpp \
hwasan_exceptions.cpp \
hwasan_flags.inc \
+ hwasan_fuchsia.cpp \
hwasan_globals.cpp \
hwasan_interceptors.cpp \
hwasan_interceptors_vfork.S \
@@ -554,9 +557,11 @@ distclean-compile:
-rm -f *.tab.c
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_allocation_functions.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_allocator.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_dynamic_shadow.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_exceptions.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_fuchsia.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_globals.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_interceptors.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_interceptors_vfork.Plo@am__quote@
diff --git a/libsanitizer/hwasan/hwasan.cpp b/libsanitizer/hwasan/hwasan.cpp
index 8d6c25261b8..cbe0dee66dc 100644
--- a/libsanitizer/hwasan/hwasan.cpp
+++ b/libsanitizer/hwasan/hwasan.cpp
@@ -50,6 +50,11 @@ bool hwasan_init_is_running;
int hwasan_report_count = 0;
+uptr kLowShadowStart;
+uptr kLowShadowEnd;
+uptr kHighShadowStart;
+uptr kHighShadowEnd;
+
void Flags::SetDefaults() {
#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
#include "hwasan_flags.inc"
@@ -177,6 +182,65 @@ void UpdateMemoryUsage() {
void UpdateMemoryUsage() {}
#endif
+void HwasanAtExit() {
+ if (common_flags()->print_module_map)
+ DumpProcessMap();
+ if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
+ ReportStats();
+ if (hwasan_report_count > 0) {
+ // ReportAtExitStatistics();
+ if (common_flags()->exitcode)
+ internal__exit(common_flags()->exitcode);
+ }
+}
+
+void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc,
+ uptr *registers_frame) {
+ InternalMmapVector<BufferedStackTrace> stack_buffer(1);
+ BufferedStackTrace *stack = stack_buffer.data();
+ stack->Reset();
+ stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
+
+ // The second stack frame contains the failure __hwasan_check function, as
+ // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
+ // we wish to ignore. This (currently) only occurs on AArch64, as x64
+ // implementations use SIGTRAP to implement the failure, and thus do not go
+ // through the stack saver.
+ if (registers_frame && stack->trace && stack->size > 0) {
+ stack->trace++;
+ stack->size--;
+ }
+
+ bool fatal = flags()->halt_on_error || !ai.recover;
+ ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
+ registers_frame);
+}
+
+void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
+ size_t outsize) {
+ __hwasan::AccessInfo ai;
+ ai.is_store = access_info & 0x10;
+ ai.is_load = !ai.is_store;
+ ai.recover = access_info & 0x20;
+ ai.addr = addr;
+ if ((access_info & 0xf) == 0xf)
+ ai.size = outsize;
+ else
+ ai.size = 1 << (access_info & 0xf);
+
+ HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
+ (uptr)__builtin_frame_address(0), nullptr, registers_frame);
+ __builtin_unreachable();
+}
+
+Thread *GetCurrentThread() {
+ uptr *ThreadLongPtr = GetCurrentThreadLongPtr();
+ if (UNLIKELY(*ThreadLongPtr == 0))
+ return nullptr;
+ auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr;
+ return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next());
+}
+
} // namespace __hwasan
using namespace __hwasan;
@@ -216,7 +280,7 @@ static void InitLoadedGlobals() {
static void InitInstrumentation() {
if (hwasan_instrumentation_inited) return;
- InitPrctl();
+ InitializeOsSupport();
if (!InitShadow()) {
Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
@@ -225,7 +289,6 @@ static void InitInstrumentation() {
}
InitThreads();
- hwasanThreadList().CreateCurrentThread();
hwasan_instrumentation_inited = 1;
}
@@ -495,7 +558,7 @@ void __hwasan_print_memory_usage() {
Printf("%s\n", s.data());
}
-static const u8 kFallbackTag = 0xBB;
+static const u8 kFallbackTag = 0xBB & kTagMask;
u8 __hwasan_generate_tag() {
Thread *t = GetCurrentThread();
@@ -516,4 +579,12 @@ void __sanitizer_print_stack_trace() {
GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
stack.Print();
}
+
+// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
+// rest of the mismatch handling code (C++).
+void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
+ size_t outsize) {
+ __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize);
+}
+
} // extern "C"
diff --git a/libsanitizer/hwasan/hwasan.h b/libsanitizer/hwasan/hwasan.h
index 8515df559f3..7338b696ad3 100644
--- a/libsanitizer/hwasan/hwasan.h
+++ b/libsanitizer/hwasan/hwasan.h
@@ -36,7 +36,10 @@
typedef u8 tag_t;
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
+# if !defined(__x86_64__)
+# error Aliasing mode is only supported on x86_64
+# endif
// Tags are done in middle bits using userspace aliasing.
constexpr unsigned kAddressTagShift = 39;
constexpr unsigned kTagBits = 3;
@@ -49,12 +52,16 @@ constexpr unsigned kTagBits = 3;
// simpler/faster shadow calculation.
constexpr unsigned kTaggableRegionCheckShift =
__sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U);
+#elif defined(__x86_64__)
+// Tags are done in upper bits using Intel LAM.
+constexpr unsigned kAddressTagShift = 57;
+constexpr unsigned kTagBits = 6;
#else
// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
// translation and can be used to store a tag.
constexpr unsigned kAddressTagShift = 56;
constexpr unsigned kTagBits = 8;
-#endif // defined(__x86_64__)
+#endif // defined(HWASAN_ALIASING_MODE)
// Mask for extracting tag bits from the lower 8 bits.
constexpr uptr kTagMask = (1UL << kTagBits) - 1;
@@ -95,7 +102,7 @@ extern bool hwasan_init_is_running;
extern int hwasan_report_count;
bool InitShadow();
-void InitPrctl();
+void InitializeOsSupport();
void InitThreads();
void InitializeInterceptors();
@@ -129,6 +136,7 @@ void InstallAtExitHandler();
void HwasanTSDInit();
void HwasanTSDThreadInit();
+void HwasanAtExit();
void HwasanOnDeadlySignal(int signo, void *info, void *context);
@@ -138,6 +146,26 @@ void AppendToErrorMessageBuffer(const char *buffer);
void AndroidTestTlsSlot();
+// This is a compiler-generated struct that can be shared between hwasan
+// implementations.
+struct AccessInfo {
+ uptr addr;
+ uptr size;
+ bool is_store;
+ bool is_load;
+ bool recover;
+};
+
+// Given access info and frame information, unwind the stack and report the tag
+// mismatch.
+void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc,
+ uptr *registers_frame = nullptr);
+
+// This dispatches to HandleTagMismatch but sets up the AccessInfo, program
+// counter, and frame pointer.
+void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
+ size_t outsize);
+
} // namespace __hwasan
#define HWASAN_MALLOC_HOOK(ptr, size) \
@@ -175,4 +203,12 @@ typedef struct __hw_jmp_buf_struct __hw_jmp_buf[1];
typedef struct __hw_jmp_buf_struct __hw_sigjmp_buf[1];
#endif // HWASAN_WITH_INTERCEPTORS && __aarch64__
+#define ENSURE_HWASAN_INITED() \
+ do { \
+ CHECK(!hwasan_init_is_running); \
+ if (!hwasan_inited) { \
+ __hwasan_init(); \
+ } \
+ } while (0)
+
#endif // HWASAN_H
diff --git a/libsanitizer/hwasan/hwasan_allocation_functions.cpp b/libsanitizer/hwasan/hwasan_allocation_functions.cpp
new file mode 100644
index 00000000000..6c2a6077866
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_allocation_functions.cpp
@@ -0,0 +1,172 @@
+//===-- hwasan_allocation_functions.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Definitions for __sanitizer allocation functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+using namespace __hwasan;
+
+static uptr allocated_for_dlsym;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static bool IsInDlsymAllocPool(const void *ptr) {
+ uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ return off < sizeof(alloc_memory_for_dlsym);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+ uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+ void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ allocated_for_dlsym += size_in_words;
+ CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+ return mem;
+}
+
+int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ CHECK_NE(memptr, 0);
+ int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
+ return res;
+}
+
+void *__sanitizer_memalign(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_memalign(alignment, size, &stack);
+}
+
+void *__sanitizer_aligned_alloc(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_aligned_alloc(alignment, size, &stack);
+}
+
+void *__sanitizer___libc_memalign(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ void *ptr = hwasan_memalign(alignment, size, &stack);
+ if (ptr)
+ DTLS_on_libc_memalign(ptr, size);
+ return ptr;
+}
+
+void *__sanitizer_valloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_valloc(size, &stack);
+}
+
+void *__sanitizer_pvalloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_pvalloc(size, &stack);
+}
+
+void __sanitizer_free(void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return;
+ hwasan_free(ptr, &stack);
+}
+
+void __sanitizer_cfree(void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return;
+ hwasan_free(ptr, &stack);
+}
+
+uptr __sanitizer_malloc_usable_size(const void *ptr) {
+ return __sanitizer_get_allocated_size(ptr);
+}
+
+struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
+ __sanitizer_struct_mallinfo sret;
+ internal_memset(&sret, 0, sizeof(sret));
+ return sret;
+}
+
+int __sanitizer_mallopt(int cmd, int value) { return 0; }
+
+void __sanitizer_malloc_stats(void) {
+ // FIXME: implement, but don't call REAL(malloc_stats)!
+}
+
+void *__sanitizer_calloc(uptr nmemb, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(nmemb * size);
+ return hwasan_calloc(nmemb, size, &stack);
+}
+
+void *__sanitizer_realloc(void *ptr, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+ uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+ void *new_ptr;
+ if (UNLIKELY(!hwasan_inited)) {
+ new_ptr = AllocateFromLocalPool(copy_size);
+ } else {
+ copy_size = size;
+ new_ptr = hwasan_malloc(copy_size, &stack);
+ }
+ internal_memcpy(new_ptr, ptr, copy_size);
+ return new_ptr;
+ }
+ return hwasan_realloc(ptr, size, &stack);
+}
+
+void *__sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_reallocarray(ptr, nmemb, size, &stack);
+}
+
+void *__sanitizer_malloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_init_is_running))
+ ENSURE_HWASAN_INITED();
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(size);
+ return hwasan_malloc(size, &stack);
+}
+
+#if HWASAN_WITH_INTERCEPTORS
+# define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \
+ ALIAS("__sanitizer_" #FN); \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \
+ ARGS) ALIAS("__sanitizer_" #FN)
+
+INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment,
+ SIZE_T size);
+INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size);
+INTERCEPTOR_ALIAS(void, free, void *ptr);
+INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr);
+INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size);
+
+# if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size);
+INTERCEPTOR_ALIAS(void, cfree, void *ptr);
+INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo);
+INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value);
+INTERCEPTOR_ALIAS(void, malloc_stats, void);
+# endif
+#endif // #if HWASAN_WITH_INTERCEPTORS
diff --git a/libsanitizer/hwasan/hwasan_allocator.cpp b/libsanitizer/hwasan/hwasan_allocator.cpp
index a6fc794082a..ef6d4d6c767 100644
--- a/libsanitizer/hwasan/hwasan_allocator.cpp
+++ b/libsanitizer/hwasan/hwasan_allocator.cpp
@@ -80,12 +80,29 @@ void GetAllocatorStats(AllocatorStatCounters s) {
allocator.GetStats(s);
}
+uptr GetAliasRegionStart() {
+#if defined(HWASAN_ALIASING_MODE)
+ constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
+ uptr AliasRegionStart =
+ __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
+
+ CHECK_EQ(AliasRegionStart >> kTaggableRegionCheckShift,
+ __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+ CHECK_EQ(
+ (AliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
+ __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+ return AliasRegionStart;
+#else
+ return 0;
+#endif
+}
+
void HwasanAllocatorInit() {
atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
!flags()->disable_allocator_tagging);
SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
allocator.Init(common_flags()->allocator_release_to_os_interval_ms,
- kAliasRegionStart);
+ GetAliasRegionStart());
for (uptr i = 0; i < sizeof(tail_magic); i++)
tail_magic[i] = GetCurrentThread()->GenerateRandomTag();
}
@@ -196,6 +213,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
: tagged_ptr;
void *aligned_ptr = reinterpret_cast<void *>(
RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
+ tag_t pointer_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr));
Metadata *meta =
reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr));
uptr orig_size = meta->get_requested_size();
@@ -229,7 +247,20 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
flags()->tag_in_free && malloc_bisect(stack, 0) &&
atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
// Always store full 8-bit tags on free to maximize UAF detection.
- tag_t tag = t ? t->GenerateRandomTag(/*num_bits=*/8) : kFallbackFreeTag;
+ tag_t tag;
+ if (t) {
+ // Make sure we are not using a short granule tag as a poison tag. This
+ // would make us attempt to read the memory on a UaF.
+ // The tag can be zero if tagging is disabled on this thread.
+ do {
+ tag = t->GenerateRandomTag(/*num_bits=*/8);
+ } while (
+ UNLIKELY((tag < kShadowAlignment || tag == pointer_tag) && tag != 0));
+ } else {
+ static_assert(kFallbackFreeTag >= kShadowAlignment,
+ "fallback tag must not be a short granule tag.");
+ tag = kFallbackFreeTag;
+ }
TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size),
tag);
}
diff --git a/libsanitizer/hwasan/hwasan_allocator.h b/libsanitizer/hwasan/hwasan_allocator.h
index 03bbcff3f0f..35c3d6b4bf4 100644
--- a/libsanitizer/hwasan/hwasan_allocator.h
+++ b/libsanitizer/hwasan/hwasan_allocator.h
@@ -15,6 +15,7 @@
#include "hwasan.h"
#include "hwasan_interface_internal.h"
+#include "hwasan_mapping.h"
#include "hwasan_poisoning.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
@@ -58,7 +59,7 @@ static const uptr kMaxAllowedMallocSize = 1UL << 40; // 1T
struct AP64 {
static const uptr kSpaceBeg = ~0ULL;
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
static const uptr kSpaceSize = 1ULL << kAddressTagShift;
#else
static const uptr kSpaceSize = 0x2000000000ULL;
@@ -110,11 +111,11 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
void GetAllocatorStats(AllocatorStatCounters s);
inline bool InTaggableRegion(uptr addr) {
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
// Aliases are mapped next to shadow so that the upper bits match the shadow
// base.
return (addr >> kTaggableRegionCheckShift) ==
- (__hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+ (GetShadowOffset() >> kTaggableRegionCheckShift);
#endif
return true;
}
diff --git a/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp b/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp
index f53276e330d..bde22dfa4bc 100644
--- a/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp
+++ b/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp
@@ -119,12 +119,12 @@ namespace __hwasan {
void InitShadowGOT() {}
uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
-#if defined(__x86_64__)
+# if defined(HWASAN_ALIASING_MODE)
constexpr uptr kAliasSize = 1ULL << kAddressTagShift;
constexpr uptr kNumAliases = 1ULL << kTagBits;
return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases,
RingBufferSize());
-#endif
+# endif
return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment,
kHighMemEnd);
}
diff --git a/libsanitizer/hwasan/hwasan_fuchsia.cpp b/libsanitizer/hwasan/hwasan_fuchsia.cpp
new file mode 100644
index 00000000000..e61f6ada72f
--- /dev/null
+++ b/libsanitizer/hwasan/hwasan_fuchsia.cpp
@@ -0,0 +1,192 @@
+//===-- hwasan_fuchsia.cpp --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and contains Fuchsia-specific
+/// code.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_fuchsia.h"
+#if SANITIZER_FUCHSIA
+
+#include "hwasan.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_report.h"
+#include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
+
+// This TLS variable contains the location of the stack ring buffer and can be
+// used to always find the hwasan thread object associated with the current
+// running thread.
+[[gnu::tls_model("initial-exec")]]
+SANITIZER_INTERFACE_ATTRIBUTE
+THREADLOCAL uptr __hwasan_tls;
+
+namespace __hwasan {
+
+bool InitShadow() {
+ __sanitizer::InitShadowBounds();
+ CHECK_NE(__sanitizer::ShadowBounds.shadow_limit, 0);
+
+ return true;
+}
+
+bool MemIsApp(uptr p) {
+ CHECK(GetTagFromPointer(p) == 0);
+ return __sanitizer::ShadowBounds.shadow_limit <= p &&
+ p <= (__sanitizer::ShadowBounds.memory_limit - 1);
+}
+
+// These are known parameters passed to the hwasan runtime on thread creation.
+struct Thread::InitState {
+ uptr stack_bottom, stack_top;
+};
+
+static void FinishThreadInitialization(Thread *thread);
+
+void InitThreads() {
+ // This is the minimal alignment needed for the storage where hwasan threads
+ // and their stack ring buffers are placed. This alignment is necessary so the
+ // stack ring buffer can perform a simple calculation to get the next element
+ // in the RB. The instructions for this calculation are emitted by the
+ // compiler. (Full explanation in hwasan_thread_list.h.)
+ uptr alloc_size = UINT64_C(1) << kShadowBaseAlignment;
+ uptr thread_start = reinterpret_cast<uptr>(
+ MmapAlignedOrDieOnFatalError(alloc_size, alloc_size, __func__));
+
+ InitThreadList(thread_start, alloc_size);
+
+ // Create the hwasan thread object for the current (main) thread. Stack info
+ // for this thread is known from information passed via
+ // __sanitizer_startup_hook.
+ const Thread::InitState state = {
+ .stack_bottom = __sanitizer::MainThreadStackBase,
+ .stack_top =
+ __sanitizer::MainThreadStackBase + __sanitizer::MainThreadStackSize,
+ };
+ FinishThreadInitialization(hwasanThreadList().CreateCurrentThread(&state));
+}
+
+uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; }
+
+// This is called from the parent thread before the new thread is created. Here
+// we can propagate known info like the stack bounds to Thread::Init before
+// jumping into the thread. We cannot initialize the stack ring buffer yet since
+// we have not entered the new thread.
+static void *BeforeThreadCreateHook(uptr user_id, bool detached,
+ const char *name, uptr stack_bottom,
+ uptr stack_size) {
+ const Thread::InitState state = {
+ .stack_bottom = stack_bottom,
+ .stack_top = stack_bottom + stack_size,
+ };
+ return hwasanThreadList().CreateCurrentThread(&state);
+}
+
+// This sets the stack top and bottom according to the InitState passed to
+// CreateCurrentThread above.
+void Thread::InitStackAndTls(const InitState *state) {
+ CHECK_NE(state->stack_bottom, 0);
+ CHECK_NE(state->stack_top, 0);
+ stack_bottom_ = state->stack_bottom;
+ stack_top_ = state->stack_top;
+ tls_end_ = tls_begin_ = 0;
+}
+
+// This is called after creating a new thread with the pointer returned by
+// BeforeThreadCreateHook. We are still in the creating thread and should check
+// if it was actually created correctly.
+static void ThreadCreateHook(void *hook, bool aborted) {
+ Thread *thread = static_cast<Thread *>(hook);
+ if (!aborted) {
+ // The thread was created successfully.
+ // ThreadStartHook can already be running in the new thread.
+ } else {
+ // The thread wasn't created after all.
+ // Clean up everything we set up in BeforeThreadCreateHook.
+ atomic_signal_fence(memory_order_seq_cst);
+ hwasanThreadList().ReleaseThread(thread);
+ }
+}
+
+// This is called in the newly-created thread before it runs anything else,
+// with the pointer returned by BeforeThreadCreateHook (above). Here we can
+// setup the stack ring buffer.
+static void ThreadStartHook(void *hook, thrd_t self) {
+ Thread *thread = static_cast<Thread *>(hook);
+ FinishThreadInitialization(thread);
+ thread->InitRandomState();
+}
+
+// This is the function that sets up the stack ring buffer and enables us to use
+// GetCurrentThread. This function should only be called while IN the thread
+// that we want to create the hwasan thread object for so __hwasan_tls can be
+// properly referenced.
+static void FinishThreadInitialization(Thread *thread) {
+ CHECK_NE(thread, nullptr);
+
+ // The ring buffer is located immediately before the thread object.
+ uptr stack_buffer_size = hwasanThreadList().GetRingBufferSize();
+ uptr stack_buffer_start = reinterpret_cast<uptr>(thread) - stack_buffer_size;
+ thread->InitStackRingBuffer(stack_buffer_start, stack_buffer_size);
+}
+
+static void ThreadExitHook(void *hook, thrd_t self) {
+ Thread *thread = static_cast<Thread *>(hook);
+ atomic_signal_fence(memory_order_seq_cst);
+ hwasanThreadList().ReleaseThread(thread);
+}
+
+// Not implemented because Fuchsia does not use signal handlers.
+void HwasanOnDeadlySignal(int signo, void *info, void *context) {}
+
+// Not implemented because Fuchsia does not use interceptors.
+void InitializeInterceptors() {}
+
+// Not implemented because this is only relevant for Android.
+void AndroidTestTlsSlot() {}
+
+// TSD was normally used on linux as a means of calling the hwasan thread exit
+// handler passed to pthread_key_create. This is not needed on Fuchsia because
+// we will be using __sanitizer_thread_exit_hook.
+void HwasanTSDInit() {}
+void HwasanTSDThreadInit() {}
+
+// On linux, this just would call `atexit(HwasanAtExit)`. The functions in
+// HwasanAtExit are unimplemented for Fuchsia and effectively no-ops, so this
+// function is unneeded.
+void InstallAtExitHandler() {}
+
+} // namespace __hwasan
+
+extern "C" {
+
+void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
+ const char *name, void *stack_base,
+ size_t stack_size) {
+ return __hwasan::BeforeThreadCreateHook(
+ reinterpret_cast<uptr>(thread), detached, name,
+ reinterpret_cast<uptr>(stack_base), stack_size);
+}
+
+void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) {
+ __hwasan::ThreadCreateHook(hook, error != thrd_success);
+}
+
+void __sanitizer_thread_start_hook(void *hook, thrd_t self) {
+ __hwasan::ThreadStartHook(hook, reinterpret_cast<uptr>(self));
+}
+
+void __sanitizer_thread_exit_hook(void *hook, thrd_t self) {
+ __hwasan::ThreadExitHook(hook, self);
+}
+
+} // extern "C"
+
+#endif // SANITIZER_FUCHSIA
diff --git a/libsanitizer/hwasan/hwasan_interceptors.cpp b/libsanitizer/hwasan/hwasan_interceptors.cpp
index ad67e2787d3..68f8adec077 100644
--- a/libsanitizer/hwasan/hwasan_interceptors.cpp
+++ b/libsanitizer/hwasan/hwasan_interceptors.cpp
@@ -16,192 +16,14 @@
#include "interception/interception.h"
#include "hwasan.h"
-#include "hwasan_allocator.h"
-#include "hwasan_mapping.h"
#include "hwasan_thread.h"
-#include "hwasan_poisoning.h"
-#include "hwasan_report.h"
-#include "sanitizer_common/sanitizer_platform_limits_posix.h"
-#include "sanitizer_common/sanitizer_allocator.h"
-#include "sanitizer_common/sanitizer_allocator_interface.h"
-#include "sanitizer_common/sanitizer_allocator_internal.h"
-#include "sanitizer_common/sanitizer_atomic.h"
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
-#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_linux.h"
-#include "sanitizer_common/sanitizer_tls_get_addr.h"
-#include <stdarg.h>
-// ACHTUNG! No other system header includes in this file.
-// Ideally, we should get rid of stdarg.h as well.
+#if !SANITIZER_FUCHSIA
using namespace __hwasan;
-using __sanitizer::memory_order;
-using __sanitizer::atomic_load;
-using __sanitizer::atomic_store;
-using __sanitizer::atomic_uintptr_t;
-
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
-
-#define ENSURE_HWASAN_INITED() do { \
- CHECK(!hwasan_init_is_running); \
- if (!hwasan_inited) { \
- __hwasan_init(); \
- } \
-} while (0)
-
-
-int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
- GET_MALLOC_STACK_TRACE;
- CHECK_NE(memptr, 0);
- int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
- return res;
-}
-
-void * __sanitizer_memalign(uptr alignment, uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_memalign(alignment, size, &stack);
-}
-
-void * __sanitizer_aligned_alloc(uptr alignment, uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_aligned_alloc(alignment, size, &stack);
-}
-
-void * __sanitizer___libc_memalign(uptr alignment, uptr size) {
- GET_MALLOC_STACK_TRACE;
- void *ptr = hwasan_memalign(alignment, size, &stack);
- if (ptr)
- DTLS_on_libc_memalign(ptr, size);
- return ptr;
-}
-
-void * __sanitizer_valloc(uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_valloc(size, &stack);
-}
-
-void * __sanitizer_pvalloc(uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_pvalloc(size, &stack);
-}
-
-void __sanitizer_free(void *ptr) {
- GET_MALLOC_STACK_TRACE;
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
- hwasan_free(ptr, &stack);
-}
-
-void __sanitizer_cfree(void *ptr) {
- GET_MALLOC_STACK_TRACE;
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
- hwasan_free(ptr, &stack);
-}
-
-uptr __sanitizer_malloc_usable_size(const void *ptr) {
- return __sanitizer_get_allocated_size(ptr);
-}
-
-struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
- __sanitizer_struct_mallinfo sret;
- internal_memset(&sret, 0, sizeof(sret));
- return sret;
-}
-
-int __sanitizer_mallopt(int cmd, int value) {
- return 0;
-}
-
-void __sanitizer_malloc_stats(void) {
- // FIXME: implement, but don't call REAL(malloc_stats)!
-}
-
-void * __sanitizer_calloc(uptr nmemb, uptr size) {
- GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(!hwasan_inited))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
- return hwasan_calloc(nmemb, size, &stack);
-}
-
-void * __sanitizer_realloc(void *ptr, uptr size) {
- GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(!hwasan_inited)) {
- new_ptr = AllocateFromLocalPool(copy_size);
- } else {
- copy_size = size;
- new_ptr = hwasan_malloc(copy_size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
- return hwasan_realloc(ptr, size, &stack);
-}
-
-void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_reallocarray(ptr, nmemb, size, &stack);
-}
-
-void * __sanitizer_malloc(uptr size) {
- GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(!hwasan_init_is_running))
- ENSURE_HWASAN_INITED();
- if (UNLIKELY(!hwasan_inited))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
- return hwasan_malloc(size, &stack);
-}
-
#if HWASAN_WITH_INTERCEPTORS
-#define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \
- extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \
- ALIAS("__sanitizer_" #FN); \
- extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \
- ARGS) ALIAS("__sanitizer_" #FN)
-
-INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment,
- SIZE_T size);
-INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size);
-INTERCEPTOR_ALIAS(void, free, void *ptr);
-INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr);
-INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size);
-
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
-INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size);
-INTERCEPTOR_ALIAS(void, cfree, void *ptr);
-INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo);
-INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value);
-INTERCEPTOR_ALIAS(void, malloc_stats, void);
-#endif
struct ThreadStartArg {
thread_callback_t callback;
@@ -346,3 +168,5 @@ void InitializeInterceptors() {
inited = 1;
}
} // namespace __hwasan
+
+#endif // #if !SANITIZER_FUCHSIA
diff --git a/libsanitizer/hwasan/hwasan_linux.cpp b/libsanitizer/hwasan/hwasan_linux.cpp
index 8ce0ff7da95..e22723529f4 100644
--- a/libsanitizer/hwasan/hwasan_linux.cpp
+++ b/libsanitizer/hwasan/hwasan_linux.cpp
@@ -69,15 +69,9 @@ static void ProtectGap(uptr addr, uptr size) {
uptr kLowMemStart;
uptr kLowMemEnd;
-uptr kLowShadowEnd;
-uptr kLowShadowStart;
-uptr kHighShadowStart;
-uptr kHighShadowEnd;
uptr kHighMemStart;
uptr kHighMemEnd;
-uptr kAliasRegionStart; // Always 0 on non-x86.
-
static void PrintRange(uptr start, uptr end, const char *name) {
Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
}
@@ -116,7 +110,7 @@ static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
FindDynamicShadowStart(shadow_size_bytes);
}
-void InitPrctl() {
+void InitializeOsSupport() {
#define PR_SET_TAGGED_ADDR_CTRL 55
#define PR_GET_TAGGED_ADDR_CTRL 56
#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
@@ -125,33 +119,50 @@ void InitPrctl() {
if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
&local_errno) &&
local_errno == EINVAL) {
-#if SANITIZER_ANDROID || defined(__x86_64__)
+# if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
// Some older Android kernels have the tagged pointer ABI on
// unconditionally, and hence don't have the tagged-addr prctl while still
// allow the ABI.
// If targeting Android and the prctl is not around we assume this is the
// case.
return;
-#else
+# else
if (flags()->fail_without_syscall_abi) {
Printf(
"FATAL: "
"HWAddressSanitizer requires a kernel with tagged address ABI.\n");
Die();
}
-#endif
+# endif
}
// Turn on the tagged address ABI.
if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) ||
- !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) &&
- flags()->fail_without_syscall_abi) {
- Printf(
- "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
- "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
- "configuration.\n");
- Die();
+ !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) {
+# if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+ // Try the new prctl API for Intel LAM. The API is based on a currently
+ // unsubmitted patch to the Linux kernel (as of May 2021) and is thus
+ // subject to change. Patch is here:
+ // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/
+ int tag_bits = kTagBits;
+ int tag_shift = kAddressTagShift;
+ if (!internal_iserror(
+ internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE,
+ reinterpret_cast<unsigned long>(&tag_bits),
+ reinterpret_cast<unsigned long>(&tag_shift), 0))) {
+ CHECK_EQ(tag_bits, kTagBits);
+ CHECK_EQ(tag_shift, kAddressTagShift);
+ return;
+ }
+# endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+ if (flags()->fail_without_syscall_abi) {
+ Printf(
+ "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
+ "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
+ "configuration.\n");
+ Die();
+ }
}
#undef PR_SET_TAGGED_ADDR_CTRL
#undef PR_GET_TAGGED_ADDR_CTRL
@@ -181,18 +192,6 @@ bool InitShadow() {
// High memory starts where allocated shadow allows.
kHighMemStart = ShadowToMem(kHighShadowStart);
-#if defined(__x86_64__)
- constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
- kAliasRegionStart =
- __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
-
- CHECK_EQ(kAliasRegionStart >> kTaggableRegionCheckShift,
- __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
- CHECK_EQ(
- (kAliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
- __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
-#endif
-
// Check the sanity of the defined memory ranges (there might be gaps).
CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
CHECK_GT(kHighMemStart, kHighShadowEnd);
@@ -233,25 +232,16 @@ void InitThreads() {
ProtectGap(thread_space_end,
__hwasan_shadow_memory_dynamic_address - thread_space_end);
InitThreadList(thread_space_start, thread_space_end - thread_space_start);
+ hwasanThreadList().CreateCurrentThread();
}
bool MemIsApp(uptr p) {
-#if !defined(__x86_64__) // Memory outside the alias range has non-zero tags.
+// Memory outside the alias range has non-zero tags.
+# if !defined(HWASAN_ALIASING_MODE)
CHECK(GetTagFromPointer(p) == 0);
-#endif
- return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
-}
+# endif
-static void HwasanAtExit(void) {
- if (common_flags()->print_module_map)
- DumpProcessMap();
- if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
- ReportStats();
- if (hwasan_report_count > 0) {
- // ReportAtExitStatistics();
- if (common_flags()->exitcode)
- internal__exit(common_flags()->exitcode);
- }
+ return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
}
void InstallAtExitHandler() {
@@ -330,22 +320,6 @@ void AndroidTestTlsSlot() {
void AndroidTestTlsSlot() {}
#endif
-Thread *GetCurrentThread() {
- uptr *ThreadLongPtr = GetCurrentThreadLongPtr();
- if (UNLIKELY(*ThreadLongPtr == 0))
- return nullptr;
- auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr;
- return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next());
-}
-
-struct AccessInfo {
- uptr addr;
- uptr size;
- bool is_store;
- bool is_load;
- bool recover;
-};
-
static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
// Access type is passed in a platform dependent way (see below) and encoded
// as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
@@ -396,28 +370,6 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
return AccessInfo{addr, size, is_store, !is_store, recover};
}
-static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame,
- ucontext_t *uc, uptr *registers_frame = nullptr) {
- InternalMmapVector<BufferedStackTrace> stack_buffer(1);
- BufferedStackTrace *stack = stack_buffer.data();
- stack->Reset();
- stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
-
- // The second stack frame contains the failure __hwasan_check function, as
- // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
- // we wish to ignore. This (currently) only occurs on AArch64, as x64
- // implementations use SIGTRAP to implement the failure, and thus do not go
- // through the stack saver.
- if (registers_frame && stack->trace && stack->size > 0) {
- stack->trace++;
- stack->size--;
- }
-
- bool fatal = flags()->halt_on_error || !ai.recover;
- ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
- registers_frame);
-}
-
static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
AccessInfo ai = GetAccessInfo(info, uc);
if (!ai.is_store && !ai.is_load)
@@ -450,27 +402,39 @@ void HwasanOnDeadlySignal(int signo, void *info, void *context) {
HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
}
+void Thread::InitStackAndTls(const InitState *) {
+ uptr tls_size;
+ uptr stack_size;
+ GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
+ &tls_size);
+ stack_top_ = stack_bottom_ + stack_size;
+ tls_end_ = tls_begin_ + tls_size;
+}
-} // namespace __hwasan
-
-// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
-// rest of the mismatch handling code (C++).
-void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
- size_t outsize) {
- __hwasan::AccessInfo ai;
- ai.is_store = access_info & 0x10;
- ai.is_load = !ai.is_store;
- ai.recover = access_info & 0x20;
- ai.addr = addr;
- if ((access_info & 0xf) == 0xf)
- ai.size = outsize;
- else
- ai.size = 1 << (access_info & 0xf);
-
- __hwasan::HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
- (uptr)__builtin_frame_address(0), nullptr,
- registers_frame);
- __builtin_unreachable();
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+ CHECK(IsAligned(p, kShadowAlignment));
+ CHECK(IsAligned(size, kShadowAlignment));
+ uptr shadow_start = MemToShadow(p);
+ uptr shadow_size = MemToShadowSize(size);
+
+ uptr page_size = GetPageSizeCached();
+ uptr page_start = RoundUpTo(shadow_start, page_size);
+ uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
+ uptr threshold = common_flags()->clear_shadow_mmap_threshold;
+ if (SANITIZER_LINUX &&
+ UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
+ internal_memset((void *)shadow_start, tag, page_start - shadow_start);
+ internal_memset((void *)page_end, tag,
+ shadow_start + shadow_size - page_end);
+ // For an anonymous private mapping MADV_DONTNEED will return a zero page on
+ // Linux.
+ ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
+ } else {
+ internal_memset((void *)shadow_start, tag, shadow_size);
+ }
+ return AddTagToPointer(p, tag);
}
+} // namespace __hwasan
+
#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/libsanitizer/hwasan/hwasan_mapping.h b/libsanitizer/hwasan/hwasan_mapping.h
index 8243d1ec7ed..79a143632f6 100644
--- a/libsanitizer/hwasan/hwasan_mapping.h
+++ b/libsanitizer/hwasan/hwasan_mapping.h
@@ -48,14 +48,14 @@ extern uptr kHighShadowEnd;
extern uptr kHighMemStart;
extern uptr kHighMemEnd;
-extern uptr kAliasRegionStart;
-
+inline uptr GetShadowOffset() {
+ return SANITIZER_FUCHSIA ? 0 : __hwasan_shadow_memory_dynamic_address;
+}
inline uptr MemToShadow(uptr untagged_addr) {
- return (untagged_addr >> kShadowScale) +
- __hwasan_shadow_memory_dynamic_address;
+ return (untagged_addr >> kShadowScale) + GetShadowOffset();
}
inline uptr ShadowToMem(uptr shadow_addr) {
- return (shadow_addr - __hwasan_shadow_memory_dynamic_address) << kShadowScale;
+ return (shadow_addr - GetShadowOffset()) << kShadowScale;
}
inline uptr MemToShadowSize(uptr size) {
return size >> kShadowScale;
@@ -63,6 +63,13 @@ inline uptr MemToShadowSize(uptr size) {
bool MemIsApp(uptr p);
+inline bool MemIsShadow(uptr p) {
+ return (kLowShadowStart <= p && p <= kLowShadowEnd) ||
+ (kHighShadowStart <= p && p <= kHighShadowEnd);
+}
+
+uptr GetAliasRegionStart();
+
} // namespace __hwasan
#endif // HWASAN_MAPPING_H
diff --git a/libsanitizer/hwasan/hwasan_new_delete.cpp b/libsanitizer/hwasan/hwasan_new_delete.cpp
index 69cddda736e..4e057a651e1 100644
--- a/libsanitizer/hwasan/hwasan_new_delete.cpp
+++ b/libsanitizer/hwasan/hwasan_new_delete.cpp
@@ -56,7 +56,6 @@ using namespace __hwasan;
// Fake std::nothrow_t to avoid including <new>.
namespace std {
struct nothrow_t {};
- enum class align_val_t : size_t {};
} // namespace std
@@ -73,6 +72,32 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void *operator new[](size_t size, std::nothrow_t const&) {
OPERATOR_NEW_BODY(true /*nothrow*/);
}
+
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr)
+ NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+ void *ptr) NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
+ void *ptr, std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+ void *ptr, std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY;
+}
+
+#endif // OPERATOR_NEW_BODY
+
+#ifdef OPERATOR_NEW_ALIGN_BODY
+
+namespace std {
+enum class align_val_t : size_t {};
+} // namespace std
+
INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
size_t size, std::align_val_t align) {
OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/);
@@ -90,16 +115,6 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/);
}
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const&) {
- OPERATOR_DELETE_BODY;
-}
INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
void *ptr, std::align_val_t align) NOEXCEPT {
OPERATOR_DELETE_BODY;
@@ -117,4 +132,4 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
OPERATOR_DELETE_BODY;
}
-#endif // OPERATOR_NEW_BODY
+#endif // OPERATOR_NEW_ALIGN_BODY
diff --git a/libsanitizer/hwasan/hwasan_poisoning.cpp b/libsanitizer/hwasan/hwasan_poisoning.cpp
index 2a0816428e7..5aafdb1884b 100644
--- a/libsanitizer/hwasan/hwasan_poisoning.cpp
+++ b/libsanitizer/hwasan/hwasan_poisoning.cpp
@@ -19,30 +19,6 @@
namespace __hwasan {
-uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
- CHECK(IsAligned(p, kShadowAlignment));
- CHECK(IsAligned(size, kShadowAlignment));
- uptr shadow_start = MemToShadow(p);
- uptr shadow_size = MemToShadowSize(size);
-
- uptr page_size = GetPageSizeCached();
- uptr page_start = RoundUpTo(shadow_start, page_size);
- uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
- uptr threshold = common_flags()->clear_shadow_mmap_threshold;
- if (SANITIZER_LINUX &&
- UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
- internal_memset((void *)shadow_start, tag, page_start - shadow_start);
- internal_memset((void *)page_end, tag,
- shadow_start + shadow_size - page_end);
- // For an anonymous private mapping MADV_DONTNEED will return a zero page on
- // Linux.
- ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
- } else {
- internal_memset((void *)shadow_start, tag, shadow_size);
- }
- return AddTagToPointer(p, tag);
-}
-
uptr TagMemory(uptr p, uptr size, tag_t tag) {
uptr start = RoundDownTo(p, kShadowAlignment);
uptr end = RoundUpTo(p + size, kShadowAlignment);
diff --git a/libsanitizer/hwasan/hwasan_report.cpp b/libsanitizer/hwasan/hwasan_report.cpp
index c0217799391..44047c9fdaf 100644
--- a/libsanitizer/hwasan/hwasan_report.cpp
+++ b/libsanitizer/hwasan/hwasan_report.cpp
@@ -236,12 +236,12 @@ static void PrintStackAllocations(StackAllocationsRingBuffer *sa,
frame_desc.append(" record_addr:0x%zx record:0x%zx",
reinterpret_cast<uptr>(record_addr), record);
if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) {
- RenderFrame(&frame_desc, " %F %L\n", 0, frame->info.address, &frame->info,
+ RenderFrame(&frame_desc, " %F %L", 0, frame->info.address, &frame->info,
common_flags()->symbolize_vs_style,
common_flags()->strip_path_prefix);
frame->ClearAll();
}
- Printf("%s", frame_desc.data());
+ Printf("%s\n", frame_desc.data());
frame_desc.clear();
}
}
@@ -296,6 +296,75 @@ static uptr GetGlobalSizeFromDescriptor(uptr ptr) {
return 0;
}
+static void ShowHeapOrGlobalCandidate(uptr untagged_addr, tag_t *candidate,
+ tag_t *left, tag_t *right) {
+ Decorator d;
+ uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate));
+ HwasanChunkView chunk = FindHeapChunkByAddress(mem);
+ if (chunk.IsAllocated()) {
+ uptr offset;
+ const char *whence;
+ if (untagged_addr < chunk.End() && untagged_addr >= chunk.Beg()) {
+ offset = untagged_addr - chunk.Beg();
+ whence = "inside";
+ } else if (candidate == left) {
+ offset = untagged_addr - chunk.End();
+ whence = "to the right of";
+ } else {
+ offset = chunk.Beg() - untagged_addr;
+ whence = "to the left of";
+ }
+ Printf("%s", d.Error());
+ Printf("\nCause: heap-buffer-overflow\n");
+ Printf("%s", d.Default());
+ Printf("%s", d.Location());
+ Printf("%p is located %zd bytes %s %zd-byte region [%p,%p)\n",
+ untagged_addr, offset, whence, chunk.UsedSize(), chunk.Beg(),
+ chunk.End());
+ Printf("%s", d.Allocation());
+ Printf("allocated here:\n");
+ Printf("%s", d.Default());
+ GetStackTraceFromId(chunk.GetAllocStackId()).Print();
+ return;
+ }
+ // Check whether the address points into a loaded library. If so, this is
+ // most likely a global variable.
+ const char *module_name;
+ uptr module_address;
+ Symbolizer *sym = Symbolizer::GetOrInit();
+ if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, &module_address)) {
+ Printf("%s", d.Error());
+ Printf("\nCause: global-overflow\n");
+ Printf("%s", d.Default());
+ DataInfo info;
+ Printf("%s", d.Location());
+ if (sym->SymbolizeData(mem, &info) && info.start) {
+ Printf(
+ "%p is located %zd bytes to the %s of %zd-byte global variable "
+ "%s [%p,%p) in %s\n",
+ untagged_addr,
+ candidate == left ? untagged_addr - (info.start + info.size)
+ : info.start - untagged_addr,
+ candidate == left ? "right" : "left", info.size, info.name,
+ info.start, info.start + info.size, module_name);
+ } else {
+ uptr size = GetGlobalSizeFromDescriptor(mem);
+ if (size == 0)
+ // We couldn't find the size of the global from the descriptors.
+ Printf("%p is located to the %s of a global variable in (%s+0x%x)\n",
+ untagged_addr, candidate == left ? "right" : "left", module_name,
+ module_address);
+ else
+ Printf(
+ "%p is located to the %s of a %zd-byte global variable in "
+ "(%s+0x%x)\n",
+ untagged_addr, candidate == left ? "right" : "left", size,
+ module_name, module_address);
+ }
+ Printf("%s", d.Default());
+ }
+}
+
void PrintAddressDescription(
uptr tagged_addr, uptr access_size,
StackAllocationsRingBuffer *current_stack_allocations) {
@@ -317,78 +386,59 @@ void PrintAddressDescription(
d.Default());
}
+ tag_t addr_tag = GetTagFromPointer(tagged_addr);
+
+ bool on_stack = false;
+ // Check stack first. If the address is on the stack of a live thread, we
+ // know it cannot be a heap / global overflow.
+ hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
+ if (t->AddrIsInStack(untagged_addr)) {
+ on_stack = true;
+ // TODO(fmayer): figure out how to distinguish use-after-return and
+ // stack-buffer-overflow.
+ Printf("%s", d.Error());
+ Printf("\nCause: stack tag-mismatch\n");
+ Printf("%s", d.Location());
+ Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
+ t->unique_id());
+ Printf("%s", d.Default());
+ t->Announce();
+
+ auto *sa = (t == GetCurrentThread() && current_stack_allocations)
+ ? current_stack_allocations
+ : t->stack_allocations();
+ PrintStackAllocations(sa, addr_tag, untagged_addr);
+ num_descriptions_printed++;
+ }
+ });
+
// Check if this looks like a heap buffer overflow by scanning
// the shadow left and right and looking for the first adjacent
// object with a different memory tag. If that tag matches addr_tag,
// check the allocator if it has a live chunk there.
- tag_t addr_tag = GetTagFromPointer(tagged_addr);
tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr;
- for (int i = 0; i < 1000; i++) {
- if (TagsEqual(addr_tag, left)) {
+ uptr candidate_distance = 0;
+ for (; candidate_distance < 1000; candidate_distance++) {
+ if (MemIsShadow(reinterpret_cast<uptr>(left)) &&
+ TagsEqual(addr_tag, left)) {
candidate = left;
break;
}
--left;
- if (TagsEqual(addr_tag, right)) {
+ if (MemIsShadow(reinterpret_cast<uptr>(right)) &&
+ TagsEqual(addr_tag, right)) {
candidate = right;
break;
}
++right;
}
- if (candidate) {
- uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate));
- HwasanChunkView chunk = FindHeapChunkByAddress(mem);
- if (chunk.IsAllocated()) {
- Printf("%s", d.Location());
- Printf("%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n",
- untagged_addr,
- candidate == left ? untagged_addr - chunk.End()
- : chunk.Beg() - untagged_addr,
- candidate == left ? "right" : "left", chunk.UsedSize(),
- chunk.Beg(), chunk.End());
- Printf("%s", d.Allocation());
- Printf("allocated here:\n");
- Printf("%s", d.Default());
- GetStackTraceFromId(chunk.GetAllocStackId()).Print();
- num_descriptions_printed++;
- } else {
- // Check whether the address points into a loaded library. If so, this is
- // most likely a global variable.
- const char *module_name;
- uptr module_address;
- Symbolizer *sym = Symbolizer::GetOrInit();
- if (sym->GetModuleNameAndOffsetForPC(mem, &module_name,
- &module_address)) {
- DataInfo info;
- if (sym->SymbolizeData(mem, &info) && info.start) {
- Printf(
- "%p is located %zd bytes to the %s of %zd-byte global variable "
- "%s [%p,%p) in %s\n",
- untagged_addr,
- candidate == left ? untagged_addr - (info.start + info.size)
- : info.start - untagged_addr,
- candidate == left ? "right" : "left", info.size, info.name,
- info.start, info.start + info.size, module_name);
- } else {
- uptr size = GetGlobalSizeFromDescriptor(mem);
- if (size == 0)
- // We couldn't find the size of the global from the descriptors.
- Printf(
- "%p is located to the %s of a global variable in (%s+0x%x)\n",
- untagged_addr, candidate == left ? "right" : "left",
- module_name, module_address);
- else
- Printf(
- "%p is located to the %s of a %zd-byte global variable in "
- "(%s+0x%x)\n",
- untagged_addr, candidate == left ? "right" : "left", size,
- module_name, module_address);
- }
- num_descriptions_printed++;
- }
- }
+ constexpr auto kCloseCandidateDistance = 1;
+
+ if (!on_stack && candidate && candidate_distance <= kCloseCandidateDistance) {
+ ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right);
+ num_descriptions_printed++;
}
hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
@@ -398,6 +448,8 @@ void PrintAddressDescription(
if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har,
&ring_index, &num_matching_addrs,
&num_matching_addrs_4b)) {
+ Printf("%s", d.Error());
+ Printf("\nCause: use-after-free\n");
Printf("%s", d.Location());
Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n",
untagged_addr, untagged_addr - UntagAddr(har.tagged_addr),
@@ -424,29 +476,25 @@ void PrintAddressDescription(
t->Announce();
num_descriptions_printed++;
}
-
- // Very basic check for stack memory.
- if (t->AddrIsInStack(untagged_addr)) {
- Printf("%s", d.Location());
- Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
- t->unique_id());
- Printf("%s", d.Default());
- t->Announce();
-
- auto *sa = (t == GetCurrentThread() && current_stack_allocations)
- ? current_stack_allocations
- : t->stack_allocations();
- PrintStackAllocations(sa, addr_tag, untagged_addr);
- num_descriptions_printed++;
- }
});
+ if (candidate && num_descriptions_printed == 0) {
+ ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right);
+ num_descriptions_printed++;
+ }
+
// Print the remaining threads, as an extra information, 1 line per thread.
hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); });
if (!num_descriptions_printed)
// We exhausted our possibilities. Bail out.
Printf("HWAddressSanitizer can not describe address in more detail.\n");
+ if (num_descriptions_printed > 1) {
+ Printf(
+ "There are %d potential causes, printed above in order "
+ "of likeliness.\n",
+ num_descriptions_printed);
+ }
}
void ReportStats() {}
@@ -538,6 +586,12 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size,
Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName,
bug_type, untagged_addr, untagged_addr + orig_size, orig_size);
Printf("\n%s", d.Default());
+ Printf(
+ "Stack of invalid access unknown. Issue detected at deallocation "
+ "time.\n");
+ Printf("%s", d.Allocation());
+ Printf("deallocated here:\n");
+ Printf("%s", d.Default());
stack->Print();
HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr);
if (chunk.Beg()) {
@@ -657,8 +711,10 @@ void ReportRegisters(uptr *frame, uptr pc) {
frame[20], frame[21], frame[22], frame[23]);
Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n",
frame[24], frame[25], frame[26], frame[27]);
- Printf(" x28 %016llx x29 %016llx x30 %016llx\n",
- frame[28], frame[29], frame[30]);
+ // hwasan_check* reduces the stack pointer by 256, then __hwasan_tag_mismatch
+ // passes it to this function.
+ Printf(" x28 %016llx x29 %016llx x30 %016llx sp %016llx\n", frame[28],
+ frame[29], frame[30], reinterpret_cast<u8 *>(frame) + 256);
}
} // namespace __hwasan
diff --git a/libsanitizer/hwasan/hwasan_thread.cpp b/libsanitizer/hwasan/hwasan_thread.cpp
index bb4d56abed0..ee747a3beea 100644
--- a/libsanitizer/hwasan/hwasan_thread.cpp
+++ b/libsanitizer/hwasan/hwasan_thread.cpp
@@ -34,7 +34,8 @@ void Thread::InitRandomState() {
stack_allocations_->push(0);
}
-void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) {
+void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size,
+ const InitState *state) {
CHECK_EQ(0, unique_id_); // try to catch bad stack reuse
CHECK_EQ(0, stack_top_);
CHECK_EQ(0, stack_bottom_);
@@ -44,6 +45,17 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) {
if (auto sz = flags()->heap_history_size)
heap_allocations_ = HeapAllocationsRingBuffer::New(sz);
+ InitStackAndTls(state);
+#if !SANITIZER_FUCHSIA
+ // Do not initialize the stack ring buffer just yet on Fuchsia. Threads will
+ // be initialized before we enter the thread itself, so we will instead call
+ // this later.
+ InitStackRingBuffer(stack_buffer_start, stack_buffer_size);
+#endif
+}
+
+void Thread::InitStackRingBuffer(uptr stack_buffer_start,
+ uptr stack_buffer_size) {
HwasanTSDThreadInit(); // Only needed with interceptors.
uptr *ThreadLong = GetCurrentThreadLongPtr();
// The following implicitly sets (this) as the current thread.
@@ -55,13 +67,6 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) {
// ScopedTaggingDisable needs GetCurrentThread to be set up.
ScopedTaggingDisabler disabler;
- uptr tls_size;
- uptr stack_size;
- GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
- &tls_size);
- stack_top_ = stack_bottom_ + stack_size;
- tls_end_ = tls_begin_ + tls_size;
-
if (stack_bottom_) {
int local;
CHECK(AddrIsInStack((uptr)&local));
diff --git a/libsanitizer/hwasan/hwasan_thread.h b/libsanitizer/hwasan/hwasan_thread.h
index 1c71cab41c4..9f20afe1dc7 100644
--- a/libsanitizer/hwasan/hwasan_thread.h
+++ b/libsanitizer/hwasan/hwasan_thread.h
@@ -23,8 +23,17 @@ typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer;
class Thread {
public:
- void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself.
+ // These are optional parameters that can be passed to Init.
+ struct InitState;
+
+ void Init(uptr stack_buffer_start, uptr stack_buffer_size,
+ const InitState *state = nullptr);
void InitRandomState();
+ void InitStackAndTls(const InitState *state = nullptr);
+
+ // Must be called from the thread itself.
+ void InitStackRingBuffer(uptr stack_buffer_start, uptr stack_buffer_size);
+
void Destroy();
uptr stack_top() { return stack_top_; }
diff --git a/libsanitizer/hwasan/hwasan_thread_list.cpp b/libsanitizer/hwasan/hwasan_thread_list.cpp
index a31eee84ed9..fa46e658b69 100644
--- a/libsanitizer/hwasan/hwasan_thread_list.cpp
+++ b/libsanitizer/hwasan/hwasan_thread_list.cpp
@@ -12,4 +12,4 @@ void InitThreadList(uptr storage, uptr size) {
new (thread_list_placeholder) HwasanThreadList(storage, size);
}
-} // namespace
+} // namespace __hwasan
diff --git a/libsanitizer/hwasan/hwasan_thread_list.h b/libsanitizer/hwasan/hwasan_thread_list.h
index 11c586314ce..15916a802d6 100644
--- a/libsanitizer/hwasan/hwasan_thread_list.h
+++ b/libsanitizer/hwasan/hwasan_thread_list.h
@@ -85,7 +85,7 @@ class HwasanThreadList {
RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2);
}
- Thread *CreateCurrentThread() {
+ Thread *CreateCurrentThread(const Thread::InitState *state = nullptr) {
Thread *t = nullptr;
{
SpinMutexLock l(&free_list_mutex_);
@@ -104,7 +104,7 @@ class HwasanThreadList {
SpinMutexLock l(&live_list_mutex_);
live_list_.push_back(t);
}
- t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_);
+ t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, state);
AddThreadStats(t);
return t;
}
@@ -171,6 +171,8 @@ class HwasanThreadList {
return stats_;
}
+ uptr GetRingBufferSize() const { return ring_buffer_size_; }
+
private:
Thread *AllocThread() {
SpinMutexLock l(&free_space_mutex_);
@@ -200,4 +202,4 @@ class HwasanThreadList {
void InitThreadList(uptr storage, uptr size);
HwasanThreadList &hwasanThreadList();
-} // namespace
+} // namespace __hwasan
diff --git a/libsanitizer/include/sanitizer/dfsan_interface.h b/libsanitizer/include/sanitizer/dfsan_interface.h
index 40f9379b557..cd3b6d6e2b1 100644
--- a/libsanitizer/include/sanitizer/dfsan_interface.h
+++ b/libsanitizer/include/sanitizer/dfsan_interface.h
@@ -21,34 +21,15 @@
extern "C" {
#endif
-typedef uint16_t dfsan_label;
+typedef uint8_t dfsan_label;
typedef uint32_t dfsan_origin;
-/// Stores information associated with a specific label identifier. A label
-/// may be a base label created using dfsan_create_label, with associated
-/// text description and user data, or an automatically created union label,
-/// which represents the union of two label identifiers (which may themselves
-/// be base or union labels).
-struct dfsan_label_info {
- // Fields for union labels, set to 0 for base labels.
- dfsan_label l1;
- dfsan_label l2;
-
- // Fields for base labels.
- const char *desc;
- void *userdata;
-};
-
/// Signature of the callback argument to dfsan_set_write_callback().
typedef void (*dfsan_write_callback_t)(int fd, const void *buf, size_t count);
-/// Computes the union of \c l1 and \c l2, possibly creating a union label in
-/// the process.
+/// Computes the union of \c l1 and \c l2, resulting in a union label.
dfsan_label dfsan_union(dfsan_label l1, dfsan_label l2);
-/// Creates and returns a base label with the given description and user data.
-dfsan_label dfsan_create_label(const char *desc, void *userdata);
-
/// Sets the label for each address in [addr,addr+size) to \c label.
void dfsan_set_label(dfsan_label label, void *addr, size_t size);
@@ -73,19 +54,9 @@ dfsan_origin dfsan_get_origin(long data);
/// Retrieves the label associated with the data at the given address.
dfsan_label dfsan_read_label(const void *addr, size_t size);
-/// Retrieves a pointer to the dfsan_label_info struct for the given label.
-const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label);
-
/// Returns whether the given label label contains the label elem.
int dfsan_has_label(dfsan_label label, dfsan_label elem);
-/// If the given label label contains a label with the description desc, returns
-/// that label, else returns 0.
-dfsan_label dfsan_has_label_with_desc(dfsan_label label, const char *desc);
-
-/// Returns the number of labels allocated.
-size_t dfsan_get_label_count(void);
-
/// Flushes the DFSan shadow, i.e. forgets about all labels currently associated
/// with the application memory. Use this call to start over the taint tracking
/// within the same process.
@@ -99,12 +70,6 @@ void dfsan_flush(void);
/// callback executes. Pass in NULL to remove any callback.
void dfsan_set_write_callback(dfsan_write_callback_t labeled_write_callback);
-/// Writes the labels currently used by the program to the given file
-/// descriptor. The lines of the output have the following format:
-///
-/// <label> <parent label 1> <parent label 2> <label description if any>
-void dfsan_dump_labels(int fd);
-
/// Interceptor hooks.
/// Whenever a dfsan's custom function is called the corresponding
/// hook is called it non-zero. The hooks should be defined by the user.
@@ -123,9 +88,65 @@ void dfsan_weak_hook_strncmp(void *caller_pc, const char *s1, const char *s2,
/// on, or the address is not labeled, it prints nothing.
void dfsan_print_origin_trace(const void *addr, const char *description);
+/// Prints the origin trace of the label at the address \p addr to a
+/// pre-allocated output buffer. If origin tracking is not on, or the address is
+/// not labeled, it prints nothing.
+///
+/// Typical usage:
+/// \code
+/// char kDescription[] = "...";
+/// char buf[1024];
+/// dfsan_sprint_origin_trace(&tainted_var, kDescription, buf, sizeof(buf));
+/// \endcode
+///
+/// Typical usage that handles truncation:
+/// \code
+/// char buf[1024];
+/// int len = dfsan_sprint_origin_trace(&var, nullptr, buf, sizeof(buf));
+///
+/// if (len < sizeof(buf)) {
+/// ProcessOriginTrace(buf);
+/// } else {
+/// char *tmpbuf = new char[len + 1];
+/// dfsan_sprint_origin_trace(&var, nullptr, tmpbuf, len + 1);
+/// ProcessOriginTrace(tmpbuf);
+/// delete[] tmpbuf;
+/// }
+/// \endcode
+///
+/// \param addr The tainted memory address whose origin we are printing.
+/// \param description A description printed at the beginning of the trace.
+/// \param [out] out_buf The output buffer to write the results to.
+/// \param out_buf_size The size of \p out_buf.
+///
+/// \returns The number of symbols that should have been written to \p out_buf
+/// (not including trailing null byte '\0'). Thus, the string is truncated iff
+/// return value is not less than \p out_buf_size.
+size_t dfsan_sprint_origin_trace(const void *addr, const char *description,
+ char *out_buf, size_t out_buf_size);
+
+/// Prints the stack trace leading to this call to a pre-allocated output
+/// buffer.
+///
+/// For usage examples, see dfsan_sprint_origin_trace.
+///
+/// \param [out] out_buf The output buffer to write the results to.
+/// \param out_buf_size The size of \p out_buf.
+///
+/// \returns The number of symbols that should have been written to \p out_buf
+/// (not including trailing null byte '\0'). Thus, the string is truncated iff
+/// return value is not less than \p out_buf_size.
+size_t dfsan_sprint_stack_trace(char *out_buf, size_t out_buf_size);
+
/// Retrieves the very first origin associated with the data at the given
/// address.
dfsan_origin dfsan_get_init_origin(const void *addr);
+
+/// Returns the value of -dfsan-track-origins.
+/// * 0: do not track origins.
+/// * 1: track origins at memory store operations.
+/// * 2: track origins at memory load and store operations.
+int dfsan_get_track_origins(void);
#ifdef __cplusplus
} // extern "C"
diff --git a/libsanitizer/interception/interception.h b/libsanitizer/interception/interception.h
index cb0b5284ed2..d8dc092c45f 100644
--- a/libsanitizer/interception/interception.h
+++ b/libsanitizer/interception/interception.h
@@ -16,10 +16,10 @@
#include "sanitizer_common/sanitizer_internal_defs.h"
-#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \
- !SANITIZER_NETBSD && !SANITIZER_WINDOWS && \
- !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_SOLARIS
-# error "Interception doesn't work on this operating system."
+#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \
+ !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \
+ !SANITIZER_SOLARIS
+# error "Interception doesn't work on this operating system."
#endif
// These typedefs should be used only in the interceptor definitions to replace
@@ -130,11 +130,6 @@ const interpose_substitution substitution_##func_name[] \
extern "C" ret_type func(__VA_ARGS__);
# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__);
-#elif SANITIZER_RTEMS
-# define WRAP(x) x
-# define WRAPPER_NAME(x) #x
-# define INTERCEPTOR_ATTRIBUTE
-# define DECLARE_WRAPPER(ret_type, func, ...)
#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
# define WRAP(x) __interceptor_ ## x
# define WRAPPER_NAME(x) "__interceptor_" #x
@@ -162,10 +157,6 @@ const interpose_substitution substitution_##func_name[] \
# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
# define REAL(x) __unsanitized_##x
# define DECLARE_REAL(ret_type, func, ...)
-#elif SANITIZER_RTEMS
-# define REAL(x) __real_ ## x
-# define DECLARE_REAL(ret_type, func, ...) \
- extern "C" ret_type REAL(func)(__VA_ARGS__);
#elif !SANITIZER_MAC
# define PTR_TO_REAL(x) real_##x
# define REAL(x) __interception::PTR_TO_REAL(x)
@@ -184,10 +175,10 @@ const interpose_substitution substitution_##func_name[] \
# define ASSIGN_REAL(x, y)
#endif // SANITIZER_MAC
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
-# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
- DECLARE_REAL(ret_type, func, __VA_ARGS__) \
- extern "C" ret_type WRAP(func)(__VA_ARGS__);
+#if !SANITIZER_FUCHSIA
+# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
+ DECLARE_REAL(ret_type, func, __VA_ARGS__) \
+ extern "C" ret_type WRAP(func)(__VA_ARGS__);
// Declare an interceptor and its wrapper defined in a different translation
// unit (ex. asm).
# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \
@@ -202,11 +193,11 @@ const interpose_substitution substitution_##func_name[] \
// macros does its job. In exceptional cases you may need to call REAL(foo)
// without defining INTERCEPTOR(..., foo, ...). For example, if you override
// foo with an interceptor for other function.
-#if !SANITIZER_MAC && !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
-# define DEFINE_REAL(ret_type, func, ...) \
+#if !SANITIZER_MAC && !SANITIZER_FUCHSIA
+# define DEFINE_REAL(ret_type, func, ...) \
typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
- namespace __interception { \
- FUNC_TYPE(func) PTR_TO_REAL(func); \
+ namespace __interception { \
+ FUNC_TYPE(func) PTR_TO_REAL(func); \
}
#else
# define DEFINE_REAL(ret_type, func, ...)
diff --git a/libsanitizer/lsan/lsan.cpp b/libsanitizer/lsan/lsan.cpp
index 2c0a3bf0787..b6adc248157 100644
--- a/libsanitizer/lsan/lsan.cpp
+++ b/libsanitizer/lsan/lsan.cpp
@@ -35,18 +35,14 @@ void __sanitizer::BufferedStackTrace::UnwindImpl(
uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
using namespace __lsan;
uptr stack_top = 0, stack_bottom = 0;
- ThreadContext *t;
- if (StackTrace::WillUseFastUnwind(request_fast) &&
- (t = CurrentThreadContext())) {
+ if (ThreadContext *t = CurrentThreadContext()) {
stack_top = t->stack_end();
stack_bottom = t->stack_begin();
}
- if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) {
- if (StackTrace::WillUseFastUnwind(request_fast))
- Unwind(max_depth, pc, bp, nullptr, stack_top, stack_bottom, true);
- else
- Unwind(max_depth, pc, 0, context, 0, 0, false);
- }
+ if (SANITIZER_MIPS && !IsValidFrame(bp, stack_top, stack_bottom))
+ return;
+ bool fast = StackTrace::WillUseFastUnwind(request_fast);
+ Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast);
}
using namespace __lsan;
diff --git a/libsanitizer/lsan/lsan_common.h b/libsanitizer/lsan/lsan_common.h
index fe855cf3754..776ca60b1e9 100644
--- a/libsanitizer/lsan/lsan_common.h
+++ b/libsanitizer/lsan/lsan_common.h
@@ -221,8 +221,8 @@ void UnlockAllocator();
// Returns true if [addr, addr + sizeof(void *)) is poisoned.
bool WordIsPoisoned(uptr addr);
// Wrappers for ThreadRegistry access.
-void LockThreadRegistry();
-void UnlockThreadRegistry();
+void LockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS;
+void UnlockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS;
ThreadRegistry *GetThreadRegistryLocked();
bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
diff --git a/libsanitizer/lsan/lsan_thread.cpp b/libsanitizer/lsan/lsan_thread.cpp
index 8efb54a6fb7..1d224ebca69 100644
--- a/libsanitizer/lsan/lsan_thread.cpp
+++ b/libsanitizer/lsan/lsan_thread.cpp
@@ -30,13 +30,10 @@ static ThreadContextBase *CreateThreadContext(u32 tid) {
return new (mem) ThreadContext(tid);
}
-static const uptr kMaxThreads = 1 << 13;
-static const uptr kThreadQuarantineSize = 64;
-
void InitializeThreadRegistry() {
static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];
- thread_registry = new (thread_registry_placeholder)
- ThreadRegistry(CreateThreadContext, kMaxThreads, kThreadQuarantineSize);
+ thread_registry =
+ new (thread_registry_placeholder) ThreadRegistry(CreateThreadContext);
}
ThreadContextLsanBase::ThreadContextLsanBase(int tid)
diff --git a/libsanitizer/sanitizer_common/Makefile.am b/libsanitizer/sanitizer_common/Makefile.am
index 7cba5342647..d04f2d8bd16 100644
--- a/libsanitizer/sanitizer_common/Makefile.am
+++ b/libsanitizer/sanitizer_common/Makefile.am
@@ -41,6 +41,7 @@ sanitizer_common_files = \
sanitizer_linux_s390.cpp \
sanitizer_mac.cpp \
sanitizer_mac_libcdep.cpp \
+ sanitizer_mutex.cpp \
sanitizer_netbsd.cpp \
sanitizer_openbsd.cpp \
sanitizer_persistent_allocator.cpp \
@@ -57,7 +58,6 @@ sanitizer_common_files = \
sanitizer_procmaps_linux.cpp \
sanitizer_procmaps_mac.cpp \
sanitizer_procmaps_solaris.cpp \
- sanitizer_rtems.cpp \
sanitizer_solaris.cpp \
sanitizer_stackdepot.cpp \
sanitizer_stacktrace.cpp \
diff --git a/libsanitizer/sanitizer_common/Makefile.in b/libsanitizer/sanitizer_common/Makefile.in
index 7e5555cc9df..2856894d62b 100644
--- a/libsanitizer/sanitizer_common/Makefile.in
+++ b/libsanitizer/sanitizer_common/Makefile.in
@@ -128,8 +128,9 @@ am__objects_1 = sancov_flags.lo sanitizer_allocator.lo \
sanitizer_file.lo sanitizer_flags.lo sanitizer_flag_parser.lo \
sanitizer_libc.lo sanitizer_libignore.lo sanitizer_linux.lo \
sanitizer_linux_libcdep.lo sanitizer_linux_s390.lo \
- sanitizer_mac.lo sanitizer_mac_libcdep.lo sanitizer_netbsd.lo \
- sanitizer_openbsd.lo sanitizer_persistent_allocator.lo \
+ sanitizer_mac.lo sanitizer_mac_libcdep.lo sanitizer_mutex.lo \
+ sanitizer_netbsd.lo sanitizer_openbsd.lo \
+ sanitizer_persistent_allocator.lo \
sanitizer_platform_limits_freebsd.lo \
sanitizer_platform_limits_linux.lo \
sanitizer_platform_limits_openbsd.lo \
@@ -138,11 +139,11 @@ am__objects_1 = sancov_flags.lo sanitizer_allocator.lo \
sanitizer_posix_libcdep.lo sanitizer_printf.lo \
sanitizer_procmaps_bsd.lo sanitizer_procmaps_common.lo \
sanitizer_procmaps_linux.lo sanitizer_procmaps_mac.lo \
- sanitizer_procmaps_solaris.lo sanitizer_rtems.lo \
- sanitizer_solaris.lo sanitizer_stackdepot.lo \
- sanitizer_stacktrace.lo sanitizer_stacktrace_libcdep.lo \
- sanitizer_stacktrace_sparc.lo sanitizer_symbolizer_mac.lo \
- sanitizer_symbolizer_report.lo sanitizer_stacktrace_printer.lo \
+ sanitizer_procmaps_solaris.lo sanitizer_solaris.lo \
+ sanitizer_stackdepot.lo sanitizer_stacktrace.lo \
+ sanitizer_stacktrace_libcdep.lo sanitizer_stacktrace_sparc.lo \
+ sanitizer_symbolizer_mac.lo sanitizer_symbolizer_report.lo \
+ sanitizer_stacktrace_printer.lo \
sanitizer_stoptheworld_linux_libcdep.lo \
sanitizer_stoptheworld_mac.lo sanitizer_suppressions.lo \
sanitizer_symbolizer.lo sanitizer_symbolizer_libbacktrace.lo \
@@ -400,6 +401,7 @@ sanitizer_common_files = \
sanitizer_linux_s390.cpp \
sanitizer_mac.cpp \
sanitizer_mac_libcdep.cpp \
+ sanitizer_mutex.cpp \
sanitizer_netbsd.cpp \
sanitizer_openbsd.cpp \
sanitizer_persistent_allocator.cpp \
@@ -416,7 +418,6 @@ sanitizer_common_files = \
sanitizer_procmaps_linux.cpp \
sanitizer_procmaps_mac.cpp \
sanitizer_procmaps_solaris.cpp \
- sanitizer_rtems.cpp \
sanitizer_solaris.cpp \
sanitizer_stackdepot.cpp \
sanitizer_stacktrace.cpp \
@@ -557,6 +558,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux_s390.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mac.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mac_libcdep.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mutex.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_netbsd.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_openbsd.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_persistent_allocator.Plo@am__quote@
@@ -573,7 +575,6 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_linux.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_mac.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_solaris.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_rtems.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_solaris.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stackdepot.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stacktrace.Plo@am__quote@
diff --git a/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h b/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
index a033e788cbf..15f81a04350 100644
--- a/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
+++ b/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
@@ -162,8 +162,8 @@ AddrHashMap<T, kSize>::AddrHashMap() {
table_ = (Bucket*)MmapOrDie(kSize * sizeof(table_[0]), "AddrHashMap");
}
-template<typename T, uptr kSize>
-void AddrHashMap<T, kSize>::acquire(Handle *h) {
+template <typename T, uptr kSize>
+void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
uptr addr = h->addr_;
uptr hash = calcHash(addr);
Bucket *b = &table_[hash];
@@ -289,57 +289,57 @@ void AddrHashMap<T, kSize>::acquire(Handle *h) {
CHECK_EQ(atomic_load(&c->addr, memory_order_relaxed), 0);
h->addidx_ = i;
h->cell_ = c;
-}
-
-template<typename T, uptr kSize>
-void AddrHashMap<T, kSize>::release(Handle *h) {
- if (!h->cell_)
- return;
- Bucket *b = h->bucket_;
- Cell *c = h->cell_;
- uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
- if (h->created_) {
- // Denote completion of insertion.
- CHECK_EQ(addr1, 0);
- // After the following store, the element becomes available
- // for lock-free reads.
- atomic_store(&c->addr, h->addr_, memory_order_release);
- b->mtx.Unlock();
- } else if (h->remove_) {
- // Denote that the cell is empty now.
- CHECK_EQ(addr1, h->addr_);
- atomic_store(&c->addr, 0, memory_order_release);
- // See if we need to compact the bucket.
- AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed);
- if (h->addidx_ == -1U) {
- // Removed from embed array, move an add element into the freed cell.
- if (add && add->size != 0) {
- uptr last = --add->size;
- Cell *c1 = &add->cells[last];
- c->val = c1->val;
- uptr addr1 = atomic_load(&c1->addr, memory_order_relaxed);
- atomic_store(&c->addr, addr1, memory_order_release);
- atomic_store(&c1->addr, 0, memory_order_release);
- }
- } else {
- // Removed from add array, compact it.
- uptr last = --add->size;
- Cell *c1 = &add->cells[last];
- if (c != c1) {
- *c = *c1;
- atomic_store(&c1->addr, 0, memory_order_relaxed);
- }
- }
- if (add && add->size == 0) {
- // FIXME(dvyukov): free add?
- }
- b->mtx.Unlock();
- } else {
- CHECK_EQ(addr1, h->addr_);
- if (h->addidx_ != -1U)
- b->mtx.ReadUnlock();
- }
-}
+ }
+
+ template <typename T, uptr kSize>
+ void AddrHashMap<T, kSize>::release(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
+ if (!h->cell_)
+ return;
+ Bucket *b = h->bucket_;
+ Cell *c = h->cell_;
+ uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
+ if (h->created_) {
+ // Denote completion of insertion.
+ CHECK_EQ(addr1, 0);
+ // After the following store, the element becomes available
+ // for lock-free reads.
+ atomic_store(&c->addr, h->addr_, memory_order_release);
+ b->mtx.Unlock();
+ } else if (h->remove_) {
+ // Denote that the cell is empty now.
+ CHECK_EQ(addr1, h->addr_);
+ atomic_store(&c->addr, 0, memory_order_release);
+ // See if we need to compact the bucket.
+ AddBucket *add = (AddBucket *)atomic_load(&b->add, memory_order_relaxed);
+ if (h->addidx_ == -1U) {
+ // Removed from embed array, move an add element into the freed cell.
+ if (add && add->size != 0) {
+ uptr last = --add->size;
+ Cell *c1 = &add->cells[last];
+ c->val = c1->val;
+ uptr addr1 = atomic_load(&c1->addr, memory_order_relaxed);
+ atomic_store(&c->addr, addr1, memory_order_release);
+ atomic_store(&c1->addr, 0, memory_order_release);
+ }
+ } else {
+ // Removed from add array, compact it.
+ uptr last = --add->size;
+ Cell *c1 = &add->cells[last];
+ if (c != c1) {
+ *c = *c1;
+ atomic_store(&c1->addr, 0, memory_order_relaxed);
+ }
+ }
+ if (add && add->size == 0) {
+ // FIXME(dvyukov): free add?
+ }
+ b->mtx.Unlock();
+ } else {
+ CHECK_EQ(addr1, h->addr_);
+ if (h->addidx_ != -1U)
+ b->mtx.ReadUnlock();
+ }
+ }
template<typename T, uptr kSize>
uptr AddrHashMap<T, kSize>::calcHash(uptr addr) {
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.cpp b/libsanitizer/sanitizer_common/sanitizer_allocator.cpp
index 3157b35ffaf..bcb7370a790 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator.cpp
@@ -137,14 +137,6 @@ static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
#endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
-namespace {
-const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull;
-
-struct BlockHeader {
- u64 magic;
-};
-} // namespace
-
static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
SetAllocatorOutOfMemory();
Report("FATAL: %s: internal allocator is out of memory trying to allocate "
@@ -153,28 +145,17 @@ static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
}
void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) {
- uptr s = size + sizeof(BlockHeader);
- if (s < size)
- return nullptr;
- BlockHeader *p = (BlockHeader *)RawInternalAlloc(s, cache, alignment);
+ void *p = RawInternalAlloc(size, cache, alignment);
if (UNLIKELY(!p))
- ReportInternalAllocatorOutOfMemory(s);
- p->magic = kBlockMagic;
- return p + 1;
+ ReportInternalAllocatorOutOfMemory(size);
+ return p;
}
void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
- if (!addr)
- return InternalAlloc(size, cache);
- uptr s = size + sizeof(BlockHeader);
- if (s < size)
- return nullptr;
- BlockHeader *p = (BlockHeader *)addr - 1;
- CHECK_EQ(kBlockMagic, p->magic);
- p = (BlockHeader *)RawInternalRealloc(p, s, cache);
+ void *p = RawInternalRealloc(addr, size, cache);
if (UNLIKELY(!p))
- ReportInternalAllocatorOutOfMemory(s);
- return p + 1;
+ ReportInternalAllocatorOutOfMemory(size);
+ return p;
}
void *InternalReallocArray(void *addr, uptr count, uptr size,
@@ -203,12 +184,7 @@ void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {
}
void InternalFree(void *addr, InternalAllocatorCache *cache) {
- if (!addr)
- return;
- BlockHeader *p = (BlockHeader *)addr - 1;
- CHECK_EQ(kBlockMagic, p->magic);
- p->magic = 0;
- RawInternalFree(p, cache);
+ RawInternalFree(addr, cache);
}
// LowLevelAllocator
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
index eb836bc4787..0e81e6764f9 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
@@ -177,12 +177,12 @@ class CombinedAllocator {
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() {
+ void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
primary_.ForceLock();
secondary_.ForceLock();
}
- void ForceUnlock() {
+ void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
secondary_.ForceUnlock();
primary_.ForceUnlock();
}
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h b/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h
index 108dfc231a2..e495c56f037 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h
@@ -17,6 +17,7 @@
template <class SizeClassAllocator>
struct SizeClassAllocator64LocalCache {
typedef SizeClassAllocator Allocator;
+ typedef MemoryMapper<Allocator> MemoryMapperT;
void Init(AllocatorGlobalStats *s) {
stats_.Init();
@@ -53,7 +54,7 @@ struct SizeClassAllocator64LocalCache {
PerClass *c = &per_class_[class_id];
InitCache(c);
if (UNLIKELY(c->count == c->max_count))
- Drain(c, allocator, class_id, c->max_count / 2);
+ DrainHalfMax(c, allocator, class_id);
CompactPtrT chunk = allocator->PointerToCompactPtr(
allocator->GetRegionBeginBySizeClass(class_id),
reinterpret_cast<uptr>(p));
@@ -62,10 +63,10 @@ struct SizeClassAllocator64LocalCache {
}
void Drain(SizeClassAllocator *allocator) {
+ MemoryMapperT memory_mapper(*allocator);
for (uptr i = 1; i < kNumClasses; i++) {
PerClass *c = &per_class_[i];
- while (c->count > 0)
- Drain(c, allocator, i, c->count);
+ while (c->count > 0) Drain(&memory_mapper, c, allocator, i, c->count);
}
}
@@ -106,12 +107,18 @@ struct SizeClassAllocator64LocalCache {
return true;
}
- NOINLINE void Drain(PerClass *c, SizeClassAllocator *allocator, uptr class_id,
- uptr count) {
+ NOINLINE void DrainHalfMax(PerClass *c, SizeClassAllocator *allocator,
+ uptr class_id) {
+ MemoryMapperT memory_mapper(*allocator);
+ Drain(&memory_mapper, c, allocator, class_id, c->max_count / 2);
+ }
+
+ void Drain(MemoryMapperT *memory_mapper, PerClass *c,
+ SizeClassAllocator *allocator, uptr class_id, uptr count) {
CHECK_GE(c->count, count);
const uptr first_idx_to_drain = c->count - count;
c->count -= count;
- allocator->ReturnToAllocator(&stats_, class_id,
+ allocator->ReturnToAllocator(memory_mapper, &stats_, class_id,
&c->chunks[first_idx_to_drain], count);
}
};
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
index fb5394cd39c..38d2a7d117f 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
@@ -237,13 +237,13 @@ class SizeClassAllocator32 {
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() {
+ void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
for (uptr i = 0; i < kNumClasses; i++) {
GetSizeClassInfo(i)->mutex.Lock();
}
}
- void ForceUnlock() {
+ void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
for (int i = kNumClasses - 1; i >= 0; i--) {
GetSizeClassInfo(i)->mutex.Unlock();
}
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
index db30e138154..b142ee0131b 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
@@ -42,6 +42,44 @@ struct SizeClassAllocator64FlagMasks { // Bit masks.
};
};
+template <typename Allocator>
+class MemoryMapper {
+ public:
+ typedef typename Allocator::CompactPtrT CompactPtrT;
+
+ explicit MemoryMapper(const Allocator &allocator) : allocator_(allocator) {}
+
+ bool GetAndResetStats(uptr &ranges, uptr &bytes) {
+ ranges = released_ranges_count_;
+ released_ranges_count_ = 0;
+ bytes = released_bytes_;
+ released_bytes_ = 0;
+ return ranges != 0;
+ }
+
+ u64 *MapPackedCounterArrayBuffer(uptr count) {
+ buffer_.clear();
+ buffer_.resize(count);
+ return buffer_.data();
+ }
+
+ // Releases [from, to) range of pages back to OS.
+ void ReleasePageRangeToOS(uptr class_id, CompactPtrT from, CompactPtrT to) {
+ const uptr region_base = allocator_.GetRegionBeginBySizeClass(class_id);
+ const uptr from_page = allocator_.CompactPtrToPointer(region_base, from);
+ const uptr to_page = allocator_.CompactPtrToPointer(region_base, to);
+ ReleaseMemoryPagesToOS(from_page, to_page);
+ released_ranges_count_++;
+ released_bytes_ += to_page - from_page;
+ }
+
+ private:
+ const Allocator &allocator_;
+ uptr released_ranges_count_ = 0;
+ uptr released_bytes_ = 0;
+ InternalMmapVector<u64> buffer_;
+};
+
template <class Params>
class SizeClassAllocator64 {
public:
@@ -57,6 +95,7 @@ class SizeClassAllocator64 {
typedef SizeClassAllocator64<Params> ThisT;
typedef SizeClassAllocator64LocalCache<ThisT> AllocatorCache;
+ typedef MemoryMapper<ThisT> MemoryMapperT;
// When we know the size class (the region base) we can represent a pointer
// as a 4-byte integer (offset from the region start shifted right by 4).
@@ -120,9 +159,10 @@ class SizeClassAllocator64 {
}
void ForceReleaseToOS() {
+ MemoryMapperT memory_mapper(*this);
for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
BlockingMutexLock l(&GetRegionInfo(class_id)->mutex);
- MaybeReleaseToOS(class_id, true /*force*/);
+ MaybeReleaseToOS(&memory_mapper, class_id, true /*force*/);
}
}
@@ -131,7 +171,8 @@ class SizeClassAllocator64 {
alignment <= SizeClassMap::kMaxSize;
}
- NOINLINE void ReturnToAllocator(AllocatorStats *stat, uptr class_id,
+ NOINLINE void ReturnToAllocator(MemoryMapperT *memory_mapper,
+ AllocatorStats *stat, uptr class_id,
const CompactPtrT *chunks, uptr n_chunks) {
RegionInfo *region = GetRegionInfo(class_id);
uptr region_beg = GetRegionBeginBySizeClass(class_id);
@@ -154,7 +195,7 @@ class SizeClassAllocator64 {
region->num_freed_chunks = new_num_freed_chunks;
region->stats.n_freed += n_chunks;
- MaybeReleaseToOS(class_id, false /*force*/);
+ MaybeReleaseToOS(memory_mapper, class_id, false /*force*/);
}
NOINLINE bool GetFromAllocator(AllocatorStats *stat, uptr class_id,
@@ -312,13 +353,13 @@ class SizeClassAllocator64 {
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() {
+ void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
for (uptr i = 0; i < kNumClasses; i++) {
GetRegionInfo(i)->mutex.Lock();
}
}
- void ForceUnlock() {
+ void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
for (int i = (int)kNumClasses - 1; i >= 0; i--) {
GetRegionInfo(i)->mutex.Unlock();
}
@@ -362,11 +403,11 @@ class SizeClassAllocator64 {
// For the performance sake, none of the accessors check the validity of the
// arguments, it is assumed that index is always in [0, n) range and the value
// is not incremented past max_value.
- template<class MemoryMapperT>
class PackedCounterArray {
public:
- PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper)
- : n(num_counters), memory_mapper(mapper) {
+ template <typename MemoryMapper>
+ PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapper *mapper)
+ : n(num_counters) {
CHECK_GT(num_counters, 0);
CHECK_GT(max_value, 0);
constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL;
@@ -383,16 +424,8 @@ class SizeClassAllocator64 {
packing_ratio_log = Log2(packing_ratio);
bit_offset_mask = packing_ratio - 1;
- buffer_size =
- (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) *
- sizeof(*buffer);
- buffer = reinterpret_cast<u64*>(
- memory_mapper->MapPackedCounterArrayBuffer(buffer_size));
- }
- ~PackedCounterArray() {
- if (buffer) {
- memory_mapper->UnmapPackedCounterArrayBuffer(buffer, buffer_size);
- }
+ buffer = mapper->MapPackedCounterArrayBuffer(
+ RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log);
}
bool IsAllocated() const {
@@ -429,19 +462,16 @@ class SizeClassAllocator64 {
u64 counter_mask;
u64 packing_ratio_log;
u64 bit_offset_mask;
-
- MemoryMapperT* const memory_mapper;
- u64 buffer_size;
u64* buffer;
};
- template<class MemoryMapperT>
+ template <class MemoryMapperT>
class FreePagesRangeTracker {
public:
- explicit FreePagesRangeTracker(MemoryMapperT* mapper)
+ FreePagesRangeTracker(MemoryMapperT *mapper, uptr class_id)
: memory_mapper(mapper),
- page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)),
- in_the_range(false), current_page(0), current_range_start_page(0) {}
+ class_id(class_id),
+ page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)) {}
void NextPage(bool freed) {
if (freed) {
@@ -463,28 +493,30 @@ class SizeClassAllocator64 {
void CloseOpenedRange() {
if (in_the_range) {
memory_mapper->ReleasePageRangeToOS(
- current_range_start_page << page_size_scaled_log,
+ class_id, current_range_start_page << page_size_scaled_log,
current_page << page_size_scaled_log);
in_the_range = false;
}
}
- MemoryMapperT* const memory_mapper;
- const uptr page_size_scaled_log;
- bool in_the_range;
- uptr current_page;
- uptr current_range_start_page;
+ MemoryMapperT *const memory_mapper = nullptr;
+ const uptr class_id = 0;
+ const uptr page_size_scaled_log = 0;
+ bool in_the_range = false;
+ uptr current_page = 0;
+ uptr current_range_start_page = 0;
};
// Iterates over the free_array to identify memory pages containing freed
// chunks only and returns these pages back to OS.
// allocated_pages_count is the total number of pages allocated for the
// current bucket.
- template<class MemoryMapperT>
+ template <typename MemoryMapper>
static void ReleaseFreeMemoryToOS(CompactPtrT *free_array,
uptr free_array_count, uptr chunk_size,
uptr allocated_pages_count,
- MemoryMapperT *memory_mapper) {
+ MemoryMapper *memory_mapper,
+ uptr class_id) {
const uptr page_size = GetPageSizeCached();
// Figure out the number of chunks per page and whether we can take a fast
@@ -520,9 +552,8 @@ class SizeClassAllocator64 {
UNREACHABLE("All chunk_size/page_size ratios must be handled.");
}
- PackedCounterArray<MemoryMapperT> counters(allocated_pages_count,
- full_pages_chunk_count_max,
- memory_mapper);
+ PackedCounterArray counters(allocated_pages_count,
+ full_pages_chunk_count_max, memory_mapper);
if (!counters.IsAllocated())
return;
@@ -547,7 +578,7 @@ class SizeClassAllocator64 {
// Iterate over pages detecting ranges of pages with chunk counters equal
// to the expected number of chunks for the particular page.
- FreePagesRangeTracker<MemoryMapperT> range_tracker(memory_mapper);
+ FreePagesRangeTracker<MemoryMapper> range_tracker(memory_mapper, class_id);
if (same_chunk_count_per_page) {
// Fast path, every page has the same number of chunks affecting it.
for (uptr i = 0; i < counters.GetCount(); i++)
@@ -586,7 +617,7 @@ class SizeClassAllocator64 {
}
private:
- friend class MemoryMapper;
+ friend class MemoryMapper<ThisT>;
ReservedAddressRange address_range;
@@ -820,57 +851,13 @@ class SizeClassAllocator64 {
return true;
}
- class MemoryMapper {
- public:
- MemoryMapper(const ThisT& base_allocator, uptr class_id)
- : allocator(base_allocator),
- region_base(base_allocator.GetRegionBeginBySizeClass(class_id)),
- released_ranges_count(0),
- released_bytes(0) {
- }
-
- uptr GetReleasedRangesCount() const {
- return released_ranges_count;
- }
-
- uptr GetReleasedBytes() const {
- return released_bytes;
- }
-
- void *MapPackedCounterArrayBuffer(uptr buffer_size) {
- // TODO(alekseyshl): The idea to explore is to check if we have enough
- // space between num_freed_chunks*sizeof(CompactPtrT) and
- // mapped_free_array to fit buffer_size bytes and use that space instead
- // of mapping a temporary one.
- return MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters");
- }
-
- void UnmapPackedCounterArrayBuffer(void *buffer, uptr buffer_size) {
- UnmapOrDie(buffer, buffer_size);
- }
-
- // Releases [from, to) range of pages back to OS.
- void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) {
- const uptr from_page = allocator.CompactPtrToPointer(region_base, from);
- const uptr to_page = allocator.CompactPtrToPointer(region_base, to);
- ReleaseMemoryPagesToOS(from_page, to_page);
- released_ranges_count++;
- released_bytes += to_page - from_page;
- }
-
- private:
- const ThisT& allocator;
- const uptr region_base;
- uptr released_ranges_count;
- uptr released_bytes;
- };
-
// Attempts to release RAM occupied by freed chunks back to OS. The region is
// expected to be locked.
//
// TODO(morehouse): Support a callback on memory release so HWASan can release
// aliases as well.
- void MaybeReleaseToOS(uptr class_id, bool force) {
+ void MaybeReleaseToOS(MemoryMapperT *memory_mapper, uptr class_id,
+ bool force) {
RegionInfo *region = GetRegionInfo(class_id);
const uptr chunk_size = ClassIdToSize(class_id);
const uptr page_size = GetPageSizeCached();
@@ -894,17 +881,16 @@ class SizeClassAllocator64 {
}
}
- MemoryMapper memory_mapper(*this, class_id);
-
- ReleaseFreeMemoryToOS<MemoryMapper>(
+ ReleaseFreeMemoryToOS(
GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size,
- RoundUpTo(region->allocated_user, page_size) / page_size,
- &memory_mapper);
+ RoundUpTo(region->allocated_user, page_size) / page_size, memory_mapper,
+ class_id);
- if (memory_mapper.GetReleasedRangesCount() > 0) {
+ uptr ranges, bytes;
+ if (memory_mapper->GetAndResetStats(ranges, bytes)) {
region->rtoi.n_freed_at_last_release = region->stats.n_freed;
- region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount();
- region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes();
+ region->rtoi.num_releases += ranges;
+ region->rtoi.last_released_bytes = bytes;
}
region->rtoi.last_release_at_ns = MonotonicNanoTime();
}
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
index 61fb9874237..dd34fe85cc3 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
@@ -267,13 +267,9 @@ class LargeMmapAllocator {
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() {
- mutex_.Lock();
- }
+ void ForceLock() ACQUIRE(mutex_) { mutex_.Lock(); }
- void ForceUnlock() {
- mutex_.Unlock();
- }
+ void ForceUnlock() RELEASE(mutex_) { mutex_.Unlock(); }
// Iterate over all existing chunks.
// The allocator must be locked when calling this function.
diff --git a/libsanitizer/sanitizer_common/sanitizer_common.cpp b/libsanitizer/sanitizer_common/sanitizer_common.cpp
index 33960d94a2f..5fae8e33b90 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_common.cpp
@@ -37,10 +37,9 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
const char *mmap_type, error_t err,
bool raw_report) {
static int recursion_count;
- if (SANITIZER_RTEMS || raw_report || recursion_count) {
- // If we are on RTEMS or raw report is requested or we went into recursion,
- // just die. The Report() and CHECK calls below may call mmap recursively
- // and fail.
+ if (raw_report || recursion_count) {
+ // If raw report is requested or we went into recursion just die. The
+ // Report() and CHECK calls below may call mmap recursively and fail.
RawWrite("ERROR: Failed to mmap\n");
Die();
}
@@ -331,6 +330,14 @@ static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr),
return 0;
}
+void internal_sleep(unsigned seconds) {
+ internal_usleep((u64)seconds * 1000 * 1000);
+}
+void SleepForSeconds(unsigned seconds) {
+ internal_usleep((u64)seconds * 1000 * 1000);
+}
+void SleepForMillis(unsigned millis) { internal_usleep((u64)millis * 1000); }
+
} // namespace __sanitizer
using namespace __sanitizer;
diff --git a/libsanitizer/sanitizer_common/sanitizer_common.h b/libsanitizer/sanitizer_common/sanitizer_common.h
index 7b65dd7dfb8..cbdbb0c4c4b 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common.h
+++ b/libsanitizer/sanitizer_common/sanitizer_common.h
@@ -237,10 +237,16 @@ void SetPrintfAndReportCallback(void (*callback)(const char *));
// Lock sanitizer error reporting and protects against nested errors.
class ScopedErrorReportLock {
public:
- ScopedErrorReportLock();
- ~ScopedErrorReportLock();
+ ScopedErrorReportLock() ACQUIRE(mutex_) { Lock(); }
+ ~ScopedErrorReportLock() RELEASE(mutex_) { Unlock(); }
- static void CheckLocked();
+ static void Lock() ACQUIRE(mutex_);
+ static void Unlock() RELEASE(mutex_);
+ static void CheckLocked() CHECK_LOCKED(mutex_);
+
+ private:
+ static atomic_uintptr_t reporting_thread_;
+ static StaticSpinMutex mutex_;
};
extern uptr stoptheworld_tracer_pid;
@@ -288,8 +294,8 @@ void InitTlsSize();
uptr GetTlsSize();
// Other
-void SleepForSeconds(int seconds);
-void SleepForMillis(int millis);
+void SleepForSeconds(unsigned seconds);
+void SleepForMillis(unsigned millis);
u64 NanoTime();
u64 MonotonicNanoTime();
int Atexit(void (*function)(void));
@@ -1057,6 +1063,13 @@ class ArrayRef {
T *end_ = nullptr;
};
+#define PRINTF_128(v) \
+ (*((u8 *)&v + 0)), (*((u8 *)&v + 1)), (*((u8 *)&v + 2)), (*((u8 *)&v + 3)), \
+ (*((u8 *)&v + 4)), (*((u8 *)&v + 5)), (*((u8 *)&v + 6)), \
+ (*((u8 *)&v + 7)), (*((u8 *)&v + 8)), (*((u8 *)&v + 9)), \
+ (*((u8 *)&v + 10)), (*((u8 *)&v + 11)), (*((u8 *)&v + 12)), \
+ (*((u8 *)&v + 13)), (*((u8 *)&v + 14)), (*((u8 *)&v + 15))
+
} // namespace __sanitizer
inline void *operator new(__sanitizer::operator_new_size_type size,
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
index 7867fccde39..6205d853a4c 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
@@ -134,11 +134,11 @@ extern const short *_tolower_tab_;
// Platform-specific options.
#if SANITIZER_MAC
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
#elif SANITIZER_WINDOWS64
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
#else
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE true
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
#endif // SANITIZER_MAC
#ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
@@ -823,11 +823,11 @@ INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
// N.B.: If we switch this to internal_ we'll have to use internal_memmove
// due to memcpy being an alias of memmove on OS X.
void *ctx;
- if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) {
+#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
- } else {
+#else
COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
- }
+#endif
}
#define INIT_MEMCPY \
@@ -957,6 +957,7 @@ INTERCEPTOR(double, frexp, double x, int *exp) {
// Assuming frexp() always writes to |exp|.
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
double res = REAL(frexp)(x, exp);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp));
return res;
}
@@ -969,22 +970,18 @@ INTERCEPTOR(double, frexp, double x, int *exp) {
INTERCEPTOR(float, frexpf, float x, int *exp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, frexpf, x, exp);
- // FIXME: under ASan the call below may write to freed memory and corrupt
- // its metadata. See
- // https://github.com/google/sanitizers/issues/321.
- float res = REAL(frexpf)(x, exp);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
+ float res = REAL(frexpf)(x, exp);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp));
return res;
}
INTERCEPTOR(long double, frexpl, long double x, int *exp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, frexpl, x, exp);
- // FIXME: under ASan the call below may write to freed memory and corrupt
- // its metadata. See
- // https://github.com/google/sanitizers/issues/321.
- long double res = REAL(frexpl)(x, exp);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
+ long double res = REAL(frexpl)(x, exp);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp));
return res;
}
@@ -5303,6 +5300,12 @@ INTERCEPTOR(__sanitizer_clock_t, times, void *tms) {
#define INIT_TIMES
#endif
+#if SANITIZER_S390 && \
+ (SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET)
+extern "C" uptr __tls_get_offset_wrapper(void *arg, uptr (*fn)(void *arg));
+DEFINE_REAL(uptr, __tls_get_offset, void *arg)
+#endif
+
#if SANITIZER_INTERCEPT_TLS_GET_ADDR
#if !SANITIZER_S390
#define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_addr)
@@ -5342,11 +5345,7 @@ void *__tls_get_addr_opt(void *arg);
// descriptor offset as an argument instead of a pointer. GOT address
// is passed in r12, so it's necessary to write it in assembly. This is
// the function used by the compiler.
-extern "C" uptr __tls_get_offset_wrapper(void *arg, uptr (*fn)(void *arg));
#define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_offset)
-DEFINE_REAL(uptr, __tls_get_offset, void *arg)
-extern "C" uptr __tls_get_offset(void *arg);
-extern "C" uptr __interceptor___tls_get_offset(void *arg);
INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, __tls_get_addr_internal, arg);
@@ -5362,6 +5361,15 @@ INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
}
return res;
}
+#endif // SANITIZER_S390
+#else
+#define INIT_TLS_GET_ADDR
+#endif
+
+#if SANITIZER_S390 && \
+ (SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET)
+extern "C" uptr __tls_get_offset(void *arg);
+extern "C" uptr __interceptor___tls_get_offset(void *arg);
// We need a hidden symbol aliasing the above, so that we can jump
// directly to it from the assembly below.
extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"),
@@ -5400,9 +5408,6 @@ asm(
"br %r3\n"
".size __tls_get_offset_wrapper, .-__tls_get_offset_wrapper\n"
);
-#endif // SANITIZER_S390
-#else
-#define INIT_TLS_GET_ADDR
#endif
#if SANITIZER_INTERCEPT_LISTXATTR
@@ -6099,6 +6104,40 @@ INTERCEPTOR(__sanitizer_FILE *, freopen, const char *path, const char *mode,
#define INIT_FOPEN
#endif
+#if SANITIZER_INTERCEPT_FLOPEN
+INTERCEPTOR(int, flopen, const char *path, int flags, ...) {
+ void *ctx;
+ va_list ap;
+ va_start(ap, flags);
+ u16 mode = static_cast<u16>(va_arg(ap, u32));
+ va_end(ap);
+ COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
+ if (path) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ }
+ return REAL(flopen)(path, flags, mode);
+}
+
+INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
+ void *ctx;
+ va_list ap;
+ va_start(ap, flags);
+ u16 mode = static_cast<u16>(va_arg(ap, u32));
+ va_end(ap);
+ COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
+ if (path) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ }
+ return REAL(flopenat)(dirfd, path, flags, mode);
+}
+
+#define INIT_FLOPEN \
+ COMMON_INTERCEPT_FUNCTION(flopen); \
+ COMMON_INTERCEPT_FUNCTION(flopenat);
+#else
+#define INIT_FLOPEN
+#endif
+
#if SANITIZER_INTERCEPT_FOPEN64
INTERCEPTOR(__sanitizer_FILE *, fopen64, const char *path, const char *mode) {
void *ctx;
@@ -6463,7 +6502,7 @@ INTERCEPTOR(int, sem_wait, __sanitizer_sem_t *s) {
INTERCEPTOR(int, sem_trywait, __sanitizer_sem_t *s) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, sem_trywait, s);
- int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_trywait)(s);
+ int res = REAL(sem_trywait)(s);
if (res == 0) {
COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s);
}
@@ -10264,6 +10303,7 @@ static void InitializeCommonInterceptors() {
INIT_LIBIO_INTERNALS;
INIT_FOPEN;
INIT_FOPEN64;
+ INIT_FLOPEN;
INIT_OPEN_MEMSTREAM;
INIT_OBSTACK;
INIT_FFLUSH;
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp
index 1037938f3d3..01ccacc6f32 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp
@@ -138,7 +138,7 @@ uptr ReservedAddressRange::InitAligned(uptr size, uptr align,
return start;
}
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#if !SANITIZER_FUCHSIA
// Reserve memory range [beg, end].
// We need to use inclusive range because end+1 may not be representable.
@@ -189,7 +189,7 @@ void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,
Die();
}
-#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#endif // !SANITIZER_FUCHSIA
} // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp b/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp
index 487a634a165..9a4e5388f24 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp
@@ -25,7 +25,6 @@ void LogMessageOnPrintf(const char *str) {}
#endif
void WriteToSyslog(const char *buffer) {}
void Abort() { internal__exit(1); }
-void SleepForSeconds(int seconds) { internal_sleep(seconds); }
#endif // !SANITIZER_WINDOWS
#if !SANITIZER_WINDOWS && !SANITIZER_MAC
diff --git a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp
index 2c924f5d396..ccb7065b07a 100644
--- a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp
@@ -136,7 +136,7 @@ void DD::ReportDeadlock(DDCallback *cb, DDMutex *m) {
DDMutex *m0 = (DDMutex*)dd.getData(from);
DDMutex *m1 = (DDMutex*)dd.getData(to);
- u32 stk_from = -1U, stk_to = -1U;
+ u32 stk_from = 0, stk_to = 0;
int unique_tid = 0;
dd.findEdge(from, to, &stk_from, &stk_to, &unique_tid);
// Printf("Edge: %zd=>%zd: %u/%u T%d\n", from, to, stk_from, stk_to,
diff --git a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector2.cpp b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector2.cpp
index e3f8e1b1276..1fbbbcccfa9 100644
--- a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector2.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector2.cpp
@@ -73,7 +73,7 @@ struct DDLogicalThread {
int nlocked;
};
-struct Mutex {
+struct MutexState {
StaticSpinMutex mtx;
u32 seq;
int nlink;
@@ -101,12 +101,12 @@ struct DD final : public DDetector {
void CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, DDMutex *mtx);
void Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath);
u32 allocateId(DDCallback *cb);
- Mutex *getMutex(u32 id);
- u32 getMutexId(Mutex *m);
+ MutexState *getMutex(u32 id);
+ u32 getMutexId(MutexState *m);
DDFlags flags;
- Mutex* mutex[kL1Size];
+ MutexState *mutex[kL1Size];
SpinMutex mtx;
InternalMmapVector<u32> free_id;
@@ -152,13 +152,11 @@ void DD::MutexInit(DDCallback *cb, DDMutex *m) {
atomic_store(&m->owner, 0, memory_order_relaxed);
}
-Mutex *DD::getMutex(u32 id) {
- return &mutex[id / kL2Size][id % kL2Size];
-}
+MutexState *DD::getMutex(u32 id) { return &mutex[id / kL2Size][id % kL2Size]; }
-u32 DD::getMutexId(Mutex *m) {
+u32 DD::getMutexId(MutexState *m) {
for (int i = 0; i < kL1Size; i++) {
- Mutex *tab = mutex[i];
+ MutexState *tab = mutex[i];
if (tab == 0)
break;
if (m >= tab && m < tab + kL2Size)
@@ -176,8 +174,8 @@ u32 DD::allocateId(DDCallback *cb) {
} else {
CHECK_LT(id_gen, kMaxMutex);
if ((id_gen % kL2Size) == 0) {
- mutex[id_gen / kL2Size] = (Mutex*)MmapOrDie(kL2Size * sizeof(Mutex),
- "deadlock detector (mutex table)");
+ mutex[id_gen / kL2Size] = (MutexState *)MmapOrDie(
+ kL2Size * sizeof(MutexState), "deadlock detector (mutex table)");
}
id = id_gen++;
}
@@ -216,11 +214,11 @@ void DD::MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock) {
}
bool added = false;
- Mutex *mtx = getMutex(m->id);
+ MutexState *mtx = getMutex(m->id);
for (int i = 0; i < lt->nlocked - 1; i++) {
u32 id1 = lt->locked[i].id;
u32 stk1 = lt->locked[i].stk;
- Mutex *mtx1 = getMutex(id1);
+ MutexState *mtx1 = getMutex(id1);
SpinMutexLock l(&mtx1->mtx);
if (mtx1->nlink == kMaxLink) {
// FIXME(dvyukov): check stale links
@@ -342,7 +340,7 @@ void DD::MutexDestroy(DDCallback *cb, DDMutex *m) {
// Clear and invalidate the mutex descriptor.
{
- Mutex *mtx = getMutex(m->id);
+ MutexState *mtx = getMutex(m->id);
SpinMutexLock l(&mtx->mtx);
mtx->seq++;
mtx->nlink = 0;
@@ -361,7 +359,7 @@ void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt,
int npath = 0;
int npending = 0;
{
- Mutex *mtx = getMutex(m->id);
+ MutexState *mtx = getMutex(m->id);
SpinMutexLock l(&mtx->mtx);
for (int li = 0; li < mtx->nlink; li++)
pt->pending[npending++] = mtx->link[li];
@@ -374,7 +372,7 @@ void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt,
}
if (pt->visited[link.id])
continue;
- Mutex *mtx1 = getMutex(link.id);
+ MutexState *mtx1 = getMutex(link.id);
SpinMutexLock l(&mtx1->mtx);
if (mtx1->seq != link.seq)
continue;
@@ -387,7 +385,7 @@ void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt,
return Report(pt, lt, npath); // Bingo!
for (int li = 0; li < mtx1->nlink; li++) {
Link *link1 = &mtx1->link[li];
- // Mutex *mtx2 = getMutex(link->id);
+ // MutexState *mtx2 = getMutex(link->id);
// FIXME(dvyukov): fast seq check
// FIXME(dvyukov): fast nlink != 0 check
// FIXME(dvyukov): fast pending check?
diff --git a/libsanitizer/sanitizer_common/sanitizer_errno.h b/libsanitizer/sanitizer_common/sanitizer_errno.h
index 94f16b6e873..70a6e88dbaa 100644
--- a/libsanitizer/sanitizer_common/sanitizer_errno.h
+++ b/libsanitizer/sanitizer_common/sanitizer_errno.h
@@ -23,8 +23,7 @@
#if SANITIZER_FREEBSD || SANITIZER_MAC
# define __errno_location __error
-#elif SANITIZER_ANDROID || SANITIZER_NETBSD || \
- SANITIZER_RTEMS
+#elif SANITIZER_ANDROID || SANITIZER_NETBSD
# define __errno_location __errno
#elif SANITIZER_SOLARIS
# define __errno_location ___errno
diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp
index 4f692f99c20..65bc398656c 100644
--- a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp
@@ -36,16 +36,11 @@ uptr internal_sched_yield() {
return 0; // Why doesn't this return void?
}
-static void internal_nanosleep(zx_time_t ns) {
- zx_status_t status = _zx_nanosleep(_zx_deadline_after(ns));
+void internal_usleep(u64 useconds) {
+ zx_status_t status = _zx_nanosleep(_zx_deadline_after(ZX_USEC(useconds)));
CHECK_EQ(status, ZX_OK);
}
-unsigned int internal_sleep(unsigned int seconds) {
- internal_nanosleep(ZX_SEC(seconds));
- return 0;
-}
-
u64 NanoTime() {
zx_handle_t utc_clock = _zx_utc_reference_get();
CHECK_NE(utc_clock, ZX_HANDLE_INVALID);
@@ -78,10 +73,6 @@ void Abort() { abort(); }
int Atexit(void (*function)(void)) { return atexit(function); }
-void SleepForSeconds(int seconds) { internal_sleep(seconds); }
-
-void SleepForMillis(int millis) { internal_nanosleep(ZX_MSEC(millis)); }
-
void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) {
pthread_attr_t attr;
CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
@@ -109,6 +100,18 @@ bool SignalContext::IsStackOverflow() const { return false; }
void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
const char *SignalContext::Describe() const { UNIMPLEMENTED(); }
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+ zx_status_t status = _zx_futex_wait(reinterpret_cast<zx_futex_t *>(p), cmp,
+ ZX_HANDLE_INVALID, ZX_TIME_INFINITE);
+ if (status != ZX_ERR_BAD_STATE) // Normal race.
+ CHECK_EQ(status, ZX_OK);
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {
+ zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(p), count);
+ CHECK_EQ(status, ZX_OK);
+}
+
enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
BlockingMutex::BlockingMutex() {
@@ -145,8 +148,8 @@ void BlockingMutex::Unlock() {
}
}
-void BlockingMutex::CheckLocked() {
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+void BlockingMutex::CheckLocked() const {
+ auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
}
@@ -156,8 +159,10 @@ uptr GetMmapGranularity() { return _zx_system_get_page_size(); }
sanitizer_shadow_bounds_t ShadowBounds;
+void InitShadowBounds() { ShadowBounds = __sanitizer_shadow_bounds(); }
+
uptr GetMaxUserVirtualAddress() {
- ShadowBounds = __sanitizer_shadow_bounds();
+ InitShadowBounds();
return ShadowBounds.memory_limit - 1;
}
diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.h b/libsanitizer/sanitizer_common/sanitizer_fuchsia.h
index 96f9cde7ef1..26c1deab9e5 100644
--- a/libsanitizer/sanitizer_common/sanitizer_fuchsia.h
+++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.h
@@ -30,6 +30,8 @@ struct MemoryMappingLayoutData {
size_t current; // Current index into the vector.
};
+void InitShadowBounds();
+
} // namespace __sanitizer
#endif // SANITIZER_FUCHSIA
diff --git a/libsanitizer/sanitizer_common/sanitizer_libc.h b/libsanitizer/sanitizer_common/sanitizer_libc.h
index ec0a6ded009..bcb81ebbc80 100644
--- a/libsanitizer/sanitizer_common/sanitizer_libc.h
+++ b/libsanitizer/sanitizer_common/sanitizer_libc.h
@@ -67,7 +67,8 @@ uptr internal_ftruncate(fd_t fd, uptr size);
// OS
void NORETURN internal__exit(int exitcode);
-unsigned int internal_sleep(unsigned int seconds);
+void internal_sleep(unsigned seconds);
+void internal_usleep(u64 useconds);
uptr internal_getpid();
uptr internal_getppid();
diff --git a/libsanitizer/sanitizer_common/sanitizer_libignore.cpp b/libsanitizer/sanitizer_common/sanitizer_libignore.cpp
index a65d3d896e3..431efc574fa 100644
--- a/libsanitizer/sanitizer_common/sanitizer_libignore.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_libignore.cpp
@@ -84,6 +84,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
ignored_code_ranges_[idx].begin = range.beg;
ignored_code_ranges_[idx].end = range.end;
atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release);
+ atomic_store(&enabled_, 1, memory_order_release);
break;
}
}
@@ -114,6 +115,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
instrumented_code_ranges_[idx].end = range.end;
atomic_store(&instrumented_ranges_count_, idx + 1,
memory_order_release);
+ atomic_store(&enabled_, 1, memory_order_release);
}
}
}
@@ -123,6 +125,29 @@ void LibIgnore::OnLibraryUnloaded() {
OnLibraryLoaded(nullptr);
}
+bool LibIgnore::IsIgnoredSlow(uptr pc, bool *pc_in_ignored_lib) const {
+ const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire);
+ for (uptr i = 0; i < n; i++) {
+ if (IsInRange(pc, ignored_code_ranges_[i])) {
+ *pc_in_ignored_lib = true;
+ return true;
+ }
+ }
+ *pc_in_ignored_lib = false;
+ if (track_instrumented_libs_ && !IsPcInstrumented(pc))
+ return true;
+ return false;
+}
+
+bool LibIgnore::IsPcInstrumented(uptr pc) const {
+ const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire);
+ for (uptr i = 0; i < n; i++) {
+ if (IsInRange(pc, instrumented_code_ranges_[i]))
+ return true;
+ }
+ return false;
+}
+
} // namespace __sanitizer
#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC ||
diff --git a/libsanitizer/sanitizer_common/sanitizer_libignore.h b/libsanitizer/sanitizer_common/sanitizer_libignore.h
index 256f685979f..85452e57ba3 100644
--- a/libsanitizer/sanitizer_common/sanitizer_libignore.h
+++ b/libsanitizer/sanitizer_common/sanitizer_libignore.h
@@ -45,9 +45,6 @@ class LibIgnore {
// "pc_in_ignored_lib" if the PC is in an ignored library, false otherwise.
bool IsIgnored(uptr pc, bool *pc_in_ignored_lib) const;
- // Checks whether the provided PC belongs to an instrumented module.
- bool IsPcInstrumented(uptr pc) const;
-
private:
struct Lib {
char *templ;
@@ -61,6 +58,10 @@ class LibIgnore {
uptr end;
};
+ // Checks whether the provided PC belongs to an instrumented module.
+ bool IsPcInstrumented(uptr pc) const;
+ bool IsIgnoredSlow(uptr pc, bool *pc_in_ignored_lib) const;
+
inline bool IsInRange(uptr pc, const LibCodeRange &range) const {
return (pc >= range.begin && pc < range.end);
}
@@ -70,6 +71,8 @@ class LibIgnore {
static const uptr kMaxLibs = 1024;
// Hot part:
+ atomic_uintptr_t enabled_;
+
atomic_uintptr_t ignored_ranges_count_;
LibCodeRange ignored_code_ranges_[kMaxIgnoredRanges];
@@ -87,27 +90,11 @@ class LibIgnore {
void operator = (const LibIgnore&); // not implemented
};
-inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {
- const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire);
- for (uptr i = 0; i < n; i++) {
- if (IsInRange(pc, ignored_code_ranges_[i])) {
- *pc_in_ignored_lib = true;
- return true;
- }
- }
- *pc_in_ignored_lib = false;
- if (track_instrumented_libs_ && !IsPcInstrumented(pc))
- return true;
- return false;
-}
-
-inline bool LibIgnore::IsPcInstrumented(uptr pc) const {
- const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire);
- for (uptr i = 0; i < n; i++) {
- if (IsInRange(pc, instrumented_code_ranges_[i]))
- return true;
- }
- return false;
+ALWAYS_INLINE
+bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {
+ if (LIKELY(atomic_load(&enabled_, memory_order_acquire) == 0))
+ return false;
+ return IsIgnoredSlow(pc, pc_in_ignored_lib);
}
} // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_linux.cpp
index b371477755f..9b7d87eb85e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_linux.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_linux.cpp
@@ -430,13 +430,11 @@ uptr internal_sched_yield() {
return internal_syscall(SYSCALL(sched_yield));
}
-unsigned int internal_sleep(unsigned int seconds) {
+void internal_usleep(u64 useconds) {
struct timespec ts;
- ts.tv_sec = seconds;
- ts.tv_nsec = 0;
- int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts);
- if (res) return ts.tv_sec;
- return 0;
+ ts.tv_sec = useconds / 1000000;
+ ts.tv_nsec = (useconds % 1000000) * 1000;
+ internal_syscall(SYSCALL(nanosleep), &ts, &ts);
}
uptr internal_execve(const char *filename, char *const argv[],
@@ -641,11 +639,27 @@ char **GetEnviron() {
}
#if !SANITIZER_SOLARIS
-enum MutexState {
- MtxUnlocked = 0,
- MtxLocked = 1,
- MtxSleeping = 2
-};
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+# if SANITIZER_FREEBSD
+ _umtx_op(p, UMTX_OP_WAIT_UINT, cmp, 0, 0);
+# elif SANITIZER_NETBSD
+ sched_yield(); /* No userspace futex-like synchronization */
+# else
+ internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAIT_PRIVATE, cmp, 0, 0, 0);
+# endif
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {
+# if SANITIZER_FREEBSD
+ _umtx_op(p, UMTX_OP_WAKE, count, 0, 0);
+# elif SANITIZER_NETBSD
+ /* No userspace futex-like synchronization */
+# else
+ internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAKE_PRIVATE, count, 0, 0, 0);
+# endif
+}
+
+enum { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
BlockingMutex::BlockingMutex() {
internal_memset(this, 0, sizeof(*this));
@@ -683,11 +697,11 @@ void BlockingMutex::Unlock() {
}
}
-void BlockingMutex::CheckLocked() {
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+void BlockingMutex::CheckLocked() const {
+ auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
}
-#endif // !SANITIZER_SOLARIS
+# endif // !SANITIZER_SOLARIS
// ----------------- sanitizer_linux.h
// The actual size of this structure is specified by d_reclen.
@@ -884,7 +898,7 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
__sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
const uptr idx = signum / (sizeof(k_set->sig[0]) * 8);
const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);
- k_set->sig[idx] &= ~(1 << bit);
+ k_set->sig[idx] &= ~((uptr)1 << bit);
}
bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
@@ -894,7 +908,7 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
__sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
const uptr idx = signum / (sizeof(k_set->sig[0]) * 8);
const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);
- return k_set->sig[idx] & (1 << bit);
+ return k_set->sig[idx] & ((uptr)1 << bit);
}
#elif SANITIZER_FREEBSD
void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp
index 4f9577a97e2..7ce9e25da34 100644
--- a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -203,7 +203,7 @@ void InitTlsSize() {
g_use_dlpi_tls_data =
GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
-#if defined(__x86_64__) || defined(__powerpc64__)
+#if defined(__aarch64__) || defined(__x86_64__) || defined(__powerpc64__)
void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
size_t tls_align;
((void (*)(size_t *, size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
@@ -317,21 +317,44 @@ struct TlsBlock {
};
} // namespace
+#ifdef __s390__
+extern "C" uptr __tls_get_offset(void *arg);
+
+static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
+ // The __tls_get_offset ABI requires %r12 to point to GOT and %r2 to be an
+ // offset of a struct tls_index inside GOT. We don't possess either of the
+ // two, so violate the letter of the "ELF Handling For Thread-Local
+ // Storage" document and assume that the implementation just dereferences
+ // %r2 + %r12.
+ uptr tls_index[2] = {ti_module, ti_offset};
+ register uptr r2 asm("2") = 0;
+ register void *r12 asm("12") = tls_index;
+ asm("basr %%r14, %[__tls_get_offset]"
+ : "+r"(r2)
+ : [__tls_get_offset] "r"(__tls_get_offset), "r"(r12)
+ : "memory", "cc", "0", "1", "3", "4", "5", "14");
+ return r2;
+}
+#else
extern "C" void *__tls_get_addr(size_t *);
+#endif
static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
void *data) {
if (!info->dlpi_tls_modid)
return 0;
uptr begin = (uptr)info->dlpi_tls_data;
-#ifndef __s390__
if (!g_use_dlpi_tls_data) {
// Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
// and FreeBSD.
+#ifdef __s390__
+ begin = (uptr)__builtin_thread_pointer() +
+ TlsGetOffset(info->dlpi_tls_modid, 0);
+#else
size_t mod_and_off[2] = {info->dlpi_tls_modid, 0};
begin = (uptr)__tls_get_addr(mod_and_off);
- }
#endif
+ }
for (unsigned i = 0; i != info->dlpi_phnum; ++i)
if (info->dlpi_phdr[i].p_type == PT_TLS) {
static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back(
@@ -427,12 +450,16 @@ static void GetTls(uptr *addr, uptr *size) {
*size = 0;
}
#elif SANITIZER_GLIBC && defined(__x86_64__)
- // For x86-64, use an O(1) approach which requires precise
- // ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
+ // For aarch64 and x86-64, use an O(1) approach which requires relatively
+ // precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
asm("mov %%fs:16,%0" : "=r"(*addr));
*size = g_tls_size;
*addr -= *size;
*addr += ThreadDescriptorSize();
+#elif SANITIZER_GLIBC && defined(__aarch64__)
+ *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
+ ThreadDescriptorSize();
+ *size = g_tls_size + ThreadDescriptorSize();
#elif SANITIZER_GLIBC && defined(__powerpc64__)
// Workaround for glibc<2.25(?). 2.27 is known to not need this.
uptr tp;
@@ -732,13 +759,9 @@ u32 GetNumberOfCPUs() {
#elif SANITIZER_SOLARIS
return sysconf(_SC_NPROCESSORS_ONLN);
#else
-#if defined(CPU_COUNT)
cpu_set_t CPUs;
CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
return CPU_COUNT(&CPUs);
-#else
- return 1;
-#endif
#endif
}
diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_mac.cpp
index 30a94fcba14..125ecac8b12 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mac.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_mac.cpp
@@ -37,7 +37,7 @@
extern char **environ;
#endif
-#if defined(__has_include) && __has_include(<os/trace.h>) && defined(__BLOCKS__)
+#if defined(__has_include) && __has_include(<os/trace.h>)
#define SANITIZER_OS_TRACE 1
#include <os/trace.h>
#else
@@ -70,15 +70,7 @@ extern "C" {
#include <mach/mach_time.h>
#include <mach/vm_statistics.h>
#include <malloc/malloc.h>
-#if defined(__has_builtin) && __has_builtin(__builtin_os_log_format)
-# include <os/log.h>
-#else
- /* Without support for __builtin_os_log_format, fall back to the older
- method. */
-# define OS_LOG_DEFAULT 0
-# define os_log_error(A,B,C) \
- asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", (C));
-#endif
+#include <os/log.h>
#include <pthread.h>
#include <sched.h>
#include <signal.h>
@@ -227,9 +219,7 @@ void internal__exit(int exitcode) {
_exit(exitcode);
}
-unsigned int internal_sleep(unsigned int seconds) {
- return sleep(seconds);
-}
+void internal_usleep(u64 useconds) { usleep(useconds); }
uptr internal_getpid() {
return getpid();
@@ -519,6 +509,13 @@ void MprotectMallocZones(void *addr, int prot) {
}
}
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+ // FIXME: implement actual blocking.
+ sched_yield();
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {}
+
BlockingMutex::BlockingMutex() {
internal_memset(this, 0, sizeof(*this));
}
@@ -534,7 +531,7 @@ void BlockingMutex::Unlock() {
OSSpinLockUnlock((OSSpinLock*)&opaque_storage_);
}
-void BlockingMutex::CheckLocked() {
+void BlockingMutex::CheckLocked() const {
CHECK_NE(*(OSSpinLock*)&opaque_storage_, 0);
}
diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.h b/libsanitizer/sanitizer_common/sanitizer_mac.h
index 96a5986a47a..0b6af5a3c0e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mac.h
+++ b/libsanitizer/sanitizer_common/sanitizer_mac.h
@@ -14,26 +14,6 @@
#include "sanitizer_common.h"
#include "sanitizer_platform.h"
-
-/* TARGET_OS_OSX is not present in SDKs before Darwin16 (macOS 10.12) use
- TARGET_OS_MAC (we have no support for iOS in any form for these versions,
- so there's no ambiguity). */
-#if !defined(TARGET_OS_OSX) && TARGET_OS_MAC
-# define TARGET_OS_OSX 1
-#endif
-
-/* Other TARGET_OS_xxx are not present on earlier versions, define them to
- 0 (we have no support for them; they are not valid targets anyway). */
-#ifndef TARGET_OS_IOS
-#define TARGET_OS_IOS 0
-#endif
-#ifndef TARGET_OS_TV
-#define TARGET_OS_TV 0
-#endif
-#ifndef TARGET_OS_WATCH
-#define TARGET_OS_WATCH 0
-#endif
-
#if SANITIZER_MAC
#include "sanitizer_posix.h"
diff --git a/libsanitizer/sanitizer_common/sanitizer_mutex.cpp b/libsanitizer/sanitizer_common/sanitizer_mutex.cpp
new file mode 100644
index 00000000000..bc2d83c42c1
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_mutex.cpp
@@ -0,0 +1,39 @@
+//===-- sanitizer_mutex.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_mutex.h"
+
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+void Semaphore::Wait() {
+ u32 count = atomic_load(&state_, memory_order_relaxed);
+ for (;;) {
+ if (count == 0) {
+ FutexWait(&state_, 0);
+ count = atomic_load(&state_, memory_order_relaxed);
+ continue;
+ }
+ if (atomic_compare_exchange_weak(&state_, &count, count - 1,
+ memory_order_acquire))
+ break;
+ }
+}
+
+void Semaphore::Post(u32 count) {
+ CHECK_NE(count, 0);
+ atomic_fetch_add(&state_, count, memory_order_release);
+ FutexWake(&state_, count);
+}
+
+} // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_mutex.h b/libsanitizer/sanitizer_common/sanitizer_mutex.h
index 40a65914299..e3ff650b2c5 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mutex.h
+++ b/libsanitizer/sanitizer_common/sanitizer_mutex.h
@@ -16,30 +16,29 @@
#include "sanitizer_atomic.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
+#include "sanitizer_thread_safety.h"
namespace __sanitizer {
-class StaticSpinMutex {
+class MUTEX StaticSpinMutex {
public:
void Init() {
atomic_store(&state_, 0, memory_order_relaxed);
}
- void Lock() {
+ void Lock() ACQUIRE() {
if (TryLock())
return;
LockSlow();
}
- bool TryLock() {
+ bool TryLock() TRY_ACQUIRE(true) {
return atomic_exchange(&state_, 1, memory_order_acquire) == 0;
}
- void Unlock() {
- atomic_store(&state_, 0, memory_order_release);
- }
+ void Unlock() RELEASE() { atomic_store(&state_, 0, memory_order_release); }
- void CheckLocked() {
+ void CheckLocked() const CHECK_LOCKED() {
CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1);
}
@@ -59,24 +58,223 @@ class StaticSpinMutex {
}
};
-class SpinMutex : public StaticSpinMutex {
+class MUTEX SpinMutex : public StaticSpinMutex {
public:
SpinMutex() {
Init();
}
private:
- SpinMutex(const SpinMutex&);
- void operator=(const SpinMutex&);
+ SpinMutex(const SpinMutex &) = delete;
+ void operator=(const SpinMutex &) = delete;
+};
+
+// Semaphore provides an OS-dependent way to park/unpark threads.
+// The last thread returned from Wait can destroy the object
+// (destruction-safety).
+class Semaphore {
+ public:
+ constexpr Semaphore() {}
+ Semaphore(const Semaphore &) = delete;
+ void operator=(const Semaphore &) = delete;
+
+ void Wait();
+ void Post(u32 count = 1);
+
+ private:
+ atomic_uint32_t state_ = {0};
+};
+
+// Reader-writer mutex.
+class MUTEX Mutex2 {
+ public:
+ constexpr Mutex2() {}
+
+ void Lock() ACQUIRE() {
+ u64 reset_mask = ~0ull;
+ u64 state = atomic_load_relaxed(&state_);
+ const uptr kMaxSpinIters = 1500;
+ for (uptr spin_iters = 0;; spin_iters++) {
+ u64 new_state;
+ bool locked = (state & (kWriterLock | kReaderLockMask)) != 0;
+ if (LIKELY(!locked)) {
+ // The mutex is not read-/write-locked, try to lock.
+ new_state = (state | kWriterLock) & reset_mask;
+ } else if (spin_iters > kMaxSpinIters) {
+ // We've spun enough, increment waiting writers count and block.
+ // The counter will be decremented by whoever wakes us.
+ new_state = (state + kWaitingWriterInc) & reset_mask;
+ } else if ((state & kWriterSpinWait) == 0) {
+ // Active spinning, but denote our presence so that unlocking
+ // thread does not wake up other threads.
+ new_state = state | kWriterSpinWait;
+ } else {
+ // Active spinning.
+ state = atomic_load(&state_, memory_order_relaxed);
+ continue;
+ }
+ if (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_acquire)))
+ continue;
+ if (LIKELY(!locked))
+ return; // We've locked the mutex.
+ if (spin_iters > kMaxSpinIters) {
+ // We've incremented waiting writers, so now block.
+ writers_.Wait();
+ spin_iters = 0;
+ state = atomic_load(&state_, memory_order_relaxed);
+ DCHECK_NE(state & kWriterSpinWait, 0);
+ } else {
+ // We've set kWriterSpinWait, but we are still in active spinning.
+ }
+ // We either blocked and were unblocked,
+ // or we just spun but set kWriterSpinWait.
+ // Either way we need to reset kWriterSpinWait
+ // next time we take the lock or block again.
+ reset_mask = ~kWriterSpinWait;
+ }
+ }
+
+ void Unlock() RELEASE() {
+ bool wake_writer;
+ u64 wake_readers;
+ u64 new_state;
+ u64 state = atomic_load_relaxed(&state_);
+ do {
+ DCHECK_NE(state & kWriterLock, 0);
+ DCHECK_EQ(state & kReaderLockMask, 0);
+ new_state = state & ~kWriterLock;
+ wake_writer =
+ (state & kWriterSpinWait) == 0 && (state & kWaitingWriterMask) != 0;
+ if (wake_writer)
+ new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
+ wake_readers =
+ (state & (kWriterSpinWait | kWaitingWriterMask)) != 0
+ ? 0
+ : ((state & kWaitingReaderMask) >> kWaitingReaderShift);
+ if (wake_readers)
+ new_state = (new_state & ~kWaitingReaderMask) +
+ (wake_readers << kReaderLockShift);
+ } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_release)));
+ if (UNLIKELY(wake_writer))
+ writers_.Post();
+ else if (UNLIKELY(wake_readers))
+ readers_.Post(wake_readers);
+ }
+
+ void ReadLock() ACQUIRE_SHARED() {
+ bool locked;
+ u64 new_state;
+ u64 state = atomic_load_relaxed(&state_);
+ do {
+ locked =
+ (state & kReaderLockMask) == 0 &&
+ (state & (kWriterLock | kWriterSpinWait | kWaitingWriterMask)) != 0;
+ if (LIKELY(!locked))
+ new_state = state + kReaderLockInc;
+ else
+ new_state = state + kWaitingReaderInc;
+ } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_acquire)));
+ if (UNLIKELY(locked))
+ readers_.Wait();
+ DCHECK_EQ(atomic_load_relaxed(&state_) & kWriterLock, 0);
+ DCHECK_NE(atomic_load_relaxed(&state_) & kReaderLockMask, 0);
+ }
+
+ void ReadUnlock() RELEASE_SHARED() {
+ bool wake;
+ u64 new_state;
+ u64 state = atomic_load_relaxed(&state_);
+ do {
+ DCHECK_NE(state & kReaderLockMask, 0);
+ DCHECK_EQ(state & (kWaitingReaderMask | kWriterLock), 0);
+ new_state = state - kReaderLockInc;
+ wake = (new_state & (kReaderLockMask | kWriterSpinWait)) == 0 &&
+ (new_state & kWaitingWriterMask) != 0;
+ if (wake)
+ new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
+ } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_release)));
+ if (UNLIKELY(wake))
+ writers_.Post();
+ }
+
+ // This function does not guarantee an explicit check that the calling thread
+ // is the thread which owns the mutex. This behavior, while more strictly
+ // correct, causes problems in cases like StopTheWorld, where a parent thread
+ // owns the mutex but a child checks that it is locked. Rather than
+ // maintaining complex state to work around those situations, the check only
+ // checks that the mutex is owned.
+ void CheckWriteLocked() const CHECK_LOCKED() {
+ CHECK(atomic_load(&state_, memory_order_relaxed) & kWriterLock);
+ }
+
+ void CheckLocked() const CHECK_LOCKED() { CheckWriteLocked(); }
+
+ void CheckReadLocked() const CHECK_LOCKED() {
+ CHECK(atomic_load(&state_, memory_order_relaxed) & kReaderLockMask);
+ }
+
+ private:
+ atomic_uint64_t state_ = {0};
+ Semaphore writers_;
+ Semaphore readers_;
+
+ // The state has 3 counters:
+ // - number of readers holding the lock,
+ // if non zero, the mutex is read-locked
+ // - number of waiting readers,
+ // if not zero, the mutex is write-locked
+ // - number of waiting writers,
+ // if non zero, the mutex is read- or write-locked
+ // And 2 flags:
+ // - writer lock
+ // if set, the mutex is write-locked
+ // - a writer is awake and spin-waiting
+ // the flag is used to prevent thundering herd problem
+ // (new writers are not woken if this flag is set)
+ //
+ // Writer support active spinning, readers does not.
+ // But readers are more aggressive and always take the mutex
+ // if there are any other readers.
+ // Writers hand off the mutex to readers: after wake up readers
+ // already assume ownership of the mutex (don't need to do any
+ // state updates). But the mutex is not handed off to writers,
+ // after wake up writers compete to lock the mutex again.
+ // This is needed to allow repeated write locks even in presence
+ // of other blocked writers.
+ static constexpr u64 kCounterWidth = 20;
+ static constexpr u64 kReaderLockShift = 0;
+ static constexpr u64 kReaderLockInc = 1ull << kReaderLockShift;
+ static constexpr u64 kReaderLockMask = ((1ull << kCounterWidth) - 1)
+ << kReaderLockShift;
+ static constexpr u64 kWaitingReaderShift = kCounterWidth;
+ static constexpr u64 kWaitingReaderInc = 1ull << kWaitingReaderShift;
+ static constexpr u64 kWaitingReaderMask = ((1ull << kCounterWidth) - 1)
+ << kWaitingReaderShift;
+ static constexpr u64 kWaitingWriterShift = 2 * kCounterWidth;
+ static constexpr u64 kWaitingWriterInc = 1ull << kWaitingWriterShift;
+ static constexpr u64 kWaitingWriterMask = ((1ull << kCounterWidth) - 1)
+ << kWaitingWriterShift;
+ static constexpr u64 kWriterLock = 1ull << (3 * kCounterWidth);
+ static constexpr u64 kWriterSpinWait = 1ull << (3 * kCounterWidth + 1);
+
+ Mutex2(const Mutex2 &) = delete;
+ void operator=(const Mutex2 &) = delete;
};
-class BlockingMutex {
+void FutexWait(atomic_uint32_t *p, u32 cmp);
+void FutexWake(atomic_uint32_t *p, u32 count);
+
+class MUTEX BlockingMutex {
public:
explicit constexpr BlockingMutex(LinkerInitialized)
: opaque_storage_ {0, }, owner_ {0} {}
BlockingMutex();
- void Lock();
- void Unlock();
+ void Lock() ACQUIRE();
+ void Unlock() RELEASE();
// This function does not guarantee an explicit check that the calling thread
// is the thread which owns the mutex. This behavior, while more strictly
@@ -85,7 +283,7 @@ class BlockingMutex {
// maintaining complex state to work around those situations, the check only
// checks that the mutex is owned, and assumes callers to be generally
// well-behaved.
- void CheckLocked();
+ void CheckLocked() const CHECK_LOCKED();
private:
// Solaris mutex_t has a member that requires 64-bit alignment.
@@ -94,7 +292,7 @@ class BlockingMutex {
};
// Reader-writer spin mutex.
-class RWMutex {
+class MUTEX RWMutex {
public:
RWMutex() {
atomic_store(&state_, kUnlocked, memory_order_relaxed);
@@ -104,7 +302,7 @@ class RWMutex {
CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked);
}
- void Lock() {
+ void Lock() ACQUIRE() {
u32 cmp = kUnlocked;
if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock,
memory_order_acquire))
@@ -112,27 +310,27 @@ class RWMutex {
LockSlow();
}
- void Unlock() {
+ void Unlock() RELEASE() {
u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release);
DCHECK_NE(prev & kWriteLock, 0);
(void)prev;
}
- void ReadLock() {
+ void ReadLock() ACQUIRE_SHARED() {
u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire);
if ((prev & kWriteLock) == 0)
return;
ReadLockSlow();
}
- void ReadUnlock() {
+ void ReadUnlock() RELEASE_SHARED() {
u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release);
DCHECK_EQ(prev & kWriteLock, 0);
DCHECK_GT(prev & ~kWriteLock, 0);
(void)prev;
}
- void CheckLocked() {
+ void CheckLocked() const CHECK_LOCKED() {
CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked);
}
@@ -171,46 +369,40 @@ class RWMutex {
}
}
- RWMutex(const RWMutex&);
- void operator = (const RWMutex&);
+ RWMutex(const RWMutex &) = delete;
+ void operator=(const RWMutex &) = delete;
};
-template<typename MutexType>
-class GenericScopedLock {
+template <typename MutexType>
+class SCOPED_LOCK GenericScopedLock {
public:
- explicit GenericScopedLock(MutexType *mu)
- : mu_(mu) {
+ explicit GenericScopedLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
mu_->Lock();
}
- ~GenericScopedLock() {
- mu_->Unlock();
- }
+ ~GenericScopedLock() RELEASE() { mu_->Unlock(); }
private:
MutexType *mu_;
- GenericScopedLock(const GenericScopedLock&);
- void operator=(const GenericScopedLock&);
+ GenericScopedLock(const GenericScopedLock &) = delete;
+ void operator=(const GenericScopedLock &) = delete;
};
-template<typename MutexType>
-class GenericScopedReadLock {
+template <typename MutexType>
+class SCOPED_LOCK GenericScopedReadLock {
public:
- explicit GenericScopedReadLock(MutexType *mu)
- : mu_(mu) {
+ explicit GenericScopedReadLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
mu_->ReadLock();
}
- ~GenericScopedReadLock() {
- mu_->ReadUnlock();
- }
+ ~GenericScopedReadLock() RELEASE() { mu_->ReadUnlock(); }
private:
MutexType *mu_;
- GenericScopedReadLock(const GenericScopedReadLock&);
- void operator=(const GenericScopedReadLock&);
+ GenericScopedReadLock(const GenericScopedReadLock &) = delete;
+ void operator=(const GenericScopedReadLock &) = delete;
};
typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock;
diff --git a/libsanitizer/sanitizer_common/sanitizer_netbsd.cpp b/libsanitizer/sanitizer_common/sanitizer_netbsd.cpp
index ac20f915fef..5e601bdcde1 100644
--- a/libsanitizer/sanitizer_common/sanitizer_netbsd.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_netbsd.cpp
@@ -215,15 +215,12 @@ void internal__exit(int exitcode) {
Die(); // Unreachable.
}
-unsigned int internal_sleep(unsigned int seconds) {
+void internal_usleep(u64 useconds) {
struct timespec ts;
- ts.tv_sec = seconds;
- ts.tv_nsec = 0;
+ ts.tv_sec = useconds / 1000000;
+ ts.tv_nsec = (useconds % 1000000) * 1000;
CHECK(&_sys___nanosleep50);
- int res = _sys___nanosleep50(&ts, &ts);
- if (res)
- return ts.tv_sec;
- return 0;
+ _sys___nanosleep50(&ts, &ts);
}
uptr internal_execve(const char *filename, char *const argv[],
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform.h b/libsanitizer/sanitizer_common/sanitizer_platform.h
index 2f6458431c8..4d3c08893c1 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform.h
+++ b/libsanitizer/sanitizer_common/sanitizer_platform.h
@@ -13,10 +13,9 @@
#define SANITIZER_PLATFORM_H
#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \
- !defined(__APPLE__) && !defined(_WIN32) && \
- !defined(__Fuchsia__) && !defined(__rtems__) && \
- !(defined(__sun__) && defined(__svr4__))
-# error "This operating system is not supported"
+ !defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__) && \
+ !(defined(__sun__) && defined(__svr4__))
+# error "This operating system is not supported"
#endif
// Get __GLIBC__ on a glibc platform. Exclude Android: features.h includes C
@@ -117,12 +116,6 @@
# define SANITIZER_FUCHSIA 0
#endif
-#if defined(__rtems__)
-# define SANITIZER_RTEMS 1
-#else
-# define SANITIZER_RTEMS 0
-#endif
-
#define SANITIZER_POSIX \
(SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
SANITIZER_NETBSD || SANITIZER_SOLARIS)
@@ -226,12 +219,6 @@
# define SANITIZER_SOLARIS32 0
#endif
-#if defined(__myriad2__)
-# define SANITIZER_MYRIAD2 1
-#else
-# define SANITIZER_MYRIAD2 0
-#endif
-
#if defined(__riscv) && (__riscv_xlen == 64)
#define SANITIZER_RISCV64 1
#else
@@ -374,9 +361,9 @@
# define SANITIZER_CACHE_LINE_SIZE 64
#endif
-// Enable offline markup symbolizer for Fuchsia and RTEMS.
-#if SANITIZER_FUCHSIA || SANITIZER_RTEMS
-#define SANITIZER_SYMBOLIZER_MARKUP 1
+// Enable offline markup symbolizer for Fuchsia.
+#if SANITIZER_FUCHSIA
+# define SANITIZER_SYMBOLIZER_MARKUP 1
#else
#define SANITIZER_SYMBOLIZER_MARKUP 0
#endif
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
index 731df710df5..5b710c23fd0 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
@@ -114,12 +114,6 @@
#define SI_NOT_FUCHSIA 1
#endif
-#if SANITIZER_RTEMS
-#define SI_NOT_RTEMS 0
-#else
-#define SI_NOT_RTEMS 1
-#endif
-
#if SANITIZER_SOLARIS
#define SI_SOLARIS 1
#else
@@ -482,13 +476,12 @@
#define SANITIZER_INTERCEPT_MMAP SI_POSIX
#define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO (SI_GLIBC || SI_ANDROID)
-#define SANITIZER_INTERCEPT_MEMALIGN \
- (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_RTEMS)
+#define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
#define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC
#define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC || SI_ANDROID)
#define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64)
#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX
-#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC && SI_NOT_RTEMS)
+#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
@@ -584,6 +577,7 @@
(SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386))
#define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD)
#define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
+#define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD
// This macro gives a way for downstream users to override the above
// interceptor macros irrespective of the platform they are on. They have
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp
index f22f5039128..c51327e1269 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp
@@ -26,12 +26,9 @@
// With old kernels (and even new kernels on powerpc) asm/stat.h uses types that
// are not defined anywhere in userspace headers. Fake them. This seems to work
-// fine with newer headers, too. Beware that with <sys/stat.h>, struct stat
-// takes the form of struct stat64 on 32-bit platforms if _FILE_OFFSET_BITS=64.
-// Also, for some platforms (e.g. mips) there are additional members in the
-// <sys/stat.h> struct stat:s.
+// fine with newer headers, too.
#include <linux/posix_types.h>
-#if defined(__x86_64__)
+#if defined(__x86_64__) || defined(__mips__)
#include <sys/stat.h>
#else
#define ino_t __kernel_ino_t
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
index cba41ba5494..4dd27644ed1 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -83,7 +83,7 @@ const unsigned struct_kernel_stat64_sz = 104;
#elif defined(__mips__)
const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
? FIRST_32_SECOND_64(104, 128)
- : FIRST_32_SECOND_64(144, 216);
+ : FIRST_32_SECOND_64(160, 216);
const unsigned struct_kernel_stat64_sz = 104;
#elif defined(__s390__) && !defined(__s390x__)
const unsigned struct_kernel_stat_sz = 64;
@@ -650,14 +650,14 @@ struct __sanitizer_sigaction {
#endif // !SANITIZER_ANDROID
#if defined(__mips__)
-struct __sanitizer_kernel_sigset_t {
- uptr sig[2];
-};
+#define __SANITIZER_KERNEL_NSIG 128
#else
+#define __SANITIZER_KERNEL_NSIG 64
+#endif
+
struct __sanitizer_kernel_sigset_t {
- u8 sig[8];
+ uptr sig[__SANITIZER_KERNEL_NSIG / (sizeof(uptr) * 8)];
};
-#endif
// Linux system headers define the 'sa_handler' and 'sa_sigaction' macros.
#if SANITIZER_MIPS
diff --git a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp
index d1d8e509c4d..ddf6844bed1 100644
--- a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp
@@ -128,14 +128,6 @@ void SetAddressSpaceUnlimited() {
CHECK(AddressSpaceIsUnlimited());
}
-void SleepForSeconds(int seconds) {
- sleep(seconds);
-}
-
-void SleepForMillis(int millis) {
- usleep(millis * 1000);
-}
-
void Abort() {
#if !SANITIZER_GO
// If we are handling SIGABRT, unhandle it first.
@@ -166,9 +158,10 @@ bool SupportsColoredOutput(fd_t fd) {
#if !SANITIZER_GO
// TODO(glider): different tools may require different altstack size.
static uptr GetAltStackSize() {
- // SIGSTKSZ is not enough.
- static const uptr kAltStackSize = SIGSTKSZ * 4;
- return kAltStackSize;
+ // Note: since GLIBC_2.31, SIGSTKSZ may be a function call, so this may be
+ // more costly that you think. However GetAltStackSize is only call 2-3 times
+ // per thread so don't cache the evaluation.
+ return SIGSTKSZ * 4;
}
void SetAlternateSignalStack() {
diff --git a/libsanitizer/sanitizer_common/sanitizer_printf.cpp b/libsanitizer/sanitizer_common/sanitizer_printf.cpp
index 5d16dfde678..b913c92e16f 100644
--- a/libsanitizer/sanitizer_common/sanitizer_printf.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_printf.cpp
@@ -20,6 +20,10 @@
#include <stdio.h>
#include <stdarg.h>
+#if defined(__x86_64__)
+# include <emmintrin.h>
+#endif
+
#if SANITIZER_WINDOWS && defined(_MSC_VER) && _MSC_VER < 1800 && \
!defined(va_copy)
# define va_copy(dst, src) ((dst) = (src))
@@ -128,7 +132,7 @@ static int AppendPointer(char **buff, const char *buff_end, u64 ptr_value) {
int VSNPrintf(char *buff, int buff_length,
const char *format, va_list args) {
static const char *kPrintfFormatsHelp =
- "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; "
+ "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X,V}; %p; "
"%[-]([0-9]*)?(\\.\\*)?s; %c\n";
RAW_CHECK(format);
RAW_CHECK(buff_length > 0);
@@ -162,17 +166,15 @@ int VSNPrintf(char *buff, int buff_length,
cur += have_z;
bool have_ll = !have_z && (cur[0] == 'l' && cur[1] == 'l');
cur += have_ll * 2;
- s64 dval;
- u64 uval;
const bool have_length = have_z || have_ll;
const bool have_flags = have_width || have_length;
// At the moment only %s supports precision and left-justification.
CHECK(!((precision >= 0 || left_justified) && *cur != 's'));
switch (*cur) {
case 'd': {
- dval = have_ll ? va_arg(args, s64)
- : have_z ? va_arg(args, sptr)
- : va_arg(args, int);
+ s64 dval = have_ll ? va_arg(args, s64)
+ : have_z ? va_arg(args, sptr)
+ : va_arg(args, int);
result += AppendSignedDecimal(&buff, buff_end, dval, width,
pad_with_zero);
break;
@@ -180,14 +182,21 @@ int VSNPrintf(char *buff, int buff_length,
case 'u':
case 'x':
case 'X': {
- uval = have_ll ? va_arg(args, u64)
- : have_z ? va_arg(args, uptr)
- : va_arg(args, unsigned);
+ u64 uval = have_ll ? va_arg(args, u64)
+ : have_z ? va_arg(args, uptr)
+ : va_arg(args, unsigned);
bool uppercase = (*cur == 'X');
result += AppendUnsigned(&buff, buff_end, uval, (*cur == 'u') ? 10 : 16,
width, pad_with_zero, uppercase);
break;
}
+ case 'V': {
+ for (uptr i = 0; i < 16; i++) {
+ unsigned x = va_arg(args, unsigned);
+ result += AppendUnsigned(&buff, buff_end, x, 16, 2, true, false);
+ }
+ break;
+ }
case 'p': {
RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
result += AppendPointer(&buff, buff_end, va_arg(args, uptr));
diff --git a/libsanitizer/sanitizer_common/sanitizer_quarantine.h b/libsanitizer/sanitizer_common/sanitizer_quarantine.h
index 992f23152c6..1a074d2bb70 100644
--- a/libsanitizer/sanitizer_common/sanitizer_quarantine.h
+++ b/libsanitizer/sanitizer_common/sanitizer_quarantine.h
@@ -149,7 +149,8 @@ class Quarantine {
Cache cache_;
char pad2_[kCacheLineSize];
- void NOINLINE Recycle(uptr min_size, Callback cb) {
+ void NOINLINE Recycle(uptr min_size, Callback cb) REQUIRES(recycle_mutex_)
+ RELEASE(recycle_mutex_) {
Cache tmp;
{
SpinMutexLock l(&cache_mutex_);
diff --git a/libsanitizer/sanitizer_common/sanitizer_rtems.cpp b/libsanitizer/sanitizer_common/sanitizer_rtems.cpp
deleted file mode 100644
index d58bd08fb1a..00000000000
--- a/libsanitizer/sanitizer_common/sanitizer_rtems.cpp
+++ /dev/null
@@ -1,281 +0,0 @@
-//===-- sanitizer_rtems.cpp -----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is shared between various sanitizers' runtime libraries and
-// implements RTEMS-specific functions.
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_rtems.h"
-#if SANITIZER_RTEMS
-
-#define posix_memalign __real_posix_memalign
-#define free __real_free
-#define memset __real_memset
-
-#include "sanitizer_file.h"
-#include "sanitizer_symbolizer.h"
-#include <errno.h>
-#include <fcntl.h>
-#include <pthread.h>
-#include <sched.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-// There is no mmap on RTEMS. Use memalign, etc.
-#define __mmap_alloc_aligned posix_memalign
-#define __mmap_free free
-#define __mmap_memset memset
-
-namespace __sanitizer {
-
-#include "sanitizer_syscall_generic.inc"
-
-void NORETURN internal__exit(int exitcode) {
- _exit(exitcode);
-}
-
-uptr internal_sched_yield() {
- return sched_yield();
-}
-
-uptr internal_getpid() {
- return getpid();
-}
-
-int internal_dlinfo(void *handle, int request, void *p) {
- UNIMPLEMENTED();
-}
-
-bool FileExists(const char *filename) {
- struct stat st;
- if (stat(filename, &st))
- return false;
- // Sanity check: filename is a regular file.
- return S_ISREG(st.st_mode);
-}
-
-uptr GetThreadSelf() { return static_cast<uptr>(pthread_self()); }
-
-tid_t GetTid() { return GetThreadSelf(); }
-
-void Abort() { abort(); }
-
-int Atexit(void (*function)(void)) { return atexit(function); }
-
-void SleepForSeconds(int seconds) { sleep(seconds); }
-
-void SleepForMillis(int millis) { usleep(millis * 1000); }
-
-bool SupportsColoredOutput(fd_t fd) { return false; }
-
-void GetThreadStackTopAndBottom(bool at_initialization,
- uptr *stack_top, uptr *stack_bottom) {
- pthread_attr_t attr;
- pthread_attr_init(&attr);
- CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
- void *base = nullptr;
- size_t size = 0;
- CHECK_EQ(pthread_attr_getstack(&attr, &base, &size), 0);
- CHECK_EQ(pthread_attr_destroy(&attr), 0);
-
- *stack_bottom = reinterpret_cast<uptr>(base);
- *stack_top = *stack_bottom + size;
-}
-
-void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
- uptr *tls_addr, uptr *tls_size) {
- uptr stack_top, stack_bottom;
- GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
- *stk_addr = stack_bottom;
- *stk_size = stack_top - stack_bottom;
- *tls_addr = *tls_size = 0;
-}
-
-void InitializePlatformEarly() {}
-void MaybeReexec() {}
-void CheckASLR() {}
-void CheckMPROTECT() {}
-void DisableCoreDumperIfNecessary() {}
-void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
-void SetAlternateSignalStack() {}
-void UnsetAlternateSignalStack() {}
-void InitTlsSize() {}
-
-void SignalContext::DumpAllRegisters(void *context) {}
-const char *DescribeSignalOrException(int signo) { UNIMPLEMENTED(); }
-
-enum MutexState { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
-
-BlockingMutex::BlockingMutex() {
- internal_memset(this, 0, sizeof(*this));
-}
-
-void BlockingMutex::Lock() {
- CHECK_EQ(owner_, 0);
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
- if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
- return;
- while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
- internal_sched_yield();
- }
-}
-
-void BlockingMutex::Unlock() {
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
- u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
- CHECK_NE(v, MtxUnlocked);
-}
-
-void BlockingMutex::CheckLocked() {
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
- CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
-}
-
-uptr GetPageSize() { return getpagesize(); }
-
-uptr GetMmapGranularity() { return GetPageSize(); }
-
-uptr GetMaxVirtualAddress() {
- return (1ULL << 32) - 1; // 0xffffffff
-}
-
-void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
- void* ptr = 0;
- int res = __mmap_alloc_aligned(&ptr, GetPageSize(), size);
- if (UNLIKELY(res))
- ReportMmapFailureAndDie(size, mem_type, "allocate", res, raw_report);
- __mmap_memset(ptr, 0, size);
- IncreaseTotalMmap(size);
- return ptr;
-}
-
-void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
- void* ptr = 0;
- int res = __mmap_alloc_aligned(&ptr, GetPageSize(), size);
- if (UNLIKELY(res)) {
- if (res == ENOMEM)
- return nullptr;
- ReportMmapFailureAndDie(size, mem_type, "allocate", false);
- }
- __mmap_memset(ptr, 0, size);
- IncreaseTotalMmap(size);
- return ptr;
-}
-
-void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
- const char *mem_type) {
- CHECK(IsPowerOfTwo(size));
- CHECK(IsPowerOfTwo(alignment));
- void* ptr = 0;
- int res = __mmap_alloc_aligned(&ptr, alignment, size);
- if (res)
- ReportMmapFailureAndDie(size, mem_type, "align allocate", res, false);
- __mmap_memset(ptr, 0, size);
- IncreaseTotalMmap(size);
- return ptr;
-}
-
-void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
- return MmapOrDie(size, mem_type, false);
-}
-
-void UnmapOrDie(void *addr, uptr size) {
- if (!addr || !size) return;
- __mmap_free(addr);
- DecreaseTotalMmap(size);
-}
-
-fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) {
- int flags;
- switch (mode) {
- case RdOnly: flags = O_RDONLY; break;
- case WrOnly: flags = O_WRONLY | O_CREAT | O_TRUNC; break;
- case RdWr: flags = O_RDWR | O_CREAT; break;
- }
- fd_t res = open(filename, flags, 0660);
- if (internal_iserror(res, errno_p))
- return kInvalidFd;
- return res;
-}
-
-void CloseFile(fd_t fd) {
- close(fd);
-}
-
-bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, uptr *bytes_read,
- error_t *error_p) {
- uptr res = read(fd, buff, buff_size);
- if (internal_iserror(res, error_p))
- return false;
- if (bytes_read)
- *bytes_read = res;
- return true;
-}
-
-bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written,
- error_t *error_p) {
- uptr res = write(fd, buff, buff_size);
- if (internal_iserror(res, error_p))
- return false;
- if (bytes_written)
- *bytes_written = res;
- return true;
-}
-
-void ReleaseMemoryPagesToOS(uptr beg, uptr end) {}
-void DumpProcessMap() {}
-
-// There is no page protection so everything is "accessible."
-bool IsAccessibleMemoryRange(uptr beg, uptr size) {
- return true;
-}
-
-char **GetArgv() { return nullptr; }
-char **GetEnviron() { return nullptr; }
-
-const char *GetEnv(const char *name) {
- return getenv(name);
-}
-
-uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
- internal_strncpy(buf, "StubBinaryName", buf_len);
- return internal_strlen(buf);
-}
-
-uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
- internal_strncpy(buf, "StubProcessName", buf_len);
- return internal_strlen(buf);
-}
-
-bool IsPathSeparator(const char c) {
- return c == '/';
-}
-
-bool IsAbsolutePath(const char *path) {
- return path != nullptr && IsPathSeparator(path[0]);
-}
-
-void ReportFile::Write(const char *buffer, uptr length) {
- SpinMutexLock l(mu);
- static const char *kWriteError =
- "ReportFile::Write() can't output requested buffer!\n";
- ReopenIfNecessary();
- if (length != write(fd, buffer, length)) {
- write(fd, kWriteError, internal_strlen(kWriteError));
- Die();
- }
-}
-
-uptr MainThreadStackBase, MainThreadStackSize;
-uptr MainThreadTlsBase, MainThreadTlsSize;
-
-} // namespace __sanitizer
-
-#endif // SANITIZER_RTEMS
diff --git a/libsanitizer/sanitizer_common/sanitizer_rtems.h b/libsanitizer/sanitizer_common/sanitizer_rtems.h
deleted file mode 100644
index e8adfd500df..00000000000
--- a/libsanitizer/sanitizer_common/sanitizer_rtems.h
+++ /dev/null
@@ -1,20 +0,0 @@
-//===-- sanitizer_rtems.h ---------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is shared between various sanitizers' runtime libraries and
-// provides definitions for RTEMS-specific functions.
-//===----------------------------------------------------------------------===//
-#ifndef SANITIZER_RTEMS_H
-#define SANITIZER_RTEMS_H
-
-#include "sanitizer_platform.h"
-#if SANITIZER_RTEMS
-#include "sanitizer_common.h"
-
-#endif // SANITIZER_RTEMS
-#endif // SANITIZER_RTEMS_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_solaris.cpp b/libsanitizer/sanitizer_common/sanitizer_solaris.cpp
index 8789dcd10a9..cb53eab8da1 100644
--- a/libsanitizer/sanitizer_common/sanitizer_solaris.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_solaris.cpp
@@ -160,6 +160,13 @@ DECLARE__REAL_AND_INTERNAL(uptr, sched_yield, void) {
return sched_yield();
}
+DECLARE__REAL_AND_INTERNAL(void, usleep, u64 useconds) {
+ struct timespec ts;
+ ts.tv_sec = useconds / 1000000;
+ ts.tv_nsec = (useconds % 1000000) * 1000;
+ nanosleep(&ts, nullptr);
+}
+
DECLARE__REAL_AND_INTERNAL(uptr, execve, const char *filename,
char *const argv[], char *const envp[]) {
return _REAL(execve)(filename, argv, envp);
@@ -211,6 +218,13 @@ uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
}
// ----------------- sanitizer_common.h
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+ // FIXME: implement actual blocking.
+ sched_yield();
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {}
+
BlockingMutex::BlockingMutex() {
CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
internal_memset(this, 0, sizeof(*this));
@@ -231,9 +245,7 @@ void BlockingMutex::Unlock() {
CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0);
}
-void BlockingMutex::CheckLocked() {
- CHECK_EQ((uptr)thr_self(), owner_);
-}
+void BlockingMutex::CheckLocked() const { CHECK_EQ((uptr)thr_self(), owner_); }
} // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp
index 515deddf6b2..07e4409f4a5 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp
@@ -85,8 +85,8 @@ static inline uhwptr *GetCanonicFrame(uptr bp,
// Nope, this does not look right either. This means the frame after next does
// not have a valid frame pointer, but we can still extract the caller PC.
// Unfortunately, there is no way to decide between GCC and LLVM frame
- // layouts. Assume GCC.
- return bp_prev - 1;
+ // layouts. Assume LLVM.
+ return bp_prev;
#else
return (uhwptr*)bp;
#endif
@@ -109,21 +109,14 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
IsAligned((uptr)frame, sizeof(*frame)) &&
size < max_depth) {
#ifdef __powerpc__
- // PowerPC ABIs specify that the return address is saved on the
- // *caller's* stack frame. Thus we must dereference the back chain
- // to find the caller frame before extracting it.
+ // PowerPC ABIs specify that the return address is saved at offset
+ // 16 of the *caller's* stack frame. Thus we must dereference the
+ // back chain to find the caller frame before extracting it.
uhwptr *caller_frame = (uhwptr*)frame[0];
if (!IsValidFrame((uptr)caller_frame, stack_top, bottom) ||
!IsAligned((uptr)caller_frame, sizeof(uhwptr)))
break;
- // For most ABIs the offset where the return address is saved is two
- // register sizes. The exception is the SVR4 ABI, which uses an
- // offset of only one register size.
-#ifdef _CALL_SYSV
- uhwptr pc1 = caller_frame[1];
-#else
uhwptr pc1 = caller_frame[2];
-#endif
#elif defined(__s390__)
uhwptr pc1 = frame[14];
#elif defined(__riscv)
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
index 15616f899d0..ea330f36f7d 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
@@ -12,6 +12,7 @@
#ifndef SANITIZER_STACKTRACE_H
#define SANITIZER_STACKTRACE_H
+#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_platform.h"
@@ -32,8 +33,8 @@ static const u32 kStackTraceMax = 256;
// Fast unwind is the only option on Mac for now; we will need to
// revisit this macro when slow unwind works on Mac, see
// https://github.com/google/sanitizers/issues/137
-#if SANITIZER_MAC || SANITIZER_RTEMS
-# define SANITIZER_CAN_SLOW_UNWIND 0
+#if SANITIZER_MAC
+# define SANITIZER_CAN_SLOW_UNWIND 0
#else
# define SANITIZER_CAN_SLOW_UNWIND 1
#endif
@@ -56,6 +57,16 @@ struct StackTrace {
// Prints a symbolized stacktrace, followed by an empty line.
void Print() const;
+ // Prints a symbolized stacktrace to the output string, followed by an empty
+ // line.
+ void PrintTo(InternalScopedString *output) const;
+
+ // Prints a symbolized stacktrace to the output buffer, followed by an empty
+ // line. Returns the number of symbols that should have been written to buffer
+ // (not including trailing '\0'). Thus, the string is truncated iff return
+ // value is not less than "out_buf_size".
+ uptr PrintTo(char *out_buf, uptr out_buf_size) const;
+
static bool WillUseFastUnwind(bool request_fast_unwind) {
if (!SANITIZER_CAN_FAST_UNWIND)
return false;
@@ -185,5 +196,26 @@ static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) {
uptr local_stack; \
uptr sp = (uptr)&local_stack
+// GET_CURRENT_PC() is equivalent to StackTrace::GetCurrentPc().
+// Optimized x86 version is faster than GetCurrentPc because
+// it does not involve a function call, instead it reads RIP register.
+// Reads of RIP by an instruction return RIP pointing to the next
+// instruction, which is exactly what we want here, thus 0 offset.
+// It needs to be a macro because otherwise we will get the name
+// of this function on the top of most stacks. Attribute artificial
+// does not do what it claims to do, unfortunatley. And attribute
+// __nodebug__ is clang-only. If we would have an attribute that
+// would remove this function from debug info, we could simply make
+// StackTrace::GetCurrentPc() faster.
+#if defined(__x86_64__)
+# define GET_CURRENT_PC() \
+ ({ \
+ uptr pc; \
+ asm("lea 0(%%rip), %0" : "=r"(pc)); \
+ pc; \
+ })
+#else
+# define GET_CURRENT_PC() StackTrace::GetCurrentPc()
+#endif
#endif // SANITIZER_STACKTRACE_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
index 738633209f0..f60ea773174 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
@@ -18,46 +18,119 @@
namespace __sanitizer {
-void StackTrace::Print() const {
+namespace {
+
+class StackTraceTextPrinter {
+ public:
+ StackTraceTextPrinter(const char *stack_trace_fmt, char frame_delimiter,
+ InternalScopedString *output,
+ InternalScopedString *dedup_token)
+ : stack_trace_fmt_(stack_trace_fmt),
+ frame_delimiter_(frame_delimiter),
+ output_(output),
+ dedup_token_(dedup_token),
+ symbolize_(RenderNeedsSymbolization(stack_trace_fmt)) {}
+
+ bool ProcessAddressFrames(uptr pc) {
+ SymbolizedStack *frames = symbolize_
+ ? Symbolizer::GetOrInit()->SymbolizePC(pc)
+ : SymbolizedStack::New(pc);
+ if (!frames)
+ return false;
+
+ for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
+ uptr prev_len = output_->length();
+ RenderFrame(output_, stack_trace_fmt_, frame_num_++, cur->info.address,
+ symbolize_ ? &cur->info : nullptr,
+ common_flags()->symbolize_vs_style,
+ common_flags()->strip_path_prefix);
+
+ if (prev_len != output_->length())
+ output_->append("%c", frame_delimiter_);
+
+ ExtendDedupToken(cur);
+ }
+ frames->ClearAll();
+ return true;
+ }
+
+ private:
+ // Extend the dedup token by appending a new frame.
+ void ExtendDedupToken(SymbolizedStack *stack) {
+ if (!dedup_token_)
+ return;
+
+ if (dedup_frames_-- > 0) {
+ if (dedup_token_->length())
+ dedup_token_->append("--");
+ if (stack->info.function != nullptr)
+ dedup_token_->append(stack->info.function);
+ }
+ }
+
+ const char *stack_trace_fmt_;
+ const char frame_delimiter_;
+ int dedup_frames_ = common_flags()->dedup_token_length;
+ uptr frame_num_ = 0;
+ InternalScopedString *output_;
+ InternalScopedString *dedup_token_;
+ const bool symbolize_ = false;
+};
+
+static void CopyStringToBuffer(const InternalScopedString &str, char *out_buf,
+ uptr out_buf_size) {
+ if (!out_buf_size)
+ return;
+
+ CHECK_GT(out_buf_size, 0);
+ uptr copy_size = Min(str.length(), out_buf_size - 1);
+ internal_memcpy(out_buf, str.data(), copy_size);
+ out_buf[copy_size] = '\0';
+}
+
+} // namespace
+
+void StackTrace::PrintTo(InternalScopedString *output) const {
+ CHECK(output);
+
+ InternalScopedString dedup_token;
+ StackTraceTextPrinter printer(common_flags()->stack_trace_format, '\n',
+ output, &dedup_token);
+
if (trace == nullptr || size == 0) {
- Printf(" <empty stack>\n\n");
+ output->append(" <empty stack>\n\n");
return;
}
- InternalScopedString frame_desc;
- InternalScopedString dedup_token;
- int dedup_frames = common_flags()->dedup_token_length;
- bool symbolize = RenderNeedsSymbolization(common_flags()->stack_trace_format);
- uptr frame_num = 0;
+
for (uptr i = 0; i < size && trace[i]; i++) {
// PCs in stack traces are actually the return addresses, that is,
// addresses of the next instructions after the call.
uptr pc = GetPreviousInstructionPc(trace[i]);
- SymbolizedStack *frames;
- if (symbolize)
- frames = Symbolizer::GetOrInit()->SymbolizePC(pc);
- else
- frames = SymbolizedStack::New(pc);
- CHECK(frames);
- for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
- frame_desc.clear();
- RenderFrame(&frame_desc, common_flags()->stack_trace_format, frame_num++,
- cur->info.address, symbolize ? &cur->info : nullptr,
- common_flags()->symbolize_vs_style,
- common_flags()->strip_path_prefix);
- Printf("%s\n", frame_desc.data());
- if (dedup_frames-- > 0) {
- if (dedup_token.length())
- dedup_token.append("--");
- if (cur->info.function != nullptr)
- dedup_token.append(cur->info.function);
- }
- }
- frames->ClearAll();
+ CHECK(printer.ProcessAddressFrames(pc));
}
- // Always print a trailing empty line after stack trace.
- Printf("\n");
+
+ // Always add a trailing empty line after stack trace.
+ output->append("\n");
+
+ // Append deduplication token, if non-empty.
if (dedup_token.length())
- Printf("DEDUP_TOKEN: %s\n", dedup_token.data());
+ output->append("DEDUP_TOKEN: %s\n", dedup_token.data());
+}
+
+uptr StackTrace::PrintTo(char *out_buf, uptr out_buf_size) const {
+ CHECK(out_buf);
+
+ InternalScopedString output;
+ PrintTo(&output);
+ CopyStringToBuffer(output, out_buf, out_buf_size);
+
+ return output.length();
+}
+
+void StackTrace::Print() const {
+ InternalScopedString output;
+ PrintTo(&output);
+ Printf("%s", output.data());
}
void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
@@ -82,12 +155,15 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
UnwindSlow(pc, context, max_depth);
else
UnwindSlow(pc, max_depth);
+ // If there are too few frames, the program may be built with
+ // -fno-asynchronous-unwind-tables. Fall back to fast unwinder below.
+ if (size > 2 || size >= max_depth)
+ return;
#else
UNREACHABLE("slow unwind requested but not available");
#endif
- } else {
- UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
}
+ UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
}
static int GetModuleAndOffsetForPc(uptr pc, char *module_name,
@@ -112,41 +188,18 @@ extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf,
uptr out_buf_size) {
- if (!out_buf_size) return;
- pc = StackTrace::GetPreviousInstructionPc(pc);
- SymbolizedStack *frame;
- bool symbolize = RenderNeedsSymbolization(fmt);
- if (symbolize)
- frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
- else
- frame = SymbolizedStack::New(pc);
- if (!frame) {
- internal_strncpy(out_buf, "<can't symbolize>", out_buf_size);
- out_buf[out_buf_size - 1] = 0;
+ if (!out_buf_size)
return;
+
+ pc = StackTrace::GetPreviousInstructionPc(pc);
+
+ InternalScopedString output;
+ StackTraceTextPrinter printer(fmt, '\0', &output, nullptr);
+ if (!printer.ProcessAddressFrames(pc)) {
+ output.clear();
+ output.append("<can't symbolize>");
}
- InternalScopedString frame_desc;
- uptr frame_num = 0;
- // Reserve one byte for the final 0.
- char *out_end = out_buf + out_buf_size - 1;
- for (SymbolizedStack *cur = frame; cur && out_buf < out_end;
- cur = cur->next) {
- frame_desc.clear();
- RenderFrame(&frame_desc, fmt, frame_num++, cur->info.address,
- symbolize ? &cur->info : nullptr,
- common_flags()->symbolize_vs_style,
- common_flags()->strip_path_prefix);
- if (!frame_desc.length())
- continue;
- // Reserve one byte for the terminating 0.
- uptr n = out_end - out_buf - 1;
- internal_strncpy(out_buf, frame_desc.data(), n);
- out_buf += __sanitizer::Min<uptr>(n, frame_desc.length());
- *out_buf++ = 0;
- }
- CHECK(out_buf <= out_end);
- *out_buf = 0;
- frame->ClearAll();
+ CopyStringToBuffer(output, out_buf, out_buf_size);
}
SANITIZER_INTERFACE_ATTRIBUTE
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp
index 01edef9c1aa..9a5b4a8c54c 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp
@@ -16,14 +16,13 @@
#if SANITIZER_FUCHSIA
#include "sanitizer_symbolizer_fuchsia.h"
-#elif SANITIZER_RTEMS
-#include "sanitizer_symbolizer_rtems.h"
-#endif
-#include "sanitizer_stacktrace.h"
-#include "sanitizer_symbolizer.h"
-
-#include <limits.h>
-#include <unwind.h>
+# endif
+
+# include <limits.h>
+# include <unwind.h>
+
+# include "sanitizer_stacktrace.h"
+# include "sanitizer_symbolizer.h"
namespace __sanitizer {
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp
index 9287993e665..f330ed36640 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp
@@ -120,7 +120,7 @@ void ReportMmapWriteExec(int prot) {
#endif
}
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_GO
+#if !SANITIZER_FUCHSIA && !SANITIZER_GO
void StartReportDeadlySignal() {
// Write the first message using fd=2, just in case.
// It may actually fail to write in case stderr is closed.
@@ -250,17 +250,17 @@ void HandleDeadlySignal(void *siginfo, void *context, u32 tid,
#endif // !SANITIZER_FUCHSIA && !SANITIZER_GO
-static atomic_uintptr_t reporting_thread = {0};
-static StaticSpinMutex CommonSanitizerReportMutex;
+atomic_uintptr_t ScopedErrorReportLock::reporting_thread_ = {0};
+StaticSpinMutex ScopedErrorReportLock::mutex_;
-ScopedErrorReportLock::ScopedErrorReportLock() {
+void ScopedErrorReportLock::Lock() {
uptr current = GetThreadSelf();
for (;;) {
uptr expected = 0;
- if (atomic_compare_exchange_strong(&reporting_thread, &expected, current,
+ if (atomic_compare_exchange_strong(&reporting_thread_, &expected, current,
memory_order_relaxed)) {
// We've claimed reporting_thread so proceed.
- CommonSanitizerReportMutex.Lock();
+ mutex_.Lock();
return;
}
@@ -282,13 +282,11 @@ ScopedErrorReportLock::ScopedErrorReportLock() {
}
}
-ScopedErrorReportLock::~ScopedErrorReportLock() {
- CommonSanitizerReportMutex.Unlock();
- atomic_store_relaxed(&reporting_thread, 0);
+void ScopedErrorReportLock::Unlock() {
+ mutex_.Unlock();
+ atomic_store_relaxed(&reporting_thread_, 0);
}
-void ScopedErrorReportLock::CheckLocked() {
- CommonSanitizerReportMutex.CheckLocked();
-}
+void ScopedErrorReportLock::CheckLocked() { mutex_.CheckLocked(); }
} // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_rtems.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer_rtems.h
deleted file mode 100644
index 3371092e068..00000000000
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_rtems.h
+++ /dev/null
@@ -1,40 +0,0 @@
-//===-- sanitizer_symbolizer_rtems.h -----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is shared between various sanitizers' runtime libraries.
-//
-// Define RTEMS's string formats and limits for the markup symbolizer.
-//===----------------------------------------------------------------------===//
-#ifndef SANITIZER_SYMBOLIZER_RTEMS_H
-#define SANITIZER_SYMBOLIZER_RTEMS_H
-
-#include "sanitizer_internal_defs.h"
-
-namespace __sanitizer {
-
-// The Myriad RTEMS symbolizer currently only parses backtrace lines,
-// so use a format that the symbolizer understands. For other
-// markups, keep them the same as the Fuchsia's.
-
-// This is used by UBSan for type names, and by ASan for global variable names.
-constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
-constexpr uptr kFormatDemangleMax = 1024; // Arbitrary.
-
-// Function name or equivalent from PC location.
-constexpr const char *kFormatFunction = "{{{pc:%p}}}";
-constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex.
-
-// Global variable name or equivalent from data memory address.
-constexpr const char *kFormatData = "{{{data:%p}}}";
-
-// One frame in a backtrace (printed on a line by itself).
-constexpr const char *kFormatFrame = " [%u] IP: %p";
-
-} // namespace __sanitizer
-
-#endif // SANITIZER_SYMBOLIZER_RTEMS_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_registry.cpp b/libsanitizer/sanitizer_common/sanitizer_thread_registry.cpp
index 3273da38bfd..745fbf76b01 100644
--- a/libsanitizer/sanitizer_common/sanitizer_thread_registry.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_thread_registry.cpp
@@ -99,6 +99,9 @@ void ThreadContextBase::Reset() {
// ThreadRegistry implementation.
+ThreadRegistry::ThreadRegistry(ThreadContextFactory factory)
+ : ThreadRegistry(factory, UINT32_MAX, UINT32_MAX, 0) {}
+
ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
u32 thread_quarantine_size, u32 max_reuse)
: context_factory_(factory),
@@ -106,13 +109,10 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
thread_quarantine_size_(thread_quarantine_size),
max_reuse_(max_reuse),
mtx_(),
- n_contexts_(0),
total_threads_(0),
alive_threads_(0),
max_alive_threads_(0),
running_threads_(0) {
- threads_ = (ThreadContextBase **)MmapOrDie(max_threads_ * sizeof(threads_[0]),
- "ThreadRegistry");
dead_threads_.clear();
invalid_threads_.clear();
}
@@ -120,7 +120,8 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running,
uptr *alive) {
BlockingMutexLock l(&mtx_);
- if (total) *total = n_contexts_;
+ if (total)
+ *total = threads_.size();
if (running) *running = running_threads_;
if (alive) *alive = alive_threads_;
}
@@ -137,11 +138,11 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
ThreadContextBase *tctx = QuarantinePop();
if (tctx) {
tid = tctx->tid;
- } else if (n_contexts_ < max_threads_) {
+ } else if (threads_.size() < max_threads_) {
// Allocate new thread context and tid.
- tid = n_contexts_++;
+ tid = threads_.size();
tctx = context_factory_(tid);
- threads_[tid] = tctx;
+ threads_.push_back(tctx);
} else {
#if !SANITIZER_GO
Report("%s: Thread limit (%u threads) exceeded. Dying.\n",
@@ -169,7 +170,7 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
void ThreadRegistry::RunCallbackForEachThreadLocked(ThreadCallback cb,
void *arg) {
CheckLocked();
- for (u32 tid = 0; tid < n_contexts_; tid++) {
+ for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx == 0)
continue;
@@ -179,7 +180,7 @@ void ThreadRegistry::RunCallbackForEachThreadLocked(ThreadCallback cb,
u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
BlockingMutexLock l(&mtx_);
- for (u32 tid = 0; tid < n_contexts_; tid++) {
+ for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx != 0 && cb(tctx, arg))
return tctx->tid;
@@ -190,7 +191,7 @@ u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
ThreadContextBase *
ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) {
CheckLocked();
- for (u32 tid = 0; tid < n_contexts_; tid++) {
+ for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx != 0 && cb(tctx, arg))
return tctx;
@@ -211,7 +212,6 @@ ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) {
void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning,
@@ -221,7 +221,7 @@ void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) {
BlockingMutexLock l(&mtx_);
- for (u32 tid = 0; tid < n_contexts_; tid++) {
+ for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx != 0 && tctx->user_id == user_id &&
tctx->status != ThreadStatusInvalid) {
@@ -233,7 +233,6 @@ void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) {
void ThreadRegistry::DetachThread(u32 tid, void *arg) {
BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
if (tctx->status == ThreadStatusInvalid) {
@@ -254,7 +253,6 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) {
do {
{
BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
if (tctx->status == ThreadStatusInvalid) {
@@ -280,7 +278,6 @@ ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
BlockingMutexLock l(&mtx_);
CHECK_GT(alive_threads_, 0);
alive_threads_--;
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
bool dead = tctx->detached;
@@ -306,7 +303,6 @@ void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type,
void *arg) {
BlockingMutexLock l(&mtx_);
running_threads_++;
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
CHECK_EQ(ThreadStatusCreated, tctx->status);
@@ -339,7 +335,6 @@ ThreadContextBase *ThreadRegistry::QuarantinePop() {
void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) {
BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
CHECK_NE(tctx->status, ThreadStatusInvalid);
diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_registry.h b/libsanitizer/sanitizer_common/sanitizer_thread_registry.h
index dcd445c28ae..0b28bbe6ddf 100644
--- a/libsanitizer/sanitizer_common/sanitizer_thread_registry.h
+++ b/libsanitizer/sanitizer_common/sanitizer_thread_registry.h
@@ -85,22 +85,22 @@ class ThreadContextBase {
typedef ThreadContextBase* (*ThreadContextFactory)(u32 tid);
-class ThreadRegistry {
+class MUTEX ThreadRegistry {
public:
+ ThreadRegistry(ThreadContextFactory factory);
ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
- u32 thread_quarantine_size, u32 max_reuse = 0);
+ u32 thread_quarantine_size, u32 max_reuse);
void GetNumberOfThreads(uptr *total = nullptr, uptr *running = nullptr,
uptr *alive = nullptr);
uptr GetMaxAliveThreads();
- void Lock() { mtx_.Lock(); }
- void CheckLocked() { mtx_.CheckLocked(); }
- void Unlock() { mtx_.Unlock(); }
+ void Lock() ACQUIRE() { mtx_.Lock(); }
+ void CheckLocked() const CHECK_LOCKED() { mtx_.CheckLocked(); }
+ void Unlock() RELEASE() { mtx_.Unlock(); }
// Should be guarded by ThreadRegistryLock.
ThreadContextBase *GetThreadLocked(u32 tid) {
- DCHECK_LT(tid, n_contexts_);
- return threads_[tid];
+ return threads_.empty() ? nullptr : threads_[tid];
}
u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
@@ -137,15 +137,13 @@ class ThreadRegistry {
BlockingMutex mtx_;
- u32 n_contexts_; // Number of created thread contexts,
- // at most max_threads_.
u64 total_threads_; // Total number of created threads. May be greater than
// max_threads_ if contexts were reused.
uptr alive_threads_; // Created or running.
uptr max_alive_threads_;
uptr running_threads_;
- ThreadContextBase **threads_; // Array of thread contexts is leaked.
+ InternalMmapVector<ThreadContextBase *> threads_;
IntrusiveList<ThreadContextBase> dead_threads_;
IntrusiveList<ThreadContextBase> invalid_threads_;
diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_safety.h b/libsanitizer/sanitizer_common/sanitizer_thread_safety.h
new file mode 100644
index 00000000000..52b25edaa7a
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_thread_safety.h
@@ -0,0 +1,42 @@
+//===-- sanitizer_thread_safety.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizer tools.
+//
+// Wrappers around thread safety annotations.
+// https://clang.llvm.org/docs/ThreadSafetyAnalysis.html
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_THREAD_SAFETY_H
+#define SANITIZER_THREAD_SAFETY_H
+
+#if defined(__clang__)
+# define THREAD_ANNOTATION(x) __attribute__((x))
+#else
+# define THREAD_ANNOTATION(x)
+#endif
+
+#define MUTEX THREAD_ANNOTATION(capability("mutex"))
+#define SCOPED_LOCK THREAD_ANNOTATION(scoped_lockable)
+#define GUARDED_BY(x) THREAD_ANNOTATION(guarded_by(x))
+#define PT_GUARDED_BY(x) THREAD_ANNOTATION(pt_guarded_by(x))
+#define REQUIRES(...) THREAD_ANNOTATION(requires_capability(__VA_ARGS__))
+#define REQUIRES_SHARED(...) \
+ THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__))
+#define ACQUIRE(...) THREAD_ANNOTATION(acquire_capability(__VA_ARGS__))
+#define ACQUIRE_SHARED(...) \
+ THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__))
+#define TRY_ACQUIRE(...) THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__))
+#define RELEASE(...) THREAD_ANNOTATION(release_capability(__VA_ARGS__))
+#define RELEASE_SHARED(...) \
+ THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__))
+#define EXCLUDES(...) THREAD_ANNOTATION(locks_excluded(__VA_ARGS__))
+#define CHECK_LOCKED(...) THREAD_ANNOTATION(assert_capability(__VA_ARGS__))
+#define NO_THREAD_SAFETY_ANALYSIS THREAD_ANNOTATION(no_thread_safety_analysis)
+
+#endif
diff --git a/libsanitizer/sanitizer_common/sanitizer_win.cpp b/libsanitizer/sanitizer_common/sanitizer_win.cpp
index f383e130fa5..dddd885a45d 100644
--- a/libsanitizer/sanitizer_common/sanitizer_win.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_win.cpp
@@ -44,6 +44,9 @@ TRACELOGGING_DEFINE_PROVIDER(g_asan_provider, "AddressSanitizerLoggingProvider",
#define TraceLoggingUnregister(x)
#endif
+// For WaitOnAddress
+# pragma comment(lib, "synchronization.lib")
+
// A macro to tell the compiler that this part of the code cannot be reached,
// if the compiler supports this feature. Since we're using this in
// code that is called when terminating the process, the expansion of the
@@ -541,13 +544,7 @@ bool IsAbsolutePath(const char *path) {
IsPathSeparator(path[2]);
}
-void SleepForSeconds(int seconds) {
- Sleep(seconds * 1000);
-}
-
-void SleepForMillis(int millis) {
- Sleep(millis);
-}
+void internal_usleep(u64 useconds) { Sleep(useconds / 1000); }
u64 NanoTime() {
static LARGE_INTEGER frequency = {};
@@ -819,6 +816,17 @@ uptr GetRSS() {
void *internal_start_thread(void *(*func)(void *arg), void *arg) { return 0; }
void internal_join_thread(void *th) { }
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+ WaitOnAddress(p, &cmp, sizeof(cmp), INFINITE);
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {
+ if (count == 1)
+ WakeByAddressSingle(p);
+ else
+ WakeByAddressAll(p);
+}
+
// ---------------------- BlockingMutex ---------------- {{{1
BlockingMutex::BlockingMutex() {
@@ -838,9 +846,7 @@ void BlockingMutex::Unlock() {
ReleaseSRWLockExclusive((PSRWLOCK)opaque_storage_);
}
-void BlockingMutex::CheckLocked() {
- CHECK_EQ(owner_, GetThreadSelf());
-}
+void BlockingMutex::CheckLocked() const { CHECK_EQ(owner_, GetThreadSelf()); }
uptr GetTlsSize() {
return 0;
diff --git a/libsanitizer/tsan/Makefile.am b/libsanitizer/tsan/Makefile.am
index fb20ff7e311..dcb247ec032 100644
--- a/libsanitizer/tsan/Makefile.am
+++ b/libsanitizer/tsan/Makefile.am
@@ -43,13 +43,12 @@ tsan_files = \
tsan_rtl_report.cpp \
tsan_rtl_thread.cpp \
tsan_stack_trace.cpp \
- tsan_stat.cpp \
tsan_suppressions.cpp \
tsan_symbolize.cpp \
tsan_sync.cpp
libtsan_la_SOURCES = $(tsan_files)
-EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S
+EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S tsan_rtl_s390x.S
libtsan_la_LIBADD = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la $(TSAN_TARGET_DEPENDENT_OBJECTS)
libtsan_la_DEPENDENCIES = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la $(TSAN_TARGET_DEPENDENT_OBJECTS)
if LIBBACKTRACE_SUPPORTED
diff --git a/libsanitizer/tsan/Makefile.in b/libsanitizer/tsan/Makefile.in
index ce11d2497b7..83617cf6ab7 100644
--- a/libsanitizer/tsan/Makefile.in
+++ b/libsanitizer/tsan/Makefile.in
@@ -155,8 +155,7 @@ am__objects_1 = tsan_clock.lo tsan_debugging.lo tsan_external.lo \
tsan_platform_posix.lo tsan_platform_windows.lo tsan_report.lo \
tsan_rtl.lo tsan_rtl_mutex.lo tsan_rtl_proc.lo \
tsan_rtl_report.lo tsan_rtl_thread.lo tsan_stack_trace.lo \
- tsan_stat.lo tsan_suppressions.lo tsan_symbolize.lo \
- tsan_sync.lo
+ tsan_suppressions.lo tsan_symbolize.lo tsan_sync.lo
am_libtsan_la_OBJECTS = $(am__objects_1)
libtsan_la_OBJECTS = $(am_libtsan_la_OBJECTS)
AM_V_lt = $(am__v_lt_@AM_V@)
@@ -446,13 +445,12 @@ tsan_files = \
tsan_rtl_report.cpp \
tsan_rtl_thread.cpp \
tsan_stack_trace.cpp \
- tsan_stat.cpp \
tsan_suppressions.cpp \
tsan_symbolize.cpp \
tsan_sync.cpp
libtsan_la_SOURCES = $(tsan_files)
-EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S
+EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S tsan_rtl_s390x.S
libtsan_la_LIBADD = \
$(top_builddir)/sanitizer_common/libsanitizer_common.la \
$(top_builddir)/interception/libinterception.la \
@@ -612,9 +610,9 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_ppc64.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_proc.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_report.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_s390x.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_thread.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_stack_trace.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_stat.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_suppressions.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_symbolize.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_sync.Plo@am__quote@
diff --git a/libsanitizer/tsan/tsan_clock.cpp b/libsanitizer/tsan/tsan_clock.cpp
index 8e5188392ca..61848c21d16 100644
--- a/libsanitizer/tsan/tsan_clock.cpp
+++ b/libsanitizer/tsan/tsan_clock.cpp
@@ -80,14 +80,6 @@
// release-store operation by the thread with release_store_tid_ index.
// release_store_reused_ - reuse count of release_store_tid_.
-// We don't have ThreadState in these methods, so this is an ugly hack that
-// works only in C++.
-#if !SANITIZER_GO
-# define CPP_STAT_INC(typ) StatInc(cur_thread(), typ)
-#else
-# define CPP_STAT_INC(typ) (void)0
-#endif
-
namespace __tsan {
static atomic_uint32_t *ref_ptr(ClockBlock *cb) {
@@ -138,14 +130,11 @@ void ThreadClock::ResetCached(ClockCache *c) {
void ThreadClock::acquire(ClockCache *c, SyncClock *src) {
DCHECK_LE(nclk_, kMaxTid);
DCHECK_LE(src->size_, kMaxTid);
- CPP_STAT_INC(StatClockAcquire);
// Check if it's empty -> no need to do anything.
const uptr nclk = src->size_;
- if (nclk == 0) {
- CPP_STAT_INC(StatClockAcquireEmpty);
+ if (nclk == 0)
return;
- }
bool acquired = false;
for (unsigned i = 0; i < kDirtyTids; i++) {
@@ -162,7 +151,6 @@ void ThreadClock::acquire(ClockCache *c, SyncClock *src) {
// Check if we've already acquired src after the last release operation on src
if (tid_ >= nclk || src->elem(tid_).reused != reused_) {
// O(N) acquire.
- CPP_STAT_INC(StatClockAcquireFull);
nclk_ = max(nclk_, nclk);
u64 *dst_pos = &clk_[0];
for (ClockElem &src_elem : *src) {
@@ -180,7 +168,6 @@ void ThreadClock::acquire(ClockCache *c, SyncClock *src) {
}
if (acquired) {
- CPP_STAT_INC(StatClockAcquiredSomething);
last_acquire_ = clk_[tid_];
ResetCached(c);
}
@@ -223,7 +210,6 @@ void ThreadClock::releaseStoreAcquire(ClockCache *c, SyncClock *sc) {
sc->release_store_reused_ = 0;
if (acquired) {
- CPP_STAT_INC(StatClockAcquiredSomething);
last_acquire_ = clk_[tid_];
ResetCached(c);
}
@@ -240,7 +226,6 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) {
return;
}
- CPP_STAT_INC(StatClockRelease);
// Check if we need to resize dst.
if (dst->size_ < nclk_)
dst->Resize(c, nclk_);
@@ -257,12 +242,9 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) {
}
// O(N) release.
- CPP_STAT_INC(StatClockReleaseFull);
dst->Unshare(c);
// First, remember whether we've acquired dst.
bool acquired = IsAlreadyAcquired(dst);
- if (acquired)
- CPP_STAT_INC(StatClockReleaseAcquired);
// Update dst->clk_.
dst->FlushDirty();
uptr i = 0;
@@ -272,8 +254,6 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) {
i++;
}
// Clear 'acquired' flag in the remaining elements.
- if (nclk_ < dst->size_)
- CPP_STAT_INC(StatClockReleaseClearTail);
dst->release_store_tid_ = kInvalidTid;
dst->release_store_reused_ = 0;
// If we've acquired dst, remember this fact,
@@ -285,7 +265,6 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) {
void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) {
DCHECK_LE(nclk_, kMaxTid);
DCHECK_LE(dst->size_, kMaxTid);
- CPP_STAT_INC(StatClockStore);
if (dst->size_ == 0 && cached_idx_ != 0) {
// Reuse the cached clock.
@@ -320,13 +299,11 @@ void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) {
if (dst->release_store_tid_ == tid_ &&
dst->release_store_reused_ == reused_ &&
!HasAcquiredAfterRelease(dst)) {
- CPP_STAT_INC(StatClockStoreFast);
UpdateCurrentThread(c, dst);
return;
}
// O(N) release-store.
- CPP_STAT_INC(StatClockStoreFull);
dst->Unshare(c);
// Note: dst can be larger than this ThreadClock.
// This is fine since clk_ beyond size is all zeros.
@@ -358,7 +335,6 @@ void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) {
}
void ThreadClock::acq_rel(ClockCache *c, SyncClock *dst) {
- CPP_STAT_INC(StatClockAcquireRelease);
acquire(c, dst);
ReleaseStore(c, dst);
}
@@ -370,7 +346,6 @@ void ThreadClock::UpdateCurrentThread(ClockCache *c, SyncClock *dst) const {
SyncClock::Dirty *dirty = &dst->dirty_[i];
const unsigned tid = dirty->tid();
if (tid == tid_ || tid == kInvalidTid) {
- CPP_STAT_INC(StatClockReleaseFast);
dirty->set_tid(tid_);
dirty->epoch = clk_[tid_];
return;
@@ -379,7 +354,6 @@ void ThreadClock::UpdateCurrentThread(ClockCache *c, SyncClock *dst) const {
// Reset all 'acquired' flags, O(N).
// We are going to touch dst elements, so we need to unshare it.
dst->Unshare(c);
- CPP_STAT_INC(StatClockReleaseSlow);
dst->elem(tid_).epoch = clk_[tid_];
for (uptr i = 0; i < dst->size_; i++)
dst->elem(i).reused = 0;
@@ -456,7 +430,6 @@ void SyncClock::ResetImpl() {
}
void SyncClock::Resize(ClockCache *c, uptr nclk) {
- CPP_STAT_INC(StatClockReleaseResize);
Unshare(c);
if (nclk <= capacity()) {
// Memory is already allocated, just increase the size.
diff --git a/libsanitizer/tsan/tsan_defs.h b/libsanitizer/tsan/tsan_defs.h
index f53787aeba9..5c8f2801b0c 100644
--- a/libsanitizer/tsan/tsan_defs.h
+++ b/libsanitizer/tsan/tsan_defs.h
@@ -15,7 +15,6 @@
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_libc.h"
-#include "tsan_stat.h"
#include "ubsan/ubsan_platform.h"
// Setup defaults for compile definitions.
@@ -23,10 +22,6 @@
# define TSAN_NO_HISTORY 0
#endif
-#ifndef TSAN_COLLECT_STATS
-# define TSAN_COLLECT_STATS 0
-#endif
-
#ifndef TSAN_CONTAINS_UBSAN
# if CAN_SANITIZE_UB && !SANITIZER_GO
# define TSAN_CONTAINS_UBSAN 1
@@ -107,23 +102,12 @@ void build_consistency_debug();
void build_consistency_release();
#endif
-#if TSAN_COLLECT_STATS
-void build_consistency_stats();
-#else
-void build_consistency_nostats();
-#endif
-
static inline void USED build_consistency() {
#if SANITIZER_DEBUG
build_consistency_debug();
#else
build_consistency_release();
#endif
-#if TSAN_COLLECT_STATS
- build_consistency_stats();
-#else
- build_consistency_nostats();
-#endif
}
template<typename T>
diff --git a/libsanitizer/tsan/tsan_interceptors.h b/libsanitizer/tsan/tsan_interceptors.h
index 29576ea2d49..c5716f53a32 100644
--- a/libsanitizer/tsan/tsan_interceptors.h
+++ b/libsanitizer/tsan/tsan_interceptors.h
@@ -30,14 +30,14 @@ inline bool in_symbolizer() {
} // namespace __tsan
-#define SCOPED_INTERCEPTOR_RAW(func, ...) \
- cur_thread_init(); \
- ThreadState *thr = cur_thread(); \
- const uptr caller_pc = GET_CALLER_PC(); \
- ScopedInterceptor si(thr, #func, caller_pc); \
- const uptr pc = StackTrace::GetCurrentPc(); \
- (void)pc; \
-/**/
+#define SCOPED_INTERCEPTOR_RAW(func, ...) \
+ cur_thread_init(); \
+ ThreadState *thr = cur_thread(); \
+ const uptr caller_pc = GET_CALLER_PC(); \
+ ScopedInterceptor si(thr, #func, caller_pc); \
+ const uptr pc = GET_CURRENT_PC(); \
+ (void)pc; \
+ /**/
#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
diff --git a/libsanitizer/tsan/tsan_interceptors_mac.cpp b/libsanitizer/tsan/tsan_interceptors_mac.cpp
index ed10fccc980..2d400c7e709 100644
--- a/libsanitizer/tsan/tsan_interceptors_mac.cpp
+++ b/libsanitizer/tsan/tsan_interceptors_mac.cpp
@@ -44,8 +44,9 @@ namespace __tsan {
// actually aliases of each other, and we cannot have different interceptors for
// them, because they're actually the same function. Thus, we have to stay
// conservative and treat the non-barrier versions as mo_acq_rel.
-static const morder kMacOrderBarrier = mo_acq_rel;
-static const morder kMacOrderNonBarrier = mo_acq_rel;
+static constexpr morder kMacOrderBarrier = mo_acq_rel;
+static constexpr morder kMacOrderNonBarrier = mo_acq_rel;
+static constexpr morder kMacFailureOrder = mo_relaxed;
#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
@@ -110,7 +111,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
return tsan_atomic_f##_compare_exchange_strong( \
(volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
- kMacOrderNonBarrier, kMacOrderNonBarrier); \
+ kMacOrderNonBarrier, kMacFailureOrder); \
} \
\
TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
@@ -118,7 +119,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
return tsan_atomic_f##_compare_exchange_strong( \
(volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
- kMacOrderBarrier, kMacOrderNonBarrier); \
+ kMacOrderBarrier, kMacFailureOrder); \
}
OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
diff --git a/libsanitizer/tsan/tsan_interceptors_posix.cpp b/libsanitizer/tsan/tsan_interceptors_posix.cpp
index 2651e22c39f..6808f2e0e2d 100644
--- a/libsanitizer/tsan/tsan_interceptors_posix.cpp
+++ b/libsanitizer/tsan/tsan_interceptors_posix.cpp
@@ -71,7 +71,8 @@ struct ucontext_t {
};
#endif
-#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1
+#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \
+ defined(__s390x__)
#define PTHREAD_ABI_BASE "GLIBC_2.3.2"
#elif defined(__aarch64__) || SANITIZER_PPC64V2
#define PTHREAD_ABI_BASE "GLIBC_2.17"
@@ -2270,6 +2271,7 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
#define NEED_TLS_GET_ADDR
#endif
#undef SANITIZER_INTERCEPT_TLS_GET_ADDR
+#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1
#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
@@ -2585,6 +2587,20 @@ static void syscall_post_fork(uptr pc, int pid) {
#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
#ifdef NEED_TLS_GET_ADDR
+
+static void handle_tls_addr(void *arg, void *res) {
+ ThreadState *thr = cur_thread();
+ if (!thr)
+ return;
+ DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr,
+ thr->tls_addr + thr->tls_size);
+ if (!dtv)
+ return;
+ // New DTLS block has been allocated.
+ MemoryResetRange(thr, 0, dtv->beg, dtv->size);
+}
+
+#if !SANITIZER_S390
// Define own interceptor instead of sanitizer_common's for three reasons:
// 1. It must not process pending signals.
// Signal handlers may contain MOVDQA instruction (see below).
@@ -2597,18 +2613,18 @@ static void syscall_post_fork(uptr pc, int pid) {
// execute MOVDQA with stack addresses.
TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) {
void *res = REAL(__tls_get_addr)(arg);
- ThreadState *thr = cur_thread();
- if (!thr)
- return res;
- DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr,
- thr->tls_addr + thr->tls_size);
- if (!dtv)
- return res;
- // New DTLS block has been allocated.
- MemoryResetRange(thr, 0, dtv->beg, dtv->size);
+ handle_tls_addr(arg, res);
+ return res;
+}
+#else // SANITIZER_S390
+TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
+ uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset));
+ char *tp = static_cast<char *>(__builtin_thread_pointer());
+ handle_tls_addr(arg, res + tp);
return res;
}
#endif
+#endif
#if SANITIZER_NETBSD
TSAN_INTERCEPTOR(void, _lwp_exit) {
@@ -2830,7 +2846,12 @@ void InitializeInterceptors() {
TSAN_INTERCEPT(_exit);
#ifdef NEED_TLS_GET_ADDR
+#if !SANITIZER_S390
TSAN_INTERCEPT(__tls_get_addr);
+#else
+ TSAN_INTERCEPT(__tls_get_addr_internal);
+ TSAN_INTERCEPT(__tls_get_offset);
+#endif
#endif
TSAN_MAYBE_INTERCEPT__LWP_EXIT;
diff --git a/libsanitizer/tsan/tsan_interface.h b/libsanitizer/tsan/tsan_interface.h
index 6e022b56850..124aa2fd214 100644
--- a/libsanitizer/tsan/tsan_interface.h
+++ b/libsanitizer/tsan/tsan_interface.h
@@ -196,7 +196,8 @@ typedef unsigned short a16;
typedef unsigned int a32;
typedef unsigned long long a64;
#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \
- || (__clang_major__ * 100 + __clang_minor__ >= 302)) && !defined(__mips64)
+ || (__clang_major__ * 100 + __clang_minor__ >= 302)) && \
+ !defined(__mips64) && !defined(__s390x__)
__extension__ typedef __int128 a128;
# define __TSAN_HAS_INT128 1
#else
diff --git a/libsanitizer/tsan/tsan_interface_ann.cpp b/libsanitizer/tsan/tsan_interface_ann.cpp
index 99516d94bba..175855f66c8 100644
--- a/libsanitizer/tsan/tsan_interface_ann.cpp
+++ b/libsanitizer/tsan/tsan_interface_ann.cpp
@@ -49,8 +49,6 @@ class ScopedAnnotation {
return ret; \
ThreadState *thr = cur_thread(); \
const uptr caller_pc = (uptr)__builtin_return_address(0); \
- StatInc(thr, StatAnnotation); \
- StatInc(thr, Stat##typ); \
ScopedAnnotation sa(thr, __func__, caller_pc); \
const uptr pc = StackTrace::GetCurrentPc(); \
(void)pc; \
@@ -77,9 +75,7 @@ struct DynamicAnnContext {
ExpectRace expect;
ExpectRace benign;
- DynamicAnnContext()
- : mtx(MutexTypeAnnotations, StatMtxAnnotations) {
- }
+ DynamicAnnContext() : mtx(MutexTypeAnnotations) {}
};
static DynamicAnnContext *dyn_ann_ctx;
diff --git a/libsanitizer/tsan/tsan_interface_atomic.cpp b/libsanitizer/tsan/tsan_interface_atomic.cpp
index 3f459aff532..21fe4a19619 100644
--- a/libsanitizer/tsan/tsan_interface_atomic.cpp
+++ b/libsanitizer/tsan/tsan_interface_atomic.cpp
@@ -402,34 +402,45 @@ static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
template<typename T>
static bool AtomicCAS(ThreadState *thr, uptr pc,
volatile T *a, T *c, T v, morder mo, morder fmo) {
- (void)fmo; // Unused because llvm does not pass it yet.
+ // 31.7.2.18: "The failure argument shall not be memory_order_release
+ // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
+ // (mo_relaxed) when those are used.
+ CHECK(IsLoadOrder(fmo));
+
MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
SyncVar *s = 0;
- bool write_lock = mo != mo_acquire && mo != mo_consume;
- if (mo != mo_relaxed) {
+ bool write_lock = IsReleaseOrder(mo);
+
+ if (mo != mo_relaxed || fmo != mo_relaxed)
s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
+
+ T cc = *c;
+ T pr = func_cas(a, cc, v);
+ bool success = pr == cc;
+ if (!success) {
+ *c = pr;
+ mo = fmo;
+ }
+
+ if (s) {
thr->fast_state.IncrementEpoch();
// Can't increment epoch w/o writing to the trace as well.
TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
- if (IsAcqRelOrder(mo))
+
+ if (success && IsAcqRelOrder(mo))
AcquireReleaseImpl(thr, pc, &s->clock);
- else if (IsReleaseOrder(mo))
+ else if (success && IsReleaseOrder(mo))
ReleaseImpl(thr, pc, &s->clock);
else if (IsAcquireOrder(mo))
AcquireImpl(thr, pc, &s->clock);
- }
- T cc = *c;
- T pr = func_cas(a, cc, v);
- if (s) {
+
if (write_lock)
s->mtx.Unlock();
else
s->mtx.ReadUnlock();
}
- if (pr == cc)
- return true;
- *c = pr;
- return false;
+
+ return success;
}
template<typename T>
@@ -481,7 +492,6 @@ static morder convert_morder(morder mo) {
const uptr callpc = (uptr)__builtin_return_address(0); \
uptr pc = StackTrace::GetCurrentPc(); \
mo = convert_morder(mo); \
- AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
ScopedAtomic sa(thr, callpc, a, mo, __func__); \
return Atomic##func(thr, pc, __VA_ARGS__); \
/**/
@@ -502,22 +512,6 @@ class ScopedAtomic {
ThreadState *thr_;
};
-static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
- StatInc(thr, StatAtomic);
- StatInc(thr, t);
- StatInc(thr, size == 1 ? StatAtomic1
- : size == 2 ? StatAtomic2
- : size == 4 ? StatAtomic4
- : size == 8 ? StatAtomic8
- : StatAtomic16);
- StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
- : mo == mo_consume ? StatAtomicConsume
- : mo == mo_acquire ? StatAtomicAcquire
- : mo == mo_release ? StatAtomicRelease
- : mo == mo_acq_rel ? StatAtomicAcq_Rel
- : StatAtomicSeq_Cst);
-}
-
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
diff --git a/libsanitizer/tsan/tsan_mman.cpp b/libsanitizer/tsan/tsan_mman.cpp
index 45a39f0f8ec..7765bc07052 100644
--- a/libsanitizer/tsan/tsan_mman.cpp
+++ b/libsanitizer/tsan/tsan_mman.cpp
@@ -70,10 +70,7 @@ struct GlobalProc {
Mutex mtx;
Processor *proc;
- GlobalProc()
- : mtx(MutexTypeGlobalProc, StatMtxGlobalProc)
- , proc(ProcCreate()) {
- }
+ GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {}
};
static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
diff --git a/libsanitizer/tsan/tsan_mutex.cpp b/libsanitizer/tsan/tsan_mutex.cpp
index 7a0918f2a2c..d8b1826ee47 100644
--- a/libsanitizer/tsan/tsan_mutex.cpp
+++ b/libsanitizer/tsan/tsan_mutex.cpp
@@ -207,14 +207,11 @@ class Backoff {
static const int kActiveSpinCnt = 20;
};
-Mutex::Mutex(MutexType type, StatType stat_type) {
+Mutex::Mutex(MutexType type) {
CHECK_GT(type, MutexTypeInvalid);
CHECK_LT(type, MutexTypeCount);
#if SANITIZER_DEBUG
type_ = type;
-#endif
-#if TSAN_COLLECT_STATS
- stat_type_ = stat_type;
#endif
atomic_store(&state_, kUnlocked, memory_order_relaxed);
}
@@ -236,9 +233,6 @@ void Mutex::Lock() {
cmp = kUnlocked;
if (atomic_compare_exchange_weak(&state_, &cmp, kWriteLock,
memory_order_acquire)) {
-#if TSAN_COLLECT_STATS && !SANITIZER_GO
- StatInc(cur_thread(), stat_type_, backoff.Contention());
-#endif
return;
}
}
@@ -264,9 +258,6 @@ void Mutex::ReadLock() {
for (Backoff backoff; backoff.Do();) {
prev = atomic_load(&state_, memory_order_acquire);
if ((prev & kWriteLock) == 0) {
-#if TSAN_COLLECT_STATS && !SANITIZER_GO
- StatInc(cur_thread(), stat_type_, backoff.Contention());
-#endif
return;
}
}
diff --git a/libsanitizer/tsan/tsan_mutex.h b/libsanitizer/tsan/tsan_mutex.h
index 80fdc6ed57b..9a579eaf914 100644
--- a/libsanitizer/tsan/tsan_mutex.h
+++ b/libsanitizer/tsan/tsan_mutex.h
@@ -41,7 +41,7 @@ enum MutexType {
class Mutex {
public:
- explicit Mutex(MutexType type, StatType stat_type);
+ explicit Mutex(MutexType type);
~Mutex();
void Lock();
@@ -57,9 +57,6 @@ class Mutex {
#if SANITIZER_DEBUG
MutexType type_;
#endif
-#if TSAN_COLLECT_STATS
- StatType stat_type_;
-#endif
Mutex(const Mutex&);
void operator = (const Mutex&);
diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h
index 101522d8fa4..8bd218e25fd 100644
--- a/libsanitizer/tsan/tsan_platform.h
+++ b/libsanitizer/tsan/tsan_platform.h
@@ -365,6 +365,38 @@ struct Mapping47 {
// Indicates the runtime will define the memory regions at runtime.
#define TSAN_RUNTIME_VMA 1
+#elif defined(__s390x__)
+/*
+C/C++ on linux/s390x
+While the kernel provides a 64-bit address space, we have to restrict ourselves
+to 48 bits due to how e.g. SyncVar::GetId() works.
+0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
+0e00 0000 0000 - 4000 0000 0000: -
+4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
+8000 0000 0000 - 9000 0000 0000: -
+9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
+9800 0000 0000 - a000 0000 0000: -
+a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
+b000 0000 0000 - be00 0000 0000: -
+be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
+*/
+struct Mapping {
+ static const uptr kMetaShadowBeg = 0x900000000000ull;
+ static const uptr kMetaShadowEnd = 0x980000000000ull;
+ static const uptr kTraceMemBeg = 0xa00000000000ull;
+ static const uptr kTraceMemEnd = 0xb00000000000ull;
+ static const uptr kShadowBeg = 0x400000000000ull;
+ static const uptr kShadowEnd = 0x800000000000ull;
+ static const uptr kHeapMemBeg = 0xbe0000000000ull;
+ static const uptr kHeapMemEnd = 0xc00000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x0e0000000000ull;
+ static const uptr kHiAppMemBeg = 0xc00000004000ull;
+ static const uptr kHiAppMemEnd = 0xc00000004000ull;
+ static const uptr kAppMemMsk = 0xb00000000000ull;
+ static const uptr kAppMemXor = 0x100000000000ull;
+ static const uptr kVdsoBeg = 0xfffffffff000ull;
+};
#endif
#elif SANITIZER_GO && !SANITIZER_WINDOWS && HAS_48_BIT_ADDRESS_SPACE
@@ -528,6 +560,28 @@ struct Mapping47 {
#define TSAN_RUNTIME_VMA 1
+#elif SANITIZER_GO && defined(__s390x__)
+/*
+Go on linux/s390x
+0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
+1000 0000 0000 - 4000 0000 0000: -
+4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
+8000 0000 0000 - 9000 0000 0000: -
+9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
+9800 0000 0000 - a000 0000 0000: -
+a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
+*/
+struct Mapping {
+ static const uptr kMetaShadowBeg = 0x900000000000ull;
+ static const uptr kMetaShadowEnd = 0x980000000000ull;
+ static const uptr kTraceMemBeg = 0xa00000000000ull;
+ static const uptr kTraceMemEnd = 0xb00000000000ull;
+ static const uptr kShadowBeg = 0x400000000000ull;
+ static const uptr kShadowEnd = 0x800000000000ull;
+ static const uptr kAppMemBeg = 0x000000001000ull;
+ static const uptr kAppMemEnd = 0x100000000000ull;
+};
+
#else
# error "Unknown platform"
#endif
diff --git a/libsanitizer/tsan/tsan_platform_linux.cpp b/libsanitizer/tsan/tsan_platform_linux.cpp
index e5b6690edfd..cfe597e5380 100644
--- a/libsanitizer/tsan/tsan_platform_linux.cpp
+++ b/libsanitizer/tsan/tsan_platform_linux.cpp
@@ -391,6 +391,10 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
return mangled_sp ^ xor_key;
#elif defined(__mips__)
return mangled_sp;
+#elif defined(__s390x__)
+ // tcbhead_t.stack_guard
+ uptr xor_key = ((uptr *)__builtin_thread_pointer())[5];
+ return mangled_sp ^ xor_key;
#else
#error "Unknown platform"
#endif
@@ -411,6 +415,8 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
# define LONG_JMP_SP_ENV_SLOT 13
# elif defined(__mips64)
# define LONG_JMP_SP_ENV_SLOT 1
+# elif defined(__s390x__)
+# define LONG_JMP_SP_ENV_SLOT 9
# else
# define LONG_JMP_SP_ENV_SLOT 6
# endif
diff --git a/libsanitizer/tsan/tsan_platform_posix.cpp b/libsanitizer/tsan/tsan_platform_posix.cpp
index 73e1d4577c2..1c6198cefcd 100644
--- a/libsanitizer/tsan/tsan_platform_posix.cpp
+++ b/libsanitizer/tsan/tsan_platform_posix.cpp
@@ -72,11 +72,15 @@ void InitializeShadowMemory() {
InitializeShadowMemoryPlatform();
}
-static void ProtectRange(uptr beg, uptr end) {
+static bool TryProtectRange(uptr beg, uptr end) {
CHECK_LE(beg, end);
if (beg == end)
- return;
- if (beg != (uptr)MmapFixedNoAccess(beg, end - beg)) {
+ return true;
+ return beg == (uptr)MmapFixedNoAccess(beg, end - beg);
+}
+
+static void ProtectRange(uptr beg, uptr end) {
+ if (!TryProtectRange(beg, end)) {
Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end);
Printf("FATAL: Make sure you are not using unlimited stack\n");
Die();
@@ -118,6 +122,16 @@ void CheckAndProtect() {
ProtectRange(TraceMemEnd(), HeapMemBeg());
ProtectRange(HeapEnd(), HiAppMemBeg());
#endif
+
+#if defined(__s390x__)
+ // Protect the rest of the address space.
+ const uptr user_addr_max_l4 = 0x0020000000000000ull;
+ const uptr user_addr_max_l5 = 0xfffffffffffff000ull;
+ // All the maintained s390x kernels support at least 4-level page tables.
+ ProtectRange(HiAppMemEnd(), user_addr_max_l4);
+ // Older s390x kernels may not support 5-level page tables.
+ TryProtectRange(user_addr_max_l4, user_addr_max_l5);
+#endif
}
#endif
diff --git a/libsanitizer/tsan/tsan_rtl.cpp b/libsanitizer/tsan/tsan_rtl.cpp
index 0efa99788ab..bcf489a71d5 100644
--- a/libsanitizer/tsan/tsan_rtl.cpp
+++ b/libsanitizer/tsan/tsan_rtl.cpp
@@ -77,7 +77,7 @@ void OnInitialize() {
}
#endif
-static char thread_registry_placeholder[sizeof(ThreadRegistry)];
+static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];
static ThreadContextBase *CreateThreadContext(u32 tid) {
// Map thread trace when context is created.
@@ -115,15 +115,15 @@ static const u32 kThreadQuarantineSize = 64;
Context::Context()
: initialized(),
- report_mtx(MutexTypeReport, StatMtxReport),
+ report_mtx(MutexTypeReport),
nreported(),
nmissed_expected(),
thread_registry(new (thread_registry_placeholder) ThreadRegistry(
CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)),
- racy_mtx(MutexTypeRacy, StatMtxRacy),
+ racy_mtx(MutexTypeRacy),
racy_stacks(),
racy_addresses(),
- fired_suppressions_mtx(MutexTypeFired, StatMtxFired),
+ fired_suppressions_mtx(MutexTypeFired),
clock_alloc(LINKER_INITIALIZED, "clock allocator") {
fired_suppressions.reserve(8);
}
@@ -522,18 +522,14 @@ int Finalize(ThreadState *thr) {
failed = OnFinalize(failed);
-#if TSAN_COLLECT_STATS
- StatAggregate(ctx->stat, thr->stat);
- StatOutput(ctx->stat);
-#endif
-
return failed ? common_flags()->exitcode : 0;
}
#if !SANITIZER_GO
-void ForkBefore(ThreadState *thr, uptr pc) {
+void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
ctx->thread_registry->Lock();
ctx->report_mtx.Lock();
+ ScopedErrorReportLock::Lock();
// Suppress all reports in the pthread_atfork callbacks.
// Reports will deadlock on the report_mtx.
// We could ignore sync operations as well,
@@ -545,16 +541,18 @@ void ForkBefore(ThreadState *thr, uptr pc) {
thr->ignore_interceptors++;
}
-void ForkParentAfter(ThreadState *thr, uptr pc) {
+void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
thr->suppress_reports--; // Enabled in ForkBefore.
thr->ignore_interceptors--;
+ ScopedErrorReportLock::Unlock();
ctx->report_mtx.Unlock();
ctx->thread_registry->Unlock();
}
-void ForkChildAfter(ThreadState *thr, uptr pc) {
+void ForkChildAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
thr->suppress_reports--; // Enabled in ForkBefore.
thr->ignore_interceptors--;
+ ScopedErrorReportLock::Unlock();
ctx->report_mtx.Unlock();
ctx->thread_registry->Unlock();
@@ -693,9 +691,6 @@ ALWAYS_INLINE
void MemoryAccessImpl1(ThreadState *thr, uptr addr,
int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
u64 *shadow_mem, Shadow cur) {
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
// This potentially can live in an MMX/SSE scratch register.
// The required intrinsics are:
@@ -752,7 +747,6 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr,
return;
// choose a random candidate slot and replace it
StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
- StatInc(thr, StatShadowReplace);
return;
RACE:
HandleRace(thr, shadow_mem, cur, old);
@@ -891,19 +885,11 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
// Access to .rodata section, no races here.
// Measurements show that it can be 10-20% of all memory accesses.
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
- StatInc(thr, StatMopRodata);
return;
}
FastState fast_state = thr->fast_state;
if (UNLIKELY(fast_state.GetIgnoreBit())) {
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
- StatInc(thr, StatMopIgnored);
return;
}
@@ -914,10 +900,6 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
thr->fast_synch_epoch, kAccessIsWrite))) {
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
- StatInc(thr, StatMopSame);
return;
}
@@ -939,10 +921,6 @@ void MemoryAccessImpl(ThreadState *thr, uptr addr,
u64 *shadow_mem, Shadow cur) {
if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
thr->fast_synch_epoch, kAccessIsWrite))) {
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
- StatInc(thr, StatMopSame);
return;
}
@@ -999,7 +977,6 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
// Reset middle part.
u64 *p1 = p;
p = RoundDown(end, kPageSize);
- UnmapOrDie((void*)p1, (uptr)p - (uptr)p1);
if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
Die();
// Set the ending.
@@ -1059,7 +1036,6 @@ void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
ALWAYS_INLINE USED
void FuncEntry(ThreadState *thr, uptr pc) {
- StatInc(thr, StatFuncEnter);
DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
if (kCollectHistory) {
thr->fast_state.IncrementEpoch();
@@ -1081,7 +1057,6 @@ void FuncEntry(ThreadState *thr, uptr pc) {
ALWAYS_INLINE USED
void FuncExit(ThreadState *thr) {
- StatInc(thr, StatFuncExit);
DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
if (kCollectHistory) {
thr->fast_state.IncrementEpoch();
@@ -1156,12 +1131,6 @@ void build_consistency_debug() {}
void build_consistency_release() {}
#endif
-#if TSAN_COLLECT_STATS
-void build_consistency_stats() {}
-#else
-void build_consistency_nostats() {}
-#endif
-
} // namespace __tsan
#if !SANITIZER_GO
diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h
index 3ae519d34da..6576d40aa48 100644
--- a/libsanitizer/tsan/tsan_rtl.h
+++ b/libsanitizer/tsan/tsan_rtl.h
@@ -399,9 +399,6 @@ struct ThreadState {
#if !SANITIZER_GO
Vector<JmpBuf> jmp_bufs;
int ignore_interceptors;
-#endif
-#if TSAN_COLLECT_STATS
- u64 stat[StatCnt];
#endif
const u32 tid;
const int unique_id;
@@ -550,7 +547,6 @@ struct Context {
Flags flags;
- u64 stat[StatCnt];
u64 int_alloc_cnt[MBlockTypeCount];
u64 int_alloc_siz[MBlockTypeCount];
};
@@ -658,22 +654,6 @@ void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
ObtainCurrentStack(thr, pc, &stack); \
stack.ReverseOrder();
-#if TSAN_COLLECT_STATS
-void StatAggregate(u64 *dst, u64 *src);
-void StatOutput(u64 *stat);
-#endif
-
-void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) {
-#if TSAN_COLLECT_STATS
- thr->stat[typ] += n;
-#endif
-}
-void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) {
-#if TSAN_COLLECT_STATS
- thr->stat[typ] = n;
-#endif
-}
-
void MapShadow(uptr addr, uptr size);
void MapThreadTrace(uptr addr, uptr size, const char *name);
void DontNeedShadowFor(uptr addr, uptr size);
@@ -854,7 +834,6 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
DCHECK_GE((int)typ, 0);
DCHECK_LE((int)typ, 7);
DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
- StatInc(thr, StatEvents);
u64 pos = fs.GetTracePos();
if (UNLIKELY((pos % kTracePartSize) == 0)) {
#if !SANITIZER_GO
diff --git a/libsanitizer/tsan/tsan_rtl_mutex.cpp b/libsanitizer/tsan/tsan_rtl_mutex.cpp
index 0a8f3aa3ddb..a214a336d69 100644
--- a/libsanitizer/tsan/tsan_rtl_mutex.cpp
+++ b/libsanitizer/tsan/tsan_rtl_mutex.cpp
@@ -65,7 +65,6 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
- StatInc(thr, StatMutexCreate);
if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
CHECK(!thr->is_freeing);
thr->is_freeing = true;
@@ -81,7 +80,6 @@ void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
- StatInc(thr, StatMutexDestroy);
SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
if (s == 0)
return;
@@ -183,11 +181,9 @@ void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
const bool first = s->recursion == 0;
s->recursion += rec;
if (first) {
- StatInc(thr, StatMutexLock);
AcquireImpl(thr, pc, &s->clock);
AcquireImpl(thr, pc, &s->read_clock);
} else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
- StatInc(thr, StatMutexRecLock);
}
thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
bool pre_lock = false;
@@ -229,11 +225,9 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
s->recursion -= rec;
if (s->recursion == 0) {
- StatInc(thr, StatMutexUnlock);
s->owner_tid = kInvalidTid;
ReleaseStoreImpl(thr, pc, &s->clock);
} else {
- StatInc(thr, StatMutexRecUnlock);
}
}
thr->mset.Del(s->GetId(), true);
@@ -268,7 +262,6 @@ void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
- StatInc(thr, StatMutexReadLock);
if (IsAppMem(addr))
MemoryReadAtomic(thr, pc, addr, kSizeLog1);
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
@@ -308,7 +301,6 @@ void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
- StatInc(thr, StatMutexReadUnlock);
if (IsAppMem(addr))
MemoryReadAtomic(thr, pc, addr, kSizeLog1);
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
@@ -348,7 +340,6 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
if (s->owner_tid == kInvalidTid) {
// Seems to be read unlock.
write = false;
- StatInc(thr, StatMutexReadUnlock);
thr->fast_state.IncrementEpoch();
TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
ReleaseImpl(thr, pc, &s->read_clock);
@@ -359,11 +350,9 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
CHECK_GT(s->recursion, 0);
s->recursion--;
if (s->recursion == 0) {
- StatInc(thr, StatMutexUnlock);
s->owner_tid = kInvalidTid;
ReleaseStoreImpl(thr, pc, &s->clock);
} else {
- StatInc(thr, StatMutexRecUnlock);
}
} else if (!s->IsFlagSet(MutexFlagBroken)) {
s->SetFlags(MutexFlagBroken);
@@ -494,7 +483,6 @@ void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
return;
thr->clock.set(thr->fast_state.epoch());
thr->clock.acquire(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncAcquire);
}
void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
@@ -503,7 +491,6 @@ void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
thr->clock.set(thr->fast_state.epoch());
thr->fast_synch_epoch = thr->fast_state.epoch();
thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncReleaseStoreAcquire);
}
void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
@@ -512,7 +499,6 @@ void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
thr->clock.set(thr->fast_state.epoch());
thr->fast_synch_epoch = thr->fast_state.epoch();
thr->clock.release(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncRelease);
}
void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) {
@@ -521,7 +507,6 @@ void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) {
thr->clock.set(thr->fast_state.epoch());
thr->fast_synch_epoch = thr->fast_state.epoch();
thr->clock.ReleaseStore(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncRelease);
}
void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
@@ -530,8 +515,6 @@ void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
thr->clock.set(thr->fast_state.epoch());
thr->fast_synch_epoch = thr->fast_state.epoch();
thr->clock.acq_rel(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncAcquire);
- StatInc(thr, StatSyncRelease);
}
void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
diff --git a/libsanitizer/tsan/tsan_rtl_ppc64.S b/libsanitizer/tsan/tsan_rtl_ppc64.S
index 9e533a71a9c..8285e21aa1e 100644
--- a/libsanitizer/tsan/tsan_rtl_ppc64.S
+++ b/libsanitizer/tsan/tsan_rtl_ppc64.S
@@ -1,6 +1,5 @@
#include "tsan_ppc_regs.h"
- .machine altivec
.section .text
.hidden __tsan_setjmp
.globl _setjmp
diff --git a/libsanitizer/tsan/tsan_rtl_s390x.S b/libsanitizer/tsan/tsan_rtl_s390x.S
new file mode 100644
index 00000000000..fcff35fbc7e
--- /dev/null
+++ b/libsanitizer/tsan/tsan_rtl_s390x.S
@@ -0,0 +1,47 @@
+#include "sanitizer_common/sanitizer_asm.h"
+
+#define CFA_OFFSET 160
+#define R2_REL_OFFSET 16
+#define R3_REL_OFFSET 24
+#define R14_REL_OFFSET 112
+#define R15_REL_OFFSET 120
+#define FRAME_SIZE 160
+
+.text
+
+ASM_HIDDEN(__tsan_setjmp)
+
+.macro intercept symbol, real
+.comm \real, 8, 8
+.globl ASM_SYMBOL_INTERCEPTOR(\symbol)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(\symbol))
+ASM_SYMBOL_INTERCEPTOR(\symbol):
+ CFI_STARTPROC
+ stmg %r2, %r3, R2_REL_OFFSET(%r15)
+ CFI_REL_OFFSET(%r2, R2_REL_OFFSET)
+ CFI_REL_OFFSET(%r3, R3_REL_OFFSET)
+ stmg %r14, %r15, R14_REL_OFFSET(%r15)
+ CFI_REL_OFFSET(%r14, R14_REL_OFFSET)
+ CFI_REL_OFFSET(%r15, R15_REL_OFFSET)
+ aghi %r15, -FRAME_SIZE
+ CFI_ADJUST_CFA_OFFSET(FRAME_SIZE)
+ la %r2, FRAME_SIZE(%r15)
+ brasl %r14, ASM_SYMBOL(__tsan_setjmp)
+ lmg %r14, %r15, FRAME_SIZE + R14_REL_OFFSET(%r15)
+ CFI_RESTORE(%r14)
+ CFI_RESTORE(%r15)
+ CFI_DEF_CFA_OFFSET(CFA_OFFSET)
+ lmg %r2, %r3, R2_REL_OFFSET(%r15)
+ CFI_RESTORE(%r2)
+ CFI_RESTORE(%r3)
+ larl %r1, \real
+ lg %r1, 0(%r1)
+ br %r1
+ CFI_ENDPROC
+ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(\symbol))
+.endm
+
+intercept setjmp, _ZN14__interception11real_setjmpE
+intercept _setjmp, _ZN14__interception12real__setjmpE
+intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE
+intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE
diff --git a/libsanitizer/tsan/tsan_rtl_thread.cpp b/libsanitizer/tsan/tsan_rtl_thread.cpp
index 6d1ccd8c9c7..cdb6e60ebbd 100644
--- a/libsanitizer/tsan/tsan_rtl_thread.cpp
+++ b/libsanitizer/tsan/tsan_rtl_thread.cpp
@@ -61,8 +61,6 @@ void ThreadContext::OnCreated(void *arg) {
TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0);
ReleaseImpl(args->thr, 0, &sync);
creation_stack_id = CurrentStackId(args->thr, args->pc);
- if (reuse_count == 0)
- StatInc(args->thr, StatThreadMaxTid);
}
void ThreadContext::OnReset() {
@@ -115,7 +113,6 @@ void ThreadContext::OnStarted(void *arg) {
thr->fast_synch_epoch = epoch0;
AcquireImpl(thr, 0, &sync);
- StatInc(thr, StatSyncAcquire);
sync.Reset(&thr->proc()->clock_cache);
thr->is_inited = true;
DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
@@ -149,9 +146,6 @@ void ThreadContext::OnFinished() {
PlatformCleanUpThreadState(thr);
#endif
thr->~ThreadState();
-#if TSAN_COLLECT_STATS
- StatAggregate(ctx->stat, thr->stat);
-#endif
thr = 0;
}
@@ -232,13 +226,11 @@ int ThreadCount(ThreadState *thr) {
}
int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
- StatInc(thr, StatThreadCreate);
OnCreatedArgs args = { thr, pc };
u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers.
int tid =
ctx->thread_registry->CreateThread(uid, detached, parent_tid, &args);
DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid);
- StatSet(thr, StatThreadMaxAlive, ctx->thread_registry->GetMaxAliveThreads());
return tid;
}
@@ -280,7 +272,6 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
void ThreadFinish(ThreadState *thr) {
ThreadCheckIgnore(thr);
- StatInc(thr, StatThreadFinish);
if (thr->stk_addr && thr->stk_size)
DontNeedShadowFor(thr->stk_addr, thr->stk_size);
if (thr->tls_addr && thr->tls_size)
@@ -372,13 +363,10 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
}
#endif
- StatInc(thr, StatMopRange);
-
if (*shadow_mem == kShadowRodata) {
DCHECK(!is_write);
// Access to .rodata section, no races here.
// Measurements show that it can be 10-20% of all memory accesses.
- StatInc(thr, StatMopRangeRodata);
return;
}
diff --git a/libsanitizer/tsan/tsan_stack_trace.cpp b/libsanitizer/tsan/tsan_stack_trace.cpp
index 403a21ae4ae..6c703d7f2b1 100644
--- a/libsanitizer/tsan/tsan_stack_trace.cpp
+++ b/libsanitizer/tsan/tsan_stack_trace.cpp
@@ -54,10 +54,8 @@ void __sanitizer::BufferedStackTrace::UnwindImpl(
uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
uptr top = 0;
uptr bottom = 0;
- if (StackTrace::WillUseFastUnwind(request_fast)) {
- GetThreadStackTopAndBottom(false, &top, &bottom);
- Unwind(max_depth, pc, bp, nullptr, top, bottom, true);
- } else
- Unwind(max_depth, pc, 0, context, 0, 0, false);
+ GetThreadStackTopAndBottom(false, &top, &bottom);
+ bool fast = StackTrace::WillUseFastUnwind(request_fast);
+ Unwind(max_depth, pc, bp, context, top, bottom, fast);
}
#endif // SANITIZER_GO
diff --git a/libsanitizer/tsan/tsan_stat.cpp b/libsanitizer/tsan/tsan_stat.cpp
deleted file mode 100644
index 78f3cce9138..00000000000
--- a/libsanitizer/tsan/tsan_stat.cpp
+++ /dev/null
@@ -1,186 +0,0 @@
-//===-- tsan_stat.cpp -----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of ThreadSanitizer (TSan), a race detector.
-//
-//===----------------------------------------------------------------------===//
-#include "tsan_stat.h"
-#include "tsan_rtl.h"
-
-namespace __tsan {
-
-#if TSAN_COLLECT_STATS
-
-void StatAggregate(u64 *dst, u64 *src) {
- for (int i = 0; i < StatCnt; i++)
- dst[i] += src[i];
-}
-
-void StatOutput(u64 *stat) {
- stat[StatShadowNonZero] = stat[StatShadowProcessed] - stat[StatShadowZero];
-
- static const char *name[StatCnt] = {};
- name[StatMop] = "Memory accesses ";
- name[StatMopRead] = " Including reads ";
- name[StatMopWrite] = " writes ";
- name[StatMop1] = " Including size 1 ";
- name[StatMop2] = " size 2 ";
- name[StatMop4] = " size 4 ";
- name[StatMop8] = " size 8 ";
- name[StatMopSame] = " Including same ";
- name[StatMopIgnored] = " Including ignored ";
- name[StatMopRange] = " Including range ";
- name[StatMopRodata] = " Including .rodata ";
- name[StatMopRangeRodata] = " Including .rodata range ";
- name[StatShadowProcessed] = "Shadow processed ";
- name[StatShadowZero] = " Including empty ";
- name[StatShadowNonZero] = " Including non empty ";
- name[StatShadowSameSize] = " Including same size ";
- name[StatShadowIntersect] = " intersect ";
- name[StatShadowNotIntersect] = " not intersect ";
- name[StatShadowSameThread] = " Including same thread ";
- name[StatShadowAnotherThread] = " another thread ";
- name[StatShadowReplace] = " Including evicted ";
-
- name[StatFuncEnter] = "Function entries ";
- name[StatFuncExit] = "Function exits ";
- name[StatEvents] = "Events collected ";
-
- name[StatThreadCreate] = "Total threads created ";
- name[StatThreadFinish] = " threads finished ";
- name[StatThreadReuse] = " threads reused ";
- name[StatThreadMaxTid] = " max tid ";
- name[StatThreadMaxAlive] = " max alive threads ";
-
- name[StatMutexCreate] = "Mutexes created ";
- name[StatMutexDestroy] = " destroyed ";
- name[StatMutexLock] = " lock ";
- name[StatMutexUnlock] = " unlock ";
- name[StatMutexRecLock] = " recursive lock ";
- name[StatMutexRecUnlock] = " recursive unlock ";
- name[StatMutexReadLock] = " read lock ";
- name[StatMutexReadUnlock] = " read unlock ";
-
- name[StatSyncCreated] = "Sync objects created ";
- name[StatSyncDestroyed] = " destroyed ";
- name[StatSyncAcquire] = " acquired ";
- name[StatSyncRelease] = " released ";
-
- name[StatClockAcquire] = "Clock acquire ";
- name[StatClockAcquireEmpty] = " empty clock ";
- name[StatClockAcquireFastRelease] = " fast from release-store ";
- name[StatClockAcquireFull] = " full (slow) ";
- name[StatClockAcquiredSomething] = " acquired something ";
- name[StatClockRelease] = "Clock release ";
- name[StatClockReleaseResize] = " resize ";
- name[StatClockReleaseFast] = " fast ";
- name[StatClockReleaseSlow] = " dirty overflow (slow) ";
- name[StatClockReleaseFull] = " full (slow) ";
- name[StatClockReleaseAcquired] = " was acquired ";
- name[StatClockReleaseClearTail] = " clear tail ";
- name[StatClockStore] = "Clock release store ";
- name[StatClockStoreResize] = " resize ";
- name[StatClockStoreFast] = " fast ";
- name[StatClockStoreFull] = " slow ";
- name[StatClockStoreTail] = " clear tail ";
- name[StatClockAcquireRelease] = "Clock acquire-release ";
-
- name[StatAtomic] = "Atomic operations ";
- name[StatAtomicLoad] = " Including load ";
- name[StatAtomicStore] = " store ";
- name[StatAtomicExchange] = " exchange ";
- name[StatAtomicFetchAdd] = " fetch_add ";
- name[StatAtomicFetchSub] = " fetch_sub ";
- name[StatAtomicFetchAnd] = " fetch_and ";
- name[StatAtomicFetchOr] = " fetch_or ";
- name[StatAtomicFetchXor] = " fetch_xor ";
- name[StatAtomicFetchNand] = " fetch_nand ";
- name[StatAtomicCAS] = " compare_exchange ";
- name[StatAtomicFence] = " fence ";
- name[StatAtomicRelaxed] = " Including relaxed ";
- name[StatAtomicConsume] = " consume ";
- name[StatAtomicAcquire] = " acquire ";
- name[StatAtomicRelease] = " release ";
- name[StatAtomicAcq_Rel] = " acq_rel ";
- name[StatAtomicSeq_Cst] = " seq_cst ";
- name[StatAtomic1] = " Including size 1 ";
- name[StatAtomic2] = " size 2 ";
- name[StatAtomic4] = " size 4 ";
- name[StatAtomic8] = " size 8 ";
- name[StatAtomic16] = " size 16 ";
-
- name[StatAnnotation] = "Dynamic annotations ";
- name[StatAnnotateHappensBefore] = " HappensBefore ";
- name[StatAnnotateHappensAfter] = " HappensAfter ";
- name[StatAnnotateCondVarSignal] = " CondVarSignal ";
- name[StatAnnotateCondVarSignalAll] = " CondVarSignalAll ";
- name[StatAnnotateMutexIsNotPHB] = " MutexIsNotPHB ";
- name[StatAnnotateCondVarWait] = " CondVarWait ";
- name[StatAnnotateRWLockCreate] = " RWLockCreate ";
- name[StatAnnotateRWLockCreateStatic] = " StatAnnotateRWLockCreateStatic ";
- name[StatAnnotateRWLockDestroy] = " RWLockDestroy ";
- name[StatAnnotateRWLockAcquired] = " RWLockAcquired ";
- name[StatAnnotateRWLockReleased] = " RWLockReleased ";
- name[StatAnnotateTraceMemory] = " TraceMemory ";
- name[StatAnnotateFlushState] = " FlushState ";
- name[StatAnnotateNewMemory] = " NewMemory ";
- name[StatAnnotateNoOp] = " NoOp ";
- name[StatAnnotateFlushExpectedRaces] = " FlushExpectedRaces ";
- name[StatAnnotateEnableRaceDetection] = " EnableRaceDetection ";
- name[StatAnnotateMutexIsUsedAsCondVar] = " MutexIsUsedAsCondVar ";
- name[StatAnnotatePCQGet] = " PCQGet ";
- name[StatAnnotatePCQPut] = " PCQPut ";
- name[StatAnnotatePCQDestroy] = " PCQDestroy ";
- name[StatAnnotatePCQCreate] = " PCQCreate ";
- name[StatAnnotateExpectRace] = " ExpectRace ";
- name[StatAnnotateBenignRaceSized] = " BenignRaceSized ";
- name[StatAnnotateBenignRace] = " BenignRace ";
- name[StatAnnotateIgnoreReadsBegin] = " IgnoreReadsBegin ";
- name[StatAnnotateIgnoreReadsEnd] = " IgnoreReadsEnd ";
- name[StatAnnotateIgnoreWritesBegin] = " IgnoreWritesBegin ";
- name[StatAnnotateIgnoreWritesEnd] = " IgnoreWritesEnd ";
- name[StatAnnotateIgnoreSyncBegin] = " IgnoreSyncBegin ";
- name[StatAnnotateIgnoreSyncEnd] = " IgnoreSyncEnd ";
- name[StatAnnotatePublishMemoryRange] = " PublishMemoryRange ";
- name[StatAnnotateUnpublishMemoryRange] = " UnpublishMemoryRange ";
- name[StatAnnotateThreadName] = " ThreadName ";
- name[Stat__tsan_mutex_create] = " __tsan_mutex_create ";
- name[Stat__tsan_mutex_destroy] = " __tsan_mutex_destroy ";
- name[Stat__tsan_mutex_pre_lock] = " __tsan_mutex_pre_lock ";
- name[Stat__tsan_mutex_post_lock] = " __tsan_mutex_post_lock ";
- name[Stat__tsan_mutex_pre_unlock] = " __tsan_mutex_pre_unlock ";
- name[Stat__tsan_mutex_post_unlock] = " __tsan_mutex_post_unlock ";
- name[Stat__tsan_mutex_pre_signal] = " __tsan_mutex_pre_signal ";
- name[Stat__tsan_mutex_post_signal] = " __tsan_mutex_post_signal ";
- name[Stat__tsan_mutex_pre_divert] = " __tsan_mutex_pre_divert ";
- name[Stat__tsan_mutex_post_divert] = " __tsan_mutex_post_divert ";
-
- name[StatMtxTotal] = "Contentionz ";
- name[StatMtxTrace] = " Trace ";
- name[StatMtxThreads] = " Threads ";
- name[StatMtxReport] = " Report ";
- name[StatMtxSyncVar] = " SyncVar ";
- name[StatMtxSyncTab] = " SyncTab ";
- name[StatMtxSlab] = " Slab ";
- name[StatMtxAtExit] = " Atexit ";
- name[StatMtxAnnotations] = " Annotations ";
- name[StatMtxMBlock] = " MBlock ";
- name[StatMtxDeadlockDetector] = " DeadlockDetector ";
- name[StatMtxFired] = " FiredSuppressions ";
- name[StatMtxRacy] = " RacyStacks ";
- name[StatMtxFD] = " FD ";
- name[StatMtxGlobalProc] = " GlobalProc ";
-
- Printf("Statistics:\n");
- for (int i = 0; i < StatCnt; i++)
- Printf("%s: %16zu\n", name[i], (uptr)stat[i]);
-}
-
-#endif
-
-} // namespace __tsan
diff --git a/libsanitizer/tsan/tsan_stat.h b/libsanitizer/tsan/tsan_stat.h
deleted file mode 100644
index 8b26a59bb2e..00000000000
--- a/libsanitizer/tsan/tsan_stat.h
+++ /dev/null
@@ -1,191 +0,0 @@
-//===-- tsan_stat.h ---------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of ThreadSanitizer (TSan), a race detector.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef TSAN_STAT_H
-#define TSAN_STAT_H
-
-namespace __tsan {
-
-enum StatType {
- // Memory access processing related stuff.
- StatMop,
- StatMopRead,
- StatMopWrite,
- StatMop1, // These must be consequtive.
- StatMop2,
- StatMop4,
- StatMop8,
- StatMopSame,
- StatMopIgnored,
- StatMopRange,
- StatMopRodata,
- StatMopRangeRodata,
- StatShadowProcessed,
- StatShadowZero,
- StatShadowNonZero, // Derived.
- StatShadowSameSize,
- StatShadowIntersect,
- StatShadowNotIntersect,
- StatShadowSameThread,
- StatShadowAnotherThread,
- StatShadowReplace,
-
- // Func processing.
- StatFuncEnter,
- StatFuncExit,
-
- // Trace processing.
- StatEvents,
-
- // Threads.
- StatThreadCreate,
- StatThreadFinish,
- StatThreadReuse,
- StatThreadMaxTid,
- StatThreadMaxAlive,
-
- // Mutexes.
- StatMutexCreate,
- StatMutexDestroy,
- StatMutexLock,
- StatMutexUnlock,
- StatMutexRecLock,
- StatMutexRecUnlock,
- StatMutexReadLock,
- StatMutexReadUnlock,
-
- // Synchronization.
- StatSyncCreated,
- StatSyncDestroyed,
- StatSyncAcquire,
- StatSyncRelease,
- StatSyncReleaseStoreAcquire,
-
- // Clocks - acquire.
- StatClockAcquire,
- StatClockAcquireEmpty,
- StatClockAcquireFastRelease,
- StatClockAcquireFull,
- StatClockAcquiredSomething,
- // Clocks - release.
- StatClockRelease,
- StatClockReleaseResize,
- StatClockReleaseFast,
- StatClockReleaseSlow,
- StatClockReleaseFull,
- StatClockReleaseAcquired,
- StatClockReleaseClearTail,
- // Clocks - release store.
- StatClockStore,
- StatClockStoreResize,
- StatClockStoreFast,
- StatClockStoreFull,
- StatClockStoreTail,
- // Clocks - acquire-release.
- StatClockAcquireRelease,
-
- // Atomics.
- StatAtomic,
- StatAtomicLoad,
- StatAtomicStore,
- StatAtomicExchange,
- StatAtomicFetchAdd,
- StatAtomicFetchSub,
- StatAtomicFetchAnd,
- StatAtomicFetchOr,
- StatAtomicFetchXor,
- StatAtomicFetchNand,
- StatAtomicCAS,
- StatAtomicFence,
- StatAtomicRelaxed,
- StatAtomicConsume,
- StatAtomicAcquire,
- StatAtomicRelease,
- StatAtomicAcq_Rel,
- StatAtomicSeq_Cst,
- StatAtomic1,
- StatAtomic2,
- StatAtomic4,
- StatAtomic8,
- StatAtomic16,
-
- // Dynamic annotations.
- StatAnnotation,
- StatAnnotateHappensBefore,
- StatAnnotateHappensAfter,
- StatAnnotateCondVarSignal,
- StatAnnotateCondVarSignalAll,
- StatAnnotateMutexIsNotPHB,
- StatAnnotateCondVarWait,
- StatAnnotateRWLockCreate,
- StatAnnotateRWLockCreateStatic,
- StatAnnotateRWLockDestroy,
- StatAnnotateRWLockAcquired,
- StatAnnotateRWLockReleased,
- StatAnnotateTraceMemory,
- StatAnnotateFlushState,
- StatAnnotateNewMemory,
- StatAnnotateNoOp,
- StatAnnotateFlushExpectedRaces,
- StatAnnotateEnableRaceDetection,
- StatAnnotateMutexIsUsedAsCondVar,
- StatAnnotatePCQGet,
- StatAnnotatePCQPut,
- StatAnnotatePCQDestroy,
- StatAnnotatePCQCreate,
- StatAnnotateExpectRace,
- StatAnnotateBenignRaceSized,
- StatAnnotateBenignRace,
- StatAnnotateIgnoreReadsBegin,
- StatAnnotateIgnoreReadsEnd,
- StatAnnotateIgnoreWritesBegin,
- StatAnnotateIgnoreWritesEnd,
- StatAnnotateIgnoreSyncBegin,
- StatAnnotateIgnoreSyncEnd,
- StatAnnotatePublishMemoryRange,
- StatAnnotateUnpublishMemoryRange,
- StatAnnotateThreadName,
- Stat__tsan_mutex_create,
- Stat__tsan_mutex_destroy,
- Stat__tsan_mutex_pre_lock,
- Stat__tsan_mutex_post_lock,
- Stat__tsan_mutex_pre_unlock,
- Stat__tsan_mutex_post_unlock,
- Stat__tsan_mutex_pre_signal,
- Stat__tsan_mutex_post_signal,
- Stat__tsan_mutex_pre_divert,
- Stat__tsan_mutex_post_divert,
-
- // Internal mutex contentionz.
- StatMtxTotal,
- StatMtxTrace,
- StatMtxThreads,
- StatMtxReport,
- StatMtxSyncVar,
- StatMtxSyncTab,
- StatMtxSlab,
- StatMtxAnnotations,
- StatMtxAtExit,
- StatMtxMBlock,
- StatMtxDeadlockDetector,
- StatMtxFired,
- StatMtxRacy,
- StatMtxFD,
- StatMtxGlobalProc,
-
- // This must be the last.
- StatCnt
-};
-
-} // namespace __tsan
-
-#endif // TSAN_STAT_H
diff --git a/libsanitizer/tsan/tsan_sync.cpp b/libsanitizer/tsan/tsan_sync.cpp
index ba24f98ae9f..d25434af52a 100644
--- a/libsanitizer/tsan/tsan_sync.cpp
+++ b/libsanitizer/tsan/tsan_sync.cpp
@@ -18,10 +18,7 @@ namespace __tsan {
void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);
-SyncVar::SyncVar()
- : mtx(MutexTypeSyncVar, StatMtxSyncVar) {
- Reset(0);
-}
+SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); }
void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) {
this->addr = addr;
diff --git a/libsanitizer/tsan/tsan_trace.h b/libsanitizer/tsan/tsan_trace.h
index fbd0f72db6e..9f2677b778d 100644
--- a/libsanitizer/tsan/tsan_trace.h
+++ b/libsanitizer/tsan/tsan_trace.h
@@ -65,9 +65,7 @@ struct Trace {
// CreateThreadContext.
TraceHeader headers[kTraceParts];
- Trace()
- : mtx(MutexTypeTrace, StatMtxTrace) {
- }
+ Trace() : mtx(MutexTypeTrace) {}
};
} // namespace __tsan
diff --git a/libsanitizer/tsan/tsan_update_shadow_word_inl.h b/libsanitizer/tsan/tsan_update_shadow_word_inl.h
index 056c3aa2032..d23dfb0ba06 100644
--- a/libsanitizer/tsan/tsan_update_shadow_word_inl.h
+++ b/libsanitizer/tsan/tsan_update_shadow_word_inl.h
@@ -13,12 +13,10 @@
// produce sligtly less efficient code.
//===----------------------------------------------------------------------===//
do {
- StatInc(thr, StatShadowProcessed);
const unsigned kAccessSize = 1 << kAccessSizeLog;
u64 *sp = &shadow_mem[idx];
old = LoadShadow(sp);
if (LIKELY(old.IsZero())) {
- StatInc(thr, StatShadowZero);
if (!stored) {
StoreIfNotYetStored(sp, &store_word);
stored = true;
@@ -27,17 +25,14 @@ do {
}
// is the memory access equal to the previous?
if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) {
- StatInc(thr, StatShadowSameSize);
// same thread?
if (LIKELY(Shadow::TidsAreEqual(old, cur))) {
- StatInc(thr, StatShadowSameThread);
if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) {
StoreIfNotYetStored(sp, &store_word);
stored = true;
}
break;
}
- StatInc(thr, StatShadowAnotherThread);
if (HappensBefore(old, thr)) {
if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) {
StoreIfNotYetStored(sp, &store_word);
@@ -51,12 +46,8 @@ do {
}
// Do the memory access intersect?
if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
- StatInc(thr, StatShadowIntersect);
- if (Shadow::TidsAreEqual(old, cur)) {
- StatInc(thr, StatShadowSameThread);
+ if (Shadow::TidsAreEqual(old, cur))
break;
- }
- StatInc(thr, StatShadowAnotherThread);
if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
break;
if (LIKELY(HappensBefore(old, thr)))
@@ -64,6 +55,5 @@ do {
goto RACE;
}
// The accesses do not intersect.
- StatInc(thr, StatShadowNotIntersect);
break;
} while (0);
diff --git a/libsanitizer/ubsan/ubsan_diag_standalone.cpp b/libsanitizer/ubsan/ubsan_diag_standalone.cpp
index 300179adae2..5526ae05165 100644
--- a/libsanitizer/ubsan/ubsan_diag_standalone.cpp
+++ b/libsanitizer/ubsan/ubsan_diag_standalone.cpp
@@ -20,11 +20,9 @@ void __sanitizer::BufferedStackTrace::UnwindImpl(
uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
uptr top = 0;
uptr bottom = 0;
- if (StackTrace::WillUseFastUnwind(request_fast)) {
- GetThreadStackTopAndBottom(false, &top, &bottom);
- Unwind(max_depth, pc, bp, nullptr, top, bottom, true);
- } else
- Unwind(max_depth, pc, bp, context, 0, 0, false);
+ GetThreadStackTopAndBottom(false, &top, &bottom);
+ bool fast = StackTrace::WillUseFastUnwind(request_fast);
+ Unwind(max_depth, pc, bp, context, top, bottom, fast);
}
extern "C" {
diff --git a/libsanitizer/ubsan/ubsan_flags.cpp b/libsanitizer/ubsan/ubsan_flags.cpp
index 9a66bd37518..25cefd46ce2 100644
--- a/libsanitizer/ubsan/ubsan_flags.cpp
+++ b/libsanitizer/ubsan/ubsan_flags.cpp
@@ -50,7 +50,6 @@ void InitializeFlags() {
{
CommonFlags cf;
cf.CopyFrom(*common_flags());
- cf.print_summary = false;
cf.external_symbolizer_path = GetFlag("UBSAN_SYMBOLIZER_PATH");
OverrideCommonFlags(cf);
}
diff --git a/libsanitizer/ubsan/ubsan_handlers.cpp b/libsanitizer/ubsan/ubsan_handlers.cpp
index 2184625aa6e..e201e6bba22 100644
--- a/libsanitizer/ubsan/ubsan_handlers.cpp
+++ b/libsanitizer/ubsan/ubsan_handlers.cpp
@@ -894,21 +894,6 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable,
} // namespace __ubsan
-void __ubsan::__ubsan_handle_cfi_bad_icall(CFIBadIcallData *CallData,
- ValueHandle Function) {
- GET_REPORT_OPTIONS(false);
- CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type};
- handleCFIBadIcall(&Data, Function, Opts);
-}
-
-void __ubsan::__ubsan_handle_cfi_bad_icall_abort(CFIBadIcallData *CallData,
- ValueHandle Function) {
- GET_REPORT_OPTIONS(true);
- CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type};
- handleCFIBadIcall(&Data, Function, Opts);
- Die();
-}
-
void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data,
ValueHandle Value,
uptr ValidVtable) {
diff --git a/libsanitizer/ubsan/ubsan_handlers.h b/libsanitizer/ubsan/ubsan_handlers.h
index 9f412353fc0..219fb15de55 100644
--- a/libsanitizer/ubsan/ubsan_handlers.h
+++ b/libsanitizer/ubsan/ubsan_handlers.h
@@ -215,20 +215,12 @@ enum CFITypeCheckKind : unsigned char {
CFITCK_VMFCall,
};
-struct CFIBadIcallData {
- SourceLocation Loc;
- const TypeDescriptor &Type;
-};
-
struct CFICheckFailData {
CFITypeCheckKind CheckKind;
SourceLocation Loc;
const TypeDescriptor &Type;
};
-/// \brief Handle control flow integrity failure for indirect function calls.
-RECOVERABLE(cfi_bad_icall, CFIBadIcallData *Data, ValueHandle Function)
-
/// \brief Handle control flow integrity failures.
RECOVERABLE(cfi_check_fail, CFICheckFailData *Data, ValueHandle Function,
uptr VtableIsValid)
diff --git a/libsanitizer/ubsan/ubsan_platform.h b/libsanitizer/ubsan/ubsan_platform.h
index 42944a0b3d0..d2cc2e10bd2 100644
--- a/libsanitizer/ubsan/ubsan_platform.h
+++ b/libsanitizer/ubsan/ubsan_platform.h
@@ -12,16 +12,14 @@
#ifndef UBSAN_PLATFORM_H
#define UBSAN_PLATFORM_H
-#ifndef CAN_SANITIZE_UB
// Other platforms should be easy to add, and probably work as-is.
#if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || \
- defined(__NetBSD__) || defined(__DragonFly__) || \
- (defined(__sun__) && defined(__svr4__)) || \
- defined(_WIN32) || defined(__Fuchsia__) || defined(__rtems__)
-# define CAN_SANITIZE_UB 1
+ defined(__NetBSD__) || defined(__DragonFly__) || \
+ (defined(__sun__) && defined(__svr4__)) || defined(_WIN32) || \
+ defined(__Fuchsia__)
+#define CAN_SANITIZE_UB 1
#else
# define CAN_SANITIZE_UB 0
#endif
-#endif //CAN_SANITIZE_UB
#endif
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2021-07-20 21:22 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-07-20 21:22 [gcc r12-2423] libsanitizer: Merge with upstream H.J. Lu
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).