public inbox for glibc-cvs@sourceware.org help / color / mirror / Atom feed
From: Szabolcs Nagy <nsz@sourceware.org> To: glibc-cvs@sourceware.org Subject: [glibc/nsz/mtag] malloc: Use global flag instead of function pointer dispatch for mtag Date: Fri, 19 Mar 2021 11:57:28 +0000 (GMT) [thread overview] Message-ID: <20210319115728.822CD386F436@sourceware.org> (raw) https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=d4f0f405ca8553dfd5a8e6b1a5debff66e9d092d commit d4f0f405ca8553dfd5a8e6b1a5debff66e9d092d Author: Szabolcs Nagy <szabolcs.nagy@arm.com> Date: Wed Jan 27 15:45:43 2021 +0000 malloc: Use global flag instead of function pointer dispatch for mtag A flag check can be faster than function pointers because of how branch prediction and speculation works and it can also remove a layer of indirection when there is a mismatch between the malloc internal tag_* api and __libc_mtag_* target hooks. Memory tagging wrapper functions are moved to malloc.c from arena.c and the logic now checks mmap_enabled. The definition of tag_new_usable is moved after chunk related definitions. This refactoring also allows using mtag_enabled checks instead of USE_MTAG ifdefs when memory tagging support only changes code logic when memory tagging is enabled at runtime. Note: an "if (false)" code block is optimized away even at -O0 by gcc. Reviewed-by: DJ Delorie <dj@redhat.com> Diff: --- malloc/arena.c | 33 +------------------------------- malloc/malloc.c | 58 +++++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 52 deletions(-) diff --git a/malloc/arena.c b/malloc/arena.c index d0778fea92..1e83bb66bd 100644 --- a/malloc/arena.c +++ b/malloc/arena.c @@ -287,34 +287,6 @@ extern struct dl_open_hook *_dl_open_hook; libc_hidden_proto (_dl_open_hook); #endif -#ifdef USE_MTAG - -/* Generate a new (random) tag value for PTR and tag the memory it - points to upto the end of the usable size for the chunk containing - it. Return the newly tagged pointer. */ -static void * -__mtag_tag_new_usable (void *ptr) -{ - if (ptr) - { - mchunkptr cp = mem2chunk(ptr); - ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), - CHUNK_AVAILABLE_SIZE (cp) - CHUNK_HDR_SZ); - } - return ptr; -} - -/* Generate a new (random) tag value for PTR, set the tags for the - memory to the new tag and initialize the memory contents to VAL. - In practice this function will only be called with VAL=0, but we - keep this parameter to maintain the same prototype as memset. */ -static void * -__mtag_tag_new_memset (void *ptr, int val, size_t size) -{ - return __libc_mtag_memset_with_tag (__libc_mtag_new_tag (ptr), val, size); -} -#endif - static void ptmalloc_init (void) { @@ -332,11 +304,8 @@ ptmalloc_init (void) if (__MTAG_SBRK_UNTAGGED) __morecore = __failing_morecore; + mtag_enabled = true; mtag_mmap_flags = __MTAG_MMAP_FLAGS; - tag_new_memset = __mtag_tag_new_memset; - tag_region = __libc_mtag_tag_region; - tag_new_usable = __mtag_tag_new_usable; - tag_at = __libc_mtag_address_get_tag; mtag_granule_mask = ~(size_t)(__MTAG_GRANULE_SIZE - 1); } #endif diff --git a/malloc/malloc.c b/malloc/malloc.c index 62d00f54cc..253a919ec5 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -441,35 +441,41 @@ void *(*__morecore)(ptrdiff_t) = __default_morecore; */ #ifdef USE_MTAG +static bool mtag_enabled = false; +static int mtag_mmap_flags = 0; +static size_t mtag_granule_mask = ~(size_t)0; +#else +# define mtag_enabled false +# define mtag_mmap_flags 0 +#endif -/* Default implementaions when memory tagging is supported, but disabled. */ -static void * -__default_tag_region (void *ptr, size_t size) +static __always_inline void * +tag_region (void *ptr, size_t size) { + if (__glibc_unlikely (mtag_enabled)) + return __libc_mtag_tag_region (ptr, size); return ptr; } -static void * -__default_tag_nop (void *ptr) +static __always_inline void * +tag_new_memset (void *ptr, int val, size_t size) { - return ptr; + if (__glibc_unlikely (mtag_enabled)) + return __libc_mtag_memset_with_tag (__libc_mtag_new_tag (ptr), val, size); + return memset (ptr, val, size); } -static int mtag_mmap_flags = 0; -static size_t mtag_granule_mask = ~(size_t)0; - -static void *(*tag_new_memset)(void *, int, size_t) = memset; -static void *(*tag_region)(void *, size_t) = __default_tag_region; -static void *(*tag_new_usable)(void *) = __default_tag_nop; -static void *(*tag_at)(void *) = __default_tag_nop; +/* Defined later. */ +static void * +tag_new_usable (void *ptr); -#else -# define mtag_mmap_flags 0 -# define tag_new_memset(ptr, val, size) memset (ptr, val, size) -# define tag_region(ptr, size) (ptr) -# define tag_new_usable(ptr) (ptr) -# define tag_at(ptr) (ptr) -#endif +static __always_inline void * +tag_at (void *ptr) +{ + if (__glibc_unlikely (mtag_enabled)) + return __libc_mtag_address_get_tag (ptr); + return ptr; +} #include <string.h> @@ -1460,6 +1466,18 @@ checked_request2size (size_t req, size_t *sz) __nonnull (1) #pragma GCC poison mchunk_size #pragma GCC poison mchunk_prev_size +static __always_inline void * +tag_new_usable (void *ptr) +{ + if (__glibc_unlikely (mtag_enabled) && ptr) + { + mchunkptr cp = mem2chunk(ptr); + ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), + CHUNK_AVAILABLE_SIZE (cp) - CHUNK_HDR_SZ); + } + return ptr; +} + /* -------------------- Internal data structures --------------------
next reply other threads:[~2021-03-19 11:57 UTC|newest] Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top 2021-03-19 11:57 Szabolcs Nagy [this message] -- strict thread matches above, loose matches on Subject: below -- 2021-03-11 17:40 Szabolcs Nagy 2021-03-11 17:39 Szabolcs Nagy 2021-03-04 16:25 Szabolcs Nagy
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210319115728.822CD386F436@sourceware.org \ --to=nsz@sourceware.org \ --cc=glibc-cvs@sourceware.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).