[glibc] libio: Improve performance of IO locks

[Wilco Dijkstra <wilco@sourceware.org>]

https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=c51c483d2b8ae66fe31a12509aedae02a6982ced

commit c51c483d2b8ae66fe31a12509aedae02a6982ced
Author: Wilco Dijkstra <wdijkstr@arm.com>
Date:   Thu Aug 11 16:47:45 2022 +0100

    libio: Improve performance of IO locks
    
    Improve performance of recursive IO locks by adding a fast path for
    the single-threaded case. To reduce the number of memory accesses for
    locking/unlocking, only increment the recursion counter if the lock
    is already taken.
    
    On Neoverse V1, a microbenchmark with many small freads improved by
    2.9x. Multithreaded performance improved by 2%.
    
    Reviewed-by: Cristian Rodríguez  <crrodriguez@opensuse.org>

Diff:
---
 sysdeps/nptl/stdio-lock.h | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/sysdeps/nptl/stdio-lock.h b/sysdeps/nptl/stdio-lock.h
index afa0b779c8..45823cd162 100644
--- a/sysdeps/nptl/stdio-lock.h
+++ b/sysdeps/nptl/stdio-lock.h
@@ -37,12 +37,18 @@ typedef struct { int lock; int cnt; void *owner; } _IO_lock_t;
 #define _IO_lock_lock(_name) \
   do {									      \
     void *__self = THREAD_SELF;						      \
-    if ((_name).owner != __self)					      \
+    if (SINGLE_THREAD_P && (_name).owner == NULL)			      \
+      {									      \
+	(_name).lock = LLL_LOCK_INITIALIZER_LOCKED;			      \
+	(_name).owner = __self;						      \
+      }									      \
+    else if ((_name).owner != __self)					      \
       {									      \
 	lll_lock ((_name).lock, LLL_PRIVATE);				      \
-        (_name).owner = __self;						      \
+	(_name).owner = __self;						      \
       }									      \
-    ++(_name).cnt;							      \
+    else								      \
+      ++(_name).cnt;							      \
   } while (0)
 
 #define _IO_lock_trylock(_name) \
@@ -52,10 +58,7 @@ typedef struct { int lock; int cnt; void *owner; } _IO_lock_t;
     if ((_name).owner != __self)					      \
       {									      \
         if (lll_trylock ((_name).lock) == 0)				      \
-          {								      \
-            (_name).owner = __self;					      \
-            (_name).cnt = 1;						      \
-          }								      \
+	  (_name).owner = __self;					      \
         else								      \
           __result = EBUSY;						      \
       }									      \
@@ -66,11 +69,18 @@ typedef struct { int lock; int cnt; void *owner; } _IO_lock_t;
 
 #define _IO_lock_unlock(_name) \
   do {									      \
-    if (--(_name).cnt == 0)						      \
+    if (SINGLE_THREAD_P && (_name).cnt == 0)				      \
+      {									      \
+	(_name).owner = NULL;						      \
+	(_name).lock = 0;						      \
+      }									      \
+    else if ((_name).cnt == 0)						      \
       {									      \
-        (_name).owner = NULL;						      \
+	(_name).owner = NULL;						      \
 	lll_unlock ((_name).lock, LLL_PRIVATE);				      \
       }									      \
+    else								      \
+      --(_name).cnt;							      \
   } while (0)