[PATCH 1/4] Mark all member functions with memory models always inline

public inbox for gcc-patches@gcc.gnu.org
 help / color / mirror / Atom feed

* [PATCH 1/4] Mark all member functions with memory models always inline
@ 2013-03-16 13:30 Andi Kleen
  2013-03-16 13:29 ` [PATCH 2/4] Avoid non constant memory model uses in libatomic Andi Kleen
                   ` (3 more replies)
  0 siblings, 4 replies; 15+ messages in thread
From: Andi Kleen @ 2013-03-16 13:30 UTC (permalink / raw)
  To: gcc-patches; +Cc: libstdc++, rth, Andi Kleen

From: Andi Kleen <ak@linux.intel.com>

When a non constant memory model is passed to __atomic_*
gcc falls back to seq_cst. This drops any HLE acquire or release bits.

This can happen when <atomic> is used with -O0
as the member functions are not always inlined then and the memory
argument passed in ends up being non-constant.

With inline __attribute__((always_inline)) these functions
get inlined even with -O0.

I hardcoded the attribute in the header for now, assuming
that all compilers that support libstdc++ have attribute
always_inline too. If not it would need to be moved
as a macro to c++config.h with appropiate ifdefs.

We still need a warning for this case too, that will be submitted
separately.

I would like to have this patch in the 4.8 series if possible, to make
sure HLE works well with <atomic>

Passed bootstrap and test on x86_64-linux.

libstdc++v3/:

2013-03-15  Andi Kleen  <ak@linux.intel.com>

	PR target/55947
	* libstdc++-v3/include/bits/atomic_base.h
	(__always_inline): Add new macro.
	(atomic_thread_fence, atomic_signal_fence, test_and_set,
	clear, store, load, exchange, compare_exchange_weak)
	compare_exchange_strong, fetch_add, fetch_sub, fetch_and,
	fetch_or, fetch_xor): Mark __always_inline.
---
 libstdc++-v3/include/bits/atomic_base.h |   88 ++++++++++++++++---------------
 1 file changed, 46 insertions(+), 42 deletions(-)

diff --git a/libstdc++-v3/include/bits/atomic_base.h b/libstdc++-v3/include/bits/atomic_base.h
index 609fe8b..475ffa0 100644
--- a/libstdc++-v3/include/bits/atomic_base.h
+++ b/libstdc++-v3/include/bits/atomic_base.h
@@ -37,6 +37,10 @@
 #include <stdint.h>
 #include <bits/atomic_lockfree_defines.h>
 
+#ifndef __always_inline
+#define __always_inline inline __attribute__((always_inline))
+#endif
+
 namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
@@ -94,11 +98,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       | (__m & __memory_order_modifier_mask));
   }
 
-  inline void
+  __always_inline void
   atomic_thread_fence(memory_order __m) noexcept
   { __atomic_thread_fence(__m); }
 
-  inline void
+  __always_inline void
   atomic_signal_fence(memory_order __m) noexcept
   { __atomic_signal_fence(__m); }
 
@@ -281,19 +285,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       : __atomic_flag_base{ _S_init(__i) }
     { }
 
-    bool
+    __always_inline bool
     test_and_set(memory_order __m = memory_order_seq_cst) noexcept
     {
       return __atomic_test_and_set (&_M_i, __m);
     }
 
-    bool
+    __always_inline bool
     test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
     {
       return __atomic_test_and_set (&_M_i, __m);
     }
 
-    void
+    __always_inline void
     clear(memory_order __m = memory_order_seq_cst) noexcept
     {
       memory_order __b = __m & __memory_order_mask;
@@ -304,7 +308,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       __atomic_clear (&_M_i, __m);
     }
 
-    void
+    __always_inline void
     clear(memory_order __m = memory_order_seq_cst) volatile noexcept
     {
       memory_order __b = __m & __memory_order_mask;
@@ -463,7 +467,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       is_lock_free() const volatile noexcept
       { return __atomic_is_lock_free(sizeof(_M_i), nullptr); }
 
-      void
+      __always_inline void
       store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
       {
         memory_order __b = __m & __memory_order_mask;
@@ -474,7 +478,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	__atomic_store_n(&_M_i, __i, __m);
       }
 
-      void
+      __always_inline void
       store(__int_type __i,
 	    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -486,7 +490,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	__atomic_store_n(&_M_i, __i, __m);
       }
 
-      __int_type
+      __always_inline __int_type
       load(memory_order __m = memory_order_seq_cst) const noexcept
       {
        memory_order __b = __m & __memory_order_mask;
@@ -496,7 +500,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_load_n(&_M_i, __m);
       }
 
-      __int_type
+      __always_inline __int_type
       load(memory_order __m = memory_order_seq_cst) const volatile noexcept
       {
         memory_order __b = __m & __memory_order_mask;
@@ -506,7 +510,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_load_n(&_M_i, __m);
       }
 
-      __int_type
+      __always_inline __int_type
       exchange(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -514,14 +518,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       }
 
 
-      __int_type
+      __always_inline __int_type
       exchange(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       {
 	return __atomic_exchange_n(&_M_i, __i, __m);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m1, memory_order __m2) noexcept
       {
@@ -534,7 +538,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m1,
 			    memory_order __m2) volatile noexcept
@@ -548,7 +552,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -556,7 +560,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 				     __cmpexch_failure_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 		   memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -564,7 +568,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 				     __cmpexch_failure_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m1, memory_order __m2) noexcept
       {
@@ -577,7 +581,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m1,
 			      memory_order __m2) volatile noexcept
@@ -592,7 +596,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -600,7 +604,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 				       __cmpexch_failure_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 		 memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -608,52 +612,52 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 				       __cmpexch_failure_order(__m));
       }
 
-      __int_type
+      __always_inline __int_type
       fetch_add(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_add(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_add(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_add(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_sub(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_sub(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_sub(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_sub(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_and(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_and(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_and(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_and(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_or(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_or(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_or(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_or(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_xor(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_xor(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_xor(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_xor(&_M_i, __i, __m); }
@@ -770,7 +774,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       is_lock_free() const volatile noexcept
       { return __atomic_is_lock_free(_M_type_size(1), nullptr); }
 
-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -783,7 +787,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	__atomic_store_n(&_M_p, __p, __m);
       }
 
-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -795,7 +799,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	__atomic_store_n(&_M_p, __p, __m);
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const noexcept
       {
         memory_order __b = __m & __memory_order_mask;
@@ -805,7 +809,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_load_n(&_M_p, __m);
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const volatile noexcept
       {
         memory_order __b = __m & __memory_order_mask;
@@ -815,7 +819,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_load_n(&_M_p, __m);
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -823,14 +827,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       }
 
 
-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       {
 	return __atomic_exchange_n(&_M_p, __p, __m);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1,
 			      memory_order __m2) noexcept
@@ -844,7 +848,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1,
 			      memory_order __m2) volatile noexcept
@@ -859,22 +863,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_add(&_M_p, _M_type_size(__d), __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_add(&_M_p, _M_type_size(__d), __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_sub(&_M_p, _M_type_size(__d), __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_sub(&_M_p, _M_type_size(__d), __m); }
-- 
1.7.10.4

^ permalink raw reply	[flat|nested] 15+ messages in thread

* [PATCH 2/4] Avoid non constant memory model uses in libatomic
  2013-03-16 13:30 [PATCH 1/4] Mark all member functions with memory models always inline Andi Kleen
@ 2013-03-16 13:29 ` Andi Kleen
  2013-03-19 21:23   ` Richard Henderson
  2013-03-16 13:29 ` [PATCH 4/4] Add warning for non-constant memory models Andi Kleen
                   ` (2 subsequent siblings)
  3 siblings, 1 reply; 15+ messages in thread
From: Andi Kleen @ 2013-03-16 13:29 UTC (permalink / raw)
  To: gcc-patches; +Cc: libstdc++, rth, Andi Kleen

From: Andi Kleen <ak@linux.intel.com>

x86 ends up using non constant memory models for some of the libatomic
functions. These all end up as __ATOMIC_SEQ_CST. Just use this
directly. This avoids a new warning for non constant memory
models, which broke the bootstrap with -Werror

Passed bootstrap and test on x86_64-linux.

libatomic/:

2013-03-15  Andi Kleen  <ak@linux.intel.com>

	* gcas.c: (EXACT_INLINE): Use __ATOMIC_SEQ_CST.
	* gexch.c: (EXACT_INLINE): Use __ATOMIC_SEQ_CST.
	* gload.c: (EXACT_INLINE): Use __ATOMIC_SEQ_CST.
	* gstore.c: (EXACT_INLINE): Use __ATOMIC_SEQ_CST.

diff --git a/libatomic/gcas.c b/libatomic/gcas.c
index edbf611..e3d77f3 100644
--- a/libatomic/gcas.c
+++ b/libatomic/gcas.c
@@ -32,7 +32,7 @@
 # define EXACT_INLINE(N)					\
   if (C2(HAVE_ATOMIC_CAS_,N))					\
     return __atomic_compare_exchange_n				\
-      (PTR(N,mptr), PTR(N,eptr), *PTR(N,dptr), false, smodel, fmodel)
+      (PTR(N,mptr), PTR(N,eptr), *PTR(N,dptr), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
 #else
 # define EXACT_INLINE(N)
 #endif
diff --git a/libatomic/gexch.c b/libatomic/gexch.c
index 1999067..c8c8658 100644
--- a/libatomic/gexch.c
+++ b/libatomic/gexch.c
@@ -33,7 +33,7 @@
   if (C2(HAVE_ATOMIC_EXCHANGE_,N))				\
     {								\
       *PTR(N,rptr) = __atomic_exchange_n			\
-	(PTR(N,mptr), *PTR(N,vptr), smodel);			\
+	(PTR(N,mptr), *PTR(N,vptr), __ATOMIC_SEQ_CST);		\
       return;							\
     }
 #else
diff --git a/libatomic/gload.c b/libatomic/gload.c
index df318d5..85865bd 100644
--- a/libatomic/gload.c
+++ b/libatomic/gload.c
@@ -32,7 +32,7 @@
 # define EXACT_INLINE(N, DEST, SRC, DONE)			\
   if (C2(HAVE_ATOMIC_LDST_,N))					\
     {								\
-      DEST = __atomic_load_n (SRC, smodel);			\
+      DEST = __atomic_load_n (SRC, __ATOMIC_SEQ_CST);		\
       DONE;							\
     }
 #else
diff --git a/libatomic/gstore.c b/libatomic/gstore.c
index d571e58..84f9a8d 100644
--- a/libatomic/gstore.c
+++ b/libatomic/gstore.c
@@ -32,7 +32,7 @@
 # define EXACT_INLINE(N)					\
   if (C2(HAVE_ATOMIC_LDST_,N))					\
     {								\
-      __atomic_store_n (PTR(N,mptr), *PTR(N,vptr), smodel);	\
+      __atomic_store_n (PTR(N,mptr), *PTR(N,vptr), __ATOMIC_SEQ_CST);	\
       return;							\
     }
 #else
---
 libatomic/gcas.c   |    2 +-
 libatomic/gexch.c  |    2 +-
 libatomic/gload.c  |    2 +-
 libatomic/gstore.c |    2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/libatomic/gcas.c b/libatomic/gcas.c
index edbf611..e3d77f3 100644
--- a/libatomic/gcas.c
+++ b/libatomic/gcas.c
@@ -32,7 +32,7 @@
 # define EXACT_INLINE(N)					\
   if (C2(HAVE_ATOMIC_CAS_,N))					\
     return __atomic_compare_exchange_n				\
-      (PTR(N,mptr), PTR(N,eptr), *PTR(N,dptr), false, smodel, fmodel)
+      (PTR(N,mptr), PTR(N,eptr), *PTR(N,dptr), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
 #else
 # define EXACT_INLINE(N)
 #endif
diff --git a/libatomic/gexch.c b/libatomic/gexch.c
index 1999067..c8c8658 100644
--- a/libatomic/gexch.c
+++ b/libatomic/gexch.c
@@ -33,7 +33,7 @@
   if (C2(HAVE_ATOMIC_EXCHANGE_,N))				\
     {								\
       *PTR(N,rptr) = __atomic_exchange_n			\
-	(PTR(N,mptr), *PTR(N,vptr), smodel);			\
+	(PTR(N,mptr), *PTR(N,vptr), __ATOMIC_SEQ_CST);		\
       return;							\
     }
 #else
diff --git a/libatomic/gload.c b/libatomic/gload.c
index df318d5..85865bd 100644
--- a/libatomic/gload.c
+++ b/libatomic/gload.c
@@ -32,7 +32,7 @@
 # define EXACT_INLINE(N, DEST, SRC, DONE)			\
   if (C2(HAVE_ATOMIC_LDST_,N))					\
     {								\
-      DEST = __atomic_load_n (SRC, smodel);			\
+      DEST = __atomic_load_n (SRC, __ATOMIC_SEQ_CST);		\
       DONE;							\
     }
 #else
diff --git a/libatomic/gstore.c b/libatomic/gstore.c
index d571e58..84f9a8d 100644
--- a/libatomic/gstore.c
+++ b/libatomic/gstore.c
@@ -32,7 +32,7 @@
 # define EXACT_INLINE(N)					\
   if (C2(HAVE_ATOMIC_LDST_,N))					\
     {								\
-      __atomic_store_n (PTR(N,mptr), *PTR(N,vptr), smodel);	\
+      __atomic_store_n (PTR(N,mptr), *PTR(N,vptr), __ATOMIC_SEQ_CST);	\
       return;							\
     }
 #else
-- 
1.7.10.4

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 2/4] Avoid non constant memory model uses in libatomic
  2013-03-16 13:29 ` [PATCH 2/4] Avoid non constant memory model uses in libatomic Andi Kleen
@ 2013-03-19 21:23   ` Richard Henderson
  0 siblings, 0 replies; 15+ messages in thread
From: Richard Henderson @ 2013-03-19 21:23 UTC (permalink / raw)
  To: Andi Kleen; +Cc: gcc-patches, libstdc++, Andi Kleen

On 03/16/2013 06:29 AM, Andi Kleen wrote:
> 2013-03-15  Andi Kleen  <ak@linux.intel.com>
> 
> 	* gcas.c: (EXACT_INLINE): Use __ATOMIC_SEQ_CST.
> 	* gexch.c: (EXACT_INLINE): Use __ATOMIC_SEQ_CST.
> 	* gload.c: (EXACT_INLINE): Use __ATOMIC_SEQ_CST.
> 	* gstore.c: (EXACT_INLINE): Use __ATOMIC_SEQ_CST.

Ok.


r~

^ permalink raw reply	[flat|nested] 15+ messages in thread

* [PATCH 4/4] Add warning for non-constant memory models
  2013-03-16 13:30 [PATCH 1/4] Mark all member functions with memory models always inline Andi Kleen
  2013-03-16 13:29 ` [PATCH 2/4] Avoid non constant memory model uses in libatomic Andi Kleen
@ 2013-03-16 13:29 ` Andi Kleen
  2013-03-19 21:23   ` Richard Henderson
  2013-03-16 13:30 ` [PATCH 3/4] Avoid nonconst memmodels in libitm's local outdated copy of <atomic> too Andi Kleen
       [not found] ` <CAH6eHdR=VhEfAes6S97CfBf0Newe1h3EeoxreRKKHpi4mpJh=w@mail.gmail.com>
  3 siblings, 1 reply; 15+ messages in thread
From: Andi Kleen @ 2013-03-16 13:29 UTC (permalink / raw)
  To: gcc-patches; +Cc: libstdc++, rth, Andi Kleen

From: Andi Kleen <ak@linux.intel.com>

Non constant memory models drop HLE attribute bits. Since we had
problems with this even in gcc code itself, like libstdc++ <atomic>
passing variables when not optimizing, add a warning for this case.

Passed bootstrap and test on x86_64-linux.

gcc/:
2013-03-15  Andi Kleen  <ak@my.domain.org>

	PR target/55947
	* gcc/builtins.c (get_memmodel): Warn for non constant
	memory model.
---
 gcc/builtins.c |   10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/gcc/builtins.c b/gcc/builtins.c
index 68b6a2c..c4efc4c 100644
--- a/gcc/builtins.c
+++ b/gcc/builtins.c
@@ -5280,10 +5280,14 @@ get_memmodel (tree exp)
   rtx op;
   unsigned HOST_WIDE_INT val;
 
-  /* If the parameter is not a constant, it's a run time value so we'll just
-     convert it to MEMMODEL_SEQ_CST to avoid annoying runtime checking.  */
+  /* Warn about non constant memory models, as we drop target specific
+     bits (like HLE). */
   if (TREE_CODE (exp) != INTEGER_CST)
-    return MEMMODEL_SEQ_CST;
+    {
+      warning (OPT_Winvalid_memory_model,
+	       "Non constant memory model: Assuming __ATOMIC_SEQ_CST");
+      return MEMMODEL_SEQ_CST;
+    }
 
   op = expand_normal (exp);
 
-- 
1.7.10.4

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 4/4] Add warning for non-constant memory models
  2013-03-16 13:29 ` [PATCH 4/4] Add warning for non-constant memory models Andi Kleen
@ 2013-03-19 21:23   ` Richard Henderson
  0 siblings, 0 replies; 15+ messages in thread
From: Richard Henderson @ 2013-03-19 21:23 UTC (permalink / raw)
  To: Andi Kleen; +Cc: gcc-patches, libstdc++, Andi Kleen

On 03/16/2013 06:29 AM, Andi Kleen wrote:
> +	       "Non constant memory model: Assuming __ATOMIC_SEQ_CST");

Warnings and error messages are all lower case.  Otherwise it seems...
well, I didn't want to say "ok", but certainly "unavoidable".


r~

^ permalink raw reply	[flat|nested] 15+ messages in thread

* [PATCH 3/4] Avoid nonconst memmodels in libitm's local outdated copy of <atomic> too
  2013-03-16 13:30 [PATCH 1/4] Mark all member functions with memory models always inline Andi Kleen
  2013-03-16 13:29 ` [PATCH 2/4] Avoid non constant memory model uses in libatomic Andi Kleen
  2013-03-16 13:29 ` [PATCH 4/4] Add warning for non-constant memory models Andi Kleen
@ 2013-03-16 13:30 ` Andi Kleen
  2013-03-19 21:23   ` Richard Henderson
       [not found] ` <CAH6eHdR=VhEfAes6S97CfBf0Newe1h3EeoxreRKKHpi4mpJh=w@mail.gmail.com>
  3 siblings, 1 reply; 15+ messages in thread
From: Andi Kleen @ 2013-03-16 13:30 UTC (permalink / raw)
  To: gcc-patches; +Cc: libstdc++, rth, Andi Kleen

From: Andi Kleen <ak@linux.intel.com>

This avoids warnings in libitm for non constant memory models,
fixing the bootstrap with -Werror

Passed bootstrap and test on x86_64-linux.

libitm/:

2013-03-15  Andi Kleen  <andi@my.domain.org>

	* local_atomic (__always_inline): Add.
	(__calculate_memory_order, atomic_thread_fence,
	 atomic_signal_fence, test_and_set, clear, store, load,
         exchange, compare_exchange_weak, compare_exchange_strong,
         fetch_add, fetch_sub, fetch_and, fetch_or, fetch_xor):
	Add __always_inline to force inlining.

diff --git a/libitm/local_atomic b/libitm/local_atomic
index 97e7d26..4cd961a 100644
--- a/libitm/local_atomic
+++ b/libitm/local_atomic
@@ -41,6 +41,10 @@
 #ifndef _GLIBCXX_ATOMIC
 #define _GLIBCXX_ATOMIC 1

+#ifndef __always_inline
+#define __always_inline inline __attribute__((always_inline))
+#endif
+
 // #pragma GCC system_header

 // #ifndef __GXX_EXPERIMENTAL_CXX0X__
@@ -71,7 +75,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       memory_order_seq_cst
     } memory_order;

-  inline memory_order
+  __always_inline memory_order
   __calculate_memory_order(memory_order __m) noexcept
   {
     const bool __cond1 = __m == memory_order_release;
@@ -81,13 +85,13 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     return __mo2;
   }

-  inline void
+  __always_inline void
   atomic_thread_fence(memory_order __m) noexcept
   {
     __atomic_thread_fence (__m);
   }

-  inline void
+  __always_inline void
   atomic_signal_fence(memory_order __m) noexcept
   {
     __atomic_thread_fence (__m);
@@ -277,19 +281,19 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     // Conversion to ATOMIC_FLAG_INIT.
     atomic_flag(bool __i) noexcept : __atomic_flag_base({ __i }) { }

-    bool
+    __always_inline bool
     test_and_set(memory_order __m = memory_order_seq_cst) noexcept
     {
       return __atomic_test_and_set (&_M_i, __m);
     }

-    bool
+    __always_inline bool
     test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
     {
       return __atomic_test_and_set (&_M_i, __m);
     }

-    void
+    __always_inline void
     clear(memory_order __m = memory_order_seq_cst) noexcept
     {
       // __glibcxx_assert(__m != memory_order_consume);
@@ -299,7 +303,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       __atomic_clear (&_M_i, __m);
     }

-    void
+    __always_inline void
     clear(memory_order __m = memory_order_seq_cst) volatile noexcept
     {
       // __glibcxx_assert(__m != memory_order_consume);
@@ -452,7 +456,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       is_lock_free() const volatile noexcept
       { return __atomic_is_lock_free (sizeof (_M_i), &_M_i); }

-      void
+      __always_inline void
       store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
       {
 	// __glibcxx_assert(__m != memory_order_acquire);
@@ -462,7 +466,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	__atomic_store_n(&_M_i, __i, __m);
       }

-      void
+      __always_inline void
       store(__int_type __i,
 	    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -473,7 +477,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	__atomic_store_n(&_M_i, __i, __m);
       }

-      __int_type
+      __always_inline __int_type
       load(memory_order __m = memory_order_seq_cst) const noexcept
       {
 	// __glibcxx_assert(__m != memory_order_release);
@@ -482,7 +486,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_load_n(&_M_i, __m);
       }

-      __int_type
+      __always_inline __int_type
       load(memory_order __m = memory_order_seq_cst) const volatile noexcept
       {
 	// __glibcxx_assert(__m != memory_order_release);
@@ -491,22 +495,21 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_load_n(&_M_i, __m);
       }

-      __int_type
+      __always_inline __int_type
       exchange(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) noexcept
       {
 	return __atomic_exchange_n(&_M_i, __i, __m);
       }

-
-      __int_type
+      __always_inline __int_type
       exchange(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       {
 	return __atomic_exchange_n(&_M_i, __i, __m);
       }

-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m1, memory_order __m2) noexcept
       {
@@ -517,7 +520,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
       }

-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m1,
 			    memory_order __m2) volatile noexcept
@@ -529,7 +532,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
       }

-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -537,7 +540,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				     __calculate_memory_order(__m));
       }

-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 		   memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -545,7 +548,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				     __calculate_memory_order(__m));
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m1, memory_order __m2) noexcept
       {
@@ -556,7 +559,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m1,
 			      memory_order __m2) volatile noexcept
@@ -568,7 +571,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -576,7 +579,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				       __calculate_memory_order(__m));
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 		 memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -584,52 +587,52 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				       __calculate_memory_order(__m));
       }

-      __int_type
+      __always_inline __int_type
       fetch_add(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_add(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_add(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_add(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_sub(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_sub(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_sub(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_sub(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_and(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_and(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_and(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_and(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_or(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_or(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_or(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_or(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_xor(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_xor(&_M_i, __i, __m); }

-      __int_type
+      __always_inline __int_type
       fetch_xor(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_xor(&_M_i, __i, __m); }
@@ -731,7 +734,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       is_lock_free() const volatile noexcept
       { return __atomic_is_lock_free (sizeof (_M_p), &_M_p); }

-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -742,7 +745,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	__atomic_store_n(&_M_p, __p, __m);
       }

-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -753,7 +756,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	__atomic_store_n(&_M_p, __p, __m);
       }

-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const noexcept
       {
 	// __glibcxx_assert(__m != memory_order_release);
@@ -762,7 +765,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_load_n(&_M_p, __m);
       }

-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const volatile noexcept
       {
 	// __glibcxx_assert(__m != memory_order_release);
@@ -771,22 +774,21 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_load_n(&_M_p, __m);
       }

-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) noexcept
       {
 	return __atomic_exchange_n(&_M_p, __p, __m);
       }

-
-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       {
 	return __atomic_exchange_n(&_M_p, __p, __m);
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1,
 			      memory_order __m2) noexcept
@@ -798,7 +800,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1,
 			      memory_order __m2) volatile noexcept
@@ -810,22 +812,22 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
       }

-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_add(&_M_p, __d, __m); }

-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_add(&_M_p, __d, __m); }

-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_sub(&_M_p, __d, __m); }

-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_sub(&_M_p, __d, __m); }
@@ -869,67 +871,67 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     bool
     is_lock_free() const volatile noexcept { return _M_base.is_lock_free(); }

-    void
+    __always_inline void
     store(bool __i, memory_order __m = memory_order_seq_cst) noexcept
     { _M_base.store(__i, __m); }

-    void
+    __always_inline void
     store(bool __i, memory_order __m = memory_order_seq_cst) volatile noexcept
     { _M_base.store(__i, __m); }

-    bool
+    __always_inline bool
     load(memory_order __m = memory_order_seq_cst) const noexcept
     { return _M_base.load(__m); }

-    bool
+    __always_inline bool
     load(memory_order __m = memory_order_seq_cst) const volatile noexcept
     { return _M_base.load(__m); }

-    bool
+    __always_inline bool
     exchange(bool __i, memory_order __m = memory_order_seq_cst) noexcept
     { return _M_base.exchange(__i, __m); }

-    bool
+    __always_inline bool
     exchange(bool __i,
 	     memory_order __m = memory_order_seq_cst) volatile noexcept
     { return _M_base.exchange(__i, __m); }

-    bool
+    __always_inline bool
     compare_exchange_weak(bool& __i1, bool __i2, memory_order __m1,
 			  memory_order __m2) noexcept
     { return _M_base.compare_exchange_weak(__i1, __i2, __m1, __m2); }

-    bool
+    __always_inline bool
     compare_exchange_weak(bool& __i1, bool __i2, memory_order __m1,
 			  memory_order __m2) volatile noexcept
     { return _M_base.compare_exchange_weak(__i1, __i2, __m1, __m2); }

-    bool
+    __always_inline bool
     compare_exchange_weak(bool& __i1, bool __i2,
 			  memory_order __m = memory_order_seq_cst) noexcept
     { return _M_base.compare_exchange_weak(__i1, __i2, __m); }

-    bool
+    __always_inline bool
     compare_exchange_weak(bool& __i1, bool __i2,
 		     memory_order __m = memory_order_seq_cst) volatile noexcept
     { return _M_base.compare_exchange_weak(__i1, __i2, __m); }

-    bool
+    __always_inline bool
     compare_exchange_strong(bool& __i1, bool __i2, memory_order __m1,
 			    memory_order __m2) noexcept
     { return _M_base.compare_exchange_strong(__i1, __i2, __m1, __m2); }

-    bool
+    __always_inline bool
     compare_exchange_strong(bool& __i1, bool __i2, memory_order __m1,
 			    memory_order __m2) volatile noexcept
     { return _M_base.compare_exchange_strong(__i1, __i2, __m1, __m2); }

-    bool
+    __always_inline bool
     compare_exchange_strong(bool& __i1, bool __i2,
 			    memory_order __m = memory_order_seq_cst) noexcept
     { return _M_base.compare_exchange_strong(__i1, __i2, __m); }

-    bool
+    __always_inline bool
     compare_exchange_strong(bool& __i1, bool __i2,
 		    memory_order __m = memory_order_seq_cst) volatile noexcept
     { return _M_base.compare_exchange_strong(__i1, __i2, __m); }
@@ -979,11 +981,11 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       store(_Tp __i, memory_order _m = memory_order_seq_cst) noexcept
       { __atomic_store(&_M_i, &__i, _m); }

-      void
+      __always_inline void
       store(_Tp __i, memory_order _m = memory_order_seq_cst) volatile noexcept
       { __atomic_store(&_M_i, &__i, _m); }

-      _Tp
+      __always_inline _Tp
       load(memory_order _m = memory_order_seq_cst) const noexcept
       {
         _Tp tmp;
@@ -991,7 +993,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return tmp;
       }

-      _Tp
+      __always_inline _Tp
       load(memory_order _m = memory_order_seq_cst) const volatile noexcept
       {
         _Tp tmp;
@@ -999,7 +1001,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return tmp;
       }

-      _Tp
+      __always_inline _Tp
       exchange(_Tp __i, memory_order _m = memory_order_seq_cst) noexcept
       {
         _Tp tmp;
@@ -1007,7 +1009,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return tmp;
       }

-      _Tp
+      __always_inline _Tp
       exchange(_Tp __i,
 	       memory_order _m = memory_order_seq_cst) volatile noexcept
       {
@@ -1016,50 +1018,50 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return tmp;
       }

-      bool
+      __always_inline bool
       compare_exchange_weak(_Tp& __e, _Tp __i, memory_order __s,
 			    memory_order __f) noexcept
       {
 	return __atomic_compare_exchange(&_M_i, &__e, &__i, true, __s, __f);
       }

-      bool
+      __always_inline bool
       compare_exchange_weak(_Tp& __e, _Tp __i, memory_order __s,
 			    memory_order __f) volatile noexcept
       {
 	return __atomic_compare_exchange(&_M_i, &__e, &__i, true, __s, __f);
       }

-      bool
+      __always_inline bool
       compare_exchange_weak(_Tp& __e, _Tp __i,
 			    memory_order __m = memory_order_seq_cst) noexcept
       { return compare_exchange_weak(__e, __i, __m, __m); }

-      bool
+      __always_inline bool
       compare_exchange_weak(_Tp& __e, _Tp __i,
 		     memory_order __m = memory_order_seq_cst) volatile noexcept
       { return compare_exchange_weak(__e, __i, __m, __m); }

-      bool
+      __always_inline bool
       compare_exchange_strong(_Tp& __e, _Tp __i, memory_order __s,
 			      memory_order __f) noexcept
       {
 	return __atomic_compare_exchange(&_M_i, &__e, &__i, false, __s, __f);
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(_Tp& __e, _Tp __i, memory_order __s,
 			      memory_order __f) volatile noexcept
       {
 	return __atomic_compare_exchange(&_M_i, &__e, &__i, false, __s, __f);
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(_Tp& __e, _Tp __i,
 			       memory_order __m = memory_order_seq_cst) noexcept
       { return compare_exchange_strong(__e, __i, __m, __m); }

-      bool
+      __always_inline bool
       compare_exchange_strong(_Tp& __e, _Tp __i,
 		     memory_order __m = memory_order_seq_cst) volatile noexcept
       { return compare_exchange_strong(__e, __i, __m, __m); }
@@ -1152,46 +1154,46 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       is_lock_free() const volatile noexcept
       { return _M_b.is_lock_free(); }

-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) noexcept
       { return _M_b.store(__p, __m); }

-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) volatile noexcept
       { return _M_b.store(__p, __m); }

-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const noexcept
       { return _M_b.load(__m); }

-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const volatile noexcept
       { return _M_b.load(__m); }

-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) noexcept
       { return _M_b.exchange(__p, __m); }

-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       { return _M_b.exchange(__p, __m); }

-      bool
+      __always_inline bool
       compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
 			    memory_order __m1, memory_order __m2) noexcept
       { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }

-      bool
+      __always_inline bool
       compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
 			    memory_order __m1,
 			    memory_order __m2) volatile noexcept
       { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }

-      bool
+      __always_inline bool
       compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
 			    memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -1199,7 +1201,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				     __calculate_memory_order(__m));
       }

-      bool
+      __always_inline bool
       compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
 		    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -1207,18 +1209,18 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				     __calculate_memory_order(__m));
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1, memory_order __m2) noexcept
       { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }

-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1,
 			      memory_order __m2) volatile noexcept
       { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }

-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -1226,7 +1228,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 					    __calculate_memory_order(__m));
       }

-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 		    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -1234,22 +1236,22 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 					    __calculate_memory_order(__m));
       }

-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return _M_b.fetch_add(__d, __m); }

-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return _M_b.fetch_add(__d, __m); }

-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return _M_b.fetch_sub(__d, __m); }

-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return _M_b.fetch_sub(__d, __m); }
@@ -1543,98 +1545,98 @@ namespace std // _GLIBCXX_VISIBILITY(default)

   // Function definitions, atomic_flag operations.
-  inline bool
+  __always_inline bool
   atomic_flag_test_and_set_explicit(atomic_flag* __a,
 				    memory_order __m) noexcept
   { return __a->test_and_set(__m); }

-  inline bool
+  __always_inline bool
   atomic_flag_test_and_set_explicit(volatile atomic_flag* __a,
 				    memory_order __m) noexcept
   { return __a->test_and_set(__m); }

-  inline void
+  __always_inline void
   atomic_flag_clear_explicit(atomic_flag* __a, memory_order __m) noexcept
   { __a->clear(__m); }

-  inline void
+  __always_inline void
   atomic_flag_clear_explicit(volatile atomic_flag* __a,
 			     memory_order __m) noexcept
   { __a->clear(__m); }

-  inline bool
+  __always_inline bool
   atomic_flag_test_and_set(atomic_flag* __a) noexcept
   { return atomic_flag_test_and_set_explicit(__a, memory_order_seq_cst); }

-  inline bool
+  __always_inline bool
   atomic_flag_test_and_set(volatile atomic_flag* __a) noexcept
   { return atomic_flag_test_and_set_explicit(__a, memory_order_seq_cst); }

-  inline void
+  __always_inline void
   atomic_flag_clear(atomic_flag* __a) noexcept
   { atomic_flag_clear_explicit(__a, memory_order_seq_cst); }

-  inline void
+  __always_inline void
   atomic_flag_clear(volatile atomic_flag* __a) noexcept
   { atomic_flag_clear_explicit(__a, memory_order_seq_cst); }

   // Function templates generally applicable to atomic types.
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_is_lock_free(const atomic<_ITp>* __a) noexcept
     { return __a->is_lock_free(); }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_is_lock_free(const volatile atomic<_ITp>* __a) noexcept
     { return __a->is_lock_free(); }

   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_init(atomic<_ITp>* __a, _ITp __i) noexcept;

   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_init(volatile atomic<_ITp>* __a, _ITp __i) noexcept;

   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_store_explicit(atomic<_ITp>* __a, _ITp __i,
 			  memory_order __m) noexcept
     { __a->store(__i, __m); }

   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_store_explicit(volatile atomic<_ITp>* __a, _ITp __i,
 			  memory_order __m) noexcept
     { __a->store(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_load_explicit(const atomic<_ITp>* __a, memory_order __m) noexcept
     { return __a->load(__m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_load_explicit(const volatile atomic<_ITp>* __a,
 			 memory_order __m) noexcept
     { return __a->load(__m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_exchange_explicit(atomic<_ITp>* __a, _ITp __i,
 			     memory_order __m) noexcept
     { return __a->exchange(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_exchange_explicit(volatile atomic<_ITp>* __a, _ITp __i,
 			     memory_order __m) noexcept
     { return __a->exchange(__i, __m); }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_weak_explicit(atomic<_ITp>* __a,
 					  _ITp* __i1, _ITp __i2,
 					  memory_order __m1,
@@ -1642,7 +1644,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     { return __a->compare_exchange_weak(*__i1, __i2, __m1, __m2); }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_weak_explicit(volatile atomic<_ITp>* __a,
 					  _ITp* __i1, _ITp __i2,
 					  memory_order __m1,
@@ -1650,7 +1652,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     { return __a->compare_exchange_weak(*__i1, __i2, __m1, __m2); }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_strong_explicit(atomic<_ITp>* __a,
 					    _ITp* __i1, _ITp __i2,
 					    memory_order __m1,
@@ -1658,7 +1660,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     { return __a->compare_exchange_strong(*__i1, __i2, __m1, __m2); }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_strong_explicit(volatile atomic<_ITp>* __a,
 					    _ITp* __i1, _ITp __i2,
 					    memory_order __m1,
@@ -1667,37 +1669,37 @@ namespace std // _GLIBCXX_VISIBILITY(default)

   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_store(atomic<_ITp>* __a, _ITp __i) noexcept
     { atomic_store_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_store(volatile atomic<_ITp>* __a, _ITp __i) noexcept
     { atomic_store_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_load(const atomic<_ITp>* __a) noexcept
     { return atomic_load_explicit(__a, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_load(const volatile atomic<_ITp>* __a) noexcept
     { return atomic_load_explicit(__a, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_exchange(atomic<_ITp>* __a, _ITp __i) noexcept
     { return atomic_exchange_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_exchange(volatile atomic<_ITp>* __a, _ITp __i) noexcept
     { return atomic_exchange_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_weak(atomic<_ITp>* __a,
 				 _ITp* __i1, _ITp __i2) noexcept
     {
@@ -1707,7 +1709,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_weak(volatile atomic<_ITp>* __a,
 				 _ITp* __i1, _ITp __i2) noexcept
     {
@@ -1717,7 +1719,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_strong(atomic<_ITp>* __a,
 				   _ITp* __i1, _ITp __i2) noexcept
     {
@@ -1727,7 +1729,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     }

   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_strong(volatile atomic<_ITp>* __a,
 				   _ITp* __i1, _ITp __i2) noexcept
     {
@@ -1741,158 +1743,158 @@ namespace std // _GLIBCXX_VISIBILITY(default)
   // intergral types as specified in the standard, excluding address
   // types.
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_add_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_add(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_add_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_add(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_sub_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_sub(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_sub_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_sub(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_and_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_and(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_and_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_and(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_or_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			     memory_order __m) noexcept
     { return __a->fetch_or(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_or_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			     memory_order __m) noexcept
     { return __a->fetch_or(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_xor_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_xor(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_xor_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_xor(__i, __m); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_add(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_add_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_add(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_add_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_sub(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_sub_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_sub(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_sub_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_and(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_and_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_and(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_and_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_or(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_or_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_or(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_or_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_xor(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_xor_explicit(__a, __i, memory_order_seq_cst); }

   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_xor(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_xor_explicit(__a, __i, memory_order_seq_cst); }

   // Partial specializations for pointers.
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_add_explicit(atomic<_ITp*>* __a, ptrdiff_t __d,
 			      memory_order __m) noexcept
     { return __a->fetch_add(__d, __m); }

   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_add_explicit(volatile atomic<_ITp*>* __a, ptrdiff_t __d,
 			      memory_order __m) noexcept
     { return __a->fetch_add(__d, __m); }

   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_add(volatile atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
     { return __a->fetch_add(__d); }

   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_add(atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
     { return __a->fetch_add(__d); }

   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_sub_explicit(volatile atomic<_ITp*>* __a,
 			      ptrdiff_t __d, memory_order __m) noexcept
     { return __a->fetch_sub(__d, __m); }

   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_sub_explicit(atomic<_ITp*>* __a, ptrdiff_t __d,
 			      memory_order __m) noexcept
     { return __a->fetch_sub(__d, __m); }

   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_sub(volatile atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
     { return __a->fetch_sub(__d); }

   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_sub(atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
     { return __a->fetch_sub(__d); }
   // @} group atomics
---
 libitm/local_atomic |  302 ++++++++++++++++++++++++++-------------------------
 1 file changed, 152 insertions(+), 150 deletions(-)

diff --git a/libitm/local_atomic b/libitm/local_atomic
index 97e7d26..4cd961a 100644
--- a/libitm/local_atomic
+++ b/libitm/local_atomic
@@ -41,6 +41,10 @@
 #ifndef _GLIBCXX_ATOMIC
 #define _GLIBCXX_ATOMIC 1
 
+#ifndef __always_inline
+#define __always_inline inline __attribute__((always_inline))
+#endif
+
 // #pragma GCC system_header
 
 // #ifndef __GXX_EXPERIMENTAL_CXX0X__
@@ -71,7 +75,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       memory_order_seq_cst
     } memory_order;
 
-  inline memory_order
+  __always_inline memory_order
   __calculate_memory_order(memory_order __m) noexcept
   {
     const bool __cond1 = __m == memory_order_release;
@@ -81,13 +85,13 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     return __mo2;
   }
 
-  inline void
+  __always_inline void
   atomic_thread_fence(memory_order __m) noexcept
   {
     __atomic_thread_fence (__m);
   }
 
-  inline void
+  __always_inline void
   atomic_signal_fence(memory_order __m) noexcept
   {
     __atomic_thread_fence (__m);
@@ -277,19 +281,19 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     // Conversion to ATOMIC_FLAG_INIT.
     atomic_flag(bool __i) noexcept : __atomic_flag_base({ __i }) { }
 
-    bool
+    __always_inline bool
     test_and_set(memory_order __m = memory_order_seq_cst) noexcept
     {
       return __atomic_test_and_set (&_M_i, __m);
     }
 
-    bool
+    __always_inline bool
     test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
     {
       return __atomic_test_and_set (&_M_i, __m);
     }
 
-    void
+    __always_inline void
     clear(memory_order __m = memory_order_seq_cst) noexcept
     {
       // __glibcxx_assert(__m != memory_order_consume);
@@ -299,7 +303,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       __atomic_clear (&_M_i, __m);
     }
 
-    void
+    __always_inline void
     clear(memory_order __m = memory_order_seq_cst) volatile noexcept
     {
       // __glibcxx_assert(__m != memory_order_consume);
@@ -452,7 +456,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       is_lock_free() const volatile noexcept
       { return __atomic_is_lock_free (sizeof (_M_i), &_M_i); }
 
-      void
+      __always_inline void
       store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
       {
 	// __glibcxx_assert(__m != memory_order_acquire);
@@ -462,7 +466,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	__atomic_store_n(&_M_i, __i, __m);
       }
 
-      void
+      __always_inline void
       store(__int_type __i,
 	    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -473,7 +477,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	__atomic_store_n(&_M_i, __i, __m);
       }
 
-      __int_type
+      __always_inline __int_type
       load(memory_order __m = memory_order_seq_cst) const noexcept
       {
 	// __glibcxx_assert(__m != memory_order_release);
@@ -482,7 +486,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_load_n(&_M_i, __m);
       }
 
-      __int_type
+      __always_inline __int_type
       load(memory_order __m = memory_order_seq_cst) const volatile noexcept
       {
 	// __glibcxx_assert(__m != memory_order_release);
@@ -491,22 +495,21 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_load_n(&_M_i, __m);
       }
 
-      __int_type
+      __always_inline __int_type
       exchange(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) noexcept
       {
 	return __atomic_exchange_n(&_M_i, __i, __m);
       }
 
-
-      __int_type
+      __always_inline __int_type
       exchange(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       {
 	return __atomic_exchange_n(&_M_i, __i, __m);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m1, memory_order __m2) noexcept
       {
@@ -517,7 +520,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m1,
 			    memory_order __m2) volatile noexcept
@@ -529,7 +532,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 			    memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -537,7 +540,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				     __calculate_memory_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__int_type& __i1, __int_type __i2,
 		   memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -545,7 +548,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				     __calculate_memory_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m1, memory_order __m2) noexcept
       {
@@ -556,7 +559,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m1,
 			      memory_order __m2) volatile noexcept
@@ -568,7 +571,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 			      memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -576,7 +579,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				       __calculate_memory_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__int_type& __i1, __int_type __i2,
 		 memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -584,52 +587,52 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				       __calculate_memory_order(__m));
       }
 
-      __int_type
+      __always_inline __int_type
       fetch_add(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_add(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_add(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_add(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_sub(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_sub(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_sub(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_sub(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_and(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_and(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_and(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_and(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_or(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_or(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_or(__int_type __i,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_or(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_xor(__int_type __i,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_xor(&_M_i, __i, __m); }
 
-      __int_type
+      __always_inline __int_type
       fetch_xor(__int_type __i,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_xor(&_M_i, __i, __m); }
@@ -731,7 +734,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       is_lock_free() const volatile noexcept
       { return __atomic_is_lock_free (sizeof (_M_p), &_M_p); }
 
-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -742,7 +745,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	__atomic_store_n(&_M_p, __p, __m);
       }
 
-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -753,7 +756,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	__atomic_store_n(&_M_p, __p, __m);
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const noexcept
       {
 	// __glibcxx_assert(__m != memory_order_release);
@@ -762,7 +765,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_load_n(&_M_p, __m);
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const volatile noexcept
       {
 	// __glibcxx_assert(__m != memory_order_release);
@@ -771,22 +774,21 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_load_n(&_M_p, __m);
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) noexcept
       {
 	return __atomic_exchange_n(&_M_p, __p, __m);
       }
 
-
-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       {
 	return __atomic_exchange_n(&_M_p, __p, __m);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1,
 			      memory_order __m2) noexcept
@@ -798,7 +800,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1,
 			      memory_order __m2) volatile noexcept
@@ -810,22 +812,22 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_add(&_M_p, __d, __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_add(&_M_p, __d, __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return __atomic_fetch_sub(&_M_p, __d, __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return __atomic_fetch_sub(&_M_p, __d, __m); }
@@ -869,67 +871,67 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     bool
     is_lock_free() const volatile noexcept { return _M_base.is_lock_free(); }
 
-    void
+    __always_inline void
     store(bool __i, memory_order __m = memory_order_seq_cst) noexcept
     { _M_base.store(__i, __m); }
 
-    void
+    __always_inline void
     store(bool __i, memory_order __m = memory_order_seq_cst) volatile noexcept
     { _M_base.store(__i, __m); }
 
-    bool
+    __always_inline bool
     load(memory_order __m = memory_order_seq_cst) const noexcept
     { return _M_base.load(__m); }
 
-    bool
+    __always_inline bool
     load(memory_order __m = memory_order_seq_cst) const volatile noexcept
     { return _M_base.load(__m); }
 
-    bool
+    __always_inline bool
     exchange(bool __i, memory_order __m = memory_order_seq_cst) noexcept
     { return _M_base.exchange(__i, __m); }
 
-    bool
+    __always_inline bool
     exchange(bool __i,
 	     memory_order __m = memory_order_seq_cst) volatile noexcept
     { return _M_base.exchange(__i, __m); }
 
-    bool
+    __always_inline bool
     compare_exchange_weak(bool& __i1, bool __i2, memory_order __m1,
 			  memory_order __m2) noexcept
     { return _M_base.compare_exchange_weak(__i1, __i2, __m1, __m2); }
 
-    bool
+    __always_inline bool
     compare_exchange_weak(bool& __i1, bool __i2, memory_order __m1,
 			  memory_order __m2) volatile noexcept
     { return _M_base.compare_exchange_weak(__i1, __i2, __m1, __m2); }
 
-    bool
+    __always_inline bool
     compare_exchange_weak(bool& __i1, bool __i2,
 			  memory_order __m = memory_order_seq_cst) noexcept
     { return _M_base.compare_exchange_weak(__i1, __i2, __m); }
 
-    bool
+    __always_inline bool
     compare_exchange_weak(bool& __i1, bool __i2,
 		     memory_order __m = memory_order_seq_cst) volatile noexcept
     { return _M_base.compare_exchange_weak(__i1, __i2, __m); }
 
-    bool
+    __always_inline bool
     compare_exchange_strong(bool& __i1, bool __i2, memory_order __m1,
 			    memory_order __m2) noexcept
     { return _M_base.compare_exchange_strong(__i1, __i2, __m1, __m2); }
 
-    bool
+    __always_inline bool
     compare_exchange_strong(bool& __i1, bool __i2, memory_order __m1,
 			    memory_order __m2) volatile noexcept
     { return _M_base.compare_exchange_strong(__i1, __i2, __m1, __m2); }
 
-    bool
+    __always_inline bool
     compare_exchange_strong(bool& __i1, bool __i2,
 			    memory_order __m = memory_order_seq_cst) noexcept
     { return _M_base.compare_exchange_strong(__i1, __i2, __m); }
 
-    bool
+    __always_inline bool
     compare_exchange_strong(bool& __i1, bool __i2,
 		    memory_order __m = memory_order_seq_cst) volatile noexcept
     { return _M_base.compare_exchange_strong(__i1, __i2, __m); }
@@ -979,11 +981,11 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       store(_Tp __i, memory_order _m = memory_order_seq_cst) noexcept
       { __atomic_store(&_M_i, &__i, _m); }
 
-      void
+      __always_inline void
       store(_Tp __i, memory_order _m = memory_order_seq_cst) volatile noexcept
       { __atomic_store(&_M_i, &__i, _m); }
 
-      _Tp
+      __always_inline _Tp
       load(memory_order _m = memory_order_seq_cst) const noexcept
       { 
         _Tp tmp;
@@ -991,7 +993,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return tmp;
       }
 
-      _Tp
+      __always_inline _Tp
       load(memory_order _m = memory_order_seq_cst) const volatile noexcept
       { 
         _Tp tmp;
@@ -999,7 +1001,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return tmp;
       }
 
-      _Tp
+      __always_inline _Tp
       exchange(_Tp __i, memory_order _m = memory_order_seq_cst) noexcept
       { 
         _Tp tmp;
@@ -1007,7 +1009,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return tmp;
       }
 
-      _Tp
+      __always_inline _Tp
       exchange(_Tp __i, 
 	       memory_order _m = memory_order_seq_cst) volatile noexcept
       { 
@@ -1016,50 +1018,50 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 	return tmp;
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(_Tp& __e, _Tp __i, memory_order __s, 
 			    memory_order __f) noexcept
       {
 	return __atomic_compare_exchange(&_M_i, &__e, &__i, true, __s, __f); 
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(_Tp& __e, _Tp __i, memory_order __s, 
 			    memory_order __f) volatile noexcept
       {
 	return __atomic_compare_exchange(&_M_i, &__e, &__i, true, __s, __f); 
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(_Tp& __e, _Tp __i,
 			    memory_order __m = memory_order_seq_cst) noexcept
       { return compare_exchange_weak(__e, __i, __m, __m); }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(_Tp& __e, _Tp __i,
 		     memory_order __m = memory_order_seq_cst) volatile noexcept
       { return compare_exchange_weak(__e, __i, __m, __m); }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(_Tp& __e, _Tp __i, memory_order __s, 
 			      memory_order __f) noexcept
       {
 	return __atomic_compare_exchange(&_M_i, &__e, &__i, false, __s, __f); 
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(_Tp& __e, _Tp __i, memory_order __s, 
 			      memory_order __f) volatile noexcept
       {
 	return __atomic_compare_exchange(&_M_i, &__e, &__i, false, __s, __f); 
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(_Tp& __e, _Tp __i,
 			       memory_order __m = memory_order_seq_cst) noexcept
       { return compare_exchange_strong(__e, __i, __m, __m); }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(_Tp& __e, _Tp __i,
 		     memory_order __m = memory_order_seq_cst) volatile noexcept
       { return compare_exchange_strong(__e, __i, __m, __m); }
@@ -1152,46 +1154,46 @@ namespace std // _GLIBCXX_VISIBILITY(default)
       is_lock_free() const volatile noexcept
       { return _M_b.is_lock_free(); }
 
-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) noexcept
       { return _M_b.store(__p, __m); }
 
-      void
+      __always_inline void
       store(__pointer_type __p,
 	    memory_order __m = memory_order_seq_cst) volatile noexcept
       { return _M_b.store(__p, __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const noexcept
       { return _M_b.load(__m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       load(memory_order __m = memory_order_seq_cst) const volatile noexcept
       { return _M_b.load(__m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) noexcept
       { return _M_b.exchange(__p, __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       exchange(__pointer_type __p,
 	       memory_order __m = memory_order_seq_cst) volatile noexcept
       { return _M_b.exchange(__p, __m); }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
 			    memory_order __m1, memory_order __m2) noexcept
       { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
 			    memory_order __m1,
 			    memory_order __m2) volatile noexcept
       { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
 			    memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -1199,7 +1201,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				     __calculate_memory_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
 		    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -1207,18 +1209,18 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 				     __calculate_memory_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1, memory_order __m2) noexcept
       { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m1,
 			      memory_order __m2) volatile noexcept
       { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 			      memory_order __m = memory_order_seq_cst) noexcept
       {
@@ -1226,7 +1228,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 					    __calculate_memory_order(__m));
       }
 
-      bool
+      __always_inline bool
       compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
 		    memory_order __m = memory_order_seq_cst) volatile noexcept
       {
@@ -1234,22 +1236,22 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 					    __calculate_memory_order(__m));
       }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return _M_b.fetch_add(__d, __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_add(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return _M_b.fetch_add(__d, __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) noexcept
       { return _M_b.fetch_sub(__d, __m); }
 
-      __pointer_type
+      __always_inline __pointer_type
       fetch_sub(ptrdiff_t __d,
 		memory_order __m = memory_order_seq_cst) volatile noexcept
       { return _M_b.fetch_sub(__d, __m); }
@@ -1543,98 +1545,98 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 
 
   // Function definitions, atomic_flag operations.
-  inline bool
+  __always_inline bool
   atomic_flag_test_and_set_explicit(atomic_flag* __a,
 				    memory_order __m) noexcept
   { return __a->test_and_set(__m); }
 
-  inline bool
+  __always_inline bool
   atomic_flag_test_and_set_explicit(volatile atomic_flag* __a,
 				    memory_order __m) noexcept
   { return __a->test_and_set(__m); }
 
-  inline void
+  __always_inline void
   atomic_flag_clear_explicit(atomic_flag* __a, memory_order __m) noexcept
   { __a->clear(__m); }
 
-  inline void
+  __always_inline void
   atomic_flag_clear_explicit(volatile atomic_flag* __a,
 			     memory_order __m) noexcept
   { __a->clear(__m); }
 
-  inline bool
+  __always_inline bool
   atomic_flag_test_and_set(atomic_flag* __a) noexcept
   { return atomic_flag_test_and_set_explicit(__a, memory_order_seq_cst); }
 
-  inline bool
+  __always_inline bool
   atomic_flag_test_and_set(volatile atomic_flag* __a) noexcept
   { return atomic_flag_test_and_set_explicit(__a, memory_order_seq_cst); }
 
-  inline void
+  __always_inline void
   atomic_flag_clear(atomic_flag* __a) noexcept
   { atomic_flag_clear_explicit(__a, memory_order_seq_cst); }
 
-  inline void
+  __always_inline void
   atomic_flag_clear(volatile atomic_flag* __a) noexcept
   { atomic_flag_clear_explicit(__a, memory_order_seq_cst); }
 
 
   // Function templates generally applicable to atomic types.
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_is_lock_free(const atomic<_ITp>* __a) noexcept
     { return __a->is_lock_free(); }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_is_lock_free(const volatile atomic<_ITp>* __a) noexcept
     { return __a->is_lock_free(); }
 
   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_init(atomic<_ITp>* __a, _ITp __i) noexcept;
 
   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_init(volatile atomic<_ITp>* __a, _ITp __i) noexcept;
 
   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_store_explicit(atomic<_ITp>* __a, _ITp __i,
 			  memory_order __m) noexcept
     { __a->store(__i, __m); }
 
   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_store_explicit(volatile atomic<_ITp>* __a, _ITp __i,
 			  memory_order __m) noexcept
     { __a->store(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_load_explicit(const atomic<_ITp>* __a, memory_order __m) noexcept
     { return __a->load(__m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_load_explicit(const volatile atomic<_ITp>* __a,
 			 memory_order __m) noexcept
     { return __a->load(__m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_exchange_explicit(atomic<_ITp>* __a, _ITp __i,
 			     memory_order __m) noexcept
     { return __a->exchange(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_exchange_explicit(volatile atomic<_ITp>* __a, _ITp __i,
 			     memory_order __m) noexcept
     { return __a->exchange(__i, __m); }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_weak_explicit(atomic<_ITp>* __a,
 					  _ITp* __i1, _ITp __i2,
 					  memory_order __m1,
@@ -1642,7 +1644,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     { return __a->compare_exchange_weak(*__i1, __i2, __m1, __m2); }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_weak_explicit(volatile atomic<_ITp>* __a,
 					  _ITp* __i1, _ITp __i2,
 					  memory_order __m1,
@@ -1650,7 +1652,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     { return __a->compare_exchange_weak(*__i1, __i2, __m1, __m2); }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_strong_explicit(atomic<_ITp>* __a,
 					    _ITp* __i1, _ITp __i2,
 					    memory_order __m1,
@@ -1658,7 +1660,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     { return __a->compare_exchange_strong(*__i1, __i2, __m1, __m2); }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_strong_explicit(volatile atomic<_ITp>* __a,
 					    _ITp* __i1, _ITp __i2,
 					    memory_order __m1,
@@ -1667,37 +1669,37 @@ namespace std // _GLIBCXX_VISIBILITY(default)
 
 
   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_store(atomic<_ITp>* __a, _ITp __i) noexcept
     { atomic_store_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline void
+    __always_inline void
     atomic_store(volatile atomic<_ITp>* __a, _ITp __i) noexcept
     { atomic_store_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_load(const atomic<_ITp>* __a) noexcept
     { return atomic_load_explicit(__a, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_load(const volatile atomic<_ITp>* __a) noexcept
     { return atomic_load_explicit(__a, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_exchange(atomic<_ITp>* __a, _ITp __i) noexcept
     { return atomic_exchange_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_exchange(volatile atomic<_ITp>* __a, _ITp __i) noexcept
     { return atomic_exchange_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_weak(atomic<_ITp>* __a,
 				 _ITp* __i1, _ITp __i2) noexcept
     {
@@ -1707,7 +1709,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_weak(volatile atomic<_ITp>* __a,
 				 _ITp* __i1, _ITp __i2) noexcept
     {
@@ -1717,7 +1719,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_strong(atomic<_ITp>* __a,
 				   _ITp* __i1, _ITp __i2) noexcept
     {
@@ -1727,7 +1729,7 @@ namespace std // _GLIBCXX_VISIBILITY(default)
     }
 
   template<typename _ITp>
-    inline bool
+    __always_inline bool
     atomic_compare_exchange_strong(volatile atomic<_ITp>* __a,
 				   _ITp* __i1, _ITp __i2) noexcept
     {
@@ -1741,158 +1743,158 @@ namespace std // _GLIBCXX_VISIBILITY(default)
   // intergral types as specified in the standard, excluding address
   // types.
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_add_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_add(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_add_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_add(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_sub_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_sub(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_sub_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_sub(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_and_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_and(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_and_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_and(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_or_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			     memory_order __m) noexcept
     { return __a->fetch_or(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_or_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			     memory_order __m) noexcept
     { return __a->fetch_or(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_xor_explicit(__atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_xor(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_xor_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
 			      memory_order __m) noexcept
     { return __a->fetch_xor(__i, __m); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_add(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_add_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_add(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_add_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_sub(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_sub_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_sub(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_sub_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_and(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_and_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_and(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_and_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_or(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_or_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_or(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_or_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_xor(__atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_xor_explicit(__a, __i, memory_order_seq_cst); }
 
   template<typename _ITp>
-    inline _ITp
+    __always_inline _ITp
     atomic_fetch_xor(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
     { return atomic_fetch_xor_explicit(__a, __i, memory_order_seq_cst); }
 
 
   // Partial specializations for pointers.
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_add_explicit(atomic<_ITp*>* __a, ptrdiff_t __d,
 			      memory_order __m) noexcept
     { return __a->fetch_add(__d, __m); }
 
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_add_explicit(volatile atomic<_ITp*>* __a, ptrdiff_t __d,
 			      memory_order __m) noexcept
     { return __a->fetch_add(__d, __m); }
 
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_add(volatile atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
     { return __a->fetch_add(__d); }
 
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_add(atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
     { return __a->fetch_add(__d); }
 
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_sub_explicit(volatile atomic<_ITp*>* __a,
 			      ptrdiff_t __d, memory_order __m) noexcept
     { return __a->fetch_sub(__d, __m); }
 
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_sub_explicit(atomic<_ITp*>* __a, ptrdiff_t __d,
 			      memory_order __m) noexcept
     { return __a->fetch_sub(__d, __m); }
 
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_sub(volatile atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
     { return __a->fetch_sub(__d); }
 
   template<typename _ITp>
-    inline _ITp*
+    __always_inline _ITp*
     atomic_fetch_sub(atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
     { return __a->fetch_sub(__d); }
   // @} group atomics
-- 
1.7.10.4

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 3/4] Avoid nonconst memmodels in libitm's local outdated copy of <atomic> too
  2013-03-16 13:30 ` [PATCH 3/4] Avoid nonconst memmodels in libitm's local outdated copy of <atomic> too Andi Kleen
@ 2013-03-19 21:23   ` Richard Henderson
  0 siblings, 0 replies; 15+ messages in thread
From: Richard Henderson @ 2013-03-19 21:23 UTC (permalink / raw)
  To: Andi Kleen; +Cc: gcc-patches, libstdc++, Andi Kleen

On 03/16/2013 06:29 AM, Andi Kleen wrote:
> 2013-03-15  Andi Kleen  <andi@my.domain.org>
> 
> 	* local_atomic (__always_inline): Add.
> 	(__calculate_memory_order, atomic_thread_fence,
> 	 atomic_signal_fence, test_and_set, clear, store, load,
>          exchange, compare_exchange_weak, compare_exchange_strong,
>          fetch_add, fetch_sub, fetch_and, fetch_or, fetch_xor):
> 	Add __always_inline to force inlining.

Ok.

One day we'll get the build fixed so this copy isn't necessary...


r~

^ permalink raw reply	[flat|nested] 15+ messages in thread

[parent not found: <CAH6eHdR=VhEfAes6S97CfBf0Newe1h3EeoxreRKKHpi4mpJh=w@mail.gmail.com>]

[parent not found: <20130319064639.GA12913@tucnak.redhat.com>]

* Re: [PATCH 1/4] Mark all member functions with memory models always inline
       [not found]   ` <20130319064639.GA12913@tucnak.redhat.com>
@ 2013-03-19 15:51     ` Andi Kleen
  2013-03-19 16:10       ` Jakub Jelinek
  0 siblings, 1 reply; 15+ messages in thread
From: Andi Kleen @ 2013-03-19 15:51 UTC (permalink / raw)
  To: Jakub Jelinek; +Cc: Jonathan Wakely, Andi Kleen, gcc-patches, libstdc++, rth

> Using __always_inline as the name of the macro is a bad idea, glibc
> headers use that macro already.  Just use something else in public headers
> that aren't part of glibc.

That's why I had the ifdef, but ok.  I'll use __force_inline then.

-Andi

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/4] Mark all member functions with memory models always inline
  2013-03-19 15:51     ` [PATCH 1/4] Mark all member functions with memory models always inline Andi Kleen
@ 2013-03-19 16:10       ` Jakub Jelinek
  2013-03-19 17:30         ` Andi Kleen
  0 siblings, 1 reply; 15+ messages in thread
From: Jakub Jelinek @ 2013-03-19 16:10 UTC (permalink / raw)
  To: Andi Kleen; +Cc: Jonathan Wakely, Andi Kleen, gcc-patches, libstdc++, rth

On Tue, Mar 19, 2013 at 08:51:21AM -0700, Andi Kleen wrote:
> > Using __always_inline as the name of the macro is a bad idea, glibc
> > headers use that macro already.  Just use something else in public headers
> > that aren't part of glibc.
> 
> That's why I had the ifdef, but ok.  I'll use __force_inline then.

I'd say Jonathan's _GLIBCXX_ALWAYS_INLINE would be better.

BTW, have you verified always_inline works fine for -O0?

int x;

inline __attribute__((always_inline)) bool
test_and_set(int __m = 5)
{
  return __atomic_test_and_set (&x, __m);
}

int
foo (void)
{
  return test_and_set (65536 | 5);
}

with -O0 -mhle doesn't result in xacquire, guess for !optimize
get_memmodel would need to look through chain of SSA_NAMEs if SSA_NAME
(that can appear because of inlining), looking for INTEGER_CSTs.
  If there is:
  _7 = 0x10005;
  _8 = _7;
  _9 = _8;
  __atomic_test_and_set (&x, _9);
still return 0x10005 rather than MEMMODEL_SEQ_CST.

	Jakub

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/4] Mark all member functions with memory models always inline
  2013-03-19 16:10       ` Jakub Jelinek
@ 2013-03-19 17:30         ` Andi Kleen
  2013-03-20 10:38           ` Richard Biener
  0 siblings, 1 reply; 15+ messages in thread
From: Andi Kleen @ 2013-03-19 17:30 UTC (permalink / raw)
  To: Jakub Jelinek
  Cc: Andi Kleen, Jonathan Wakely, Andi Kleen, gcc-patches, libstdc++, rth

On Tue, Mar 19, 2013 at 05:10:22PM +0100, Jakub Jelinek wrote:
> On Tue, Mar 19, 2013 at 08:51:21AM -0700, Andi Kleen wrote:
> > > Using __always_inline as the name of the macro is a bad idea, glibc
> > > headers use that macro already.  Just use something else in public headers
> > > that aren't part of glibc.
> > 
> > That's why I had the ifdef, but ok.  I'll use __force_inline then.
> 
> I'd say Jonathan's _GLIBCXX_ALWAYS_INLINE would be better.
> 
> BTW, have you verified always_inline works fine for -O0?

I did some simple tests and it seemed to work.

But thanks for the counter example.

> 
> with -O0 -mhle doesn't result in xacquire, guess for !optimize
> get_memmodel would need to look through chain of SSA_NAMEs if SSA_NAME
> (that can appear because of inlining), looking for INTEGER_CSTs.

interesting. so need more fixes. i'll look into that. Is there 
already a convenient helper for it?

There was an alternative approach of using a template, but I suppose
that would have the same problem.


-Andi

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/4] Mark all member functions with memory models always inline
  2013-03-19 17:30         ` Andi Kleen
@ 2013-03-20 10:38           ` Richard Biener
  2013-03-20 10:46             ` Jakub Jelinek
  0 siblings, 1 reply; 15+ messages in thread
From: Richard Biener @ 2013-03-20 10:38 UTC (permalink / raw)
  To: Andi Kleen
  Cc: Jakub Jelinek, Andi Kleen, Jonathan Wakely, gcc-patches, libstdc++, rth

On Tue, Mar 19, 2013 at 6:30 PM, Andi Kleen <andi@firstfloor.org> wrote:
> On Tue, Mar 19, 2013 at 05:10:22PM +0100, Jakub Jelinek wrote:
>> On Tue, Mar 19, 2013 at 08:51:21AM -0700, Andi Kleen wrote:
>> > > Using __always_inline as the name of the macro is a bad idea, glibc
>> > > headers use that macro already.  Just use something else in public headers
>> > > that aren't part of glibc.
>> >
>> > That's why I had the ifdef, but ok.  I'll use __force_inline then.
>>
>> I'd say Jonathan's _GLIBCXX_ALWAYS_INLINE would be better.
>>
>> BTW, have you verified always_inline works fine for -O0?
>
> I did some simple tests and it seemed to work.
>
> But thanks for the counter example.
>
>>
>> with -O0 -mhle doesn't result in xacquire, guess for !optimize
>> get_memmodel would need to look through chain of SSA_NAMEs if SSA_NAME
>> (that can appear because of inlining), looking for INTEGER_CSTs.
>
> interesting. so need more fixes. i'll look into that. Is there
> already a convenient helper for it?

Not without using information created by TER which is disabled for this
case because of different line information.  If TER would not be disabled
for this reason it would already work automagically.

Eventually we can relax this restriction for constants that are only fed
through temporaries ...

> There was an alternative approach of using a template, but I suppose
> that would have the same problem.

Possibly yes, but I guess the frontend substitutes without going through
an extra temporary (if all arguments are template parameters!)

Richard.

>
> -Andi

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/4] Mark all member functions with memory models always inline
  2013-03-20 10:38           ` Richard Biener
@ 2013-03-20 10:46             ` Jakub Jelinek
  2013-03-20 11:21               ` Richard Biener
  0 siblings, 1 reply; 15+ messages in thread
From: Jakub Jelinek @ 2013-03-20 10:46 UTC (permalink / raw)
  To: Richard Biener
  Cc: Andi Kleen, Andi Kleen, Jonathan Wakely, gcc-patches, libstdc++, rth

On Wed, Mar 20, 2013 at 11:38:03AM +0100, Richard Biener wrote:
> Not without using information created by TER which is disabled for this
> case because of different line information.  If TER would not be disabled
> for this reason it would already work automagically.

Would relaxing that in TER for constants help in the case of:
static inline __attribute__((always_inline)) ...
foo (..., int m = __ATOMIC_SEQ_CST)
{
  if (something)
    bar ();
  else
    baz ();
  __atomic_test_and_set (&x, m);
}

void
test ()
{
  foo (..., __ATOMIC_HLE_ACQUIRE | __ATOMIC_SEQ_CST);
}
though?  I'd think as the temp = 0x10005; would be in a different bb, TER
wouldn't do anything here, for -O1 of course CCP or similar would propagate
that, but for -O0 we'd still have to walk the chain of SSA_NAMEs.

	Jakub

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/4] Mark all member functions with memory models always inline
  2013-03-20 10:46             ` Jakub Jelinek
@ 2013-03-20 11:21               ` Richard Biener
  2013-03-20 11:25                 ` Jakub Jelinek
  0 siblings, 1 reply; 15+ messages in thread
From: Richard Biener @ 2013-03-20 11:21 UTC (permalink / raw)
  To: Jakub Jelinek
  Cc: Andi Kleen, Andi Kleen, Jonathan Wakely, gcc-patches, libstdc++, rth

On Wed, Mar 20, 2013 at 11:46 AM, Jakub Jelinek <jakub@redhat.com> wrote:
> On Wed, Mar 20, 2013 at 11:38:03AM +0100, Richard Biener wrote:
>> Not without using information created by TER which is disabled for this
>> case because of different line information.  If TER would not be disabled
>> for this reason it would already work automagically.
>
> Would relaxing that in TER for constants help in the case of:
> static inline __attribute__((always_inline)) ...
> foo (..., int m = __ATOMIC_SEQ_CST)
> {
>   if (something)
>     bar ();
>   else
>     baz ();
>   __atomic_test_and_set (&x, m);
> }
>
> void
> test ()
> {
>   foo (..., __ATOMIC_HLE_ACQUIRE | __ATOMIC_SEQ_CST);
> }
> though?  I'd think as the temp = 0x10005; would be in a different bb, TER
> wouldn't do anything here, for -O1 of course CCP or similar would propagate
> that, but for -O0 we'd still have to walk the chain of SSA_NAMEs.

True.  I was for a long time thinking of running a CCP pass at -O0 ...
and only restrict what we sustitute into (builtin arguments and asm operands).

Walking the SSA def chain would also be possible - preferably from
cfgexpand.c when we process the stmt.  But you'll eventually lose
debug info like for

  int i = 1;
  __builtin_foo (i);

when we then never expand the SSA name def i_2 = 1 because we are
never visiting its use ...

Richard.

>         Jakub

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/4] Mark all member functions with memory models always inline
  2013-03-20 11:21               ` Richard Biener
@ 2013-03-20 11:25                 ` Jakub Jelinek
  2013-03-20 11:29                   ` Richard Biener
  0 siblings, 1 reply; 15+ messages in thread
From: Jakub Jelinek @ 2013-03-20 11:25 UTC (permalink / raw)
  To: Richard Biener
  Cc: Andi Kleen, Andi Kleen, Jonathan Wakely, gcc-patches, libstdc++, rth

On Wed, Mar 20, 2013 at 12:20:27PM +0100, Richard Biener wrote:
> Walking the SSA def chain would also be possible - preferably from
> cfgexpand.c when we process the stmt.  But you'll eventually lose
> debug info like for
> 
>   int i = 1;
>   __builtin_foo (i);
> 
> when we then never expand the SSA name def i_2 = 1 because we are
> never visiting its use ...

If i is a user variable, then for -O0 we'd better create a memory slot for
it and expand the i = 1 store as store to that memory location.

	Jakub

^ permalink raw reply	[flat|nested] 15+ messages in thread

* Re: [PATCH 1/4] Mark all member functions with memory models always inline
  2013-03-20 11:25                 ` Jakub Jelinek
@ 2013-03-20 11:29                   ` Richard Biener
  0 siblings, 0 replies; 15+ messages in thread
From: Richard Biener @ 2013-03-20 11:29 UTC (permalink / raw)
  To: Jakub Jelinek
  Cc: Andi Kleen, Andi Kleen, Jonathan Wakely, gcc-patches, libstdc++, rth

On Wed, Mar 20, 2013 at 12:24 PM, Jakub Jelinek <jakub@redhat.com> wrote:
> On Wed, Mar 20, 2013 at 12:20:27PM +0100, Richard Biener wrote:
>> Walking the SSA def chain would also be possible - preferably from
>> cfgexpand.c when we process the stmt.  But you'll eventually lose
>> debug info like for
>>
>>   int i = 1;
>>   __builtin_foo (i);
>>
>> when we then never expand the SSA name def i_2 = 1 because we are
>> never visiting its use ...
>
> If i is a user variable, then for -O0 we'd better create a memory slot for
> it and expand the i = 1 store as store to that memory location.

Ok, we seem to do that...

Richard.

>         Jakub

^ permalink raw reply	[flat|nested] 15+ messages in thread

end of thread, other threads:[~2013-03-20 11:29 UTC | newest]

Thread overview: 15+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2013-03-16 13:30 [PATCH 1/4] Mark all member functions with memory models always inline Andi Kleen
2013-03-16 13:29 ` [PATCH 2/4] Avoid non constant memory model uses in libatomic Andi Kleen
2013-03-19 21:23   ` Richard Henderson
2013-03-16 13:29 ` [PATCH 4/4] Add warning for non-constant memory models Andi Kleen
2013-03-19 21:23   ` Richard Henderson
2013-03-16 13:30 ` [PATCH 3/4] Avoid nonconst memmodels in libitm's local outdated copy of <atomic> too Andi Kleen
2013-03-19 21:23   ` Richard Henderson
     [not found] ` <CAH6eHdR=VhEfAes6S97CfBf0Newe1h3EeoxreRKKHpi4mpJh=w@mail.gmail.com>
     [not found]   ` <20130319064639.GA12913@tucnak.redhat.com>
2013-03-19 15:51     ` [PATCH 1/4] Mark all member functions with memory models always inline Andi Kleen
2013-03-19 16:10       ` Jakub Jelinek
2013-03-19 17:30         ` Andi Kleen
2013-03-20 10:38           ` Richard Biener
2013-03-20 10:46             ` Jakub Jelinek
2013-03-20 11:21               ` Richard Biener
2013-03-20 11:25                 ` Jakub Jelinek
2013-03-20 11:29                   ` Richard Biener

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).