[Test patch] Inline hot functions in libcxx shared_ptr

Moves hot functions such as atomic add into the memory header file
so that they can be inlined, which brings performance benefits.

Patch by Kevin Hu, Aditya Kumar, Sebastian Pop

Differential Revision: https://reviews.llvm.org/D24991


git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@292184 91177308-0d34-0410-b5e6-96231b3b80d8

include/memory

@@ -3681,6 +3681,39 @@ uninitialized_move_n(_InputIt __first, _Size __n, _ForwardIt __first_res) {
 
 #endif // _LIBCPP_STD_VER > 14
 
+// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
+// should be sufficient for thread safety.
+// See https://llvm.org/bugs/show_bug.cgi?id=22803
+#if defined(__clang__) && __has_builtin(__atomic_add_fetch)          \
+                       && defined(__ATOMIC_RELAXED)                  \
+                       && defined(__ATOMIC_ACQ_REL)
+#   define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#elif !defined(__clang__) && defined(_GNUC_VER) && _GNUC_VER >= 407
+#   define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#endif
+
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY _Tp
+__libcpp_atomic_refcount_increment(_Tp& __t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+    return __atomic_add_fetch(&__t, 1, __ATOMIC_RELAXED);
+#else
+    return __t += 1;
+#endif
+}
+
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY _Tp
+__libcpp_atomic_refcount_decrement(_Tp& __t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+    return __atomic_add_fetch(&__t, -1, __ATOMIC_ACQ_REL);
+#else
+    return __t -= 1;
+#endif
+}
+
 class _LIBCPP_EXCEPTION_ABI bad_weak_ptr
     : public std::exception
 {
@@ -3717,8 +3750,23 @@ public:
     explicit __shared_count(long __refs = 0) _NOEXCEPT
         : __shared_owners_(__refs) {}
 
-    void __add_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
-    bool __release_shared() _NOEXCEPT;
+    _LIBCPP_FUNC_VIS void __add_shared() _NOEXCEPT;
+    _LIBCPP_FUNC_VIS bool __release_shared() _NOEXCEPT;
+#else
+    _LIBCPP_INLINE_VISIBILITY
+    void __add_shared() _NOEXCEPT {
+      __libcpp_atomic_refcount_increment(__shared_owners_);
+    }
+    _LIBCPP_INLINE_VISIBILITY
+    bool __release_shared() _NOEXCEPT {
+      if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) {
+        __on_zero_shared();
+        return true;
+      }
+      return false;
+    }
+#endif
     _LIBCPP_INLINE_VISIBILITY
     long use_count() const _NOEXCEPT {
         return __libcpp_relaxed_load(&__shared_owners_) + 1;
@@ -3739,9 +3787,25 @@ protected:
     virtual ~__shared_weak_count();
 
 public:
-    void __add_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
-    void __add_weak() _NOEXCEPT;
+    _LIBCPP_FUNC_VIS void __add_shared() _NOEXCEPT;
-    void __release_shared() _NOEXCEPT;
+    _LIBCPP_FUNC_VIS void __add_weak() _NOEXCEPT;
+    _LIBCPP_FUNC_VIS void __release_shared() _NOEXCEPT;
+#else
+    _LIBCPP_INLINE_VISIBILITY
+    void __add_shared() _NOEXCEPT {
+      __shared_count::__add_shared();
+    }
+    _LIBCPP_INLINE_VISIBILITY
+    void __add_weak() _NOEXCEPT {
+      __libcpp_atomic_refcount_increment(__shared_weak_owners_);
+    }
+    _LIBCPP_INLINE_VISIBILITY
+    void __release_shared() _NOEXCEPT {
+      if (__shared_count::__release_shared())
+        __release_weak();
+    }
+#endif
     void __release_weak() _NOEXCEPT;
     _LIBCPP_INLINE_VISIBILITY
     long use_count() const _NOEXCEPT {return __shared_count::use_count();}

src/memory.cpp

@@ -17,28 +17,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-namespace
-{
-
-// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
-// should be sufficient for thread safety.
-// See https://llvm.org/bugs/show_bug.cgi?id=22803
-template <class T>
-inline T
-increment(T& t) _NOEXCEPT
-{
-    return __libcpp_atomic_add(&t, 1, _AO_Relaxed);
-}
-
-template <class T>
-inline T
-decrement(T& t) _NOEXCEPT
-{
-    return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel);
-}
-
-}  // namespace
-
 const allocator_arg_t allocator_arg = allocator_arg_t();
 
 bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {}
@@ -53,16 +31,20 @@ __shared_count::~__shared_count()
 {
 }
 
+__shared_weak_count::~__shared_weak_count()
+{
+}
+
 void
 __shared_count::__add_shared() _NOEXCEPT
 {
-    increment(__shared_owners_);
+    __libcpp_atomic_refcount_increment(__shared_owners_);
 }
 
 bool
 __shared_count::__release_shared() _NOEXCEPT
 {
-    if (decrement(__shared_owners_) == -1)
+    if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1)
     {
         __on_zero_shared();
         return true;
@@ -70,10 +52,6 @@ __shared_count::__release_shared() _NOEXCEPT
     return false;
 }
 
-__shared_weak_count::~__shared_weak_count()
-{
-}
-
 void
 __shared_weak_count::__add_shared() _NOEXCEPT
 {
@@ -83,7 +61,7 @@ __shared_weak_count::__add_shared() _NOEXCEPT
 void
 __shared_weak_count::__add_weak() _NOEXCEPT
 {
-    increment(__shared_weak_owners_);
+    __libcpp_atomic_refcount_increment(__shared_weak_owners_);
 }
 
 void
@@ -124,7 +102,7 @@ __shared_weak_count::__release_weak() _NOEXCEPT
         //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
         __on_zero_shared_weak();
     }
-    else if (decrement(__shared_weak_owners_) == -1)
+    else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
         __on_zero_shared_weak();
 }