Generates better code (GCC-6.2.1): 0000000000000420 : 420: 55 push %rbp 421: 8b 97 48 01 00 00 mov 0x148(%rdi),%edx 427: 48 89 e5 mov %rsp,%rbp 42a: eb 10 jmp 43c 42c: 89 d0 mov %edx,%eax 42e: f0 0f b1 8f 48 01 00 lock cmpxchg %ecx,0x148(%rdi) 435: 00 436: 39 c2 cmp %eax,%edx 438: 74 0d je 447 43a: 89 c2 mov %eax,%edx 43c: 8d 42 ff lea -0x1(%rdx),%eax 43f: 8d 4a 01 lea 0x1(%rdx),%ecx 442: 83 f8 fd cmp $0xfffffffd,%eax 445: 76 e5 jbe 42c 447: 5d pop %rbp 448: c3 retq 0000000000001490 : 1490: 55 push %rbp 1491: 8b 87 48 01 00 00 mov 0x148(%rdi),%eax 1497: 48 89 e5 mov %rsp,%rbp 149a: eb 0a jmp 14a6 149c: f0 0f b1 97 48 01 00 lock cmpxchg %edx,0x148(%rdi) 14a3: 00 14a4: 74 0b je 14b1 14a6: 8d 48 ff lea -0x1(%rax),%ecx 14a9: 8d 50 01 lea 0x1(%rax),%edx 14ac: 83 f9 fd cmp $0xfffffffd,%ecx 14af: 76 eb jbe 149c 14b1: 5d pop %rbp 14b2: c3 retq Signed-off-by: Peter Zijlstra (Intel) --- include/linux/refcount.h | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) --- a/include/linux/refcount.h +++ b/include/linux/refcount.h @@ -69,7 +69,7 @@ static inline unsigned int refcount_read static inline __refcount_check bool refcount_add_not_zero(unsigned int i, refcount_t *r) { - unsigned int old, new, val = atomic_read(&r->refs); + unsigned int new, val = atomic_read(&r->refs); for (;;) { if (!val) @@ -81,11 +81,9 @@ bool refcount_add_not_zero(unsigned int new = val + i; if (new < val) new = UINT_MAX; - old = atomic_cmpxchg_relaxed(&r->refs, val, new); - if (old == val) - break; - val = old; + if (atomic_try_cmpxchg_relaxed(&r->refs, &val, new)) + break; } REFCOUNT_WARN(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n"); @@ -108,7 +106,7 @@ static inline void refcount_add(unsigned static inline __refcount_check bool refcount_inc_not_zero(refcount_t *r) { - unsigned int old, new, val = atomic_read(&r->refs); + unsigned int new, val = atomic_read(&r->refs); for (;;) { new = val + 1; @@ -119,11 +117,8 @@ bool refcount_inc_not_zero(refcount_t *r if (unlikely(!new)) return true; - old = atomic_cmpxchg_relaxed(&r->refs, val, new); - if (old == val) + if (atomic_try_cmpxchg_relaxed(&r->refs, &val, new)) break; - - val = old; } REFCOUNT_WARN(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n"); @@ -153,7 +148,7 @@ static inline void refcount_inc(refcount static inline __refcount_check bool refcount_sub_and_test(unsigned int i, refcount_t *r) { - unsigned int old, new, val = atomic_read(&r->refs); + unsigned int new, val = atomic_read(&r->refs); for (;;) { if (unlikely(val == UINT_MAX)) @@ -165,11 +160,8 @@ bool refcount_sub_and_test(unsigned int return false; } - old = atomic_cmpxchg_release(&r->refs, val, new); - if (old == val) + if (atomic_try_cmpxchg_release(&r->refs, &val, new)) break; - - val = old; } return !new; @@ -208,7 +200,9 @@ void refcount_dec(refcount_t *r) static inline __refcount_check bool refcount_dec_if_one(refcount_t *r) { - return atomic_cmpxchg_release(&r->refs, 1, 0) == 1; + int val = 1; + + return atomic_try_cmpxchg_release(&r->refs, &val, 0); } /* @@ -220,7 +214,7 @@ bool refcount_dec_if_one(refcount_t *r) static inline __refcount_check bool refcount_dec_not_one(refcount_t *r) { - unsigned int old, new, val = atomic_read(&r->refs); + unsigned int new, val = atomic_read(&r->refs); for (;;) { if (unlikely(val == UINT_MAX)) @@ -235,11 +229,8 @@ bool refcount_dec_not_one(refcount_t *r) return true; } - old = atomic_cmpxchg_release(&r->refs, val, new); - if (old == val) + if (atomic_try_cmpxchg_release(&r->refs, &val, new)) break; - - val = old; } return true;