[PATCH] atomic: add_unless cmpxchg optimise Without branch hints, the very unlikely chance of the loop repeating due to cmpxchg failure is unrolled with gcc-4 that I have tested. Improve this for architectures with a native cas/cmpxchg. llsc archs should try to implement this natively. Signed-off-by: Nick Piggin <npiggin@suse.de> Cc: Andi Kleen <ak@muc.de> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: "David S. Miller" <davem@davemloft.net> Cc: Roman Zippel <zippel@linux-m68k.org> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
diff --git a/include/asm-i386/atomic.h b/include/asm-i386/atomic.h index 78b0032..22d80ec 100644 --- a/include/asm-i386/atomic.h +++ b/include/asm-i386/atomic.h
@@ -225,8 +225,14 @@ ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
diff --git a/include/asm-ia64/atomic.h b/include/asm-ia64/atomic.h index d3e0dfa..569ec75 100644 --- a/include/asm-ia64/atomic.h +++ b/include/asm-ia64/atomic.h
@@ -95,8 +95,14 @@ ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
diff --git a/include/asm-m68k/atomic.h b/include/asm-m68k/atomic.h index 862e497..732d696 100644 --- a/include/asm-m68k/atomic.h +++ b/include/asm-m68k/atomic.h
@@ -175,8 +175,14 @@ ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
diff --git a/include/asm-s390/atomic.h b/include/asm-s390/atomic.h index be6fefe2..de1d992 100644 --- a/include/asm-s390/atomic.h +++ b/include/asm-s390/atomic.h
@@ -89,10 +89,15 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) { int c, old; - c = atomic_read(v); - while (c != u && (old = atomic_cmpxchg(v, c, c + a)) != c) + for (;;) { + if (unlikely(c == u)) + break; + old = atomic_cmpxchg(v, c, c + a); + if (likely(old == c)) + break; c = old; + } return c != u; } @@ -167,10 +172,15 @@ long long a, long long u) { long long c, old; - c = atomic64_read(v); - while (c != u && (old = atomic64_cmpxchg(v, c, c + a)) != c) + for (;;) { + if (unlikely(c == u)) + break; + old = atomic64_cmpxchg(v, c, c + a); + if (likely(old == c)) + break; c = old; + } return c != u; }
diff --git a/include/asm-sparc64/atomic.h b/include/asm-sparc64/atomic.h index 25256bd..468eb48 100644 --- a/include/asm-sparc64/atomic.h +++ b/include/asm-sparc64/atomic.h
@@ -78,9 +78,15 @@ ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ - c != (u); \ + } \ + likely(c != (u)); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
diff --git a/include/asm-x86_64/atomic.h b/include/asm-x86_64/atomic.h index 4b5cd55..cecbf7b 100644 --- a/include/asm-x86_64/atomic.h +++ b/include/asm-x86_64/atomic.h
@@ -405,8 +405,14 @@ ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)