mirror of
https://github.com/adulau/aha.git
synced 2024-12-27 11:16:11 +00:00
[LIB]: Consolidate _atomic_dec_and_lock()
Several implementations were essentialy a common piece of C code using the cmpxchg() macro. Put the implementation in one spot that everyone can share, and convert sparc64 over to using this. Alpha is the lone arch-specific implementation, which codes up a special fast path for the common case in order to avoid GP reloading which a pure C version would require. Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller
parent
4a805e863d
commit
4db2ce0199
26 changed files with 38 additions and 396 deletions
|
|
@ -908,11 +908,6 @@ config IRQBALANCE
|
|||
The default yes will allow the kernel to do irq load balancing.
|
||||
Saying no will keep the kernel from doing irq load balancing.
|
||||
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
depends on (SMP || PREEMPT) && X86_CMPXCHG
|
||||
default y
|
||||
|
||||
# turning this on wastes a bunch of space.
|
||||
# Summit needs it only when NUMA is on
|
||||
config BOOT_IOREMAP
|
||||
|
|
|
|||
|
|
@ -7,4 +7,3 @@ lib-y = checksum.o delay.o usercopy.o getuser.o putuser.o memcpy.o strstr.o \
|
|||
bitops.o
|
||||
|
||||
lib-$(CONFIG_X86_USE_3DNOW) += mmx.o
|
||||
lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
/*
|
||||
* x86 version of "atomic_dec_and_lock()" using
|
||||
* the atomic "cmpxchg" instruction.
|
||||
*
|
||||
* (For CPU's lacking cmpxchg, we use the slow
|
||||
* generic version, and this one never even gets
|
||||
* compiled).
|
||||
*/
|
||||
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/module.h>
|
||||
#include <asm/atomic.h>
|
||||
|
||||
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
||||
{
|
||||
int counter;
|
||||
int newcount;
|
||||
|
||||
repeat:
|
||||
counter = atomic_read(atomic);
|
||||
newcount = counter-1;
|
||||
|
||||
if (!newcount)
|
||||
goto slow_path;
|
||||
|
||||
asm volatile("lock; cmpxchgl %1,%2"
|
||||
:"=a" (newcount)
|
||||
:"r" (newcount), "m" (atomic->counter), "0" (counter));
|
||||
|
||||
/* If the above failed, "eax" will have changed */
|
||||
if (newcount != counter)
|
||||
goto repeat;
|
||||
return 0;
|
||||
|
||||
slow_path:
|
||||
spin_lock(lock);
|
||||
if (atomic_dec_and_test(atomic))
|
||||
return 1;
|
||||
spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
|
|
@ -298,11 +298,6 @@ config PREEMPT
|
|||
|
||||
source "mm/Kconfig"
|
||||
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
depends on (SMP || PREEMPT)
|
||||
default y
|
||||
|
||||
config IA32_SUPPORT
|
||||
bool "Support for Linux/x86 binaries"
|
||||
help
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ lib-$(CONFIG_ITANIUM) += copy_page.o copy_user.o memcpy.o
|
|||
lib-$(CONFIG_MCKINLEY) += copy_page_mck.o memcpy_mck.o
|
||||
lib-$(CONFIG_PERFMON) += carta_random.o
|
||||
lib-$(CONFIG_MD_RAID5) += xor.o
|
||||
lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
|
||||
|
||||
AFLAGS___divdi3.o =
|
||||
AFLAGS___udivdi3.o = -DUNSIGNED
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2003 Jerome Marchand, Bull S.A.
|
||||
* Cleaned up by David Mosberger-Tang <davidm@hpl.hp.com>
|
||||
*
|
||||
* This file is released under the GPLv2, or at your option any later version.
|
||||
*
|
||||
* ia64 version of "atomic_dec_and_lock()" using the atomic "cmpxchg" instruction. This
|
||||
* code is an adaptation of the x86 version of "atomic_dec_and_lock()".
|
||||
*/
|
||||
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <asm/atomic.h>
|
||||
|
||||
/*
|
||||
* Decrement REFCOUNT and if the count reaches zero, acquire the spinlock. Both of these
|
||||
* operations have to be done atomically, so that the count doesn't drop to zero without
|
||||
* acquiring the spinlock first.
|
||||
*/
|
||||
int
|
||||
_atomic_dec_and_lock (atomic_t *refcount, spinlock_t *lock)
|
||||
{
|
||||
int old, new;
|
||||
|
||||
do {
|
||||
old = atomic_read(refcount);
|
||||
new = old - 1;
|
||||
|
||||
if (unlikely (old == 1)) {
|
||||
/* oops, we may be decrementing to zero, do it the slow way... */
|
||||
spin_lock(lock);
|
||||
if (atomic_dec_and_test(refcount))
|
||||
return 1;
|
||||
spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
} while (cmpxchg(&refcount->counter, old, new) != old);
|
||||
return 0;
|
||||
}
|
||||
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
|
|
@ -220,11 +220,6 @@ config PREEMPT
|
|||
Say Y here if you are building a kernel for a desktop, embedded
|
||||
or real-time system. Say N if you are unsure.
|
||||
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
depends on (SMP || PREEMPT)
|
||||
default n
|
||||
|
||||
config SMP
|
||||
bool "Symmetric multi-processing support"
|
||||
---help---
|
||||
|
|
|
|||
|
|
@ -1009,10 +1009,6 @@ config GENERIC_CALIBRATE_DELAY
|
|||
bool
|
||||
default y
|
||||
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
default y
|
||||
|
||||
#
|
||||
# Select some configuration options automatically based on user selections.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# Makefile for MIPS-specific library files..
|
||||
#
|
||||
|
||||
lib-y += csum_partial_copy.o dec_and_lock.o memcpy.o promlib.o \
|
||||
lib-y += csum_partial_copy.o memcpy.o promlib.o \
|
||||
strlen_user.o strncpy_user.o strnlen_user.o
|
||||
|
||||
obj-y += iomap.o
|
||||
|
|
|
|||
|
|
@ -1,47 +0,0 @@
|
|||
/*
|
||||
* MIPS version of atomic_dec_and_lock() using cmpxchg
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version
|
||||
* 2 of the License, or (at your option) any later version.
|
||||
*/
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <asm/atomic.h>
|
||||
#include <asm/system.h>
|
||||
|
||||
/*
|
||||
* This is an implementation of the notion of "decrement a
|
||||
* reference count, and return locked if it decremented to zero".
|
||||
*
|
||||
* This implementation can be used on any architecture that
|
||||
* has a cmpxchg, and where atomic->value is an int holding
|
||||
* the value of the atomic (i.e. the high bits aren't used
|
||||
* for a lock or anything like that).
|
||||
*/
|
||||
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
||||
{
|
||||
int counter;
|
||||
int newcount;
|
||||
|
||||
for (;;) {
|
||||
counter = atomic_read(atomic);
|
||||
newcount = counter - 1;
|
||||
if (!newcount)
|
||||
break; /* do it the slow way */
|
||||
|
||||
newcount = cmpxchg(&atomic->counter, counter, newcount);
|
||||
if (newcount == counter)
|
||||
return 0;
|
||||
}
|
||||
|
||||
spin_lock(lock);
|
||||
if (atomic_dec_and_test(atomic))
|
||||
return 1;
|
||||
spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
|
|
@ -26,10 +26,6 @@ config GENERIC_CALIBRATE_DELAY
|
|||
bool
|
||||
default y
|
||||
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
default y
|
||||
|
||||
config PPC
|
||||
bool
|
||||
default y
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# Makefile for ppc-specific library files..
|
||||
#
|
||||
|
||||
obj-y := checksum.o string.o strcase.o dec_and_lock.o div64.o
|
||||
obj-y := checksum.o string.o strcase.o div64.o
|
||||
|
||||
obj-$(CONFIG_8xx) += rheap.o
|
||||
obj-$(CONFIG_CPM2) += rheap.o
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
#include <linux/module.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <asm/atomic.h>
|
||||
#include <asm/system.h>
|
||||
|
||||
/*
|
||||
* This is an implementation of the notion of "decrement a
|
||||
* reference count, and return locked if it decremented to zero".
|
||||
*
|
||||
* This implementation can be used on any architecture that
|
||||
* has a cmpxchg, and where atomic->value is an int holding
|
||||
* the value of the atomic (i.e. the high bits aren't used
|
||||
* for a lock or anything like that).
|
||||
*/
|
||||
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
||||
{
|
||||
int counter;
|
||||
int newcount;
|
||||
|
||||
for (;;) {
|
||||
counter = atomic_read(atomic);
|
||||
newcount = counter - 1;
|
||||
if (!newcount)
|
||||
break; /* do it the slow way */
|
||||
|
||||
newcount = cmpxchg(&atomic->counter, counter, newcount);
|
||||
if (newcount == counter)
|
||||
return 0;
|
||||
}
|
||||
|
||||
spin_lock(lock);
|
||||
if (atomic_dec_and_test(atomic))
|
||||
return 1;
|
||||
spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
|
|
@ -28,10 +28,6 @@ config GENERIC_ISA_DMA
|
|||
bool
|
||||
default y
|
||||
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
default y
|
||||
|
||||
config EARLY_PRINTK
|
||||
bool
|
||||
default y
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# Makefile for ppc64-specific library files..
|
||||
#
|
||||
|
||||
lib-y := checksum.o dec_and_lock.o string.o strcase.o
|
||||
lib-y := checksum.o string.o strcase.o
|
||||
lib-y += copypage.o memcpy.o copyuser.o usercopy.o
|
||||
|
||||
# Lock primitives are defined as no-ops in include/linux/spinlock.h
|
||||
|
|
|
|||
|
|
@ -1,47 +0,0 @@
|
|||
/*
|
||||
* ppc64 version of atomic_dec_and_lock() using cmpxchg
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version
|
||||
* 2 of the License, or (at your option) any later version.
|
||||
*/
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <asm/atomic.h>
|
||||
#include <asm/system.h>
|
||||
|
||||
/*
|
||||
* This is an implementation of the notion of "decrement a
|
||||
* reference count, and return locked if it decremented to zero".
|
||||
*
|
||||
* This implementation can be used on any architecture that
|
||||
* has a cmpxchg, and where atomic->value is an int holding
|
||||
* the value of the atomic (i.e. the high bits aren't used
|
||||
* for a lock or anything like that).
|
||||
*/
|
||||
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
||||
{
|
||||
int counter;
|
||||
int newcount;
|
||||
|
||||
for (;;) {
|
||||
counter = atomic_read(atomic);
|
||||
newcount = counter - 1;
|
||||
if (!newcount)
|
||||
break; /* do it the slow way */
|
||||
|
||||
newcount = cmpxchg(&atomic->counter, counter, newcount);
|
||||
if (newcount == counter)
|
||||
return 0;
|
||||
}
|
||||
|
||||
spin_lock(lock);
|
||||
if (atomic_dec_and_test(atomic))
|
||||
return 1;
|
||||
spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
|
|
@ -33,14 +33,6 @@ config DEBUG_BOOTMEM
|
|||
depends on DEBUG_KERNEL
|
||||
bool "Debug BOOTMEM initialization"
|
||||
|
||||
# We have a custom atomic_dec_and_lock() implementation but it's not
|
||||
# compatible with spinlock debugging so we need to fall back on
|
||||
# the generic version in that case.
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
depends on SMP && !DEBUG_SPINLOCK
|
||||
default y
|
||||
|
||||
config MCOUNT
|
||||
bool
|
||||
depends on STACK_DEBUG
|
||||
|
|
|
|||
|
|
@ -163,9 +163,6 @@ EXPORT_SYMBOL(atomic64_add);
|
|||
EXPORT_SYMBOL(atomic64_add_ret);
|
||||
EXPORT_SYMBOL(atomic64_sub);
|
||||
EXPORT_SYMBOL(atomic64_sub_ret);
|
||||
#ifdef CONFIG_SMP
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
#endif
|
||||
|
||||
/* Atomic bit operations. */
|
||||
EXPORT_SYMBOL(test_and_set_bit);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,4 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
|
|||
copy_in_user.o user_fixup.o memmove.o \
|
||||
mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o
|
||||
|
||||
lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
|
||||
|
||||
obj-y += iomap.o
|
||||
|
|
|
|||
|
|
@ -1,80 +0,0 @@
|
|||
/* $Id: dec_and_lock.S,v 1.5 2001/11/18 00:12:56 davem Exp $
|
||||
* dec_and_lock.S: Sparc64 version of "atomic_dec_and_lock()"
|
||||
* using cas and ldstub instructions.
|
||||
*
|
||||
* Copyright (C) 2000 David S. Miller (davem@redhat.com)
|
||||
*/
|
||||
#include <linux/config.h>
|
||||
#include <asm/thread_info.h>
|
||||
|
||||
.text
|
||||
.align 64
|
||||
|
||||
/* CAS basically works like this:
|
||||
*
|
||||
* void CAS(MEM, REG1, REG2)
|
||||
* {
|
||||
* START_ATOMIC();
|
||||
* if (*(MEM) == REG1) {
|
||||
* TMP = *(MEM);
|
||||
* *(MEM) = REG2;
|
||||
* REG2 = TMP;
|
||||
* } else
|
||||
* REG2 = *(MEM);
|
||||
* END_ATOMIC();
|
||||
* }
|
||||
*/
|
||||
|
||||
.globl _atomic_dec_and_lock
|
||||
_atomic_dec_and_lock: /* %o0 = counter, %o1 = lock */
|
||||
loop1: lduw [%o0], %g2
|
||||
subcc %g2, 1, %g7
|
||||
be,pn %icc, start_to_zero
|
||||
nop
|
||||
nzero: cas [%o0], %g2, %g7
|
||||
cmp %g2, %g7
|
||||
bne,pn %icc, loop1
|
||||
mov 0, %g1
|
||||
|
||||
out:
|
||||
membar #StoreLoad | #StoreStore
|
||||
retl
|
||||
mov %g1, %o0
|
||||
start_to_zero:
|
||||
#ifdef CONFIG_PREEMPT
|
||||
ldsw [%g6 + TI_PRE_COUNT], %g3
|
||||
add %g3, 1, %g3
|
||||
stw %g3, [%g6 + TI_PRE_COUNT]
|
||||
#endif
|
||||
to_zero:
|
||||
ldstub [%o1], %g3
|
||||
membar #StoreLoad | #StoreStore
|
||||
brnz,pn %g3, spin_on_lock
|
||||
nop
|
||||
loop2: cas [%o0], %g2, %g7 /* ASSERT(g7 == 0) */
|
||||
cmp %g2, %g7
|
||||
|
||||
be,pt %icc, out
|
||||
mov 1, %g1
|
||||
lduw [%o0], %g2
|
||||
subcc %g2, 1, %g7
|
||||
be,pn %icc, loop2
|
||||
nop
|
||||
membar #StoreStore | #LoadStore
|
||||
stb %g0, [%o1]
|
||||
#ifdef CONFIG_PREEMPT
|
||||
ldsw [%g6 + TI_PRE_COUNT], %g3
|
||||
sub %g3, 1, %g3
|
||||
stw %g3, [%g6 + TI_PRE_COUNT]
|
||||
#endif
|
||||
|
||||
b,pt %xcc, nzero
|
||||
nop
|
||||
spin_on_lock:
|
||||
ldub [%o1], %g3
|
||||
membar #LoadLoad
|
||||
brnz,pt %g3, spin_on_lock
|
||||
nop
|
||||
ba,pt %xcc, to_zero
|
||||
nop
|
||||
nop
|
||||
|
|
@ -277,11 +277,6 @@ source "mm/Kconfig"
|
|||
config HAVE_ARCH_EARLY_PFN_TO_NID
|
||||
def_bool y
|
||||
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
depends on SMP
|
||||
default y
|
||||
|
||||
config NR_CPUS
|
||||
int "Maximum number of CPUs (2-256)"
|
||||
range 2 256
|
||||
|
|
|
|||
|
|
@ -178,10 +178,6 @@ EXPORT_SYMBOL(rwsem_down_write_failed_thunk);
|
|||
|
||||
EXPORT_SYMBOL(empty_zero_page);
|
||||
|
||||
#ifdef CONFIG_HAVE_DEC_LOCK
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
#endif
|
||||
|
||||
EXPORT_SYMBOL(die_chain);
|
||||
EXPORT_SYMBOL(register_die_notifier);
|
||||
|
||||
|
|
|
|||
|
|
@ -10,5 +10,3 @@ lib-y := csum-partial.o csum-copy.o csum-wrappers.o delay.o \
|
|||
usercopy.o getuser.o putuser.o \
|
||||
thunk.o clear_page.o copy_page.o bitstr.o bitops.o
|
||||
lib-y += memcpy.o memmove.o memset.o copy_user.o
|
||||
|
||||
lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
/*
|
||||
* x86 version of "atomic_dec_and_lock()" using
|
||||
* the atomic "cmpxchg" instruction.
|
||||
*
|
||||
* (For CPU's lacking cmpxchg, we use the slow
|
||||
* generic version, and this one never even gets
|
||||
* compiled).
|
||||
*/
|
||||
|
||||
#include <linux/spinlock.h>
|
||||
#include <asm/atomic.h>
|
||||
|
||||
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
||||
{
|
||||
int counter;
|
||||
int newcount;
|
||||
|
||||
repeat:
|
||||
counter = atomic_read(atomic);
|
||||
newcount = counter-1;
|
||||
|
||||
if (!newcount)
|
||||
goto slow_path;
|
||||
|
||||
asm volatile("lock; cmpxchgl %1,%2"
|
||||
:"=a" (newcount)
|
||||
:"r" (newcount), "m" (atomic->counter), "0" (counter));
|
||||
|
||||
/* If the above failed, "eax" will have changed */
|
||||
if (newcount != counter)
|
||||
goto repeat;
|
||||
return 0;
|
||||
|
||||
slow_path:
|
||||
spin_lock(lock);
|
||||
if (atomic_dec_and_test(atomic))
|
||||
return 1;
|
||||
spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -26,10 +26,6 @@ config RWSEM_XCHGADD_ALGORITHM
|
|||
bool
|
||||
default y
|
||||
|
||||
config HAVE_DEC_LOCK
|
||||
bool
|
||||
default y
|
||||
|
||||
config GENERIC_HARDIRQS
|
||||
bool
|
||||
default y
|
||||
|
|
|
|||
|
|
@ -1,7 +1,41 @@
|
|||
#include <linux/module.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <asm/atomic.h>
|
||||
#include <asm/system.h>
|
||||
|
||||
#ifdef __HAVE_ARCH_CMPXCHG
|
||||
/*
|
||||
* This is an implementation of the notion of "decrement a
|
||||
* reference count, and return locked if it decremented to zero".
|
||||
*
|
||||
* This implementation can be used on any architecture that
|
||||
* has a cmpxchg, and where atomic->value is an int holding
|
||||
* the value of the atomic (i.e. the high bits aren't used
|
||||
* for a lock or anything like that).
|
||||
*/
|
||||
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
||||
{
|
||||
int counter;
|
||||
int newcount;
|
||||
|
||||
for (;;) {
|
||||
counter = atomic_read(atomic);
|
||||
newcount = counter - 1;
|
||||
if (!newcount)
|
||||
break; /* do it the slow way */
|
||||
|
||||
newcount = cmpxchg(&atomic->counter, counter, newcount);
|
||||
if (newcount == counter)
|
||||
return 0;
|
||||
}
|
||||
|
||||
spin_lock(lock);
|
||||
if (atomic_dec_and_test(atomic))
|
||||
return 1;
|
||||
spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
/*
|
||||
* This is an architecture-neutral, but slow,
|
||||
* implementation of the notion of "decrement
|
||||
|
|
@ -33,5 +67,6 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
|||
spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
|
|
|
|||
Loading…
Reference in a new issue