Merge branch 'x86-atomic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'x86-atomic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: x86: Fix LOCK_PREFIX_HERE for uniprocessor build x86, atomic64: In selftest, distinguish x86-64 from 586+ x86-32: Fix atomic64_inc_not_zero return value convention lib: Fix atomic64_inc_not_zero test lib: Fix atomic64_add_unless return value convention x86-32: Fix atomic64_add_unless return value convention lib: Fix atomic64_add_unless test x86: Implement atomic[64]_dec_if_positive() lib: Only test atomic64_dec_if_positive on archs having it x86-32: Rewrite 32-bit atomic64 functions in assembly lib: Add self-test for atomic64_t x86-32: Allow UP/SMP lock replacement in cmpxchg64 x86: Add support for lock prefix in alternatives
This commit is contained in:
Linus Torvalds
@@ -26,11 +26,12 @@ obj-y += msr.o msr-reg.o msr-reg-export.o
|
||||
|
||||
ifeq ($(CONFIG_X86_32),y)
|
||||
obj-y += atomic64_32.o
|
||||
lib-y += atomic64_cx8_32.o
|
||||
lib-y += checksum_32.o
|
||||
lib-y += strstr_32.o
|
||||
lib-y += semaphore_32.o string_32.o
|
||||
ifneq ($(CONFIG_X86_CMPXCHG64),y)
|
||||
lib-y += cmpxchg8b_emu.o
|
||||
lib-y += cmpxchg8b_emu.o atomic64_386_32.o
|
||||
endif
|
||||
lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o
|
||||
else
|
||||
|
||||
@@ -6,225 +6,54 @@
|
||||
#include <asm/cmpxchg.h>
|
||||
#include <asm/atomic.h>
|
||||
|
||||
static noinline u64 cmpxchg8b(u64 *ptr, u64 old, u64 new)
|
||||
{
|
||||
u32 low = new;
|
||||
u32 high = new >> 32;
|
||||
long long atomic64_read_cx8(long long, const atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_read_cx8);
|
||||
long long atomic64_set_cx8(long long, const atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_set_cx8);
|
||||
long long atomic64_xchg_cx8(long long, unsigned high);
|
||||
EXPORT_SYMBOL(atomic64_xchg_cx8);
|
||||
long long atomic64_add_return_cx8(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_add_return_cx8);
|
||||
long long atomic64_sub_return_cx8(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_sub_return_cx8);
|
||||
long long atomic64_inc_return_cx8(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_inc_return_cx8);
|
||||
long long atomic64_dec_return_cx8(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_dec_return_cx8);
|
||||
long long atomic64_dec_if_positive_cx8(atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_dec_if_positive_cx8);
|
||||
int atomic64_inc_not_zero_cx8(atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_inc_not_zero_cx8);
|
||||
int atomic64_add_unless_cx8(atomic64_t *v, long long a, long long u);
|
||||
EXPORT_SYMBOL(atomic64_add_unless_cx8);
|
||||
|
||||
asm volatile(
|
||||
LOCK_PREFIX "cmpxchg8b %1\n"
|
||||
: "+A" (old), "+m" (*ptr)
|
||||
: "b" (low), "c" (high)
|
||||
);
|
||||
return old;
|
||||
}
|
||||
|
||||
u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val)
|
||||
{
|
||||
return cmpxchg8b(&ptr->counter, old_val, new_val);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_cmpxchg);
|
||||
|
||||
/**
|
||||
* atomic64_xchg - xchg atomic64 variable
|
||||
* @ptr: pointer to type atomic64_t
|
||||
* @new_val: value to assign
|
||||
*
|
||||
* Atomically xchgs the value of @ptr to @new_val and returns
|
||||
* the old value.
|
||||
*/
|
||||
u64 atomic64_xchg(atomic64_t *ptr, u64 new_val)
|
||||
{
|
||||
/*
|
||||
* Try first with a (possibly incorrect) assumption about
|
||||
* what we have there. We'll do two loops most likely,
|
||||
* but we'll get an ownership MESI transaction straight away
|
||||
* instead of a read transaction followed by a
|
||||
* flush-for-ownership transaction:
|
||||
*/
|
||||
u64 old_val, real_val = 0;
|
||||
|
||||
do {
|
||||
old_val = real_val;
|
||||
|
||||
real_val = atomic64_cmpxchg(ptr, old_val, new_val);
|
||||
|
||||
} while (real_val != old_val);
|
||||
|
||||
return old_val;
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_xchg);
|
||||
|
||||
/**
|
||||
* atomic64_set - set atomic64 variable
|
||||
* @ptr: pointer to type atomic64_t
|
||||
* @new_val: value to assign
|
||||
*
|
||||
* Atomically sets the value of @ptr to @new_val.
|
||||
*/
|
||||
void atomic64_set(atomic64_t *ptr, u64 new_val)
|
||||
{
|
||||
atomic64_xchg(ptr, new_val);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_set);
|
||||
|
||||
/**
|
||||
EXPORT_SYMBOL(atomic64_read);
|
||||
* atomic64_add_return - add and return
|
||||
* @delta: integer value to add
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically adds @delta to @ptr and returns @delta + *@ptr
|
||||
*/
|
||||
noinline u64 atomic64_add_return(u64 delta, atomic64_t *ptr)
|
||||
{
|
||||
/*
|
||||
* Try first with a (possibly incorrect) assumption about
|
||||
* what we have there. We'll do two loops most likely,
|
||||
* but we'll get an ownership MESI transaction straight away
|
||||
* instead of a read transaction followed by a
|
||||
* flush-for-ownership transaction:
|
||||
*/
|
||||
u64 old_val, new_val, real_val = 0;
|
||||
|
||||
do {
|
||||
old_val = real_val;
|
||||
new_val = old_val + delta;
|
||||
|
||||
real_val = atomic64_cmpxchg(ptr, old_val, new_val);
|
||||
|
||||
} while (real_val != old_val);
|
||||
|
||||
return new_val;
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_add_return);
|
||||
|
||||
u64 atomic64_sub_return(u64 delta, atomic64_t *ptr)
|
||||
{
|
||||
return atomic64_add_return(-delta, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_sub_return);
|
||||
|
||||
u64 atomic64_inc_return(atomic64_t *ptr)
|
||||
{
|
||||
return atomic64_add_return(1, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_inc_return);
|
||||
|
||||
u64 atomic64_dec_return(atomic64_t *ptr)
|
||||
{
|
||||
return atomic64_sub_return(1, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_dec_return);
|
||||
|
||||
/**
|
||||
* atomic64_add - add integer to atomic64 variable
|
||||
* @delta: integer value to add
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically adds @delta to @ptr.
|
||||
*/
|
||||
void atomic64_add(u64 delta, atomic64_t *ptr)
|
||||
{
|
||||
atomic64_add_return(delta, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_add);
|
||||
|
||||
/**
|
||||
* atomic64_sub - subtract the atomic64 variable
|
||||
* @delta: integer value to subtract
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically subtracts @delta from @ptr.
|
||||
*/
|
||||
void atomic64_sub(u64 delta, atomic64_t *ptr)
|
||||
{
|
||||
atomic64_add(-delta, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_sub);
|
||||
|
||||
/**
|
||||
* atomic64_sub_and_test - subtract value from variable and test result
|
||||
* @delta: integer value to subtract
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically subtracts @delta from @ptr and returns
|
||||
* true if the result is zero, or false for all
|
||||
* other cases.
|
||||
*/
|
||||
int atomic64_sub_and_test(u64 delta, atomic64_t *ptr)
|
||||
{
|
||||
u64 new_val = atomic64_sub_return(delta, ptr);
|
||||
|
||||
return new_val == 0;
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_sub_and_test);
|
||||
|
||||
/**
|
||||
* atomic64_inc - increment atomic64 variable
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically increments @ptr by 1.
|
||||
*/
|
||||
void atomic64_inc(atomic64_t *ptr)
|
||||
{
|
||||
atomic64_add(1, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_inc);
|
||||
|
||||
/**
|
||||
* atomic64_dec - decrement atomic64 variable
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically decrements @ptr by 1.
|
||||
*/
|
||||
void atomic64_dec(atomic64_t *ptr)
|
||||
{
|
||||
atomic64_sub(1, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_dec);
|
||||
|
||||
/**
|
||||
* atomic64_dec_and_test - decrement and test
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically decrements @ptr by 1 and
|
||||
* returns true if the result is 0, or false for all other
|
||||
* cases.
|
||||
*/
|
||||
int atomic64_dec_and_test(atomic64_t *ptr)
|
||||
{
|
||||
return atomic64_sub_and_test(1, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_dec_and_test);
|
||||
|
||||
/**
|
||||
* atomic64_inc_and_test - increment and test
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically increments @ptr by 1
|
||||
* and returns true if the result is zero, or false for all
|
||||
* other cases.
|
||||
*/
|
||||
int atomic64_inc_and_test(atomic64_t *ptr)
|
||||
{
|
||||
return atomic64_sub_and_test(-1, ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_inc_and_test);
|
||||
|
||||
/**
|
||||
* atomic64_add_negative - add and test if negative
|
||||
* @delta: integer value to add
|
||||
* @ptr: pointer to type atomic64_t
|
||||
*
|
||||
* Atomically adds @delta to @ptr and returns true
|
||||
* if the result is negative, or false when
|
||||
* result is greater than or equal to zero.
|
||||
*/
|
||||
int atomic64_add_negative(u64 delta, atomic64_t *ptr)
|
||||
{
|
||||
s64 new_val = atomic64_add_return(delta, ptr);
|
||||
|
||||
return new_val < 0;
|
||||
}
|
||||
EXPORT_SYMBOL(atomic64_add_negative);
|
||||
#ifndef CONFIG_X86_CMPXCHG64
|
||||
long long atomic64_read_386(long long, const atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_read_386);
|
||||
long long atomic64_set_386(long long, const atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_set_386);
|
||||
long long atomic64_xchg_386(long long, unsigned high);
|
||||
EXPORT_SYMBOL(atomic64_xchg_386);
|
||||
long long atomic64_add_return_386(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_add_return_386);
|
||||
long long atomic64_sub_return_386(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_sub_return_386);
|
||||
long long atomic64_inc_return_386(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_inc_return_386);
|
||||
long long atomic64_dec_return_386(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_dec_return_386);
|
||||
long long atomic64_add_386(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_add_386);
|
||||
long long atomic64_sub_386(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_sub_386);
|
||||
long long atomic64_inc_386(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_inc_386);
|
||||
long long atomic64_dec_386(long long a, atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_dec_386);
|
||||
long long atomic64_dec_if_positive_386(atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_dec_if_positive_386);
|
||||
int atomic64_inc_not_zero_386(atomic64_t *v);
|
||||
EXPORT_SYMBOL(atomic64_inc_not_zero_386);
|
||||
int atomic64_add_unless_386(atomic64_t *v, long long a, long long u);
|
||||
EXPORT_SYMBOL(atomic64_add_unless_386);
|
||||
#endif
|
||||
|
||||
174
arch/x86/lib/atomic64_386_32.S
Normal file
174
arch/x86/lib/atomic64_386_32.S
Normal file
@@ -0,0 +1,174 @@
|
||||
/*
|
||||
* atomic64_t for 386/486
|
||||
*
|
||||
* Copyright © 2010 Luca Barbieri
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/alternative-asm.h>
|
||||
#include <asm/dwarf2.h>
|
||||
|
||||
/* if you want SMP support, implement these with real spinlocks */
|
||||
.macro LOCK reg
|
||||
pushfl
|
||||
CFI_ADJUST_CFA_OFFSET 4
|
||||
cli
|
||||
.endm
|
||||
|
||||
.macro UNLOCK reg
|
||||
popfl
|
||||
CFI_ADJUST_CFA_OFFSET -4
|
||||
.endm
|
||||
|
||||
.macro BEGIN func reg
|
||||
$v = \reg
|
||||
|
||||
ENTRY(atomic64_\func\()_386)
|
||||
CFI_STARTPROC
|
||||
LOCK $v
|
||||
|
||||
.macro RETURN
|
||||
UNLOCK $v
|
||||
ret
|
||||
.endm
|
||||
|
||||
.macro END_
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_\func\()_386)
|
||||
.purgem RETURN
|
||||
.purgem END_
|
||||
.purgem END
|
||||
.endm
|
||||
|
||||
.macro END
|
||||
RETURN
|
||||
END_
|
||||
.endm
|
||||
.endm
|
||||
|
||||
BEGIN read %ecx
|
||||
movl ($v), %eax
|
||||
movl 4($v), %edx
|
||||
END
|
||||
|
||||
BEGIN set %esi
|
||||
movl %ebx, ($v)
|
||||
movl %ecx, 4($v)
|
||||
END
|
||||
|
||||
BEGIN xchg %esi
|
||||
movl ($v), %eax
|
||||
movl 4($v), %edx
|
||||
movl %ebx, ($v)
|
||||
movl %ecx, 4($v)
|
||||
END
|
||||
|
||||
BEGIN add %ecx
|
||||
addl %eax, ($v)
|
||||
adcl %edx, 4($v)
|
||||
END
|
||||
|
||||
BEGIN add_return %ecx
|
||||
addl ($v), %eax
|
||||
adcl 4($v), %edx
|
||||
movl %eax, ($v)
|
||||
movl %edx, 4($v)
|
||||
END
|
||||
|
||||
BEGIN sub %ecx
|
||||
subl %eax, ($v)
|
||||
sbbl %edx, 4($v)
|
||||
END
|
||||
|
||||
BEGIN sub_return %ecx
|
||||
negl %edx
|
||||
negl %eax
|
||||
sbbl $0, %edx
|
||||
addl ($v), %eax
|
||||
adcl 4($v), %edx
|
||||
movl %eax, ($v)
|
||||
movl %edx, 4($v)
|
||||
END
|
||||
|
||||
BEGIN inc %esi
|
||||
addl $1, ($v)
|
||||
adcl $0, 4($v)
|
||||
END
|
||||
|
||||
BEGIN inc_return %esi
|
||||
movl ($v), %eax
|
||||
movl 4($v), %edx
|
||||
addl $1, %eax
|
||||
adcl $0, %edx
|
||||
movl %eax, ($v)
|
||||
movl %edx, 4($v)
|
||||
END
|
||||
|
||||
BEGIN dec %esi
|
||||
subl $1, ($v)
|
||||
sbbl $0, 4($v)
|
||||
END
|
||||
|
||||
BEGIN dec_return %esi
|
||||
movl ($v), %eax
|
||||
movl 4($v), %edx
|
||||
subl $1, %eax
|
||||
sbbl $0, %edx
|
||||
movl %eax, ($v)
|
||||
movl %edx, 4($v)
|
||||
END
|
||||
|
||||
BEGIN add_unless %ecx
|
||||
addl %eax, %esi
|
||||
adcl %edx, %edi
|
||||
addl ($v), %eax
|
||||
adcl 4($v), %edx
|
||||
cmpl %eax, %esi
|
||||
je 3f
|
||||
1:
|
||||
movl %eax, ($v)
|
||||
movl %edx, 4($v)
|
||||
movl $1, %eax
|
||||
2:
|
||||
RETURN
|
||||
3:
|
||||
cmpl %edx, %edi
|
||||
jne 1b
|
||||
xorl %eax, %eax
|
||||
jmp 2b
|
||||
END_
|
||||
|
||||
BEGIN inc_not_zero %esi
|
||||
movl ($v), %eax
|
||||
movl 4($v), %edx
|
||||
testl %eax, %eax
|
||||
je 3f
|
||||
1:
|
||||
addl $1, %eax
|
||||
adcl $0, %edx
|
||||
movl %eax, ($v)
|
||||
movl %edx, 4($v)
|
||||
movl $1, %eax
|
||||
2:
|
||||
RETURN
|
||||
3:
|
||||
testl %edx, %edx
|
||||
jne 1b
|
||||
jmp 2b
|
||||
END_
|
||||
|
||||
BEGIN dec_if_positive %esi
|
||||
movl ($v), %eax
|
||||
movl 4($v), %edx
|
||||
subl $1, %eax
|
||||
sbbl $0, %edx
|
||||
js 1f
|
||||
movl %eax, ($v)
|
||||
movl %edx, 4($v)
|
||||
1:
|
||||
END
|
||||
224
arch/x86/lib/atomic64_cx8_32.S
Normal file
224
arch/x86/lib/atomic64_cx8_32.S
Normal file
@@ -0,0 +1,224 @@
|
||||
/*
|
||||
* atomic64_t for 586+
|
||||
*
|
||||
* Copyright © 2010 Luca Barbieri
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/alternative-asm.h>
|
||||
#include <asm/dwarf2.h>
|
||||
|
||||
.macro SAVE reg
|
||||
pushl %\reg
|
||||
CFI_ADJUST_CFA_OFFSET 4
|
||||
CFI_REL_OFFSET \reg, 0
|
||||
.endm
|
||||
|
||||
.macro RESTORE reg
|
||||
popl %\reg
|
||||
CFI_ADJUST_CFA_OFFSET -4
|
||||
CFI_RESTORE \reg
|
||||
.endm
|
||||
|
||||
.macro read64 reg
|
||||
movl %ebx, %eax
|
||||
movl %ecx, %edx
|
||||
/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */
|
||||
LOCK_PREFIX
|
||||
cmpxchg8b (\reg)
|
||||
.endm
|
||||
|
||||
ENTRY(atomic64_read_cx8)
|
||||
CFI_STARTPROC
|
||||
|
||||
read64 %ecx
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_read_cx8)
|
||||
|
||||
ENTRY(atomic64_set_cx8)
|
||||
CFI_STARTPROC
|
||||
|
||||
1:
|
||||
/* we don't need LOCK_PREFIX since aligned 64-bit writes
|
||||
* are atomic on 586 and newer */
|
||||
cmpxchg8b (%esi)
|
||||
jne 1b
|
||||
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_set_cx8)
|
||||
|
||||
ENTRY(atomic64_xchg_cx8)
|
||||
CFI_STARTPROC
|
||||
|
||||
movl %ebx, %eax
|
||||
movl %ecx, %edx
|
||||
1:
|
||||
LOCK_PREFIX
|
||||
cmpxchg8b (%esi)
|
||||
jne 1b
|
||||
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_xchg_cx8)
|
||||
|
||||
.macro addsub_return func ins insc
|
||||
ENTRY(atomic64_\func\()_return_cx8)
|
||||
CFI_STARTPROC
|
||||
SAVE ebp
|
||||
SAVE ebx
|
||||
SAVE esi
|
||||
SAVE edi
|
||||
|
||||
movl %eax, %esi
|
||||
movl %edx, %edi
|
||||
movl %ecx, %ebp
|
||||
|
||||
read64 %ebp
|
||||
1:
|
||||
movl %eax, %ebx
|
||||
movl %edx, %ecx
|
||||
\ins\()l %esi, %ebx
|
||||
\insc\()l %edi, %ecx
|
||||
LOCK_PREFIX
|
||||
cmpxchg8b (%ebp)
|
||||
jne 1b
|
||||
|
||||
10:
|
||||
movl %ebx, %eax
|
||||
movl %ecx, %edx
|
||||
RESTORE edi
|
||||
RESTORE esi
|
||||
RESTORE ebx
|
||||
RESTORE ebp
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_\func\()_return_cx8)
|
||||
.endm
|
||||
|
||||
addsub_return add add adc
|
||||
addsub_return sub sub sbb
|
||||
|
||||
.macro incdec_return func ins insc
|
||||
ENTRY(atomic64_\func\()_return_cx8)
|
||||
CFI_STARTPROC
|
||||
SAVE ebx
|
||||
|
||||
read64 %esi
|
||||
1:
|
||||
movl %eax, %ebx
|
||||
movl %edx, %ecx
|
||||
\ins\()l $1, %ebx
|
||||
\insc\()l $0, %ecx
|
||||
LOCK_PREFIX
|
||||
cmpxchg8b (%esi)
|
||||
jne 1b
|
||||
|
||||
10:
|
||||
movl %ebx, %eax
|
||||
movl %ecx, %edx
|
||||
RESTORE ebx
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_\func\()_return_cx8)
|
||||
.endm
|
||||
|
||||
incdec_return inc add adc
|
||||
incdec_return dec sub sbb
|
||||
|
||||
ENTRY(atomic64_dec_if_positive_cx8)
|
||||
CFI_STARTPROC
|
||||
SAVE ebx
|
||||
|
||||
read64 %esi
|
||||
1:
|
||||
movl %eax, %ebx
|
||||
movl %edx, %ecx
|
||||
subl $1, %ebx
|
||||
sbb $0, %ecx
|
||||
js 2f
|
||||
LOCK_PREFIX
|
||||
cmpxchg8b (%esi)
|
||||
jne 1b
|
||||
|
||||
2:
|
||||
movl %ebx, %eax
|
||||
movl %ecx, %edx
|
||||
RESTORE ebx
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_dec_if_positive_cx8)
|
||||
|
||||
ENTRY(atomic64_add_unless_cx8)
|
||||
CFI_STARTPROC
|
||||
SAVE ebp
|
||||
SAVE ebx
|
||||
/* these just push these two parameters on the stack */
|
||||
SAVE edi
|
||||
SAVE esi
|
||||
|
||||
movl %ecx, %ebp
|
||||
movl %eax, %esi
|
||||
movl %edx, %edi
|
||||
|
||||
read64 %ebp
|
||||
1:
|
||||
cmpl %eax, 0(%esp)
|
||||
je 4f
|
||||
2:
|
||||
movl %eax, %ebx
|
||||
movl %edx, %ecx
|
||||
addl %esi, %ebx
|
||||
adcl %edi, %ecx
|
||||
LOCK_PREFIX
|
||||
cmpxchg8b (%ebp)
|
||||
jne 1b
|
||||
|
||||
movl $1, %eax
|
||||
3:
|
||||
addl $8, %esp
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
RESTORE ebx
|
||||
RESTORE ebp
|
||||
ret
|
||||
4:
|
||||
cmpl %edx, 4(%esp)
|
||||
jne 2b
|
||||
xorl %eax, %eax
|
||||
jmp 3b
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_add_unless_cx8)
|
||||
|
||||
ENTRY(atomic64_inc_not_zero_cx8)
|
||||
CFI_STARTPROC
|
||||
SAVE ebx
|
||||
|
||||
read64 %esi
|
||||
1:
|
||||
testl %eax, %eax
|
||||
je 4f
|
||||
2:
|
||||
movl %eax, %ebx
|
||||
movl %edx, %ecx
|
||||
addl $1, %ebx
|
||||
adcl $0, %ecx
|
||||
LOCK_PREFIX
|
||||
cmpxchg8b (%esi)
|
||||
jne 1b
|
||||
|
||||
movl $1, %eax
|
||||
3:
|
||||
RESTORE ebx
|
||||
ret
|
||||
4:
|
||||
testl %edx, %edx
|
||||
jne 2b
|
||||
jmp 3b
|
||||
CFI_ENDPROC
|
||||
ENDPROC(atomic64_inc_not_zero_cx8)
|
||||
Reference in New Issue
Block a user