aboutsummaryrefslogtreecommitdiffstats
path: root/common/atomic/i386/atomic.s
diff options
context:
space:
mode:
Diffstat (limited to 'common/atomic/i386/atomic.s')
-rw-r--r--common/atomic/i386/atomic.s720
1 files changed, 720 insertions, 0 deletions
diff --git a/common/atomic/i386/atomic.s b/common/atomic/i386/atomic.s
new file mode 100644
index 000000000000..4fa525ba20af
--- /dev/null
+++ b/common/atomic/i386/atomic.s
@@ -0,0 +1,720 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+ .file "atomic.s"
+
+#include <sys/asm_linkage.h>
+
+#if defined(_KERNEL)
+ /*
+ * Legacy kernel interfaces; they will go away (eventually).
+ */
+ ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
+ ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
+ ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
+ ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
+ ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
+ ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
+ ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
+#endif
+
+ ENTRY(atomic_inc_8)
+ ALTENTRY(atomic_inc_uchar)
+ movl 4(%esp), %eax
+ lock
+ incb (%eax)
+ ret
+ SET_SIZE(atomic_inc_uchar)
+ SET_SIZE(atomic_inc_8)
+
+ ENTRY(atomic_inc_16)
+ ALTENTRY(atomic_inc_ushort)
+ movl 4(%esp), %eax
+ lock
+ incw (%eax)
+ ret
+ SET_SIZE(atomic_inc_ushort)
+ SET_SIZE(atomic_inc_16)
+
+ ENTRY(atomic_inc_32)
+ ALTENTRY(atomic_inc_uint)
+ ALTENTRY(atomic_inc_ulong)
+ movl 4(%esp), %eax
+ lock
+ incl (%eax)
+ ret
+ SET_SIZE(atomic_inc_ulong)
+ SET_SIZE(atomic_inc_uint)
+ SET_SIZE(atomic_inc_32)
+
+ ENTRY(atomic_inc_8_nv)
+ ALTENTRY(atomic_inc_uchar_nv)
+ movl 4(%esp), %edx / %edx = target address
+ xorl %eax, %eax / clear upper bits of %eax
+ incb %al / %al = 1
+ lock
+ xaddb %al, (%edx) / %al = old value, inc (%edx)
+ incb %al / return new value
+ ret
+ SET_SIZE(atomic_inc_uchar_nv)
+ SET_SIZE(atomic_inc_8_nv)
+
+ ENTRY(atomic_inc_16_nv)
+ ALTENTRY(atomic_inc_ushort_nv)
+ movl 4(%esp), %edx / %edx = target address
+ xorl %eax, %eax / clear upper bits of %eax
+ incw %ax / %ax = 1
+ lock
+ xaddw %ax, (%edx) / %ax = old value, inc (%edx)
+ incw %ax / return new value
+ ret
+ SET_SIZE(atomic_inc_ushort_nv)
+ SET_SIZE(atomic_inc_16_nv)
+
+ ENTRY(atomic_inc_32_nv)
+ ALTENTRY(atomic_inc_uint_nv)
+ ALTENTRY(atomic_inc_ulong_nv)
+ movl 4(%esp), %edx / %edx = target address
+ xorl %eax, %eax / %eax = 0
+ incl %eax / %eax = 1
+ lock
+ xaddl %eax, (%edx) / %eax = old value, inc (%edx)
+ incl %eax / return new value
+ ret
+ SET_SIZE(atomic_inc_ulong_nv)
+ SET_SIZE(atomic_inc_uint_nv)
+ SET_SIZE(atomic_inc_32_nv)
+
+ /*
+ * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
+ * separated, you need to also edit the libc i386 platform
+ * specific mapfile and remove the NODYNSORT attribute
+ * from atomic_inc_64_nv.
+ */
+ ENTRY(atomic_inc_64)
+ ALTENTRY(atomic_inc_64_nv)
+ pushl %edi
+ pushl %ebx
+ movl 12(%esp), %edi / %edi = target address
+ movl (%edi), %eax
+ movl 4(%edi), %edx / %edx:%eax = old value
+1:
+ xorl %ebx, %ebx
+ xorl %ecx, %ecx
+ incl %ebx / %ecx:%ebx = 1
+ addl %eax, %ebx
+ adcl %edx, %ecx / add in the carry from inc
+ lock
+ cmpxchg8b (%edi) / try to stick it in
+ jne 1b
+ movl %ebx, %eax
+ movl %ecx, %edx / return new value
+ popl %ebx
+ popl %edi
+ ret
+ SET_SIZE(atomic_inc_64_nv)
+ SET_SIZE(atomic_inc_64)
+
+ ENTRY(atomic_dec_8)
+ ALTENTRY(atomic_dec_uchar)
+ movl 4(%esp), %eax
+ lock
+ decb (%eax)
+ ret
+ SET_SIZE(atomic_dec_uchar)
+ SET_SIZE(atomic_dec_8)
+
+ ENTRY(atomic_dec_16)
+ ALTENTRY(atomic_dec_ushort)
+ movl 4(%esp), %eax
+ lock
+ decw (%eax)
+ ret
+ SET_SIZE(atomic_dec_ushort)
+ SET_SIZE(atomic_dec_16)
+
+ ENTRY(atomic_dec_32)
+ ALTENTRY(atomic_dec_uint)
+ ALTENTRY(atomic_dec_ulong)
+ movl 4(%esp), %eax
+ lock
+ decl (%eax)
+ ret
+ SET_SIZE(atomic_dec_ulong)
+ SET_SIZE(atomic_dec_uint)
+ SET_SIZE(atomic_dec_32)
+
+ ENTRY(atomic_dec_8_nv)
+ ALTENTRY(atomic_dec_uchar_nv)
+ movl 4(%esp), %edx / %edx = target address
+ xorl %eax, %eax / zero upper bits of %eax
+ decb %al / %al = -1
+ lock
+ xaddb %al, (%edx) / %al = old value, dec (%edx)
+ decb %al / return new value
+ ret
+ SET_SIZE(atomic_dec_uchar_nv)
+ SET_SIZE(atomic_dec_8_nv)
+
+ ENTRY(atomic_dec_16_nv)
+ ALTENTRY(atomic_dec_ushort_nv)
+ movl 4(%esp), %edx / %edx = target address
+ xorl %eax, %eax / zero upper bits of %eax
+ decw %ax / %ax = -1
+ lock
+ xaddw %ax, (%edx) / %ax = old value, dec (%edx)
+ decw %ax / return new value
+ ret
+ SET_SIZE(atomic_dec_ushort_nv)
+ SET_SIZE(atomic_dec_16_nv)
+
+ ENTRY(atomic_dec_32_nv)
+ ALTENTRY(atomic_dec_uint_nv)
+ ALTENTRY(atomic_dec_ulong_nv)
+ movl 4(%esp), %edx / %edx = target address
+ xorl %eax, %eax / %eax = 0
+ decl %eax / %eax = -1
+ lock
+ xaddl %eax, (%edx) / %eax = old value, dec (%edx)
+ decl %eax / return new value
+ ret
+ SET_SIZE(atomic_dec_ulong_nv)
+ SET_SIZE(atomic_dec_uint_nv)
+ SET_SIZE(atomic_dec_32_nv)
+
+ /*
+ * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
+ * separated, it is important to edit the libc i386 platform
+ * specific mapfile and remove the NODYNSORT attribute
+ * from atomic_dec_64_nv.
+ */
+ ENTRY(atomic_dec_64)
+ ALTENTRY(atomic_dec_64_nv)
+ pushl %edi
+ pushl %ebx
+ movl 12(%esp), %edi / %edi = target address
+ movl (%edi), %eax
+ movl 4(%edi), %edx / %edx:%eax = old value
+1:
+ xorl %ebx, %ebx
+ xorl %ecx, %ecx
+ not %ecx
+ not %ebx / %ecx:%ebx = -1
+ addl %eax, %ebx
+ adcl %edx, %ecx / add in the carry from inc
+ lock
+ cmpxchg8b (%edi) / try to stick it in
+ jne 1b
+ movl %ebx, %eax
+ movl %ecx, %edx / return new value
+ popl %ebx
+ popl %edi
+ ret
+ SET_SIZE(atomic_dec_64_nv)
+ SET_SIZE(atomic_dec_64)
+
+ ENTRY(atomic_add_8)
+ ALTENTRY(atomic_add_char)
+ movl 4(%esp), %eax
+ movl 8(%esp), %ecx
+ lock
+ addb %cl, (%eax)
+ ret
+ SET_SIZE(atomic_add_char)
+ SET_SIZE(atomic_add_8)
+
+ ENTRY(atomic_add_16)
+ ALTENTRY(atomic_add_short)
+ movl 4(%esp), %eax
+ movl 8(%esp), %ecx
+ lock
+ addw %cx, (%eax)
+ ret
+ SET_SIZE(atomic_add_short)
+ SET_SIZE(atomic_add_16)
+
+ ENTRY(atomic_add_32)
+ ALTENTRY(atomic_add_int)
+ ALTENTRY(atomic_add_ptr)
+ ALTENTRY(atomic_add_long)
+ movl 4(%esp), %eax
+ movl 8(%esp), %ecx
+ lock
+ addl %ecx, (%eax)
+ ret
+ SET_SIZE(atomic_add_long)
+ SET_SIZE(atomic_add_ptr)
+ SET_SIZE(atomic_add_int)
+ SET_SIZE(atomic_add_32)
+
+ ENTRY(atomic_or_8)
+ ALTENTRY(atomic_or_uchar)
+ movl 4(%esp), %eax
+ movb 8(%esp), %cl
+ lock
+ orb %cl, (%eax)
+ ret
+ SET_SIZE(atomic_or_uchar)
+ SET_SIZE(atomic_or_8)
+
+ ENTRY(atomic_or_16)
+ ALTENTRY(atomic_or_ushort)
+ movl 4(%esp), %eax
+ movw 8(%esp), %cx
+ lock
+ orw %cx, (%eax)
+ ret
+ SET_SIZE(atomic_or_ushort)
+ SET_SIZE(atomic_or_16)
+
+ ENTRY(atomic_or_32)
+ ALTENTRY(atomic_or_uint)
+ ALTENTRY(atomic_or_ulong)
+ movl 4(%esp), %eax
+ movl 8(%esp), %ecx
+ lock
+ orl %ecx, (%eax)
+ ret
+ SET_SIZE(atomic_or_ulong)
+ SET_SIZE(atomic_or_uint)
+ SET_SIZE(atomic_or_32)
+
+ ENTRY(atomic_and_8)
+ ALTENTRY(atomic_and_uchar)
+ movl 4(%esp), %eax
+ movb 8(%esp), %cl
+ lock
+ andb %cl, (%eax)
+ ret
+ SET_SIZE(atomic_and_uchar)
+ SET_SIZE(atomic_and_8)
+
+ ENTRY(atomic_and_16)
+ ALTENTRY(atomic_and_ushort)
+ movl 4(%esp), %eax
+ movw 8(%esp), %cx
+ lock
+ andw %cx, (%eax)
+ ret
+ SET_SIZE(atomic_and_ushort)
+ SET_SIZE(atomic_and_16)
+
+ ENTRY(atomic_and_32)
+ ALTENTRY(atomic_and_uint)
+ ALTENTRY(atomic_and_ulong)
+ movl 4(%esp), %eax
+ movl 8(%esp), %ecx
+ lock
+ andl %ecx, (%eax)
+ ret
+ SET_SIZE(atomic_and_ulong)
+ SET_SIZE(atomic_and_uint)
+ SET_SIZE(atomic_and_32)
+
+ ENTRY(atomic_add_8_nv)
+ ALTENTRY(atomic_add_char_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movb 8(%esp), %cl / %cl = delta
+ movzbl %cl, %eax / %al = delta, zero extended
+ lock
+ xaddb %cl, (%edx) / %cl = old value, (%edx) = sum
+ addb %cl, %al / return old value plus delta
+ ret
+ SET_SIZE(atomic_add_char_nv)
+ SET_SIZE(atomic_add_8_nv)
+
+ ENTRY(atomic_add_16_nv)
+ ALTENTRY(atomic_add_short_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movw 8(%esp), %cx / %cx = delta
+ movzwl %cx, %eax / %ax = delta, zero extended
+ lock
+ xaddw %cx, (%edx) / %cx = old value, (%edx) = sum
+ addw %cx, %ax / return old value plus delta
+ ret
+ SET_SIZE(atomic_add_short_nv)
+ SET_SIZE(atomic_add_16_nv)
+
+ ENTRY(atomic_add_32_nv)
+ ALTENTRY(atomic_add_int_nv)
+ ALTENTRY(atomic_add_ptr_nv)
+ ALTENTRY(atomic_add_long_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movl 8(%esp), %eax / %eax = delta
+ movl %eax, %ecx / %ecx = delta
+ lock
+ xaddl %eax, (%edx) / %eax = old value, (%edx) = sum
+ addl %ecx, %eax / return old value plus delta
+ ret
+ SET_SIZE(atomic_add_long_nv)
+ SET_SIZE(atomic_add_ptr_nv)
+ SET_SIZE(atomic_add_int_nv)
+ SET_SIZE(atomic_add_32_nv)
+
+ /*
+ * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
+ * separated, it is important to edit the libc i386 platform
+ * specific mapfile and remove the NODYNSORT attribute
+ * from atomic_add_64_nv.
+ */
+ ENTRY(atomic_add_64)
+ ALTENTRY(atomic_add_64_nv)
+ pushl %edi
+ pushl %ebx
+ movl 12(%esp), %edi / %edi = target address
+ movl (%edi), %eax
+ movl 4(%edi), %edx / %edx:%eax = old value
+1:
+ movl 16(%esp), %ebx
+ movl 20(%esp), %ecx / %ecx:%ebx = delta
+ addl %eax, %ebx
+ adcl %edx, %ecx / %ecx:%ebx = new value
+ lock
+ cmpxchg8b (%edi) / try to stick it in
+ jne 1b
+ movl %ebx, %eax
+ movl %ecx, %edx / return new value
+ popl %ebx
+ popl %edi
+ ret
+ SET_SIZE(atomic_add_64_nv)
+ SET_SIZE(atomic_add_64)
+
+ ENTRY(atomic_or_8_nv)
+ ALTENTRY(atomic_or_uchar_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movb (%edx), %al / %al = old value
+1:
+ movl 8(%esp), %ecx / %ecx = delta
+ orb %al, %cl / %cl = new value
+ lock
+ cmpxchgb %cl, (%edx) / try to stick it in
+ jne 1b
+ movzbl %cl, %eax / return new value
+ ret
+ SET_SIZE(atomic_or_uchar_nv)
+ SET_SIZE(atomic_or_8_nv)
+
+ ENTRY(atomic_or_16_nv)
+ ALTENTRY(atomic_or_ushort_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movw (%edx), %ax / %ax = old value
+1:
+ movl 8(%esp), %ecx / %ecx = delta
+ orw %ax, %cx / %cx = new value
+ lock
+ cmpxchgw %cx, (%edx) / try to stick it in
+ jne 1b
+ movzwl %cx, %eax / return new value
+ ret
+ SET_SIZE(atomic_or_ushort_nv)
+ SET_SIZE(atomic_or_16_nv)
+
+ ENTRY(atomic_or_32_nv)
+ ALTENTRY(atomic_or_uint_nv)
+ ALTENTRY(atomic_or_ulong_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movl (%edx), %eax / %eax = old value
+1:
+ movl 8(%esp), %ecx / %ecx = delta
+ orl %eax, %ecx / %ecx = new value
+ lock
+ cmpxchgl %ecx, (%edx) / try to stick it in
+ jne 1b
+ movl %ecx, %eax / return new value
+ ret
+ SET_SIZE(atomic_or_ulong_nv)
+ SET_SIZE(atomic_or_uint_nv)
+ SET_SIZE(atomic_or_32_nv)
+
+ /*
+ * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
+ * separated, it is important to edit the libc i386 platform
+ * specific mapfile and remove the NODYNSORT attribute
+ * from atomic_or_64_nv.
+ */
+ ENTRY(atomic_or_64)
+ ALTENTRY(atomic_or_64_nv)
+ pushl %edi
+ pushl %ebx
+ movl 12(%esp), %edi / %edi = target address
+ movl (%edi), %eax
+ movl 4(%edi), %edx / %edx:%eax = old value
+1:
+ movl 16(%esp), %ebx
+ movl 20(%esp), %ecx / %ecx:%ebx = delta
+ orl %eax, %ebx
+ orl %edx, %ecx / %ecx:%ebx = new value
+ lock
+ cmpxchg8b (%edi) / try to stick it in
+ jne 1b
+ movl %ebx, %eax
+ movl %ecx, %edx / return new value
+ popl %ebx
+ popl %edi
+ ret
+ SET_SIZE(atomic_or_64_nv)
+ SET_SIZE(atomic_or_64)
+
+ ENTRY(atomic_and_8_nv)
+ ALTENTRY(atomic_and_uchar_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movb (%edx), %al / %al = old value
+1:
+ movl 8(%esp), %ecx / %ecx = delta
+ andb %al, %cl / %cl = new value
+ lock
+ cmpxchgb %cl, (%edx) / try to stick it in
+ jne 1b
+ movzbl %cl, %eax / return new value
+ ret
+ SET_SIZE(atomic_and_uchar_nv)
+ SET_SIZE(atomic_and_8_nv)
+
+ ENTRY(atomic_and_16_nv)
+ ALTENTRY(atomic_and_ushort_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movw (%edx), %ax / %ax = old value
+1:
+ movl 8(%esp), %ecx / %ecx = delta
+ andw %ax, %cx / %cx = new value
+ lock
+ cmpxchgw %cx, (%edx) / try to stick it in
+ jne 1b
+ movzwl %cx, %eax / return new value
+ ret
+ SET_SIZE(atomic_and_ushort_nv)
+ SET_SIZE(atomic_and_16_nv)
+
+ ENTRY(atomic_and_32_nv)
+ ALTENTRY(atomic_and_uint_nv)
+ ALTENTRY(atomic_and_ulong_nv)
+ movl 4(%esp), %edx / %edx = target address
+ movl (%edx), %eax / %eax = old value
+1:
+ movl 8(%esp), %ecx / %ecx = delta
+ andl %eax, %ecx / %ecx = new value
+ lock
+ cmpxchgl %ecx, (%edx) / try to stick it in
+ jne 1b
+ movl %ecx, %eax / return new value
+ ret
+ SET_SIZE(atomic_and_ulong_nv)
+ SET_SIZE(atomic_and_uint_nv)
+ SET_SIZE(atomic_and_32_nv)
+
+ /*
+ * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
+ * separated, it is important to edit the libc i386 platform
+ * specific mapfile and remove the NODYNSORT attribute
+ * from atomic_and_64_nv.
+ */
+ ENTRY(atomic_and_64)
+ ALTENTRY(atomic_and_64_nv)
+ pushl %edi
+ pushl %ebx
+ movl 12(%esp), %edi / %edi = target address
+ movl (%edi), %eax
+ movl 4(%edi), %edx / %edx:%eax = old value
+1:
+ movl 16(%esp), %ebx
+ movl 20(%esp), %ecx / %ecx:%ebx = delta
+ andl %eax, %ebx
+ andl %edx, %ecx / %ecx:%ebx = new value
+ lock
+ cmpxchg8b (%edi) / try to stick it in
+ jne 1b
+ movl %ebx, %eax
+ movl %ecx, %edx / return new value
+ popl %ebx
+ popl %edi
+ ret
+ SET_SIZE(atomic_and_64_nv)
+ SET_SIZE(atomic_and_64)
+
+ ENTRY(atomic_cas_8)
+ ALTENTRY(atomic_cas_uchar)
+ movl 4(%esp), %edx
+ movzbl 8(%esp), %eax
+ movb 12(%esp), %cl
+ lock
+ cmpxchgb %cl, (%edx)
+ ret
+ SET_SIZE(atomic_cas_uchar)
+ SET_SIZE(atomic_cas_8)
+
+ ENTRY(atomic_cas_16)
+ ALTENTRY(atomic_cas_ushort)
+ movl 4(%esp), %edx
+ movzwl 8(%esp), %eax
+ movw 12(%esp), %cx
+ lock
+ cmpxchgw %cx, (%edx)
+ ret
+ SET_SIZE(atomic_cas_ushort)
+ SET_SIZE(atomic_cas_16)
+
+ ENTRY(atomic_cas_32)
+ ALTENTRY(atomic_cas_uint)
+ ALTENTRY(atomic_cas_ulong)
+ ALTENTRY(atomic_cas_ptr)
+ movl 4(%esp), %edx
+ movl 8(%esp), %eax
+ movl 12(%esp), %ecx
+ lock
+ cmpxchgl %ecx, (%edx)
+ ret
+ SET_SIZE(atomic_cas_ptr)
+ SET_SIZE(atomic_cas_ulong)
+ SET_SIZE(atomic_cas_uint)
+ SET_SIZE(atomic_cas_32)
+
+ ENTRY(atomic_cas_64)
+ pushl %ebx
+ pushl %esi
+ movl 12(%esp), %esi
+ movl 16(%esp), %eax
+ movl 20(%esp), %edx
+ movl 24(%esp), %ebx
+ movl 28(%esp), %ecx
+ lock
+ cmpxchg8b (%esi)
+ popl %esi
+ popl %ebx
+ ret
+ SET_SIZE(atomic_cas_64)
+
+ ENTRY(atomic_swap_8)
+ ALTENTRY(atomic_swap_uchar)
+ movl 4(%esp), %edx
+ movzbl 8(%esp), %eax
+ lock
+ xchgb %al, (%edx)
+ ret
+ SET_SIZE(atomic_swap_uchar)
+ SET_SIZE(atomic_swap_8)
+
+ ENTRY(atomic_swap_16)
+ ALTENTRY(atomic_swap_ushort)
+ movl 4(%esp), %edx
+ movzwl 8(%esp), %eax
+ lock
+ xchgw %ax, (%edx)
+ ret
+ SET_SIZE(atomic_swap_ushort)
+ SET_SIZE(atomic_swap_16)
+
+ ENTRY(atomic_swap_32)
+ ALTENTRY(atomic_swap_uint)
+ ALTENTRY(atomic_swap_ptr)
+ ALTENTRY(atomic_swap_ulong)
+ movl 4(%esp), %edx
+ movl 8(%esp), %eax
+ lock
+ xchgl %eax, (%edx)
+ ret
+ SET_SIZE(atomic_swap_ulong)
+ SET_SIZE(atomic_swap_ptr)
+ SET_SIZE(atomic_swap_uint)
+ SET_SIZE(atomic_swap_32)
+
+ ENTRY(atomic_swap_64)
+ pushl %esi
+ pushl %ebx
+ movl 12(%esp), %esi
+ movl 16(%esp), %ebx
+ movl 20(%esp), %ecx
+ movl (%esi), %eax
+ movl 4(%esi), %edx / %edx:%eax = old value
+1:
+ lock
+ cmpxchg8b (%esi)
+ jne 1b
+ popl %ebx
+ popl %esi
+ ret
+ SET_SIZE(atomic_swap_64)
+
+ ENTRY(atomic_set_long_excl)
+ movl 4(%esp), %edx / %edx = target address
+ movl 8(%esp), %ecx / %ecx = bit id
+ xorl %eax, %eax
+ lock
+ btsl %ecx, (%edx)
+ jnc 1f
+ decl %eax / return -1
+1:
+ ret
+ SET_SIZE(atomic_set_long_excl)
+
+ ENTRY(atomic_clear_long_excl)
+ movl 4(%esp), %edx / %edx = target address
+ movl 8(%esp), %ecx / %ecx = bit id
+ xorl %eax, %eax
+ lock
+ btrl %ecx, (%edx)
+ jc 1f
+ decl %eax / return -1
+1:
+ ret
+ SET_SIZE(atomic_clear_long_excl)
+
+#if !defined(_KERNEL)
+
+ /*
+ * NOTE: membar_enter, membar_exit, membar_producer, and
+ * membar_consumer are all identical routines. We define them
+ * separately, instead of using ALTENTRY definitions to alias them
+ * together, so that DTrace and debuggers will see a unique address
+ * for them, allowing more accurate tracing.
+ */
+
+
+ ENTRY(membar_enter)
+ lock
+ xorl $0, (%esp)
+ ret
+ SET_SIZE(membar_enter)
+
+ ENTRY(membar_exit)
+ lock
+ xorl $0, (%esp)
+ ret
+ SET_SIZE(membar_exit)
+
+ ENTRY(membar_producer)
+ lock
+ xorl $0, (%esp)
+ ret
+ SET_SIZE(membar_producer)
+
+ ENTRY(membar_consumer)
+ lock
+ xorl $0, (%esp)
+ ret
+ SET_SIZE(membar_consumer)
+
+#endif /* !_KERNEL */