mass sync with glibc nptl

Signed-off-by: Austin Foxley <austinf@cetoncorp.com>

20 files changed, 2484 insertions, 1195 deletions

diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/Versions b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/Versions
deleted file mode 100644
index 3b111ddb5..000000000
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/Versions
+++ /dev/null
@@ -1,7 +0,0 @@
-librt {
-  GLIBC_2.3.3 {
-    # Changed timer_t.
-    timer_create; timer_delete; timer_getoverrun; timer_gettime;
-    timer_settime;
-  }
-}
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/bits/semaphore.h b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/bits/semaphore.h
index 57edbbbfb..e973bc5bf 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/bits/semaphore.h
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/bits/semaphore.h
@@ -33,9 +33,6 @@
 /* Value returned if `sem_open' failed.  */
 #define SEM_FAILED      ((sem_t *) 0)
 
-/* Maximum value the semaphore can have.  */
-#define SEM_VALUE_MAX   (2147483647)
-
 
 typedef union
 {
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
index 1e461ad41..b0d04c75b 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2006, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -19,39 +19,74 @@
 
 #include <sysdep.h>
 #include <pthread-errnos.h>
+#include <bits/kernel-features.h>
+#include <lowlevellock.h>
 
 	.text
 
-#ifndef LOCK
-# ifdef UP
-#  define LOCK
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
+	movl	$(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
+# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
+	movl	$(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
+# define LOAD_FUTEX_WAIT(reg) \
+	xorl	$(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
+# define LOAD_FUTEX_WAIT_ABS(reg) \
+	xorl	$(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
+# define LOAD_FUTEX_WAKE(reg) \
+	xorl	$(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
+#else
+# if FUTEX_WAIT == 0
+#  define LOAD_PRIVATE_FUTEX_WAIT(reg) \
+	movl    %fs:PRIVATE_FUTEX, reg
+# else
+#  define LOAD_PRIVATE_FUTEX_WAIT(reg) \
+	movl	%fs:PRIVATE_FUTEX, reg ; \
+	orl	$FUTEX_WAIT, reg
+# endif
+# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
+	movl    %fs:PRIVATE_FUTEX, reg ; \
+	orl     $FUTEX_WAKE, reg
+# if FUTEX_WAIT == 0
+#  define LOAD_FUTEX_WAIT(reg) \
+	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
+	andl	%fs:PRIVATE_FUTEX, reg
 # else
-#  define LOCK lock
+#  define LOAD_FUTEX_WAIT(reg) \
+	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
+	andl	%fs:PRIVATE_FUTEX, reg ; \
+	orl	$FUTEX_WAIT, reg
 # endif
+# define LOAD_FUTEX_WAIT_ABS(reg) \
+	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
+	andl	%fs:PRIVATE_FUTEX, reg ; \
+	orl	$FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg
+# define LOAD_FUTEX_WAKE(reg) \
+	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
+	andl	%fs:PRIVATE_FUTEX, reg ; \
+	orl	$FUTEX_WAKE, reg
 #endif
 
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
 
 /* For the calculation see asm/vsyscall.h.  */
 #define VSYSCALL_ADDR_vgettimeofday	0xffffffffff600000
 
 
-	.globl	__lll_mutex_lock_wait
-	.type	__lll_mutex_lock_wait,@function
-	.hidden	__lll_mutex_lock_wait
+	.globl	__lll_lock_wait_private
+	.type	__lll_lock_wait_private,@function
+	.hidden	__lll_lock_wait_private
 	.align	16
-__lll_mutex_lock_wait:
+__lll_lock_wait_private:
+	cfi_startproc
 	pushq	%r10
+	cfi_adjust_cfa_offset(8)
 	pushq	%rdx
-
+	cfi_adjust_cfa_offset(8)
+	cfi_offset(%r10, -16)
+	cfi_offset(%rdx, -24)
 	xorq	%r10, %r10	/* No timeout.  */
 	movl	$2, %edx
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
-#else
-	movl	$FUTEX_WAIT, %esi
-#endif
+	LOAD_PRIVATE_FUTEX_WAIT (%esi)
 
 	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
 	jne	2f
@@ -66,33 +101,144 @@ __lll_mutex_lock_wait:
 	jnz	1b
 
 	popq	%rdx
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%rdx)
 	popq	%r10
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r10)
 	retq
-	.size	__lll_mutex_lock_wait,.-__lll_mutex_lock_wait
-
+	cfi_endproc
+	.size	__lll_lock_wait_private,.-__lll_lock_wait_private
 
 #ifdef NOT_IN_libc
-	.globl	__lll_mutex_timedlock_wait
-	.type	__lll_mutex_timedlock_wait,@function
-	.hidden	__lll_mutex_timedlock_wait
+	.globl	__lll_lock_wait
+	.type	__lll_lock_wait,@function
+	.hidden	__lll_lock_wait
+	.align	16
+__lll_lock_wait:
+	cfi_startproc
+	pushq	%r10
+	cfi_adjust_cfa_offset(8)
+	pushq	%rdx
+	cfi_adjust_cfa_offset(8)
+	cfi_offset(%r10, -16)
+	cfi_offset(%rdx, -24)
+	xorq	%r10, %r10	/* No timeout.  */
+	movl	$2, %edx
+	LOAD_FUTEX_WAIT (%esi)
+
+	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
+	jne	2f
+
+1:	movl	$SYS_futex, %eax
+	syscall
+
+2:	movl	%edx, %eax
+	xchgl	%eax, (%rdi)	/* NB:	 lock is implied */
+
+	testl	%eax, %eax
+	jnz	1b
+
+	popq	%rdx
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%rdx)
+	popq	%r10
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r10)
+	retq
+	cfi_endproc
+	.size	__lll_lock_wait,.-__lll_lock_wait
+
+	/*      %rdi: futex
+		%rsi: flags
+		%rdx: timeout
+		%eax: futex value
+	*/
+	.globl	__lll_timedlock_wait
+	.type	__lll_timedlock_wait,@function
+	.hidden	__lll_timedlock_wait
 	.align	16
-__lll_mutex_timedlock_wait:
+__lll_timedlock_wait:
+	cfi_startproc
+# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+#  ifdef PIC
+	cmpl	$0, __have_futex_clock_realtime(%rip)
+#  else
+	cmpl	$0, __have_futex_clock_realtime
+#  endif
+	je	.Lreltmo
+# endif
+
+	pushq	%r9
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r9, 0)
+	movq	%rdx, %r10
+	movl	$0xffffffff, %r9d
+	LOAD_FUTEX_WAIT_ABS (%esi)
+
+	movl	$2, %edx
+	cmpl	%edx, %eax
+	jne	2f
+
+1:	movl	$SYS_futex, %eax
+	movl	$2, %edx
+	syscall
+
+2:	xchgl	%edx, (%rdi)	/* NB:   lock is implied */
+
+	testl	%edx, %edx
+	jz	3f
+
+	cmpl	$-ETIMEDOUT, %eax
+	je	4f
+	cmpl	$-EINVAL, %eax
+	jne	1b
+4:	movl	%eax, %edx
+	negl	%edx
+
+3:	movl	%edx, %eax
+	popq	%r9
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r9)
+	retq
+
+# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+.Lreltmo:
 	/* Check for a valid timeout value.  */
 	cmpq	$1000000000, 8(%rdx)
 	jae	3f
 
 	pushq	%r8
+	cfi_adjust_cfa_offset(8)
 	pushq	%r9
+	cfi_adjust_cfa_offset(8)
 	pushq	%r12
+	cfi_adjust_cfa_offset(8)
 	pushq	%r13
+	cfi_adjust_cfa_offset(8)
 	pushq	%r14
+	cfi_adjust_cfa_offset(8)
+	cfi_offset(%r8, -16)
+	cfi_offset(%r9, -24)
+	cfi_offset(%r12, -32)
+	cfi_offset(%r13, -40)
+	cfi_offset(%r14, -48)
+	pushq	%rsi
+	cfi_adjust_cfa_offset(8)
 
 	/* Stack frame for the timespec and timeval structs.  */
-	subq	$16, %rsp
+	subq	$24, %rsp
+	cfi_adjust_cfa_offset(24)
 
 	movq	%rdi, %r12
 	movq	%rdx, %r13
 
+	movl	$2, %edx
+	xchgl	%edx, (%r12)
+
+	testl	%edx, %edx
+	je	6f
+
 1:
 	/* Get current time.  */
 	movq	%rsp, %rdi
@@ -114,118 +260,137 @@ __lll_mutex_timedlock_wait:
 	addq	$1000000000, %rsi
 	decq	%rdi
 4:	testq	%rdi, %rdi
-	js	5f		/* Time is already up.  */
+	js	2f		/* Time is already up.  */
 
-	/* Futex call.  */
-	movq	%rdi, (%rsp)	/* Store relative timeout.  */
+	/* Store relative timeout.  */
+	movq	%rdi, (%rsp)
 	movq	%rsi, 8(%rsp)
 
-	movl	$1, %eax
+	/* Futex call.  */
 	movl	$2, %edx
-	LOCK
-	cmpxchgl %edx, (%r12)
-
-	testl	%eax, %eax
-	je	8f
-
+	movl	$1, %eax
 	movq	%rsp, %r10
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
-#else
-	movl	$FUTEX_WAIT, %esi
-#endif
+	movl	24(%rsp), %esi
+	LOAD_FUTEX_WAIT (%esi)
 	movq	%r12, %rdi
 	movl	$SYS_futex, %eax
 	syscall
-	movq	%rax, %rcx
 
-8:				/* NB: %edx == 2 */
-	xorl	%eax, %eax
-	LOCK
-	cmpxchgl %edx, (%rdi)
-	jnz	7f
+	/* NB: %edx == 2 */
+	xchgl	%edx, (%r12)
+
+	testl	%edx, %edx
+	je	6f
+
+	cmpl	$-ETIMEDOUT, %eax
+	jne	1b
+2:	movl	$ETIMEDOUT, %edx
 
-6:	addq	$16, %rsp
+6:	addq	$32, %rsp
+	cfi_adjust_cfa_offset(-32)
 	popq	%r14
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r14)
 	popq	%r13
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r13)
 	popq	%r12
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r12)
 	popq	%r9
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r9)
 	popq	%r8
-	retq
-
-	/* Check whether the time expired.  */
-7:	cmpq	$-ETIMEDOUT, %rcx
-	je	5f
-
-	/* Make sure the current holder knows we are going to sleep.  */
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r8)
 	movl	%edx, %eax
-	xchgl	%eax, (%rdi)
-	testl	%eax, %eax
-	jz	6b
-	jmp	1b
+	retq
 
 3:	movl	$EINVAL, %eax
 	retq
-
-5:	movl	$ETIMEDOUT, %eax
-	jmp	6b
-	.size	__lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait
+# endif
+	cfi_endproc
+	.size	__lll_timedlock_wait,.-__lll_timedlock_wait
 #endif
 
 
-#ifdef NOT_IN_libc
-	.globl	lll_unlock_wake_cb
-	.type	lll_unlock_wake_cb,@function
-	.hidden	lll_unlock_wake_cb
+	.globl	__lll_unlock_wake_private
+	.type	__lll_unlock_wake_private,@function
+	.hidden	__lll_unlock_wake_private
 	.align	16
-lll_unlock_wake_cb:
+__lll_unlock_wake_private:
+	cfi_startproc
 	pushq	%rsi
+	cfi_adjust_cfa_offset(8)
 	pushq	%rdx
+	cfi_adjust_cfa_offset(8)
+	cfi_offset(%rsi, -16)
+	cfi_offset(%rdx, -24)
 
-	LOCK
-	addl	$1, (%rdi)
-	jng	1f
+	movl	$0, (%rdi)
+	LOAD_PRIVATE_FUTEX_WAKE (%esi)
+	movl	$1, %edx	/* Wake one thread.  */
+	movl	$SYS_futex, %eax
+	syscall
 
 	popq	%rdx
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%rdx)
 	popq	%rsi
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%rsi)
 	retq
-	.size	lll_unlock_wake_cb,.-lll_unlock_wake_cb
-#endif
-
+	cfi_endproc
+	.size	__lll_unlock_wake_private,.-__lll_unlock_wake_private
 
-	.globl	__lll_mutex_unlock_wake
-	.type	__lll_mutex_unlock_wake,@function
-	.hidden	__lll_mutex_unlock_wake
+#ifdef NOT_IN_libc
+	.globl	__lll_unlock_wake
+	.type	__lll_unlock_wake,@function
+	.hidden	__lll_unlock_wake
 	.align	16
-__lll_mutex_unlock_wake:
+__lll_unlock_wake:
+	cfi_startproc
 	pushq	%rsi
+	cfi_adjust_cfa_offset(8)
 	pushq	%rdx
+	cfi_adjust_cfa_offset(8)
+	cfi_offset(%rsi, -16)
+	cfi_offset(%rdx, -24)
 
 	movl	$0, (%rdi)
-	movl	$FUTEX_WAKE, %esi
+	LOAD_FUTEX_WAKE (%esi)
 	movl	$1, %edx	/* Wake one thread.  */
 	movl	$SYS_futex, %eax
 	syscall
 
 	popq	%rdx
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%rdx)
 	popq	%rsi
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%rsi)
 	retq
-	.size	__lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake
+	cfi_endproc
+	.size	__lll_unlock_wake,.-__lll_unlock_wake
 
-
-#ifdef NOT_IN_libc
 	.globl	__lll_timedwait_tid
 	.type	__lll_timedwait_tid,@function
 	.hidden	__lll_timedwait_tid
 	.align	16
 __lll_timedwait_tid:
+	cfi_startproc
 	pushq	%r12
+	cfi_adjust_cfa_offset(8)
 	pushq	%r13
+	cfi_adjust_cfa_offset(8)
+	cfi_offset(%r12, -16)
+	cfi_offset(%r13, -24)
 
 	movq	%rdi, %r12
 	movq	%rsi, %r13
 
 	subq	$16, %rsp
+	cfi_adjust_cfa_offset(16)
 
 	/* Get current time.  */
 2:	movq	%rsp, %rdi
@@ -255,6 +420,8 @@ __lll_timedwait_tid:
 	jz	4f
 
 	movq	%rsp, %r10
+	/* XXX The kernel so far uses global futex for the wakeup at
+	   all times.  */
 #if FUTEX_WAIT == 0
 	xorl	%esi, %esi
 #else
@@ -269,14 +436,21 @@ __lll_timedwait_tid:
 4:	xorl	%eax, %eax
 
 8:	addq	$16, %rsp
+	cfi_adjust_cfa_offset(-16)
 	popq	%r13
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r13)
 	popq	%r12
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r12)
 	retq
 
+	cfi_adjust_cfa_offset(32)
 1:	cmpq	$-ETIMEDOUT, %rax
 	jne	2b
 
 6:	movl	$ETIMEDOUT, %eax
 	jmp	8b
+	cfi_endproc
 	.size	__lll_timedwait_tid,.-__lll_timedwait_tid
 #endif
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index c9f30e962..7c042fc80 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2004, 2006-2008, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -20,266 +20,541 @@
 #ifndef _LOWLEVELLOCK_H
 #define _LOWLEVELLOCK_H	1
 
-#include <time.h>
-#include <sys/param.h>
-#include <bits/pthreadtypes.h>
-#include <atomic.h>
-#include <sysdep.h>
-
-#ifndef LOCK_INSTR
-# ifdef UP
-#  define LOCK_INSTR	/* nothing */
-# else
-#  define LOCK_INSTR "lock;"
+#ifndef __ASSEMBLER__
+# include <time.h>
+# include <sys/param.h>
+# include <bits/pthreadtypes.h>
+# include <bits/kernel-features.h>
+# include <tcb-offsets.h>
+
+# ifndef LOCK_INSTR
+#  ifdef UP
+#   define LOCK_INSTR	/* nothing */
+#  else
+#   define LOCK_INSTR "lock;"
+#  endif
+# endif
+#else
+# ifndef LOCK
+#  ifdef UP
+#   define LOCK
+#  else
+#   define LOCK lock
+#  endif
 # endif
 #endif
 
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_CMP_REQUEUE	4
+#define FUTEX_WAKE_OP		5
+#define FUTEX_LOCK_PI		6
+#define FUTEX_UNLOCK_PI		7
+#define FUTEX_TRYLOCK_PI	8
+#define FUTEX_WAIT_BITSET	9
+#define FUTEX_WAKE_BITSET	10
+#define FUTEX_WAIT_REQUEUE_PI	11
+#define FUTEX_CMP_REQUEUE_PI	12
+#define FUTEX_PRIVATE_FLAG	128
+#define FUTEX_CLOCK_REALTIME	256
+
+#define FUTEX_BITSET_MATCH_ANY	0xffffffff
+
+#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE	((4 << 24) | 1)
+
+/* Values for 'private' parameter of locking macros.  Yes, the
+   definition seems to be backwards.  But it is not.  The bit will be
+   reversed before passing to the system call.  */
+#define LLL_PRIVATE	0
+#define LLL_SHARED	FUTEX_PRIVATE_FLAG
+
+#ifndef __ASSEMBLER__
+
+#if !defined NOT_IN_libc || defined IS_IN_rtld
+/* In libc.so or ld.so all futexes are private.  */
+# ifdef __ASSUME_PRIVATE_FUTEX
+#  define __lll_private_flag(fl, private) \
+  ((fl) | FUTEX_PRIVATE_FLAG)
+# else
+#  define __lll_private_flag(fl, private) \
+  ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
+# endif
+#else
+# ifdef __ASSUME_PRIVATE_FUTEX
+#  define __lll_private_flag(fl, private) \
+  (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
+# else
+#  define __lll_private_flag(fl, private) \
+  (__builtin_constant_p (private)					      \
+   ? ((private) == 0							      \
+      ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))	      \
+      : (fl))								      \
+   : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG);		      \
+	__asm__ ("andl %%fs:%P1, %0" : "+r" (__fl)				      \
+	     : "i" (offsetof (struct pthread, header.private_futex)));	      \
+	__fl | (fl); }))
+# endif
+#endif
 
-
-/* Initializer for compatibility lock.  */
-#define LLL_MUTEX_LOCK_INITIALIZER		(0)
-#define LLL_MUTEX_LOCK_INITIALIZER_LOCKED	(1)
-#define LLL_MUTEX_LOCK_INITIALIZER_WAITERS	(2)
+/* Initializer for lock.  */
+#define LLL_LOCK_INITIALIZER		(0)
+#define LLL_LOCK_INITIALIZER_LOCKED	(1)
+#define LLL_LOCK_INITIALIZER_WAITERS	(2)
 
 /* Delay in spinlock loop.  */
-#define BUSY_WAIT_NOP          __asm__ ("rep; nop")
-
-
-#define lll_futex_wait(futex, val) \
-  do {									      \
-    int __ignore;							      \
+#define BUSY_WAIT_NOP	  __asm__ ("rep; nop")
+
+
+#define LLL_STUB_UNWIND_INFO_START \
+	".section	.eh_frame,\"a\",@progbits\n" 		\
+"7:\t"	".long	9f-8f	# Length of Common Information Entry\n" \
+"8:\t"	".long	0x0	# CIE Identifier Tag\n\t" 		\
+	".byte	0x1	# CIE Version\n\t" 			\
+	".ascii \"zR\\0\"	# CIE Augmentation\n\t" 	\
+	".uleb128 0x1	# CIE Code Alignment Factor\n\t" 	\
+	".sleb128 -8	# CIE Data Alignment Factor\n\t" 	\
+	".byte	0x10	# CIE RA Column\n\t" 			\
+	".uleb128 0x1	# Augmentation size\n\t" 		\
+	".byte	0x1b	# FDE Encoding (pcrel sdata4)\n\t" 	\
+	".byte	0x12	# DW_CFA_def_cfa_sf\n\t" 		\
+	".uleb128 0x7\n\t" 					\
+	".sleb128 16\n\t" 					\
+	".align 8\n" 						\
+"9:\t"	".long	23f-10f	# FDE Length\n" 			\
+"10:\t"	".long	10b-7b	# FDE CIE offset\n\t" 			\
+	".long	1b-.	# FDE initial location\n\t" 		\
+	".long	6b-1b	# FDE address range\n\t" 		\
+	".uleb128 0x0	# Augmentation size\n\t" 		\
+	".byte	0x16	# DW_CFA_val_expression\n\t" 		\
+	".uleb128 0x10\n\t" 					\
+	".uleb128 12f-11f\n" 					\
+"11:\t"	".byte	0x80	# DW_OP_breg16\n\t" 			\
+	".sleb128 4b-1b\n"
+#define LLL_STUB_UNWIND_INFO_END \
+	".byte	0x16	# DW_CFA_val_expression\n\t" 		\
+	".uleb128 0x10\n\t" 					\
+	".uleb128 14f-13f\n" 					\
+"13:\t"	".byte	0x80	# DW_OP_breg16\n\t" 			\
+	".sleb128 4b-2b\n" 					\
+"14:\t"	".byte	0x40 + (3b-2b) # DW_CFA_advance_loc\n\t" 	\
+	".byte	0x0e	# DW_CFA_def_cfa_offset\n\t" 		\
+	".uleb128 0\n\t" 					\
+	".byte	0x16	# DW_CFA_val_expression\n\t" 		\
+	".uleb128 0x10\n\t" 					\
+	".uleb128 16f-15f\n" 					\
+"15:\t"	".byte	0x80	# DW_OP_breg16\n\t" 			\
+	".sleb128 4b-3b\n" 					\
+"16:\t"	".byte	0x40 + (4b-3b-1) # DW_CFA_advance_loc\n\t" 	\
+	".byte	0x0e	# DW_CFA_def_cfa_offset\n\t" 		\
+	".uleb128 128\n\t" 					\
+	".byte	0x16	# DW_CFA_val_expression\n\t" 		\
+	".uleb128 0x10\n\t" 					\
+	".uleb128 20f-17f\n" 					\
+"17:\t"	".byte	0x80	# DW_OP_breg16\n\t" 			\
+	".sleb128 19f-18f\n\t" 					\
+	".byte	0x0d	# DW_OP_const4s\n" 			\
+"18:\t"	".4byte	4b-.\n\t" 					\
+	".byte	0x1c	# DW_OP_minus\n\t" 			\
+	".byte	0x0d	# DW_OP_const4s\n" 			\
+"19:\t"	".4byte	24f-.\n\t" 					\
+	".byte	0x22	# DW_OP_plus\n" 			\
+"20:\t"	".byte	0x40 + (5b-4b+1) # DW_CFA_advance_loc\n\t" 	\
+	".byte	0x13	# DW_CFA_def_cfa_offset_sf\n\t" 	\
+	".sleb128 16\n\t" 					\
+	".byte	0x16	# DW_CFA_val_expression\n\t" 		\
+	".uleb128 0x10\n\t" 					\
+	".uleb128 22f-21f\n" 					\
+"21:\t"	".byte	0x80	# DW_OP_breg16\n\t" 			\
+	".sleb128 4b-5b\n" 					\
+"22:\t"	".align 8\n" 						\
+"23:\t"	".previous\n"
+
+/* Unwind info for
+   1: leaq ..., %rdi
+   2: subq $128, %rsp
+   3: callq ...
+   4: addq $128, %rsp
+   5: jmp 24f
+   6:
+   snippet.  */
+#define LLL_STUB_UNWIND_INFO_5 \
+LLL_STUB_UNWIND_INFO_START					\
+"12:\t"	".byte	0x40 + (2b-1b) # DW_CFA_advance_loc\n\t" 	\
+LLL_STUB_UNWIND_INFO_END
+
+/* Unwind info for
+   1: leaq ..., %rdi
+   0: movq ..., %rdx
+   2: subq $128, %rsp
+   3: callq ...
+   4: addq $128, %rsp
+   5: jmp 24f
+   6:
+   snippet.  */
+#define LLL_STUB_UNWIND_INFO_6 \
+LLL_STUB_UNWIND_INFO_START					\
+"12:\t"	".byte	0x40 + (0b-1b) # DW_CFA_advance_loc\n\t" 	\
+	".byte	0x16	# DW_CFA_val_expression\n\t" 		\
+	".uleb128 0x10\n\t" 					\
+	".uleb128 26f-25f\n" 					\
+"25:\t"	".byte	0x80	# DW_OP_breg16\n\t" 			\
+	".sleb128 4b-0b\n" 					\
+"26:\t"	".byte	0x40 + (2b-0b) # DW_CFA_advance_loc\n\t" 	\
+LLL_STUB_UNWIND_INFO_END
+
+
+#define lll_futex_wait(futex, val, private) \
+  lll_futex_timed_wait(futex, val, NULL, private)
+
+
+#define lll_futex_timed_wait(futex, val, timeout, private) \
+  ({									      \
+    register const struct timespec *__to __asm__ ("r10") = timeout;	      \
+    int __status;							      \
     register __typeof (val) _val __asm__ ("edx") = (val);			      \
-    __asm__ __volatile ("xorq %%r10, %%r10\n\t"				      \
-		      "syscall"						      \
-		      : "=a" (__ignore)					      \
-		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT),	      \
-			"d" (_val)					      \
-		      : "memory", "cc", "r10", "r11", "cx");		      \
-  } while (0)
+    __asm__ __volatile ("syscall"						      \
+		      : "=a" (__status)					      \
+		      : "0" (SYS_futex), "D" (futex),			      \
+			"S" (__lll_private_flag (FUTEX_WAIT, private)),	      \
+			"d" (_val), "r" (__to)				      \
+		      : "memory", "cc", "r11", "cx");			      \
+    __status;								      \
+  })
 
 
-#define lll_futex_wake(futex, nr) \
+#define lll_futex_wake(futex, nr, private) \
   do {									      \
     int __ignore;							      \
     register __typeof (nr) _nr __asm__ ("edx") = (nr);			      \
     __asm__ __volatile ("syscall"						      \
 		      : "=a" (__ignore)					      \
-		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE),	      \
+		      : "0" (SYS_futex), "D" (futex),			      \
+			"S" (__lll_private_flag (FUTEX_WAKE, private)),	      \
 			"d" (_nr)					      \
 		      : "memory", "cc", "r10", "r11", "cx");		      \
   } while (0)
 
 
-/* Does not preserve %eax and %ecx.  */
-extern int __lll_mutex_lock_wait (int *__futex, int __val) attribute_hidden;
-/* Does not preserver %eax, %ecx, and %edx.  */
-extern int __lll_mutex_timedlock_wait (int *__futex, int __val,
-				       const struct timespec *__abstime)
-     attribute_hidden;
-/* Preserves all registers but %eax.  */
-extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
-
-
-/* NB: in the lll_mutex_trylock macro we simply return the value in %eax
+/* NB: in the lll_trylock macro we simply return the value in %eax
    after the cmpxchg instruction.  In case the operation succeded this
    value is zero.  In case the operation failed, the cmpxchg instruction
    has loaded the current value of the memory work which is guaranteed
    to be nonzero.  */
-#define lll_mutex_trylock(futex) \
+#if defined NOT_IN_libc || defined UP
+# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
+#else
+# define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t"      \
+			   "je 0f\n\t"					      \
+			   "lock; cmpxchgl %2, %1\n\t"			      \
+			   "jmp 1f\n\t"					      \
+			   "0:\tcmpxchgl %2, %1\n\t"			      \
+			   "1:"
+#endif
+
+#define lll_trylock(futex) \
   ({ int ret;								      \
-     __asm__ __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
+     __asm__ __volatile (__lll_trylock_asm				      \
 		       : "=a" (ret), "=m" (futex)			      \
-		       : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
-			 "0" (LLL_MUTEX_LOCK_INITIALIZER)		      \
+		       : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
+			 "0" (LLL_LOCK_INITIALIZER)			      \
 		       : "memory");					      \
      ret; })
 
-
-#define lll_mutex_cond_trylock(futex) \
+#define lll_robust_trylock(futex, id) \
   ({ int ret;								      \
      __asm__ __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
 		       : "=a" (ret), "=m" (futex)			      \
-		       : "r" (LLL_MUTEX_LOCK_INITIALIZER_WAITERS),	      \
-			 "m" (futex), "0" (LLL_MUTEX_LOCK_INITIALIZER)	      \
+		       : "r" (id), "m" (futex),	"0" (LLL_LOCK_INITIALIZER)    \
 		       : "memory");					      \
      ret; })
 
+#define lll_cond_trylock(futex) \
+  ({ int ret;								      \
+     __asm__ __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
+		       : "=a" (ret), "=m" (futex)			      \
+		       : "r" (LLL_LOCK_INITIALIZER_WAITERS),		      \
+			 "m" (futex), "0" (LLL_LOCK_INITIALIZER)	      \
+		       : "memory");					      \
+     ret; })
 
-#define lll_mutex_lock(futex) \
-  (void) ({ int ignore1, ignore2, ignore3;				      \
-	    __asm__ __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t"		      \
-			      "jnz 1f\n\t"				      \
-			      ".subsection 1\n"				      \
-			      "1:\tleaq %2, %%rdi\n\t"			      \
-			      "subq $128, %%rsp\n\t"			      \
-			      "callq __lll_mutex_lock_wait\n\t"		      \
-			      "addq $128, %%rsp\n\t"			      \
-			      "jmp 2f\n\t"				      \
-			      ".previous\n"				      \
-			      "2:"					      \
-			      : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\
-				"=a" (ignore3)				      \
-			      : "0" (1), "m" (futex), "3" (0)		      \
-			      : "cx", "r11", "cc", "memory"); })
-
-
-#define lll_mutex_cond_lock(futex) \
-  (void) ({ int ignore1, ignore2, ignore3;				      \
-	    __asm__ __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t"		      \
+#if defined NOT_IN_libc || defined UP
+# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t"		      \
+			      "jnz 1f\n\t"
+#else
+# define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
+			      "je 0f\n\t"				      \
+			      "lock; cmpxchgl %4, %2\n\t"		      \
 			      "jnz 1f\n\t"				      \
-			      ".subsection 1\n"				      \
-			      "1:\tleaq %2, %%rdi\n\t"			      \
-			      "subq $128, %%rsp\n\t"			      \
-			      "callq __lll_mutex_lock_wait\n\t"		      \
-			      "addq $128, %%rsp\n\t"			      \
-			      "jmp 2f\n\t"				      \
-			      ".previous\n"				      \
-			      "2:"					      \
-			      : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\
-				"=a" (ignore3)				      \
-			      : "0" (2), "m" (futex), "3" (0)		      \
-			      : "cx", "r11", "cc", "memory"); })
-
-
-#define lll_mutex_timedlock(futex, timeout) \
-  ({ int _result, ignore1, ignore2, ignore3;				      \
-     __asm__ __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t"			      \
+		  	      "jmp 24f\n"				      \
+			      "0:\tcmpxchgl %4, %2\n\t"			      \
+			      "jnz 1f\n\t"
+#endif
+
+#define lll_lock(futex, private) \
+  (void)								      \
+    ({ int ignore1, ignore2, ignore3;					      \
+       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
+	 __asm__ __volatile (__lll_lock_asm_start				      \
+			   ".subsection 1\n\t"				      \
+			   ".type _L_lock_%=, @function\n"		      \
+			   "_L_lock_%=:\n"				      \
+			   "1:\tleaq %2, %%rdi\n"			      \
+			   "2:\tsubq $128, %%rsp\n"			      \
+			   "3:\tcallq __lll_lock_wait_private\n"	      \
+			   "4:\taddq $128, %%rsp\n"			      \
+			   "5:\tjmp 24f\n"				      \
+			   "6:\t.size _L_lock_%=, 6b-1b\n\t"		      \
+			   ".previous\n"				      \
+			   LLL_STUB_UNWIND_INFO_5			      \
+			   "24:"					      \
+			   : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),   \
+			     "=a" (ignore3)				      \
+			   : "0" (1), "m" (futex), "3" (0)		      \
+			   : "cx", "r11", "cc", "memory");		      \
+       else								      \
+	 __asm__ __volatile (__lll_lock_asm_start				      \
+			   ".subsection 1\n\t"				      \
+			   ".type _L_lock_%=, @function\n"		      \
+			   "_L_lock_%=:\n"				      \
+			   "1:\tleaq %2, %%rdi\n"			      \
+			   "2:\tsubq $128, %%rsp\n"			      \
+			   "3:\tcallq __lll_lock_wait\n"		      \
+			   "4:\taddq $128, %%rsp\n"			      \
+			   "5:\tjmp 24f\n"				      \
+			   "6:\t.size _L_lock_%=, 6b-1b\n\t"		      \
+			   ".previous\n"				      \
+			   LLL_STUB_UNWIND_INFO_5			      \
+			   "24:"					      \
+			   : "=S" (ignore1), "=D" (ignore2), "=m" (futex),    \
+			     "=a" (ignore3)				      \
+			   : "1" (1), "m" (futex), "3" (0), "0" (private)     \
+			   : "cx", "r11", "cc", "memory");		      \
+    })									      \
+
+#define lll_robust_lock(futex, id, private) \
+  ({ int result, ignore1, ignore2;					      \
+    __asm__ __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t"			      \
+		      "jnz 1f\n\t"					      \
+		      ".subsection 1\n\t"				      \
+		      ".type _L_robust_lock_%=, @function\n"		      \
+		      "_L_robust_lock_%=:\n"				      \
+		      "1:\tleaq %2, %%rdi\n"				      \
+		      "2:\tsubq $128, %%rsp\n"				      \
+		      "3:\tcallq __lll_robust_lock_wait\n"		      \
+		      "4:\taddq $128, %%rsp\n"				      \
+		      "5:\tjmp 24f\n"					      \
+		      "6:\t.size _L_robust_lock_%=, 6b-1b\n\t"		      \
+		      ".previous\n"					      \
+		      LLL_STUB_UNWIND_INFO_5				      \
+		      "24:"						      \
+		      : "=S" (ignore1), "=D" (ignore2), "=m" (futex),	      \
+			"=a" (result)					      \
+		      : "1" (id), "m" (futex), "3" (0), "0" (private)	      \
+		      : "cx", "r11", "cc", "memory");			      \
+    result; })
+
+#define lll_cond_lock(futex, private) \
+  (void)								      \
+    ({ int ignore1, ignore2, ignore3;					      \
+       __asm__ __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t"		      \
+			 "jnz 1f\n\t"					      \
+			 ".subsection 1\n\t"				      \
+			 ".type _L_cond_lock_%=, @function\n"		      \
+			 "_L_cond_lock_%=:\n"				      \
+			 "1:\tleaq %2, %%rdi\n"				      \
+			 "2:\tsubq $128, %%rsp\n"			      \
+			 "3:\tcallq __lll_lock_wait\n"			      \
+			 "4:\taddq $128, %%rsp\n"			      \
+			 "5:\tjmp 24f\n"				      \
+			 "6:\t.size _L_cond_lock_%=, 6b-1b\n\t"		      \
+			 ".previous\n"					      \
+			 LLL_STUB_UNWIND_INFO_5				      \
+			 "24:"						      \
+			 : "=S" (ignore1), "=D" (ignore2), "=m" (futex),      \
+			   "=a" (ignore3)				      \
+			 : "1" (2), "m" (futex), "3" (0), "0" (private)	      \
+			 : "cx", "r11", "cc", "memory");		      \
+    })
+
+#define lll_robust_cond_lock(futex, id, private) \
+  ({ int result, ignore1, ignore2;					      \
+    __asm__ __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t"			      \
+		      "jnz 1f\n\t"					      \
+		      ".subsection 1\n\t"				      \
+		      ".type _L_robust_cond_lock_%=, @function\n"	      \
+		      "_L_robust_cond_lock_%=:\n"			      \
+		      "1:\tleaq %2, %%rdi\n"				      \
+		      "2:\tsubq $128, %%rsp\n"				      \
+		      "3:\tcallq __lll_robust_lock_wait\n"		      \
+		      "4:\taddq $128, %%rsp\n"				      \
+		      "5:\tjmp 24f\n"					      \
+		      "6:\t.size _L_robust_cond_lock_%=, 6b-1b\n\t"	      \
+		      ".previous\n"					      \
+		      LLL_STUB_UNWIND_INFO_5				      \
+		      "24:"						      \
+		      : "=S" (ignore1), "=D" (ignore2), "=m" (futex),	      \
+			"=a" (result)					      \
+		      : "1" (id | FUTEX_WAITERS), "m" (futex), "3" (0),	      \
+			"0" (private)					      \
+		      : "cx", "r11", "cc", "memory");			      \
+    result; })
+
+#define lll_timedlock(futex, timeout, private) \
+  ({ int result, ignore1, ignore2, ignore3;				      \
+     __asm__ __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t"			      \
 		       "jnz 1f\n\t"					      \
-		       ".subsection 1\n"				      \
-		       "1:\tleaq %4, %%rdi\n\t"				      \
-		       "movq %8, %%rdx\n\t"				      \
-		       "subq $128, %%rsp\n\t"				      \
-		       "callq __lll_mutex_timedlock_wait\n\t"		      \
-		       "addq $128, %%rsp\n\t"				      \
-		       "jmp 2f\n\t"					      \
+		       ".subsection 1\n\t"				      \
+		       ".type _L_timedlock_%=, @function\n"		      \
+		       "_L_timedlock_%=:\n"				      \
+		       "1:\tleaq %4, %%rdi\n"				      \
+		       "0:\tmovq %8, %%rdx\n"				      \
+		       "2:\tsubq $128, %%rsp\n"				      \
+		       "3:\tcallq __lll_timedlock_wait\n"		      \
+		       "4:\taddq $128, %%rsp\n"				      \
+		       "5:\tjmp 24f\n"					      \
+		       "6:\t.size _L_timedlock_%=, 6b-1b\n\t"		      \
 		       ".previous\n"					      \
-		       "2:"						      \
-		       : "=a" (_result), "=&D" (ignore1), "=S" (ignore2),      \
+		       LLL_STUB_UNWIND_INFO_6				      \
+		       "24:"						      \
+		       : "=a" (result), "=D" (ignore1), "=S" (ignore2),	      \
 			 "=&d" (ignore3), "=m" (futex)			      \
-		       : "0" (0), "2" (1), "m" (futex), "m" (timeout)	      \
+		       : "0" (0), "1" (1), "m" (futex), "m" (timeout),	      \
+			 "2" (private)					      \
 		       : "memory", "cx", "cc", "r10", "r11");		      \
-     _result; })
-
-
-#define lll_mutex_unlock(futex) \
-  (void) ({ int ignore;							      \
-            __asm__ __volatile (LOCK_INSTR "decl %0\n\t"			      \
-			      "jne 1f\n\t"				      \
-			      ".subsection 1\n"				      \
-			      "1:\tleaq %0, %%rdi\n\t"			      \
-			      "subq $128, %%rsp\n\t"			      \
-			      "callq __lll_mutex_unlock_wake\n\t"	      \
-			      "addq $128, %%rsp\n\t"			      \
-			      "jmp 2f\n\t"				      \
-			      ".previous\n"				      \
-			      "2:"					      \
-			      : "=m" (futex), "=&D" (ignore)		      \
-			      : "m" (futex)				      \
-			      : "ax", "cx", "r11", "cc", "memory"); })
-
-
-#define lll_mutex_islocked(futex) \
-  (futex != LLL_MUTEX_LOCK_INITIALIZER)
-
-
-/* We have a separate internal lock implementation which is not tied
-   to binary compatibility.  */
-
-/* Type for lock object.  */
-typedef int lll_lock_t;
-
-/* Initializers for lock.  */
-#define LLL_LOCK_INITIALIZER		(0)
-#define LLL_LOCK_INITIALIZER_LOCKED	(1)
-
-
-extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
-
-
-/* The states of a lock are:
-    0  -  untaken
-    1  -  taken by one user
-    2  -  taken by more users */
+     result; })
 
+#define lll_robust_timedlock(futex, timeout, id, private) \
+  ({ int result, ignore1, ignore2, ignore3;				      \
+     __asm__ __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t"			      \
+		       "jnz 1f\n\t"					      \
+		       ".subsection 1\n\t"				      \
+		       ".type _L_robust_timedlock_%=, @function\n"	      \
+		       "_L_robust_timedlock_%=:\n"			      \
+		       "1:\tleaq %4, %%rdi\n"				      \
+		       "0:\tmovq %8, %%rdx\n"				      \
+		       "2:\tsubq $128, %%rsp\n"				      \
+		       "3:\tcallq __lll_robust_timedlock_wait\n"	      \
+		       "4:\taddq $128, %%rsp\n"				      \
+		       "5:\tjmp 24f\n"					      \
+		       "6:\t.size _L_robust_timedlock_%=, 6b-1b\n\t"	      \
+		       ".previous\n"					      \
+		       LLL_STUB_UNWIND_INFO_6				      \
+		       "24:"						      \
+		       : "=a" (result), "=D" (ignore1), "=S" (ignore2),       \
+			 "=&d" (ignore3), "=m" (futex)			      \
+		       : "0" (0), "1" (id), "m" (futex), "m" (timeout),	      \
+			 "2" (private)					      \
+		       : "memory", "cx", "cc", "r10", "r11");		      \
+     result; })
 
 #if defined NOT_IN_libc || defined UP
-# define lll_trylock(futex) lll_mutex_trylock (futex)
-# define lll_lock(futex) lll_mutex_lock (futex)
-# define lll_unlock(futex) lll_mutex_unlock (futex)
+# define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t"		      \
+				"jne 1f\n\t"
 #else
-/* Special versions of the macros for use in libc itself.  They avoid
-   the lock prefix when the thread library is not used.
-
-   The code sequence to avoid unnecessary lock prefixes is what the AMD
-   guys suggested.  If you do not like it, bring it up with AMD.
-
-   XXX In future we might even want to avoid it on UP machines.  */
-
-# define lll_trylock(futex) \
-  ({ unsigned char ret;							      \
-     __asm__ __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"	      \
-		       "je 0f\n\t"					      \
-		       "lock; cmpxchgl %2, %1\n\t"			      \
-		       "jmp 1f\n"					      \
-		       "0:\tcmpxchgl %2, %1\n\t"			      \
-		       "1:setne %0"					      \
-		       : "=a" (ret), "=m" (futex)			      \
-		       : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
-			 "0" (LLL_MUTEX_LOCK_INITIALIZER)		      \
-		       : "memory");					      \
-     ret; })
-
-
-# define lll_lock(futex) \
-  (void) ({ int ignore1, ignore2, ignore3;				      \
-	    __asm__ __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
-			      "je 0f\n\t"				      \
-			      "lock; cmpxchgl %0, %2\n\t"		      \
-			      "jnz 1f\n\t"				      \
-			      "jmp 2f\n"				      \
-			      "0:\tcmpxchgl %0, %2\n\t"			      \
-			      "jnz 1f\n\t"				      \
-			      ".subsection 1\n"				      \
-			      "1:\tleaq %2, %%rdi\n\t"			      \
-			      "subq $128, %%rsp\n\t"			      \
-			      "callq __lll_mutex_lock_wait\n\t"		      \
-			      "addq $128, %%rsp\n\t"			      \
-			      "jmp 2f\n\t"				      \
-			      ".previous\n"				      \
-			      "2:"					      \
-			      : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\
-				"=a" (ignore3)				      \
-			      : "0" (1), "m" (futex), "3" (0)		      \
-			      : "cx", "r11", "cc", "memory"); })
-
-
-# define lll_unlock(futex) \
-  (void) ({ int ignore;							      \
-            __asm__ __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
-			      "je 0f\n\t"				      \
-			      "lock; decl %0\n\t"			      \
-			      "jne 1f\n\t"				      \
-			      "jmp 2f\n"				      \
-			      "0:\tdecl %0\n\t"				      \
-			      "jne 1f\n\t"				      \
-			      ".subsection 1\n"				      \
-			      "1:\tleaq %0, %%rdi\n\t"			      \
-			      "subq $128, %%rsp\n\t"			      \
-			      "callq __lll_mutex_unlock_wake\n\t"	      \
-			      "addq $128, %%rsp\n\t"			      \
-			      "jmp 2f\n\t"				      \
-			      ".previous\n"				      \
-			      "2:"					      \
-			      : "=m" (futex), "=&D" (ignore)		      \
-			      : "m" (futex)				      \
-			      : "ax", "cx", "r11", "cc", "memory"); })
+# define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
+				"je 0f\n\t"				      \
+				"lock; decl %0\n\t"			      \
+				"jne 1f\n\t"				      \
+				"jmp 24f\n\t"				      \
+				"0:\tdecl %0\n\t"			      \
+				"jne 1f\n\t"
 #endif
 
+#define lll_unlock(futex, private) \
+  (void)								      \
+    ({ int ignore;							      \
+       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
+	 __asm__ __volatile (__lll_unlock_asm_start			      \
+			   ".subsection 1\n\t"				      \
+			   ".type _L_unlock_%=, @function\n"		      \
+			   "_L_unlock_%=:\n"				      \
+			   "1:\tleaq %0, %%rdi\n"			      \
+			   "2:\tsubq $128, %%rsp\n"			      \
+			   "3:\tcallq __lll_unlock_wake_private\n"	      \
+			   "4:\taddq $128, %%rsp\n"			      \
+			   "5:\tjmp 24f\n"				      \
+			   "6:\t.size _L_unlock_%=, 6b-1b\n\t"		      \
+			   ".previous\n"				      \
+			   LLL_STUB_UNWIND_INFO_5			      \
+			   "24:"					      \
+			   : "=m" (futex), "=&D" (ignore)		      \
+			   : "m" (futex)				      \
+			   : "ax", "cx", "r11", "cc", "memory");	      \
+       else								      \
+	 __asm__ __volatile (__lll_unlock_asm_start			      \
+			   ".subsection 1\n\t"				      \
+			   ".type _L_unlock_%=, @function\n"		      \
+			   "_L_unlock_%=:\n"				      \
+			   "1:\tleaq %0, %%rdi\n"			      \
+			   "2:\tsubq $128, %%rsp\n"			      \
+			   "3:\tcallq __lll_unlock_wake\n"		      \
+			   "4:\taddq $128, %%rsp\n"			      \
+			   "5:\tjmp 24f\n"				      \
+			   "6:\t.size _L_unlock_%=, 6b-1b\n\t"		      \
+			   ".previous\n"				      \
+			   LLL_STUB_UNWIND_INFO_5			      \
+			   "24:"					      \
+			   : "=m" (futex), "=&D" (ignore)		      \
+			   : "m" (futex), "S" (private)			      \
+			   : "ax", "cx", "r11", "cc", "memory");	      \
+    })
+
+#define lll_robust_unlock(futex, private) \
+  do									      \
+    {									      \
+      int ignore;							      \
+      __asm__ __volatile (LOCK_INSTR "andl %2, %0\n\t"			      \
+			"jne 1f\n\t"					      \
+			".subsection 1\n\t"				      \
+			".type _L_robust_unlock_%=, @function\n"	      \
+			"_L_robust_unlock_%=:\n"			      \
+			"1:\tleaq %0, %%rdi\n"				      \
+			"2:\tsubq $128, %%rsp\n"			      \
+			"3:\tcallq __lll_unlock_wake\n"			      \
+			"4:\taddq $128, %%rsp\n"			      \
+			"5:\tjmp 24f\n"					      \
+			"6:\t.size _L_robust_unlock_%=, 6b-1b\n\t"	      \
+			".previous\n"					      \
+			LLL_STUB_UNWIND_INFO_5				      \
+			"24:"						      \
+			: "=m" (futex), "=&D" (ignore)			      \
+			: "i" (FUTEX_WAITERS), "m" (futex),		      \
+			  "S" (private)					      \
+			: "ax", "cx", "r11", "cc", "memory");		      \
+    }									      \
+  while (0)
+
+#define lll_robust_dead(futex, private) \
+  do									      \
+    {									      \
+      int ignore;							      \
+      __asm__ __volatile (LOCK_INSTR "orl %3, (%2)\n\t"			      \
+			"syscall"					      \
+			: "=m" (futex), "=a" (ignore)			      \
+			: "D" (&(futex)), "i" (FUTEX_OWNER_DIED),	      \
+			  "S" (__lll_private_flag (FUTEX_WAKE, private)),     \
+			  "1" (__NR_futex), "d" (1)			      \
+			: "cx", "r11", "cc", "memory");			      \
+    }									      \
+  while (0)
+
+/* Returns non-zero if error happened, zero if success.  */
+#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \
+  ({ int __res;								      \
+     register int __nr_move __asm__ ("r10") = nr_move;			      \
+     register void *__mutex __asm__ ("r8") = mutex;			      \
+     register int __val __asm__ ("r9") = val;				      \
+     __asm__ __volatile ("syscall"					      \
+		       : "=a" (__res)					      \
+		       : "0" (__NR_futex), "D" ((void *) ftx),		      \
+			 "S" (__lll_private_flag (FUTEX_CMP_REQUEUE,	      \
+						  private)), "d" (nr_wake),   \
+			 "r" (__nr_move), "r" (__mutex), "r" (__val)	      \
+		       : "cx", "r11", "cc", "memory");			      \
+     __res < 0; })
 
 #define lll_islocked(futex) \
-  (futex != LLL_MUTEX_LOCK_INITIALIZER)
+  (futex != LLL_LOCK_INITIALIZER)
 
 
 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
@@ -318,25 +593,6 @@ extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
       }									      \
     __result; })
 
-
-/* Conditional variable handling.  */
-
-extern void __lll_cond_wait (pthread_cond_t *cond) attribute_hidden;
-extern int __lll_cond_timedwait (pthread_cond_t *cond,
-				 const struct timespec *abstime)
-     attribute_hidden;
-extern void __lll_cond_wake (pthread_cond_t *cond) attribute_hidden;
-extern void __lll_cond_broadcast (pthread_cond_t *cond) attribute_hidden;
-
-
-#define lll_cond_wait(cond) \
-  __lll_cond_wait (cond)
-#define lll_cond_timedwait(cond, abstime) \
-  __lll_cond_timedwait (cond, abstime)
-#define lll_cond_wake(cond) \
-  __lll_cond_wake (cond)
-#define lll_cond_broadcast(cond) \
-  __lll_cond_broadcast (cond)
-
+#endif  /* !__ASSEMBLER__ */
 
 #endif	/* lowlevellock.h */
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S
index c20ef73e7..df4949615 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S
@@ -16,6 +16,8 @@
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
    02111-1307 USA.  */
 
+#include <tcb-offsets.h>
+
 #define SAVE_PID \
 	movl	%fs:PID, %esi;						      \
 	movl	%esi, %edx;						      \
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
index f6e15a2d7..15ad534fa 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,17 +18,9 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelbarrier.h>
 
-#define FUTEX_WAIT	0
-#define FUTEX_WAKE	1
-
-#ifndef UP
-# define LOCK lock
-#else
-# define LOCK
-#endif
-
 
 	.text
 
@@ -64,9 +56,10 @@ pthread_barrier_wait:
 	   if the CURR_EVENT memory has meanwhile been changed.  */
 7:
 #if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	movl	PRIVATE(%rdi), %esi
 #else
 	movl	$FUTEX_WAIT, %esi
+	orl	PRIVATE(%rdi), %esi
 #endif
 	xorq	%r10, %r10
 8:	movl	$SYS_futex, %eax
@@ -115,6 +108,7 @@ pthread_barrier_wait:
 	   so 0x7fffffff is the highest value.  */
 	movl	$0x7fffffff, %edx
 	movl	$FUTEX_WAKE, %esi
+	orl	PRIVATE(%rdi), %esi
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -139,21 +133,29 @@ pthread_barrier_wait:
 
 	retq
 
-1:	addq	$MUTEX, %rdi
-	callq	__lll_mutex_lock_wait
+1:	movl	PRIVATE(%rdi), %esi
+	addq	$MUTEX, %rdi
+	xorl	$LLL_SHARED, %esi
+	callq	__lll_lock_wait
 	subq	$MUTEX, %rdi
 	jmp	2b
 
-4:	addq	$MUTEX, %rdi
-	callq	__lll_mutex_unlock_wake
+4:	movl	PRIVATE(%rdi), %esi
+	addq	$MUTEX, %rdi
+	xorl	$LLL_SHARED, %esi
+	callq	__lll_unlock_wake
 	jmp	5b
 
-6:	addq	$MUTEX, %rdi
-	callq	__lll_mutex_unlock_wake
+6:	movl	PRIVATE(%rdi), %esi
+	addq	$MUTEX, %rdi
+	xorl	$LLL_SHARED, %esi
+	callq	__lll_unlock_wake
 	subq	$MUTEX, %rdi
 	jmp	7b
 
-9:	addq	$MUTEX, %rdi
-	callq	__lll_mutex_unlock_wake
+9:	movl	PRIVATE(%rdi), %esi
+	addq	$MUTEX, %rdi
+	xorl	$LLL_SHARED, %esi
+	callq	__lll_unlock_wake
 	jmp	10b
 	.size	pthread_barrier_wait,.-pthread_barrier_wait
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index d8ebdfab8..0f8037ba2 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -1,4 +1,5 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2009
+   Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,21 +19,11 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <bits/kernel-features.h>
-
-#ifdef UP
-# define LOCK
-#else
-# define LOCK lock
-#endif
-
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-#define FUTEX_REQUEUE		3
-#define FUTEX_CMP_REQUEUE	4
-
-#define EINVAL			22
+#include <pthread-pi-defines.h>
+#include <pthread-errnos.h>
 
 
 	.text
@@ -78,8 +69,23 @@ __pthread_cond_broadcast:
 8:	cmpq	$-1, %r8
 	je	9f
 
+	/* Do not use requeue for pshared condvars.  */
+	testl	$PS_BIT, MUTEX_KIND(%r8)
+	jne	9f
+
+	/* Requeue to a PI mutex if the PI bit is set.  */
+	movl	MUTEX_KIND(%r8), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	je	81f
+
 	/* Wake up all threads.  */
-	movl	$FUTEX_CMP_REQUEUE, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi
+#else
+	movl	%fs:PRIVATE_FUTEX, %esi
+	orl	$FUTEX_CMP_REQUEUE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	movl	$1, %edx
 	movl	$0x7fffffff, %r10d
@@ -94,6 +100,20 @@ __pthread_cond_broadcast:
 10:	xorl	%eax, %eax
 	retq
 
+	/* Wake up all threads.  */
+81:	movl	$(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
+	movl	$SYS_futex, %eax
+	movl	$1, %edx
+	movl	$0x7fffffff, %r10d
+	syscall
+
+	/* For any kind of error, which mainly is EAGAIN, we try again
+	   with WAKE.  The general test also covers running on old
+	   kernels.  */
+	cmpq	$-4095, %rax
+	jb	10b
+	jmp	9f
+
 	.align	16
 	/* Unlock.  */
 4:	LOCK
@@ -108,7 +128,11 @@ __pthread_cond_broadcast:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
 #endif
@@ -116,21 +140,38 @@ __pthread_cond_broadcast:
 
 	/* Unlock in loop requires wakeup.  */
 5:	addq	$cond_lock-cond_futex, %rdi
-	callq	__lll_mutex_unlock_wake
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
 	jmp	6b
 
 	/* Unlock in loop requires wakeup.  */
 7:	addq	$cond_lock-cond_futex, %rdi
-	callq	__lll_mutex_unlock_wake
+	cmpq	$-1, %r8
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
 	subq	$cond_lock-cond_futex, %rdi
 	jmp	8b
 
 9:	/* The futex requeue functionality is not available.  */
+	cmpq	$-1, %r8
 	movl	$0x7fffffff, %edx
-	movl	$FUTEX_WAKE, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 	jmp	10b
 	.size	__pthread_cond_broadcast, .-__pthread_cond_broadcast
 weak_alias(__pthread_cond_broadcast, pthread_cond_broadcast)
-
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index c7cc3ddd8..568c98470 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,20 +18,11 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelcond.h>
+#include <pthread-pi-defines.h>
 #include <bits/kernel-features.h>
-
-#ifdef UP
-# define LOCK
-#else
-# define LOCK lock
-#endif
-
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-#define FUTEX_REQUEUE		3
-
-#define EINVAL			22
+#include <pthread-errnos.h>
 
 
 	.text
@@ -64,9 +55,66 @@ __pthread_cond_signal:
 	addl	$1, (%rdi)
 
 	/* Wake up one thread.  */
-	movl	$FUTEX_WAKE, %esi
-	movl	$SYS_futex, %eax
+	cmpq	$-1, dep_mutex(%r8)
+	movl	$FUTEX_WAKE_OP, %esi
 	movl	$1, %edx
+	movl	$SYS_futex, %eax
+	je	8f
+
+	/* Get the address of the mutex used.  */
+	movq    dep_mutex(%r8), %rcx
+	movl	MUTEX_KIND(%rcx), %r11d
+	andl	$(ROBUST_BIT|PI_BIT), %r11d
+	cmpl	$PI_BIT, %r11d
+	je	9f
+
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
+#else
+	orl	%fs:PRIVATE_FUTEX, %esi
+#endif
+
+8:	movl	$1, %r10d
+#if cond_lock != 0
+	addq	$cond_lock, %r8
+#endif
+	movl	$FUTEX_OP_CLEAR_WAKE_IF_GT_ONE, %r9d
+	syscall
+#if cond_lock != 0
+	subq	$cond_lock, %r8
+#endif
+	/* For any kind of error, we try again with WAKE.
+	   The general test also covers running on old kernels.  */
+	cmpq	$-4095, %rax
+	jae	7f
+
+	xorl	%eax, %eax
+	retq
+
+	/* Wake up one thread and requeue none in the PI Mutex case.  */
+9:	movl	$(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
+	movq	%rcx, %r8
+	xorq	%r10, %r10
+	movl	(%rdi), %r9d	// XXX Can this be right?
+	syscall
+
+	leaq	-cond_futex(%rdi), %r8
+
+	/* For any kind of error, we try again with WAKE.
+	   The general test also covers running on old kernels.  */
+	cmpq	$-4095, %rax
+	jb	4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %esi
+#else
+	andl	%fs:PRIVATE_FUTEX, %esi
+#endif
+	orl	$FUTEX_WAKE, %esi
+	movl	$SYS_futex, %eax
+	/* %rdx should be 1 already from $FUTEX_WAKE_OP syscall.
+	movl	$1, %edx  */
 	syscall
 
 	/* Unlock.  */
@@ -86,7 +134,11 @@ __pthread_cond_signal:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
 #endif
@@ -95,7 +147,14 @@ __pthread_cond_signal:
 	/* Unlock in loop requires wakeup.  */
 5:
 	movq	%r8, %rdi
-	callq	__lll_mutex_unlock_wake
+#if cond_lock != 0
+	addq	$cond_lock, %rdi
+#endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
 	jmp	6b
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
 weak_alias(__pthread_cond_signal, pthread_cond_signal)
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index f0dcdb750..427a723cb 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,18 +18,12 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelcond.h>
+#include <pthread-pi-defines.h>
 #include <pthread-errnos.h>
-#include <tcb-offsets.h>
 
-#ifdef UP
-# define LOCK
-#else
-# define LOCK lock
-#endif
-
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
+#include <bits/kernel-features.h>
 
 /* For the calculation see asm/vsyscall.h.  */
 #define VSYSCALL_ADDR_vgettimeofday	0xffffffffff600000
@@ -37,6 +31,7 @@
 
 	.text
 
+
 /* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
 			       const struct timespec *abstime)  */
 	.globl	__pthread_cond_timedwait
@@ -44,38 +39,57 @@
 	.align	16
 __pthread_cond_timedwait:
 .LSTARTCODE:
+	cfi_startproc
+#ifdef SHARED
+	cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
+			DW.ref.__gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
+#else
+	cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
+#endif
+
 	pushq	%r12
-.Lpush_r12:
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r12, 0)
 	pushq	%r13
-.Lpush_r13:
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r13, 0)
 	pushq	%r14
-.Lpush_r14:
-#define FRAME_SIZE 80
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r14, 0)
+	pushq	%r15
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r15, 0)
+#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
+# define FRAME_SIZE 32
+#else
+# define FRAME_SIZE 48
+#endif
 	subq	$FRAME_SIZE, %rsp
-.Lsubq:
+	cfi_adjust_cfa_offset(FRAME_SIZE)
+	cfi_remember_state
 
 	cmpq	$1000000000, 8(%rdx)
 	movl	$EINVAL, %eax
-	jae	18f
+	jae	48f
 
 	/* Stack frame:
 
-	   rsp + 80
-	            +--------------------------+
-	   rsp + 48 | cleanup buffer           |
-	            +--------------------------+
-	   rsp + 40 | old wake_seq value       |
-	            +--------------------------+
-	   rsp + 24 | timeout value            |
-	            +--------------------------+
+	   rsp + 48
+		    +--------------------------+
+	   rsp + 32 | timeout value            |
+		    +--------------------------+
+	   rsp + 24 | old wake_seq value       |
+		    +--------------------------+
 	   rsp + 16 | mutex pointer            |
-	            +--------------------------+
+		    +--------------------------+
 	   rsp +  8 | condvar pointer          |
-	            +--------------------------+
+		    +--------------------------+
 	   rsp +  4 | old broadcast_seq value  |
-	            +--------------------------+
+		    +--------------------------+
 	   rsp +  0 | old cancellation mode    |
-	            +--------------------------+
+		    +--------------------------+
 	*/
 
 	cmpq	$-1, dep_mutex(%rdi)
@@ -88,8 +102,18 @@ __pthread_cond_timedwait:
 	je	22f
 	movq	%rsi, dep_mutex(%rdi)
 
+22:
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+#  ifdef PIC
+	cmpl	$0, __have_futex_clock_realtime(%rip)
+#  else
+	cmpl	$0, __have_futex_clock_realtime
+#  endif
+	je	.Lreltmo
+#endif
+
 	/* Get internal lock.  */
-22:	movl	$1, %esi
+	movl	$1, %esi
 	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
@@ -97,89 +121,29 @@ __pthread_cond_timedwait:
 #else
 	cmpxchgl %esi, cond_lock(%rdi)
 #endif
-	jnz	1f
+	jnz	31f
 
 	/* Unlock the mutex.  */
-2:	movq	16(%rsp), %rdi
+32:	movq	16(%rsp), %rdi
 	xorl	%esi, %esi
 	callq	__pthread_mutex_unlock_usercnt
 
 	testl	%eax, %eax
-	jne	16f
+	jne	46f
 
 	movq	8(%rsp), %rdi
 	incq	total_seq(%rdi)
 	incl	cond_futex(%rdi)
-	addl	$(1 << clock_bits), cond_nwaiters(%rdi)
-
-	/* Install cancellation handler.  */
-#ifdef __PIC__
-	leaq	__condvar_cleanup(%rip), %rsi
-#else
-	leaq	__condvar_cleanup, %rsi
-#endif
-	leaq	48(%rsp), %rdi
-	movq	%rsp, %rdx
-	callq	__pthread_cleanup_push
+	addl	$(1 << nwaiters_shift), cond_nwaiters(%rdi)
 
 	/* Get and store current wakeup_seq value.  */
 	movq	8(%rsp), %rdi
 	movq	wakeup_seq(%rdi), %r9
 	movl	broadcast_seq(%rdi), %edx
-	movq	%r9, 40(%rsp)
+	movq	%r9, 24(%rsp)
 	movl	%edx, 4(%rsp)
 
-	/* Get the current time.  */
-8:
-#ifdef __NR_clock_gettime
-	/* Get the clock number.  Note that the field in the condvar
-	   structure stores the number minus 1.  */
-	movq	8(%rsp), %rdi
-	movl	cond_nwaiters(%rdi), %edi
-	andl	$((1 << clock_bits) - 1), %edi
-	/* Only clocks 0 and 1 are allowed so far.  Both are handled in the
-	   kernel.  */
-	leaq	24(%rsp), %rsi
-	movl	$__NR_clock_gettime, %eax
-	syscall
-# ifndef __ASSUME_POSIX_TIMERS
-	cmpq	$-ENOSYS, %rax
-	je	19f
-# endif
-
-	/* Compute relative timeout.  */
-	movq	(%r13), %rcx
-	movq	8(%r13), %rdx
-	subq	24(%rsp), %rcx
-	subq	32(%rsp), %rdx
-#else
-	leaq	24(%rsp), %rdi
-	xorl	%esi, %esi
-	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
-	callq	*%rax
-
-	/* Compute relative timeout.  */
-	movq	32(%rsp), %rax
-	movl	$1000, %edx
-	mul	%rdx		/* Milli seconds to nano seconds.  */
-	movq	(%r13), %rcx
-	movq	8(%r13), %rdx
-	subq	24(%rsp), %rcx
-	subq	%rax, %rdx
-#endif
-	jns	12f
-	addq	$1000000000, %rdx
-	decq	%rcx
-12:	testq	%rcx, %rcx
-	movq	8(%rsp), %rdi
-	movq	$-ETIMEDOUT, %r14
-	js	6f
-
-	/* Store relative timeout.  */
-21:	movq	%rcx, 24(%rsp)
-	movq	%rdx, 32(%rsp)
-
-	movl	cond_futex(%rdi), %r12d
+38:	movl	cond_futex(%rdi), %r12d
 
 	/* Unlock.  */
 	LOCK
@@ -188,25 +152,67 @@ __pthread_cond_timedwait:
 #else
 	decl	cond_lock(%rdi)
 #endif
-	jne	3f
+	jne	33f
 
-4:	callq	__pthread_enable_asynccancel
+.LcleanupSTART1:
+34:	callq	__pthread_enable_asynccancel
 	movl	%eax, (%rsp)
 
-	leaq	24(%rsp), %r10
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	movq	%r13, %r10
+	movl	$FUTEX_WAIT_BITSET, %esi
+	cmpq	$-1, dep_mutex(%rdi)
+	je	60f
+
+	movq	dep_mutex(%rdi), %r8
+	/* Requeue to a non-robust PI mutex if the PI bit is set and
+	the robust bit is not set.  */
+	movl	MUTEX_KIND(%r8), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	jne	61f
+
+	movl	$(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
+	xorl	%eax, %eax
+	/* The following only works like this because we only support
+	   two clocks, represented using a single bit.  */
+	testl	$1, cond_nwaiters(%rdi)
+	movl	$FUTEX_CLOCK_REALTIME, %edx
+	cmove	%edx, %eax
+	orl	%eax, %esi
+	movq	%r12, %rdx
+	addq	$cond_futex, %rdi
+	movl	$SYS_futex, %eax
+	syscall
+
+	movl	$1, %r15d
+#ifdef __ASSUME_REQUEUE_PI
+	jmp	62f
 #else
-	movl	$FUTEX_WAIT, %esi
+	cmpq	$-4095, %rax
+	jnae	62f
+
+	subq	$cond_futex, %rdi
 #endif
+
+61:	movl	$(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
+60:	xorl	%r15d, %r15d
+	xorl	%eax, %eax
+	/* The following only works like this because we only support
+	   two clocks, represented using a single bit.  */
+	testl	$1, cond_nwaiters(%rdi)
+	movl	$FUTEX_CLOCK_REALTIME, %edx
+	movl	$0xffffffff, %r9d
+	cmove	%edx, %eax
+	orl	%eax, %esi
 	movq	%r12, %rdx
 	addq	$cond_futex, %rdi
 	movl	$SYS_futex, %eax
 	syscall
-	movq	%rax, %r14
+62:	movq	%rax, %r14
 
 	movl	(%rsp), %edi
 	callq	__pthread_disable_asynccancel
+.LcleanupEND1:
 
 	/* Lock.  */
 	movq	8(%rsp), %rdi
@@ -218,120 +224,158 @@ __pthread_cond_timedwait:
 #else
 	cmpxchgl %esi, cond_lock(%rdi)
 #endif
-	jne	5f
+	jne	35f
 
-6:	movl	broadcast_seq(%rdi), %edx
+36:	movl	broadcast_seq(%rdi), %edx
 
 	movq	woken_seq(%rdi), %rax
 
 	movq	wakeup_seq(%rdi), %r9
 
 	cmpl	4(%rsp), %edx
-	jne	23f
+	jne	53f
 
-	cmpq	40(%rsp), %r9
-	jbe	15f
+	cmpq	24(%rsp), %r9
+	jbe	45f
 
 	cmpq	%rax, %r9
-	ja	9f
+	ja	39f
 
-15:	cmpq	$-ETIMEDOUT, %r14
-	jne	8b
+45:	cmpq	$-ETIMEDOUT, %r14
+	jne	38b
 
-13:	incq	wakeup_seq(%rdi)
+99:	incq	wakeup_seq(%rdi)
 	incl	cond_futex(%rdi)
 	movl	$ETIMEDOUT, %r14d
-	jmp	14f
+	jmp	44f
 
-23:	xorq	%r14, %r14
-	jmp	24f
+53:	xorq	%r14, %r14
+	jmp	54f
 
-9:	xorq	%r14, %r14
-14:	incq	woken_seq(%rdi)
+39:	xorq	%r14, %r14
+44:	incq	woken_seq(%rdi)
 
-24:	subl	$(1 << clock_bits), cond_nwaiters(%rdi)
+54:	subl	$(1 << nwaiters_shift), cond_nwaiters(%rdi)
 
 	/* Wake up a thread which wants to destroy the condvar object.  */
 	cmpq	$0xffffffffffffffff, total_seq(%rdi)
-	jne	25f
+	jne	55f
 	movl	cond_nwaiters(%rdi), %eax
-	andl	$~((1 << clock_bits) - 1), %eax
-	jne	25f
+	andl	$~((1 << nwaiters_shift) - 1), %eax
+	jne	55f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
-25:	LOCK
+55:	LOCK
 #if cond_lock == 0
 	decl	(%rdi)
 #else
 	decl	cond_lock(%rdi)
 #endif
-	jne	10f
+	jne	40f
 
-	/* Remove cancellation handler.  */
-11:	movq	48+CLEANUP_PREV(%rsp), %rdx
-	movq	%rdx, %fs:CLEANUP
+	/* If requeue_pi is used the kernel performs the locking of the
+	   mutex. */
+41:	movq	16(%rsp), %rdi
+	testl	%r15d, %r15d
+	jnz	64f
 
-	movq	16(%rsp), %rdi
 	callq	__pthread_mutex_cond_lock
 
-	testq	%rax, %rax
+63:	testq	%rax, %rax
 	cmoveq	%r14, %rax
 
-18:	addq	$FRAME_SIZE, %rsp
-.Laddq:
+48:	addq	$FRAME_SIZE, %rsp
+	cfi_adjust_cfa_offset(-FRAME_SIZE)
+	popq	%r15
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r15)
 	popq	%r14
-.Lpop_r14:
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r14)
 	popq	%r13
-.Lpop_r13:
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r13)
 	popq	%r12
-.Lpop_r12:
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r12)
 
 	retq
 
+	cfi_restore_state
+
+64:	callq	__pthread_mutex_cond_lock_adjust
+	movq	%r14, %rax
+	jmp	48b
+
 	/* Initial locking failed.  */
-1:
-.LSbl1:
+31:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
-	jmp	2b
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
+	jmp	32b
 
 	/* Unlock in loop requires wakeup.  */
-3:
+33:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
-	jmp	4b
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
+	jmp	34b
 
 	/* Locking in loop failed.  */
-5:
+35:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
 #endif
-	jmp	6b
+	jmp	36b
 
 	/* Unlock after loop requires wakeup.  */
-10:
+40:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
-	jmp	11b
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
+	jmp	41b
 
 	/* The initial unlocking of the mutex failed.  */
-16:	movq	8(%rsp), %rdi
+46:	movq	8(%rsp), %rdi
 	movq	%rax, (%rsp)
 	LOCK
 #if cond_lock == 0
@@ -339,30 +383,239 @@ __pthread_cond_timedwait:
 #else
 	decl	cond_lock(%rdi)
 #endif
-	jne	17f
+	jne	47f
 
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
+
+47:	movq	(%rsp), %rax
+	jmp	48b
+
 
-17:	movq	(%rsp), %rax
-	jmp	18b
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+.Lreltmo:
+	xorl	%r15d, %r15d
 
-#if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS
+	/* Get internal lock.  */
+	movl	$1, %esi
+	xorl	%eax, %eax
+	LOCK
+# if cond_lock == 0
+	cmpxchgl %esi, (%rdi)
+# else
+	cmpxchgl %esi, cond_lock(%rdi)
+# endif
+	jnz	1f
+
+	/* Unlock the mutex.  */
+2:	movq	16(%rsp), %rdi
+	xorl	%esi, %esi
+	callq	__pthread_mutex_unlock_usercnt
+
+	testl	%eax, %eax
+	jne	46b
+
+	movq	8(%rsp), %rdi
+	incq	total_seq(%rdi)
+	incl	cond_futex(%rdi)
+	addl	$(1 << nwaiters_shift), cond_nwaiters(%rdi)
+
+	/* Get and store current wakeup_seq value.  */
+	movq	8(%rsp), %rdi
+	movq	wakeup_seq(%rdi), %r9
+	movl	broadcast_seq(%rdi), %edx
+	movq	%r9, 24(%rsp)
+	movl	%edx, 4(%rsp)
+
+	/* Get the current time.  */
+8:
+# ifdef __NR_clock_gettime
+	/* Get the clock number.  Note that the field in the condvar
+	   structure stores the number minus 1.  */
+	movq	8(%rsp), %rdi
+	movl	cond_nwaiters(%rdi), %edi
+	andl	$((1 << nwaiters_shift) - 1), %edi
+	/* Only clocks 0 and 1 are allowed so far.  Both are handled in the
+	   kernel.  */
+	leaq	32(%rsp), %rsi
+#  ifdef SHARED
+	movq	__vdso_clock_gettime@GOTPCREL(%rip), %rax
+	movq	(%rax), %rax
+	PTR_DEMANGLE (%rax)
+	jz	26f
+	call	*%rax
+	jmp	27f
+#  endif
+26:	movl	$__NR_clock_gettime, %eax
+	syscall
+27:
+#  ifndef __ASSUME_POSIX_TIMERS
+	cmpq	$-ENOSYS, %rax
+	je	19f
+#  endif
+
+	/* Compute relative timeout.  */
+	movq	(%r13), %rcx
+	movq	8(%r13), %rdx
+	subq	32(%rsp), %rcx
+	subq	40(%rsp), %rdx
+# else
+	leaq	24(%rsp), %rdi
+	xorl	%esi, %esi
+	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
+	callq	*%rax
+
+	/* Compute relative timeout.  */
+	movq	40(%rsp), %rax
+	movl	$1000, %edx
+	mul	%rdx		/* Milli seconds to nano seconds.  */
+	movq	(%r13), %rcx
+	movq	8(%r13), %rdx
+	subq	32(%rsp), %rcx
+	subq	%rax, %rdx
+# endif
+	jns	12f
+	addq	$1000000000, %rdx
+	decq	%rcx
+12:	testq	%rcx, %rcx
+	movq	8(%rsp), %rdi
+	movq	$-ETIMEDOUT, %r14
+	js	6f
+
+	/* Store relative timeout.  */
+21:	movq	%rcx, 32(%rsp)
+	movq	%rdx, 40(%rsp)
+
+	movl	cond_futex(%rdi), %r12d
+
+	/* Unlock.  */
+	LOCK
+# if cond_lock == 0
+	decl	(%rdi)
+# else
+	decl	cond_lock(%rdi)
+# endif
+	jne	3f
+
+.LcleanupSTART2:
+4:	callq	__pthread_enable_asynccancel
+	movl	%eax, (%rsp)
+
+	leaq	32(%rsp), %r10
+	cmpq	$-1, dep_mutex(%rdi)
+	movq	%r12, %rdx
+# ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT, %eax
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+# else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+#  if FUTEX_WAIT != 0
+	orl	$FUTEX_WAIT, %esi
+#  endif
+# endif
+	addq	$cond_futex, %rdi
+	movl	$SYS_futex, %eax
+	syscall
+	movq	%rax, %r14
+
+	movl	(%rsp), %edi
+	callq	__pthread_disable_asynccancel
+.LcleanupEND2:
+
+	/* Lock.  */
+	movq	8(%rsp), %rdi
+	movl	$1, %esi
+	xorl	%eax, %eax
+	LOCK
+# if cond_lock == 0
+	cmpxchgl %esi, (%rdi)
+# else
+	cmpxchgl %esi, cond_lock(%rdi)
+# endif
+	jne	5f
+
+6:	movl	broadcast_seq(%rdi), %edx
+
+	movq	woken_seq(%rdi), %rax
+
+	movq	wakeup_seq(%rdi), %r9
+
+	cmpl	4(%rsp), %edx
+	jne	53b
+
+	cmpq	24(%rsp), %r9
+	jbe	15f
+
+	cmpq	%rax, %r9
+	ja	39b
+
+15:	cmpq	$-ETIMEDOUT, %r14
+	jne	8b
+
+	jmp	99b
+
+	/* Initial locking failed.  */
+1:
+# if cond_lock != 0
+	addq	$cond_lock, %rdi
+# endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
+	jmp	2b
+
+	/* Unlock in loop requires wakeup.  */
+3:
+# if cond_lock != 0
+	addq	$cond_lock, %rdi
+# endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
+	jmp	4b
+
+	/* Locking in loop failed.  */
+5:
+# if cond_lock != 0
+	addq	$cond_lock, %rdi
+# endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
+# if cond_lock != 0
+	subq	$cond_lock, %rdi
+# endif
+	jmp	6b
+
+# if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS
 	/* clock_gettime not available.  */
-19:	leaq	24(%rsp), %rdi
+19:	leaq	32(%rsp), %rdi
 	xorl	%esi, %esi
 	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
 	callq	*%rax
 
 	/* Compute relative timeout.  */
-	movq	32(%rsp), %rax
+	movq	40(%rsp), %rax
 	movl	$1000, %edx
 	mul	%rdx		/* Milli seconds to nano seconds.  */
 	movq	(%r13), %rcx
 	movq	8(%r13), %rdx
-	subq	24(%rsp), %rcx
+	subq	32(%rsp), %rcx
 	subq	%rax, %rdx
 	jns	20f
 	addq	$1000000000, %rdx
@@ -372,97 +625,187 @@ __pthread_cond_timedwait:
 	movq	$-ETIMEDOUT, %r14
 	js	6b
 	jmp	21b
+# endif
 #endif
-.LENDCODE:
 	.size	__pthread_cond_timedwait, .-__pthread_cond_timedwait
 weak_alias(__pthread_cond_timedwait, pthread_cond_timedwait)
 
 
-	.section .eh_frame,"a",@progbits
-.LSTARTFRAME:
-	.long	.LENDCIE-.LSTARTCIE		# Length of the CIE.
-.LSTARTCIE:
-	.long	0				# CIE ID.
-	.byte	1				# Version number.
-#ifdef SHARED
-	.string	"zR"				# NUL-terminated augmentation
-						# string.
+	.align	16
+	.type	__condvar_cleanup2, @function
+__condvar_cleanup2:
+	/* Stack frame:
+
+	   rsp + 72
+		    +--------------------------+
+	   rsp + 64 | %r12                     |
+		    +--------------------------+
+	   rsp + 56 | %r13                     |
+		    +--------------------------+
+	   rsp + 48 | %r14                     |
+		    +--------------------------+
+	   rsp + 24 | unused                   |
+		    +--------------------------+
+	   rsp + 16 | mutex pointer            |
+		    +--------------------------+
+	   rsp +  8 | condvar pointer          |
+		    +--------------------------+
+	   rsp +  4 | old broadcast_seq value  |
+		    +--------------------------+
+	   rsp +  0 | old cancellation mode    |
+		    +--------------------------+
+	*/
+
+	movq	%rax, 24(%rsp)
+
+	/* Get internal lock.  */
+	movq	8(%rsp), %rdi
+	movl	$1, %esi
+	xorl	%eax, %eax
+	LOCK
+#if cond_lock == 0
+	cmpxchgl %esi, (%rdi)
 #else
-	.ascii	"\0"				# NUL-terminated augmentation
-						# string.
+	cmpxchgl %esi, cond_lock(%rdi)
 #endif
-	.uleb128 1				# Code alignment factor.
-	.sleb128 -8				# Data alignment factor.
-	.byte	16				# Return address register
-						# column.
-#ifdef SHARED
-	.uleb128 1				# Augmentation value length.
-	.byte	0x1b				# Encoding: DW_EH_PE_pcrel
-						# + DW_EH_PE_sdata4.
+	jz	1f
+
+#if cond_lock != 0
+	addq	$cond_lock, %rdi
 #endif
-	.byte 0x0c				# DW_CFA_def_cfa
-	.uleb128 7
-	.uleb128 8
-	.byte	0x90				# DW_CFA_offset, column 0x8
-	.uleb128 1
-	.align 8
-.LENDCIE:
-
-	.long	.LENDFDE-.LSTARTFDE		# Length of the FDE.
-.LSTARTFDE:
-	.long	.LSTARTFDE-.LSTARTFRAME		# CIE pointer.
-#ifdef SHARED
-	.long	.LSTARTCODE-.			# PC-relative start address
-						# of the code
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
+#if cond_lock != 0
+	subq	$cond_lock, %rdi
+#endif
+
+1:	movl	broadcast_seq(%rdi), %edx
+	cmpl	4(%rsp), %edx
+	jne	3f
+
+	/* We increment the wakeup_seq counter only if it is lower than
+	   total_seq.  If this is not the case the thread was woken and
+	   then canceled.  In this case we ignore the signal.  */
+	movq	total_seq(%rdi), %rax
+	cmpq	wakeup_seq(%rdi), %rax
+	jbe	6f
+	incq	wakeup_seq(%rdi)
+	incl	cond_futex(%rdi)
+6:	incq	woken_seq(%rdi)
+
+3:	subl	$(1 << nwaiters_shift), cond_nwaiters(%rdi)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	xorq	%r12, %r12
+	cmpq	$0xffffffffffffffff, total_seq(%rdi)
+	jne	4f
+	movl	cond_nwaiters(%rdi), %eax
+	andl	$~((1 << nwaiters_shift) - 1), %eax
+	jne	4f
+
+	cmpq	$-1, dep_mutex(%rdi)
+	leaq	cond_nwaiters(%rdi), %rdi
+	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	.long	.LSTARTCODE			# Start address of the code.
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
 #endif
-	.long	.LENDCODE-.LSTARTCODE		# Length of the code.
-#ifdef SHARED
-	.uleb128 0				# No augmentation data.
+	movl	$SYS_futex, %eax
+	syscall
+	subq	$cond_nwaiters, %rdi
+	movl	$1, %r12d
+
+4:	LOCK
+#if cond_lock == 0
+	decl	(%rdi)
+#else
+	decl	cond_lock(%rdi)
+#endif
+	je	2f
+#if cond_lock != 0
+	addq	$cond_lock, %rdi
+#endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
+
+	/* Wake up all waiters to make sure no signal gets lost.  */
+2:	testq	%r12, %r12
+	jnz	5f
+	addq	$cond_futex, %rdi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
+	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
+	syscall
+
+5:	movq	16(%rsp), %rdi
+	callq	__pthread_mutex_cond_lock
+
+	movq	24(%rsp), %rdi
+	movq	FRAME_SIZE(%rsp), %r15
+	movq	FRAME_SIZE+8(%rsp), %r14
+	movq	FRAME_SIZE+16(%rsp), %r13
+	movq	FRAME_SIZE+24(%rsp), %r12
+.LcallUR:
+	call	_Unwind_Resume@PLT
+	hlt
+.LENDCODE:
+	cfi_endproc
+	.size	__condvar_cleanup2, .-__condvar_cleanup2
+
+
+	.section .gcc_except_table,"a",@progbits
+.LexceptSTART:
+	.byte	DW_EH_PE_omit			# @LPStart format
+	.byte	DW_EH_PE_omit			# @TType format
+	.byte	DW_EH_PE_uleb128		# call-site format
+	.uleb128 .Lcstend-.Lcstbegin
+.Lcstbegin:
+	.uleb128 .LcleanupSTART1-.LSTARTCODE
+	.uleb128 .LcleanupEND1-.LcleanupSTART1
+	.uleb128 __condvar_cleanup2-.LSTARTCODE
+	.uleb128  0
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	.uleb128 .LcleanupSTART2-.LSTARTCODE
+	.uleb128 .LcleanupEND2-.LcleanupSTART2
+	.uleb128 __condvar_cleanup2-.LSTARTCODE
+	.uleb128  0
 #endif
-	.byte	0x40+.Lpush_r12-.LSTARTCODE	# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 16
-	.byte	0x8c				# DW_CFA_offset %r12
-	.uleb128 2
-	.byte	0x40+.Lpush_r13-.Lpush_r12	# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 24
-	.byte	0x8d				# DW_CFA_offset %r13
-	.uleb128 3
-	.byte	0x40+.Lpush_r14-.Lpush_r13	# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 32
-	.byte	0x84				# DW_CFA_offset %r14
-	.uleb128 4
-	.byte	0x40+.Lsubq-.Lpush_r14		# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 32+FRAME_SIZE
-	.byte	3				# DW_CFA_advance_loc2
-	.2byte	.Laddq-.Lsubq
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 32
-	.byte	0x40+.Lpop_r14-.Laddq		# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 24
-	.byte	0xce				# DW_CFA_restore %r14
-	.byte	0x40+.Lpop_r13-.Lpop_r14	# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 16
-	.byte	0xcd				# DW_CFA_restore %r13
-	.byte	0x40+.Lpop_r12-.Lpop_r13	# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 8
-	.byte	0xcc				# DW_CFA_restore %r12
-	.byte	0x40+.LSbl1-.Lpop_r12		# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 32+FRAME_SIZE
-	.byte	0x8c				# DW_CFA_offset %r12
-	.uleb128 2
-	.byte	0x8d				# DW_CFA_offset %r13
-	.uleb128 3
-	.byte	0x84				# DW_CFA_offset %r14
-	.uleb128 4
+	.uleb128 .LcallUR-.LSTARTCODE
+	.uleb128 .LENDCODE-.LcallUR
+	.uleb128 0
+	.uleb128  0
+.Lcstend:
+
+
+#ifdef SHARED
+	.hidden	DW.ref.__gcc_personality_v0
+	.weak	DW.ref.__gcc_personality_v0
+	.section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
 	.align	8
-.LENDFDE:
+	.type	DW.ref.__gcc_personality_v0, @object
+	.size	DW.ref.__gcc_personality_v0, 8
+DW.ref.__gcc_personality_v0:
+	.quad	__gcc_personality_v0
+#endif
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 544118eb7..7c488f261 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,121 +18,39 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <tcb-offsets.h>
+#include <pthread-pi-defines.h>
 
-#ifdef UP
-# define LOCK
-#else
-# define LOCK lock
-#endif
-
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
+#include <bits/kernel-features.h>
 
 
 	.text
 
-	.align	16
-	.type	__condvar_cleanup, @function
-	.globl	__condvar_cleanup
-	.hidden	__condvar_cleanup
-__condvar_cleanup:
-	pushq	%r12
-
-	/* Get internal lock.  */
-	movq	%rdi, %r8
-	movq	8(%rdi), %rdi
-	movl	$1, %esi
-	xorl	%eax, %eax
-	LOCK
-#if cond_lock == 0
-	cmpxchgl %esi, (%rdi)
-#else
-	cmpxchgl %esi, cond_lock(%rdi)
-#endif
-	jz	1f
-
-#if cond_lock != 0
-	addq	$cond_lock, %rdi
-#endif
-	callq	__lll_mutex_lock_wait
-#if cond_lock != 0
-	subq	$cond_lock, %rdi
-#endif
-
-1:	movl	broadcast_seq(%rdi), %edx
-	cmpl	4(%r8), %edx
-	jne	3f
-
-	incq	wakeup_seq(%rdi)
-	incq	woken_seq(%rdi)
-	incl	cond_futex(%rdi)
-
-3:	subl	$(1 << clock_bits), cond_nwaiters(%rdi)
-
-	/* Wake up a thread which wants to destroy the condvar object.  */
-	xorq	%r12, %r12
-	cmpq	$0xffffffffffffffff, total_seq(%rdi)
-	jne	4f
-	movl	cond_nwaiters(%rdi), %eax
-	andl	$~((1 << clock_bits) - 1), %eax
-	jne	4f
-
-	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
-	movl	$1, %edx
-	syscall
-	subq	$cond_nwaiters, %rdi
-	movl	$1, %r12d
-
-4:	LOCK
-#if cond_lock == 0
-	decl	(%rdi)
-#else
-	decl	cond_lock(%rdi)
-#endif
-	je	2f
-#if cond_lock != 0
-	addq	$cond_lock, %rdi
-#endif
-	callq	__lll_mutex_unlock_wake
-
-	/* Wake up all waiters to make sure no signal gets lost.  */
-2:	testq	%r12, %r12
-	jnz	5f
-	addq	$cond_futex, %rdi
-	movl	$FUTEX_WAKE, %esi
-	movl	$0x7fffffff, %edx
-	movl	$SYS_futex, %eax
-	syscall
-
-5:	movq	16(%r8), %rdi
-	callq	__pthread_mutex_cond_lock
-
-	popq	%r12
-
-	retq
-	.size	__condvar_cleanup, .-__condvar_cleanup
-
-
 /* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)  */
 	.globl	__pthread_cond_wait
 	.type	__pthread_cond_wait, @function
 	.align	16
 __pthread_cond_wait:
 .LSTARTCODE:
-	pushq	%r12
-.Lpush_r12:
-#define FRAME_SIZE 64
-	subq	$FRAME_SIZE, %rsp
-.Lsubq:
+	cfi_startproc
+#ifdef SHARED
+	cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
+			DW.ref.__gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
+#else
+	cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
+#endif
+
+#define FRAME_SIZE 32
+	leaq	-FRAME_SIZE(%rsp), %rsp
+	cfi_adjust_cfa_offset(FRAME_SIZE)
+
 	/* Stack frame:
 
-	   rsp + 64
-	            +--------------------------+
-	   rsp + 32 | cleanup buffer           |
+	   rsp + 32
 		    +--------------------------+
 	   rsp + 24 | old wake_seq value       |
 	            +--------------------------+
@@ -177,17 +95,7 @@ __pthread_cond_wait:
 	movq	8(%rsp), %rdi
 	incq	total_seq(%rdi)
 	incl	cond_futex(%rdi)
-	addl	$(1 << clock_bits), cond_nwaiters(%rdi)
-
-	/* Install cancellation handler.  */
-#ifdef __PIC__
-	leaq	__condvar_cleanup(%rip), %rsi
-#else
-	leaq	__condvar_cleanup, %rsi
-#endif
-	leaq	32(%rsp), %rdi
-	movq	%rsp, %rdx
-	callq	__pthread_cleanup_push
+	addl	$(1 << nwaiters_shift), cond_nwaiters(%rdi)
 
 	/* Get and store current wakeup_seq value.  */
 	movq	8(%rsp), %rdi
@@ -197,7 +105,7 @@ __pthread_cond_wait:
 	movl	%edx, 4(%rsp)
 
 	/* Unlock.  */
-8:	movl	cond_futex(%rdi), %r12d
+8:	movl	cond_futex(%rdi), %edx
 	LOCK
 #if cond_lock == 0
 	decl	(%rdi)
@@ -206,23 +114,53 @@ __pthread_cond_wait:
 #endif
 	jne	3f
 
+.LcleanupSTART:
 4:	callq	__pthread_enable_asynccancel
 	movl	%eax, (%rsp)
 
-	movq	8(%rsp), %rdi
 	xorq	%r10, %r10
-	movq	%r12, %rdx
-	addq	$cond_futex-cond_lock, %rdi
+	cmpq	$-1, dep_mutex(%rdi)
+	leaq	cond_futex(%rdi), %rdi
+	movl	$FUTEX_WAIT, %esi
+	je	60f
+
+	movq	dep_mutex-cond_futex(%rdi), %r8
+	/* Requeue to a non-robust PI mutex if the PI bit is set and
+	the robust bit is not set.  */
+	movl	MUTEX_KIND(%r8), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	jne	61f
+
+	movl	$(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
 	movl	$SYS_futex, %eax
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	syscall
+
+	movl	$1, %r8d
+#ifdef __ASSUME_REQUEUE_PI
+	jmp	62f
 #else
+	cmpq	$-4095, %rax
+	jnae	62f
+
+# ifndef __ASSUME_PRIVATE_FUTEX
 	movl	$FUTEX_WAIT, %esi
+# endif
 #endif
+
+61:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+#else
+	orl	%fs:PRIVATE_FUTEX, %esi
+#endif
+60:	xorl	%r8d, %r8d
+	movl	$SYS_futex, %eax
 	syscall
 
-	movl	(%rsp), %edi
+62:	movl	(%rsp), %edi
 	callq	__pthread_disable_asynccancel
+.LcleanupEND:
 
 	/* Lock.  */
 	movq	8(%rsp), %rdi
@@ -254,19 +192,29 @@ __pthread_cond_wait:
 	incq	woken_seq(%rdi)
 
 	/* Unlock */
-16:	subl	$(1 << clock_bits), cond_nwaiters(%rdi)
+16:	subl	$(1 << nwaiters_shift), cond_nwaiters(%rdi)
 
 	/* Wake up a thread which wants to destroy the condvar object.  */
 	cmpq	$0xffffffffffffffff, total_seq(%rdi)
 	jne	17f
 	movl	cond_nwaiters(%rdi), %eax
-	andl	$~((1 << clock_bits) - 1), %eax
+	andl	$~((1 << nwaiters_shift) - 1), %eax
 	jne	17f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
@@ -278,28 +226,36 @@ __pthread_cond_wait:
 #endif
 	jne	10f
 
-	/* Remove cancellation handler.  */
-11:	movq	32+CLEANUP_PREV(%rsp), %rdx
-	movq	%rdx, %fs:CLEANUP
+	/* If requeue_pi is used the kernel performs the locking of the
+	   mutex. */
+11:	movq	16(%rsp), %rdi
+	testl	%r8d, %r8d
+	jnz	18f
 
-	movq	16(%rsp), %rdi
 	callq	__pthread_mutex_cond_lock
-14:	addq	$FRAME_SIZE, %rsp
-.Laddq:
 
-	popq	%r12
-.Lpop_r12:
+14:	leaq	FRAME_SIZE(%rsp), %rsp
+	cfi_adjust_cfa_offset(-FRAME_SIZE)
 
 	/* We return the result of the mutex_lock operation.  */
 	retq
 
+	cfi_adjust_cfa_offset(FRAME_SIZE)
+
+18:	callq	__pthread_mutex_cond_lock_adjust
+	xorl	%eax, %eax
+	jmp	14b
+
 	/* Initial locking failed.  */
 1:
-.LSbl1:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
 	jmp	2b
 
 	/* Unlock in loop requires wakeup.  */
@@ -307,7 +263,15 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	/* The call preserves %rdx.  */
+	callq	__lll_unlock_wake
+#if cond_lock != 0
+	subq	$cond_lock, %rdi
+#endif
 	jmp	4b
 
 	/* Locking in loop failed.  */
@@ -315,7 +279,11 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
 #endif
@@ -326,7 +294,11 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
 	jmp	11b
 
 	/* The initial unlocking of the mutex failed.  */
@@ -338,83 +310,185 @@ __pthread_cond_wait:
 #else
 	decl	cond_lock(%rdi)
 #endif
-	jne	13f
+	je	13f
 
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_unlock_wake
 
 13:	movq	%r10, %rax
 	jmp	14b
-.LENDCODE:
 	.size	__pthread_cond_wait, .-__pthread_cond_wait
 weak_alias(__pthread_cond_wait, pthread_cond_wait)
 
 
-	.section .eh_frame,"a",@progbits
-.LSTARTFRAME:
-	.long	.LENDCIE-.LSTARTCIE		# Length of the CIE.
-.LSTARTCIE:
-	.long	0				# CIE ID.
-	.byte	1				# Version number.
-#ifdef SHARED
-	.string	"zR"				# NUL-terminated augmentation
-						# string.
+	.align	16
+	.type	__condvar_cleanup1, @function
+	.globl	__condvar_cleanup1
+	.hidden	__condvar_cleanup1
+__condvar_cleanup1:
+	/* Stack frame:
+
+	   rsp + 32
+		    +--------------------------+
+	   rsp + 24 | unused                   |
+	            +--------------------------+
+	   rsp + 16 | mutex pointer            |
+	            +--------------------------+
+	   rsp +  8 | condvar pointer          |
+	            +--------------------------+
+	   rsp +  4 | old broadcast_seq value  |
+	            +--------------------------+
+	   rsp +  0 | old cancellation mode    |
+	            +--------------------------+
+	*/
+
+	movq	%rax, 24(%rsp)
+
+	/* Get internal lock.  */
+	movq	8(%rsp), %rdi
+	movl	$1, %esi
+	xorl	%eax, %eax
+	LOCK
+#if cond_lock == 0
+	cmpxchgl %esi, (%rdi)
 #else
-	.ascii	"\0"				# NUL-terminated augmentation
-						# string.
+	cmpxchgl %esi, cond_lock(%rdi)
 #endif
-	.uleb128 1				# Code alignment factor.
-	.sleb128 -8				# Data alignment factor.
-	.byte	16				# Return address register
-						# column.
-#ifdef SHARED
-	.uleb128 1				# Augmentation value length.
-	.byte	0x1b				# Encoding: DW_EH_PE_pcrel
-						# + DW_EH_PE_sdata4.
+	jz	1f
+
+#if cond_lock != 0
+	addq	$cond_lock, %rdi
 #endif
-	.byte 0x0c				# DW_CFA_def_cfa
-	.uleb128 7
-	.uleb128 8
-	.byte	0x90				# DW_CFA_offset, column 0x8
-	.uleb128 1
-	.align 8
-.LENDCIE:
-
-	.long	.LENDFDE-.LSTARTFDE		# Length of the FDE.
-.LSTARTFDE:
-	.long	.LSTARTFDE-.LSTARTFRAME		# CIE pointer.
-#ifdef SHARED
-	.long	.LSTARTCODE-.			# PC-relative start address
-						# of the code
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	callq	__lll_lock_wait
+#if cond_lock != 0
+	subq	$cond_lock, %rdi
+#endif
+
+1:	movl	broadcast_seq(%rdi), %edx
+	cmpl	4(%rsp), %edx
+	jne	3f
+
+	/* We increment the wakeup_seq counter only if it is lower than
+	   total_seq.  If this is not the case the thread was woken and
+	   then canceled.  In this case we ignore the signal.  */
+	movq	total_seq(%rdi), %rax
+	cmpq	wakeup_seq(%rdi), %rax
+	jbe	6f
+	incq	wakeup_seq(%rdi)
+	incl	cond_futex(%rdi)
+6:	incq	woken_seq(%rdi)
+
+3:	subl	$(1 << nwaiters_shift), cond_nwaiters(%rdi)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	xorl	%ecx, %ecx
+	cmpq	$0xffffffffffffffff, total_seq(%rdi)
+	jne	4f
+	movl	cond_nwaiters(%rdi), %eax
+	andl	$~((1 << nwaiters_shift) - 1), %eax
+	jne	4f
+
+	cmpq	$-1, dep_mutex(%rdi)
+	leaq	cond_nwaiters(%rdi), %rdi
+	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	.long	.LSTARTCODE			# Start address of the code.
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
 #endif
-	.long	.LENDCODE-.LSTARTCODE		# Length of the code.
-#ifdef SHARED
-	.uleb128 0				# No augmentation data.
+	movl	$SYS_futex, %eax
+	syscall
+	subq	$cond_nwaiters, %rdi
+	movl	$1, %ecx
+
+4:	LOCK
+#if cond_lock == 0
+	decl	(%rdi)
+#else
+	decl	cond_lock(%rdi)
+#endif
+	je	2f
+#if cond_lock != 0
+	addq	$cond_lock, %rdi
+#endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
+	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
+	/* The call preserves %rcx.  */
+	callq	__lll_unlock_wake
+
+	/* Wake up all waiters to make sure no signal gets lost.  */
+2:	testl	%ecx, %ecx
+	jnz	5f
+	addq	$cond_futex, %rdi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
+	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
 #endif
-	.byte	0x40+.Lpush_r12-.LSTARTCODE	# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 16
-	.byte	0x8c				# DW_CFA_offset %r12
-	.uleb128 2
-	.byte	0x40+.Lsubq-.Lpush_r12		# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 16+FRAME_SIZE
-	.byte	3				# DW_CFA_advance_loc2
-	.2byte	.Laddq-.Lsubq
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 16
-	.byte	0x40+.Lpop_r12-.Laddq		# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 8
-	.byte	0xcc				# DW_CFA_restore %r12
-	.byte	0x40+.LSbl1-.Lpop_r12		# DW_CFA_advance_loc+N
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 80
-	.byte	0x8c				# DW_CFA_offset %r12
-	.uleb128 2
+	movl	$SYS_futex, %eax
+	syscall
+
+5:	movq	16(%rsp), %rdi
+	callq	__pthread_mutex_cond_lock
+
+	movq	24(%rsp), %rdi
+.LcallUR:
+	call	_Unwind_Resume@PLT
+	hlt
+.LENDCODE:
+	cfi_endproc
+	.size	__condvar_cleanup1, .-__condvar_cleanup1
+
+
+	.section .gcc_except_table,"a",@progbits
+.LexceptSTART:
+	.byte	DW_EH_PE_omit			# @LPStart format
+	.byte	DW_EH_PE_omit			# @TType format
+	.byte	DW_EH_PE_uleb128		# call-site format
+	.uleb128 .Lcstend-.Lcstbegin
+.Lcstbegin:
+	.uleb128 .LcleanupSTART-.LSTARTCODE
+	.uleb128 .LcleanupEND-.LcleanupSTART
+	.uleb128 __condvar_cleanup1-.LSTARTCODE
+	.uleb128  0
+	.uleb128 .LcallUR-.LSTARTCODE
+	.uleb128 .LENDCODE-.LcallUR
+	.uleb128 0
+	.uleb128  0
+.Lcstend:
+
+
+#ifdef SHARED
+	.hidden	DW.ref.__gcc_personality_v0
+	.weak	DW.ref.__gcc_personality_v0
+	.section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
 	.align	8
-.LENDFDE:
+	.type	DW.ref.__gcc_personality_v0, @object
+	.size	DW.ref.__gcc_personality_v0, 8
+DW.ref.__gcc_personality_v0:
+	.quad	__gcc_personality_v0
+#endif
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
index d8bfa26c6..0ac952b66 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,15 +18,10 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <bits/kernel-features.h>
+#include <tcb-offsets.h>
+#include <lowlevellock.h>
 
-#ifndef UP
-# define LOCK lock
-#else
-# define LOCK
-#endif
-
-#define FUTEX_WAIT  0
-#define FUTEX_WAKE	1
 
 	.comm	__fork_generation, 4, 4
 
@@ -38,6 +33,15 @@
 	.align	16
 __pthread_once:
 .LSTARTCODE:
+	cfi_startproc
+#ifdef SHARED
+	cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
+			DW.ref.__gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
+#else
+	cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
+#endif
 	testl	$2, (%rdi)
 	jz	1f
 	xorl	%eax, %eax
@@ -45,7 +49,7 @@ __pthread_once:
 
 	/* Preserve the function pointer.  */
 1:	pushq	%rsi
-.Lpush_rsi:
+	cfi_adjust_cfa_offset(8)
 	xorq	%r10, %r10
 
 	/* Not yet initialized or initialization in progress.
@@ -76,10 +80,15 @@ __pthread_once:
 	jnz	3f	/* Different for generation -> run initializer.  */
 
 	/* Somebody else got here first.  Wait.  */
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
 #else
+# if FUTEX_WAIT == 0
+	movl	%fs:PRIVATE_FUTEX, %esi
+# else
 	movl	$FUTEX_WAIT, %esi
+	orl	%fs:PRIVATE_FUTEX, %esi
+# endif
 #endif
 	movl	$SYS_futex, %eax
 	syscall
@@ -87,31 +96,40 @@ __pthread_once:
 
 	/* Preserve the pointer to the control variable.  */
 3:	pushq	%rdi
-.Lpush_rdi:
+	cfi_adjust_cfa_offset(8)
+	pushq	%rdi
+	cfi_adjust_cfa_offset(8)
 
 .LcleanupSTART:
-	callq	*8(%rsp)
+	callq	*16(%rsp)
 .LcleanupEND:
 
 	/* Get the control variable address back.  */
 	popq	%rdi
-.Lpop_rdi:
+	cfi_adjust_cfa_offset(-8)
 
 	/* Sucessful run of the initializer.  Signal that we are done.  */
 	LOCK
 	incl	(%rdi)
 
+	addq	$8, %rsp
+	cfi_adjust_cfa_offset(-8)
+
 	/* Wake up all other threads.  */
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
+#else
 	movl	$FUTEX_WAKE, %esi
+	orl	%fs:PRIVATE_FUTEX, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 
 4:	addq	$8, %rsp
-.Ladd:
+	cfi_adjust_cfa_offset(-8)
 	xorl	%eax, %eax
 	retq
-
 	.size	__pthread_once,.-__pthread_once
 
 
@@ -125,12 +143,18 @@ pthread_once = __pthread_once
 	.type	clear_once_control,@function
 	.align	16
 clear_once_control:
+	cfi_adjust_cfa_offset(3 * 8)
 	movq	(%rsp), %rdi
 	movq	%rax, %r8
 	movl	$0, (%rdi)
 
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
+#else
 	movl	$FUTEX_WAKE, %esi
+	orl	%fs:PRIVATE_FUTEX, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -139,15 +163,15 @@ clear_once_control:
 	call	_Unwind_Resume@PLT
 	hlt
 .LENDCODE:
+	cfi_endproc
 	.size	clear_once_control,.-clear_once_control
 
 
 	.section .gcc_except_table,"a",@progbits
 .LexceptSTART:
-	.byte	0xff				# @LPStart format (omit)
-	.byte	0xff				# @TType format (omit)
-	.byte	0x01				# call-site format
-						# DW_EH_PE_uleb128
+	.byte	DW_EH_PE_omit			# @LPStart format
+	.byte	DW_EH_PE_omit			# @TType format
+	.byte	DW_EH_PE_uleb128		# call-site format
 	.uleb128 .Lcstend-.Lcstbegin
 .Lcstbegin:
 	.uleb128 .LcleanupSTART-.LSTARTCODE
@@ -161,92 +185,6 @@ clear_once_control:
 .Lcstend:
 
 
-	.section .eh_frame,"a",@progbits
-.LSTARTFRAME:
-	.long	.LENDCIE-.LSTARTCIE		# Length of the CIE.
-.LSTARTCIE:
-	.long	0				# CIE ID.
-	.byte	1				# Version number.
-#ifdef SHARED
-	.string	"zPLR"				# NUL-terminated augmentation
-						# string.
-#else
-	.string	"zPL"				# NUL-terminated augmentation
-						# string.
-#endif
-	.uleb128 1				# Code alignment factor.
-	.sleb128 -8				# Data alignment factor.
-	.byte	16				# Return address register
-						# column.
-#ifdef SHARED
-	.uleb128 7				# Augmentation value length.
-	.byte	0x9b				# Personality: DW_EH_PE_pcrel
-						# + DW_EH_PE_sdata4
-						# + DW_EH_PE_indirect
-	.long	DW.ref.__gcc_personality_v0-.
-	.byte	0x1b				# LSDA Encoding: DW_EH_PE_pcrel
-						# + DW_EH_PE_sdata4.
-	.byte	0x1b				# FDE Encoding: DW_EH_PE_pcrel
-						# + DW_EH_PE_sdata4.
-#else
-	.uleb128 10				# Augmentation value length.
-	.byte	0x0				# Personality: absolute
-	.quad	__gcc_personality_v0
-	.byte	0x0				# LSDA Encoding: absolute
-#endif
-	.byte 0x0c				# DW_CFA_def_cfa
-	.uleb128 7
-	.uleb128 8
-	.byte	0x90				# DW_CFA_offset, column 0x10
-	.uleb128 1
-	.align 8
-.LENDCIE:
-
-	.long	.LENDFDE-.LSTARTFDE		# Length of the FDE.
-.LSTARTFDE:
-	.long	.LSTARTFDE-.LSTARTFRAME		# CIE pointer.
-#ifdef SHARED
-	.long	.LSTARTCODE-.			# PC-relative start address
-						# of the code.
-	.long	.LENDCODE-.LSTARTCODE		# Length of the code.
-	.uleb128 4				# Augmentation size
-	.long	.LexceptSTART-.
-#else
-	.quad	.LSTARTCODE			# Start address of the code.
-	.quad	.LENDCODE-.LSTARTCODE		# Length of the code.
-	.uleb128 8				# Augmentation size
-	.quad	.LexceptSTART
-#endif
-	.byte	4				# DW_CFA_advance_loc4
-	.long	.Lpush_rsi-.LSTARTCODE
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 16
-	.byte	4				# DW_CFA_advance_loc4
-	.long	.Lpush_rdi-.Lpush_rsi
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 24
-	.byte	4				# DW_CFA_advance_loc4
-	.long	.Lpop_rdi-.Lpush_rdi
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 16
-	.byte	4				# DW_CFA_advance_loc4
-	.long	.Ladd-.Lpop_rdi
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 8
-	.byte	4				# DW_CFA_advance_loc4
-	.long	clear_once_control-.Ladd
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 24
-#if 0
-	.byte	4				# DW_CFA_advance_loc4
-	.long	.Lpop_rdi2-clear_once_control
-	.byte	14				# DW_CFA_def_cfa_offset
-	.uleb128 16
-#endif
-	.align	8
-.LENDFDE:
-
-
 #ifdef SHARED
 	.hidden	DW.ref.__gcc_personality_v0
 	.weak	DW.ref.__gcc_personality_v0
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
index d7543572a..9b8408b69 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,19 +18,10 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
-#include <tcb-offsets.h>
-
-
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-
-#ifndef UP
-# define LOCK lock
-#else
-# define LOCK
-#endif
+#include <bits/kernel-features.h>
 
 
 	.text
@@ -39,6 +30,7 @@
 	.type	__pthread_rwlock_rdlock,@function
 	.align	16
 __pthread_rwlock_rdlock:
+	cfi_startproc
 	xorq	%r10, %r10
 
 	/* Get the lock.  */
@@ -73,12 +65,20 @@ __pthread_rwlock_rdlock:
 #endif
 	jne	10f
 
-11:	addq	$READERS_WAKEUP, %rdi
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+11:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi
+	xorl	PSHARED(%rdi), %esi
 #else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%rdi), %esi
+# else
 	movl	$FUTEX_WAIT, %esi
+	orl	PSHARED(%rdi), %esi
+# endif
+	xorl	%fs:PRIVATE_FUTEX, %esi
 #endif
+	addq	$READERS_WAKEUP, %rdi
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -113,11 +113,11 @@ __pthread_rwlock_rdlock:
 	movq	%rdx, %rax
 	retq
 
-1:
+1:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 #if MUTEX != 0
 	subq	$MUTEX, %rdi
 #endif
@@ -129,11 +129,11 @@ __pthread_rwlock_rdlock:
 	movl	$EDEADLK, %edx
 	jmp	9b
 
-6:
+6:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 #if MUTEX != 0
 	subq	$MUTEX, %rdi
 #endif
@@ -149,25 +149,26 @@ __pthread_rwlock_rdlock:
 	movl	$EAGAIN, %edx
 	jmp	9b
 
-10:
+10:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 #if MUTEX != 0
 	subq	$MUTEX, %rdi
 #endif
 	jmp	11b
 
-12:
+12:	movl	PSHARED(%rdi), %esi
 #if MUTEX == 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 #if MUTEX != 0
 	subq	$MUTEX, %rdi
 #endif
 	jmp	13b
+	cfi_endproc
 	.size	__pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
 
 	.globl	pthread_rwlock_rdlock
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
index f044842e0..bb12d4941 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,35 +18,39 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
-#include <tcb-offsets.h>
+#include <bits/kernel-features.h>
 
 
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-
 /* For the calculation see asm/vsyscall.h.  */
 #define VSYSCALL_ADDR_vgettimeofday	0xffffffffff600000
 
-
-#ifndef UP
-# define LOCK lock
-#else
-# define LOCK
-#endif
-
-
 	.text
 
 	.globl	pthread_rwlock_timedrdlock
 	.type	pthread_rwlock_timedrdlock,@function
 	.align	16
 pthread_rwlock_timedrdlock:
+	cfi_startproc
 	pushq	%r12
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r12, 0)
 	pushq	%r13
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r13, 0)
+#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
+# define VALREG	%edx
+#else
 	pushq	%r14
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r14, 0)
+
 	subq	$16, %rsp
+	cfi_adjust_cfa_offset(16)
+# define VALREG %r14d
+#endif
 
 	movq	%rdi, %r12
 	movq	%rsi, %r13
@@ -77,7 +81,7 @@ pthread_rwlock_timedrdlock:
 	incl	READERS_QUEUED(%r12)
 	je	4f
 
-	movl	READERS_WAKEUP(%r12), %r14d
+	movl	READERS_WAKEUP(%r12), VALREG
 
 	/* Unlock.  */
 	LOCK
@@ -88,8 +92,33 @@ pthread_rwlock_timedrdlock:
 #endif
 	jne	10f
 
+11:
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+#  ifdef PIC
+	cmpl	$0, __have_futex_clock_realtime(%rip)
+#  else
+	cmpl	$0, __have_futex_clock_realtime
+#  endif
+	je	.Lreltmo
+#endif
+
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi
+	xorl	PSHARED(%r12), %esi
+	movq	%r13, %r10
+	movl	$0xffffffff, %r9d
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	movl	%r14d, %edx
+#endif
+21:	leaq	READERS_WAKEUP(%r12), %rdi
+	movl	$SYS_futex, %eax
+	syscall
+	movq	%rax, %rdx
+
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	.subsection 2
+.Lreltmo:
 	/* Get current time.  */
-11:	movq	%rsp, %rdi
+	movq	%rsp, %rdi
 	xorl	%esi, %esi
 	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
 	callq	*%rax
@@ -112,20 +141,26 @@ pthread_rwlock_timedrdlock:
 	movq	%rcx, (%rsp)	/* Store relative timeout.  */
 	movq	%rdi, 8(%rsp)
 
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
-#else
+# ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi
+	xorl	PSHARED(%r12), %esi
+# else
+#  if FUTEX_WAIT == 0
+	movl	PSHARED(%r12), %esi
+#  else
 	movl	$FUTEX_WAIT, %esi
-#endif
+	orl	PSHARED(%r12), %esi
+#  endif
+	xorl	%fs:PRIVATE_FUTEX, %esi
+# endif
 	movq	%rsp, %r10
 	movl	%r14d, %edx
-	leaq	READERS_WAKEUP(%r12), %rdi
-	movl	$SYS_futex, %eax
-	syscall
-	movq	%rax, %rdx
-17:
 
-	/* Reget the lock.  */
+	jmp	21b
+	.previous
+#endif
+
+17:	/* Reget the lock.  */
 	movl	$1, %esi
 	xorl	%eax, %eax
 	LOCK
@@ -157,17 +192,36 @@ pthread_rwlock_timedrdlock:
 
 7:	movq	%rdx, %rax
 
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 	addq	$16, %rsp
+	cfi_adjust_cfa_offset(-16)
 	popq	%r14
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r14)
+#endif
 	popq	%r13
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r13)
 	popq	%r12
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r12)
 	retq
 
-1:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	cfi_adjust_cfa_offset(16)
+	cfi_rel_offset(%r12, 8)
+	cfi_rel_offset(%r13, 0)
+#else
+	cfi_adjust_cfa_offset(40)
+	cfi_offset(%r12, -16)
+	cfi_offset(%r13, -24)
+	cfi_offset(%r14, -32)
+#endif
+1:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 	jmp	2b
 
 14:	cmpl	%fs:TID, %eax
@@ -175,13 +229,13 @@ pthread_rwlock_timedrdlock:
 	movl	$EDEADLK, %edx
 	jmp	9b
 
-6:
+6:	movl	PSHARED(%r12), %esi
 #if MUTEX == 0
 	movq	%r12, %rdi
 #else
 	leal	MUTEX(%r12), %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 	jmp	7b
 
 	/* Overflow.  */
@@ -194,22 +248,22 @@ pthread_rwlock_timedrdlock:
 	movl	$EAGAIN, %edx
 	jmp	9b
 
-10:
+10:	movl	PSHARED(%r12), %esi
 #if MUTEX == 0
 	movq	%r12, %rdi
 #else
 	leaq	MUTEX(%r12), %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 	jmp	11b
 
-12:
+12:	movl	PSHARED(%r12), %esi
 #if MUTEX == 0
 	movq	%r12, %rdi
 #else
 	leaq	MUTEX(%r12), %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 	jmp	13b
 
 16:	movq	$-ETIMEDOUT, %rdx
@@ -217,4 +271,5 @@ pthread_rwlock_timedrdlock:
 
 19:	movl	$EINVAL, %edx
 	jmp	9b
+	cfi_endproc
 	.size	pthread_rwlock_timedrdlock,.-pthread_rwlock_timedrdlock
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
index b479da727..401bbc5d9 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,34 +18,39 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
-#include <tcb-offsets.h>
+#include <bits/kernel-features.h>
 
 
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-
 /* For the calculation see asm/vsyscall.h.  */
 #define VSYSCALL_ADDR_vgettimeofday	0xffffffffff600000
 
-#ifndef UP
-# define LOCK lock
-#else
-# define LOCK
-#endif
-
-
 	.text
 
 	.globl	pthread_rwlock_timedwrlock
 	.type	pthread_rwlock_timedwrlock,@function
 	.align	16
 pthread_rwlock_timedwrlock:
+	cfi_startproc
 	pushq	%r12
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r12, 0)
 	pushq	%r13
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r13, 0)
+#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
+# define VALREG	%edx
+#else
 	pushq	%r14
+	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r14, 0)
+
 	subq	$16, %rsp
+	cfi_adjust_cfa_offset(16)
+# define VALREG %r14d
+#endif
 
 	movq	%rdi, %r12
 	movq	%rsi, %r13
@@ -74,7 +79,7 @@ pthread_rwlock_timedwrlock:
 	incl	WRITERS_QUEUED(%r12)
 	je	4f
 
-	movl	WRITERS_WAKEUP(%r12), %r14d
+	movl	WRITERS_WAKEUP(%r12), VALREG
 
 	LOCK
 #if MUTEX == 0
@@ -84,8 +89,33 @@ pthread_rwlock_timedwrlock:
 #endif
 	jne	10f
 
+11:
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+#  ifdef PIC
+	cmpl	$0, __have_futex_clock_realtime(%rip)
+#  else
+	cmpl	$0, __have_futex_clock_realtime
+#  endif
+	je	.Lreltmo
+#endif
+
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi
+	xorl	PSHARED(%r12), %esi
+	movq	%r13, %r10
+	movl	$0xffffffff, %r9d
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	movl	%r14d, %edx
+#endif
+21:	leaq	WRITERS_WAKEUP(%r12), %rdi
+	movl	$SYS_futex, %eax
+	syscall
+	movq	%rax, %rdx
+
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	.subsection 2
+.Lreltmo:
 	/* Get current time.  */
-11:	movq	%rsp, %rdi
+	movq	%rsp, %rdi
 	xorl	%esi, %esi
 	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
 	callq	*%rax
@@ -108,20 +138,26 @@ pthread_rwlock_timedwrlock:
 	movq	%rcx, (%rsp)	/* Store relative timeout.  */
 	movq	%rdi, 8(%rsp)
 
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
-#else
+# ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi
+	xorl	PSHARED(%r12), %esi
+# else
+#  if FUTEX_WAIT == 0
+	movl	PSHARED(%r12), %esi
+#  else
 	movl	$FUTEX_WAIT, %esi
-#endif
+	orl	PSHARED(%r12), %esi
+#  endif
+	xorl	%fs:PRIVATE_FUTEX, %esi
+# endif
 	movq	%rsp, %r10
 	movl	%r14d, %edx
-	leaq	WRITERS_WAKEUP(%r12), %rdi
-	movl	$SYS_futex, %eax
-	syscall
-	movq	%rax, %rdx
-17:
 
-	/* Reget the lock.  */
+	jmp	21b
+	.previous
+#endif
+
+17:	/* Reget the lock.  */
 	movl	$1, %esi
 	xorl	%eax, %eax
 	LOCK
@@ -153,17 +189,36 @@ pthread_rwlock_timedwrlock:
 
 7:	movq	%rdx, %rax
 
+#ifndef __ASSUME_PRIVATE_FUTEX
 	addq	$16, %rsp
+	cfi_adjust_cfa_offset(-16)
 	popq	%r14
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r14)
+#endif
 	popq	%r13
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r13)
 	popq	%r12
+	cfi_adjust_cfa_offset(-8)
+	cfi_restore(%r12)
 	retq
 
-1:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	cfi_adjust_cfa_offset(16)
+	cfi_rel_offset(%r12, 8)
+	cfi_rel_offset(%r13, 0)
+#else
+	cfi_adjust_cfa_offset(40)
+	cfi_offset(%r12, -16)
+	cfi_offset(%r13, -24)
+	cfi_offset(%r14, -32)
+#endif
+1:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 	jmp	2b
 
 14:	cmpl	%fs:TID, %eax
@@ -171,13 +226,13 @@ pthread_rwlock_timedwrlock:
 20:	movl	$EDEADLK, %edx
 	jmp	9b
 
-6:
+6:	movl	PSHARED(%r12), %esi
 #if MUTEX == 0
 	movq	%r12, %rdi
 #else
 	leal	MUTEX(%r12), %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 	jmp	7b
 
 	/* Overflow.  */
@@ -185,22 +240,22 @@ pthread_rwlock_timedwrlock:
 	movl	$EAGAIN, %edx
 	jmp	9b
 
-10:
+10:	movl	PSHARED(%r12), %esi
 #if MUTEX == 0
 	movq	%r12, %rdi
 #else
 	leaq	MUTEX(%r12), %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 	jmp	11b
 
-12:
+12:	movl	PSHARED(%r12), %esi
 #if MUTEX == 0
 	movq	%r12, %rdi
 #else
 	leaq	MUTEX(%r12), %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 	jmp	13b
 
 16:	movq	$-ETIMEDOUT, %rdx
@@ -208,4 +263,5 @@ pthread_rwlock_timedwrlock:
 
 19:	movl	$EINVAL, %edx
 	jmp	9b
+	cfi_endproc
 	.size	pthread_rwlock_timedwrlock,.-pthread_rwlock_timedwrlock
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
index a0f75226a..cfcc7a18c 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,17 +18,9 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelrwlock.h>
-
-
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-
-#ifndef UP
-# define LOCK lock
-#else
-# define LOCK
-#endif
+#include <bits/kernel-features.h>
 
 
 	.text
@@ -37,6 +29,7 @@
 	.type	__pthread_rwlock_unlock,@function
 	.align	16
 __pthread_rwlock_unlock:
+	cfi_startproc
 	/* Get the lock.  */
 	movl	$1, %esi
 	xorl	%eax, %eax
@@ -55,9 +48,8 @@ __pthread_rwlock_unlock:
 
 5:	movl	$0, WRITER(%rdi)
 
-	movl	$1, %esi
+	movl	$1, %edx
 	leaq	WRITERS_WAKEUP(%rdi), %r10
-	movq	%rsi, %rdx
 	cmpl	$0, WRITERS_QUEUED(%rdi)
 	jne	0f
 
@@ -77,7 +69,16 @@ __pthread_rwlock_unlock:
 #endif
 	jne	7f
 
-8:	movl	$SYS_futex, %eax
+8:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %esi
+	xorl	PSHARED(%rdi), %esi
+#else
+	movl	$FUTEX_WAKE, %esi
+	orl	PSHARED(%rdi), %esi
+	xorl	%fs:PRIVATE_FUTEX, %esi
+#endif
+	movl	$SYS_futex, %eax
 	movq	%r10, %rdi
 	syscall
 
@@ -96,30 +97,30 @@ __pthread_rwlock_unlock:
 4:	xorl	%eax, %eax
 	retq
 
-1:
+1:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 #if MUTEX != 0
 	subq	$MUTEX, %rdi
 #endif
 	jmp	2b
 
-3:
+3:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 	jmp	4b
 
-7:
+7:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 	jmp	8b
-
+	cfi_endproc
 	.size	__pthread_rwlock_unlock,.-__pthread_rwlock_unlock
 
 	.globl	pthread_rwlock_unlock
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
index 39b54dc6b..b7bc8522d 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,19 +18,10 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
-#include <tcb-offsets.h>
-
-
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-
-#ifndef UP
-# define LOCK lock
-#else
-# define LOCK
-#endif
+#include <bits/kernel-features.h>
 
 
 	.text
@@ -39,6 +30,7 @@
 	.type	__pthread_rwlock_wrlock,@function
 	.align	16
 __pthread_rwlock_wrlock:
+	cfi_startproc
 	xorq	%r10, %r10
 
 	/* Get the lock.  */
@@ -71,12 +63,20 @@ __pthread_rwlock_wrlock:
 #endif
 	jne	10f
 
-11:	addq	$WRITERS_WAKEUP, %rdi
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+11:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi
+	xorl	PSHARED(%rdi), %esi
 #else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%rdi), %esi
+# else
 	movl	$FUTEX_WAIT, %esi
+	orl	PSHARED(%rdi), %esi
+# endif
+	xorl	%fs:PRIVATE_FUTEX, %esi
 #endif
+	addq	$WRITERS_WAKEUP, %rdi
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -111,11 +111,11 @@ __pthread_rwlock_wrlock:
 	movq	%rdx, %rax
 	retq
 
-1:
+1:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 #if MUTEX != 0
 	subq	$MUTEX, %rdi
 #endif
@@ -126,36 +126,37 @@ __pthread_rwlock_wrlock:
 	movl	$EDEADLK, %edx
 	jmp	9b
 
-6:
+6:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 	jmp	7b
 
 4:	decl	WRITERS_QUEUED(%rdi)
 	movl	$EAGAIN, %edx
 	jmp	9b
 
-10:
+10:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_unlock_wake
+	callq	__lll_unlock_wake
 #if MUTEX != 0
 	subq	$MUTEX, %rdi
 #endif
 	jmp	11b
 
-12:
+12:	movl	PSHARED(%rdi), %esi
 #if MUTEX != 0
 	addq	$MUTEX, %rdi
 #endif
-	callq	__lll_mutex_lock_wait
+	callq	__lll_lock_wait
 #if MUTEX != 0
 	subq	$MUTEX, %rdi
 #endif
 	jmp	13b
+	cfi_endproc
 	.size	__pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
 
 	.globl	pthread_rwlock_wrlock
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S
index 5c8a858ad..7af6524fe 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007, 2008 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,15 +18,9 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <pthread-errnos.h>
-
-#ifndef UP
-# define LOCK lock
-#else
-# define
-#endif
-
-#define FUTEX_WAKE		1
+#include <structsem.h>
 
 
 	.text
@@ -35,30 +29,61 @@
 	.type	sem_post,@function
 	.align	16
 sem_post:
-	movl	$1, %edx
+#if VALUE == 0
+	movl	(%rdi), %eax
+#else
+	movl	VALUE(%rdi), %eax
+#endif
+0:	cmpl	$SEM_VALUE_MAX, %eax
+	je	3f
+	leal	1(%rax), %esi
 	LOCK
-	xaddl	%edx, (%rdi)
+#if VALUE == 0
+	cmpxchgl %esi, (%rdi)
+#else
+	cmpxchgl %esi, VALUE(%rdi)
+#endif
+	jnz	0b
+
+	cmpq	$0, NWAITERS(%rdi)
+	je	2f
 
 	movl	$SYS_futex, %eax
 	movl	$FUTEX_WAKE, %esi
-	incl	%edx
+	orl	PRIVATE(%rdi), %esi
+	movl	$1, %edx
 	syscall
 
 	testq	%rax, %rax
 	js	1f
 
-	xorl	%eax, %eax
+2:	xorl	%eax, %eax
 	retq
 
 1:
 #if USE___THREAD
-	movq	errno@gottpoff(%rip), %rdx
-	movl	$EINVAL, %fs:(%rdx)
+	movl	$EINVAL, %eax
+#else
+	callq	__errno_location@plt
+	movl	$EINVAL, %edx
+#endif
+	jmp	4f
+
+3:
+#if USE___THREAD
+	movl	$EOVERFLOW, %eax
 #else
 	callq	__errno_location@plt
-	movl	$EINVAL, (%rax)
+	movl	$EOVERFLOW, %edx
 #endif
 
+4:
+#if USE___THREAD
+	movq	errno@gottpoff(%rip), %rdx
+	movl	%eax, %fs:(%rdx)
+#else
+	movl	%edx, (%rax)
+#endif
 	orl	$-1, %eax
 	retq
 	.size	sem_post,.-sem_post
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
index 64e168099..f9af8ecc1 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,70 +18,183 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <bits/kernel-features.h>
+#include <lowlevellock.h>
 #include <pthread-errnos.h>
-#include <tcb-offsets.h>
-#include <tls.h>
+#include <structsem.h>
 
-#ifndef UP
-# define LOCK lock
-#else
-# define
-#endif
 
 /* For the calculation see asm/vsyscall.h.  */
 #define VSYSCALL_ADDR_vgettimeofday	0xffffffffff600000
 
-
 	.text
 
 	.globl	sem_timedwait
 	.type	sem_timedwait,@function
 	.align	16
-	cfi_startproc
 sem_timedwait:
-	/* First check for cancellation.  */
-	movl	%fs:CANCELHANDLING, %eax
-	andl	$0xfffffff9, %eax
-	cmpl	$8, %eax
-	je	11f
-
+.LSTARTCODE:
+	cfi_startproc
+#ifdef SHARED
+	cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
+			DW.ref.__gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
+#else
+	cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
+#endif
+#if VALUE == 0
 	movl	(%rdi), %eax
+#else
+	movl	VALUE(%rdi), %eax
+#endif
 2:	testl	%eax, %eax
 	je	1f
 
 	leaq	-1(%rax), %rdx
 	LOCK
+#if VALUE == 0
 	cmpxchgl %edx, (%rdi)
+#else
+	cmpxchgl %edx, VALUE(%rdi)
+#endif
 	jne	2b
 
 	xorl	%eax, %eax
 	retq
 
 	/* Check whether the timeout value is valid.  */
-1:	pushq	%r12
+1:	cmpq	$1000000000, 8(%rsi)
+	jae	6f
+
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+#  ifdef PIC
+	cmpl	$0, __have_futex_clock_realtime(%rip)
+#  else
+	cmpl	$0, __have_futex_clock_realtime
+#  endif
+	je	.Lreltmo
+#endif
+
+	/* This push is only needed to store the sem_t pointer for the
+	   exception handler.  */
+	pushq	%rdi
+	cfi_adjust_cfa_offset(8)
+
+	movq	%rsi, %r10
+
+	LOCK
+	addq	$1, NWAITERS(%rdi)
+
+.LcleanupSTART:
+13:	call	__pthread_enable_asynccancel
+	movl	%eax, %r8d
+
+#if VALUE != 0
+	leaq	VALUE(%rdi), %rdi
+#endif
+	movl	$0xffffffff, %r9d
+	movl	$FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi
+	orl	PRIVATE(%rdi), %esi
+	movl	$SYS_futex, %eax
+	xorl	%edx, %edx
+	syscall
+	movq	%rax, %r9
+#if VALUE != 0
+	leaq	-VALUE(%rdi), %rdi
+#endif
+
+	xchgq	%r8, %rdi
+	call	__pthread_disable_asynccancel
+.LcleanupEND:
+	movq	%r8, %rdi
+
+	testq	%r9, %r9
+	je	11f
+	cmpq	$-EWOULDBLOCK, %r9
+	jne	3f
+
+11:
+#if VALUE == 0
+	movl	(%rdi), %eax
+#else
+	movl	VALUE(%rdi), %eax
+#endif
+14:	testl	%eax, %eax
+	je	13b
+
+	leaq	-1(%rax), %rcx
+	LOCK
+#if VALUE == 0
+	cmpxchgl %ecx, (%rdi)
+#else
+	cmpxchgl %ecx, VALUE(%rdi)
+#endif
+	jne	14b
+
+	xorl	%eax, %eax
+
+15:	LOCK
+	subq	$1, NWAITERS(%rdi)
+
+	leaq	8(%rsp), %rsp
+	cfi_adjust_cfa_offset(-8)
+	retq
+
+	cfi_adjust_cfa_offset(8)
+3:	negq	%r9
+#if USE___THREAD
+	movq	errno@gottpoff(%rip), %rdx
+	movl	%r9d, %fs:(%rdx)
+#else
+	callq	__errno_location@plt
+	movl	%r9d, (%rax)
+#endif
+
+	orl	$-1, %eax
+	jmp	15b
+
+	cfi_adjust_cfa_offset(-8)
+6:
+#if USE___THREAD
+	movq	errno@gottpoff(%rip), %rdx
+	movl	$EINVAL, %fs:(%rdx)
+#else
+	callq	__errno_location@plt
+	movl	$EINVAL, (%rax)
+#endif
+
+	orl	$-1, %eax
+
+	retq
+
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+.Lreltmo:
+	pushq	%r12
 	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r12, 0)
 	pushq	%r13
 	cfi_adjust_cfa_offset(8)
+	cfi_rel_offset(%r13, 0)
 	pushq	%r14
 	cfi_adjust_cfa_offset(8)
-	subq	$24, %rsp
-	cfi_adjust_cfa_offset(24)
+	cfi_rel_offset(%r14, 0)
+
+#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
+# define STACKFRAME 8
+#else
+# define STACKFRAME 24
+#endif
+	subq	$STACKFRAME, %rsp
+	cfi_adjust_cfa_offset(STACKFRAME)
 
 	movq	%rdi, %r12
-	cfi_offset(12, -16)		/* %r12 */
 	movq	%rsi, %r13
-	cfi_offset(13, -24)		/* %r13 */
 
-	/* Check for invalid nanosecond field.  */
-	cmpq	$1000000000, 8(%r13)
-	movl	$EINVAL, %r14d
-	cfi_offset(14, -24)		/* %r14 */
-	jae	6f
-
-7:	call	__pthread_enable_asynccancel
-	movl	%eax, 16(%rsp)
+	LOCK
+	addq	$1, NWAITERS(%r12)
 
-	xorl	%esi, %esi
+7:	xorl	%esi, %esi
 	movq	%rsp, %rdi
 	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
 	callq	*%rax
@@ -99,14 +212,27 @@ sem_timedwait:
 	decq	%rdi
 5:	testq	%rdi, %rdi
 	movl	$ETIMEDOUT, %r14d
-	js	6f		/* Time is already up.  */
+	js	36f		/* Time is already up.  */
 
 	movq	%rdi, (%rsp)	/* Store relative timeout.  */
 	movq	%rsi, 8(%rsp)
 
+.LcleanupSTART2:
+	call	__pthread_enable_asynccancel
+	movl	%eax, 16(%rsp)
+
 	movq	%rsp, %r10
+# if VALUE == 0
 	movq	%r12, %rdi
-	xorl	%esi, %esi
+# else
+	leaq	VALUE(%r12), %rdi
+# endif
+# if FUTEX_WAIT == 0
+	movl	PRIVATE(%rdi), %esi
+# else
+	movl	$FUTEX_WAIT, %esi
+	orl	PRIVATE(%rdi), %esi
+# endif
 	movl	$SYS_futex, %eax
 	xorl	%edx, %edx
 	syscall
@@ -114,41 +240,55 @@ sem_timedwait:
 
 	movl	16(%rsp), %edi
 	call	__pthread_disable_asynccancel
+.LcleanupEND2:
 
 	testq	%r14, %r14
 	je	9f
 	cmpq	$-EWOULDBLOCK, %r14
-	jne	3f
+	jne	33f
 
-9:	movl	(%r12), %eax
+9:
+# if VALUE == 0
+	movl	(%r12), %eax
+# else
+	movl	VALUE(%r12), %eax
+# endif
 8:	testl	%eax, %eax
 	je	7b
 
 	leaq	-1(%rax), %rcx
 	LOCK
+# if VALUE == 0
 	cmpxchgl %ecx, (%r12)
+# else
+	cmpxchgl %ecx, VALUE(%r12)
+# endif
 	jne	8b
 
 	xorl	%eax, %eax
-10:	addq	$24, %rsp
-	cfi_adjust_cfa_offset(-24)
+
+45:	LOCK
+	subq	$1, NWAITERS(%r12)
+
+	addq	$STACKFRAME, %rsp
+	cfi_adjust_cfa_offset(-STACKFRAME)
 	popq	%r14
 	cfi_adjust_cfa_offset(-8)
-	cfi_restore(14)
+	cfi_restore(%r14)
 	popq	%r13
 	cfi_adjust_cfa_offset(-8)
-	cfi_restore(13)
+	cfi_restore(%r13)
 	popq	%r12
 	cfi_adjust_cfa_offset(-8)
-	cfi_restore(12)
+	cfi_restore(%r12)
 	retq
 
-	cfi_adjust_cfa_offset(48)
-	cfi_offset(12, -16)		/* %r12 */
-	cfi_offset(13, -24)		/* %r13 */
-	cfi_offset(14, -32)		/* %r14 */
-3:	negq	%r14
-6:
+	cfi_adjust_cfa_offset(STACKFRAME + 3 * 8)
+	cfi_rel_offset(%r12, STACKFRAME + 2 * 8)
+	cfi_rel_offset(%r13, STACKFRAME + 1 * 8)
+	cfi_rel_offset(%r14, STACKFRAME)
+33:	negq	%r14
+36:
 #if USE___THREAD
 	movq	errno@gottpoff(%rip), %rdx
 	movl	%r14d, %fs:(%rdx)
@@ -158,17 +298,90 @@ sem_timedwait:
 #endif
 
 	orl	$-1, %eax
-	jmp	10b
-	cfi_adjust_cfa_offset(-48)
-	cfi_restore(14)
-	cfi_restore(13)
-	cfi_restore(12)
-
-11:	/* Canceled.  */
-	movq	$0xffffffffffffffff, %fs:RESULT
-	LOCK
-	orl	$0x10, %fs:CANCELHANDLING
-	movq	%fs:CLEANUP_JMP_BUF, %rdi
-	jmp	HIDDEN_JUMPTARGET (__pthread_unwind)
+	jmp	45b
+#endif
 	cfi_endproc
 	.size	sem_timedwait,.-sem_timedwait
+
+
+	.type	sem_timedwait_cleanup,@function
+sem_timedwait_cleanup:
+	cfi_startproc
+	cfi_adjust_cfa_offset(8)
+
+	movq	(%rsp), %rdi
+	LOCK
+	subq	$1, NWAITERS(%rdi)
+	movq	%rax, %rdi
+.LcallUR:
+	call	_Unwind_Resume@PLT
+	hlt
+.LENDCODE:
+	cfi_endproc
+	.size	sem_timedwait_cleanup,.-sem_timedwait_cleanup
+
+
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	.type	sem_timedwait_cleanup2,@function
+sem_timedwait_cleanup2:
+	cfi_startproc
+	cfi_adjust_cfa_offset(STACKFRAME + 3 * 8)
+	cfi_rel_offset(%r12, STACKFRAME + 2 * 8)
+	cfi_rel_offset(%r13, STACKFRAME + 1 * 8)
+	cfi_rel_offset(%r14, STACKFRAME)
+
+	LOCK
+	subq	$1, NWAITERS(%r12)
+	movq	%rax, %rdi
+	movq	STACKFRAME(%rsp), %r14
+	movq	STACKFRAME+8(%rsp), %r13
+	movq	STACKFRAME+16(%rsp), %r12
+.LcallUR2:
+	call	_Unwind_Resume@PLT
+	hlt
+.LENDCODE2:
+	cfi_endproc
+	.size	sem_timedwait_cleanup2,.-sem_timedwait_cleanup2
+#endif
+
+
+	.section .gcc_except_table,"a",@progbits
+.LexceptSTART:
+	.byte	DW_EH_PE_omit			# @LPStart format
+	.byte	DW_EH_PE_omit			# @TType format
+	.byte	DW_EH_PE_uleb128		# call-site format
+	.uleb128 .Lcstend-.Lcstbegin
+.Lcstbegin:
+	.uleb128 .LcleanupSTART-.LSTARTCODE
+	.uleb128 .LcleanupEND-.LcleanupSTART
+	.uleb128 sem_timedwait_cleanup-.LSTARTCODE
+	.uleb128  0
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	.uleb128 .LcleanupSTART2-.LSTARTCODE
+	.uleb128 .LcleanupEND2-.LcleanupSTART2
+	.uleb128 sem_timedwait_cleanup2-.LSTARTCODE
+	.uleb128  0
+#endif
+	.uleb128 .LcallUR-.LSTARTCODE
+	.uleb128 .LENDCODE-.LcallUR
+	.uleb128 0
+	.uleb128  0
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	.uleb128 .LcallUR2-.LSTARTCODE
+	.uleb128 .LENDCODE2-.LcallUR2
+	.uleb128 0
+	.uleb128  0
+#endif
+.Lcstend:
+
+
+#ifdef SHARED
+	.hidden	DW.ref.__gcc_personality_v0
+	.weak	DW.ref.__gcc_personality_v0
+	.section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
+	.align	8
+	.type	DW.ref.__gcc_personality_v0, @object
+	.size	DW.ref.__gcc_personality_v0, 8
+DW.ref.__gcc_personality_v0:
+	.quad	__gcc_personality_v0
+#endif
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S
index 08edc390c..7b7f63ddb 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,14 +18,9 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <pthread-errnos.h>
 
-#ifndef UP
-# define LOCK lock
-#else
-# define
-#endif
-
 	.text
 
 	.globl	sem_trywait
@@ -36,7 +31,7 @@ sem_trywait:
 2:	testl	%eax, %eax
 	jz	1f
 
-	leaq	-1(%rax), %rdx
+	leal	-1(%rax), %edx
 	LOCK
 	cmpxchgl %edx, (%rdi)
 	jne	2b
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S
index c2f94d47f..73d1d1633 100644
--- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S
+++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,15 +18,9 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <lowlevellock.h>
 #include <pthread-errnos.h>
-#include <tcb-offsets.h>
-#include <tls.h>
-
-#ifndef UP
-# define LOCK lock
-#else
-# define
-#endif
+#include <structsem.h>
 
 
 	.text
@@ -34,86 +28,155 @@
 	.globl	sem_wait
 	.type	sem_wait,@function
 	.align	16
-	cfi_startproc
 sem_wait:
-	/* First check for cancellation.  */
-	movl	%fs:CANCELHANDLING, %eax
-	andl	$0xfffffff9, %eax
-	cmpl	$8, %eax
-	je	4f
-
-	pushq	%r12
-	cfi_adjust_cfa_offset(8)
-	cfi_offset(12, -16)
-	pushq	%r13
-	cfi_adjust_cfa_offset(8)
-	movq	%rdi, %r13
-	cfi_offset(13, -24)
+.LSTARTCODE:
+	cfi_startproc
+#ifdef SHARED
+	cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
+			DW.ref.__gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
+#else
+	cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
+	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
+#endif
 
-3:	movl	(%r13), %eax
+#if VALUE == 0
+	movl	(%rdi), %eax
+#else
+	movl	VALUE(%rdi), %eax
+#endif
 2:	testl	%eax, %eax
 	je	1f
 
-	leaq	-1(%rax), %rdx
+	leal	-1(%rax), %edx
 	LOCK
-	cmpxchgl %edx, (%r13)
+#if VALUE == 0
+	cmpxchgl %edx, (%rdi)
+#else
+	cmpxchgl %edx, VALUE(%rdi)
+#endif
 	jne	2b
+
 	xorl	%eax, %eax
+	retq
 
-	popq	%r13
-	cfi_adjust_cfa_offset(-8)
-	cfi_restore(13)
-	popq	%r12
-	cfi_adjust_cfa_offset(-8)
-	cfi_restore(12)
+	/* This push is only needed to store the sem_t pointer for the
+	   exception handler.  */
+1:	pushq	%rdi
+	cfi_adjust_cfa_offset(8)
 
-	retq
+	LOCK
+	addq	$1, NWAITERS(%rdi)
 
-	cfi_adjust_cfa_offset(16)
-	cfi_offset(12, -16)
-	cfi_offset(13, -24)
-1:	call	__pthread_enable_asynccancel
+.LcleanupSTART:
+6:	call	__pthread_enable_asynccancel
 	movl	%eax, %r8d
 
 	xorq	%r10, %r10
 	movl	$SYS_futex, %eax
-	movq	%r13, %rdi
-	movq	%r10, %rsi
-	movq	%r10, %rdx
+#if FUTEX_WAIT == 0
+	movl	PRIVATE(%rdi), %esi
+#else
+	movl	$FUTEX_WAIT, %esi
+	orl	PRIVATE(%rdi), %esi
+#endif
+	xorl	%edx, %edx
 	syscall
-	movq	%rax, %r12
+	movq	%rax, %rcx
 
-	movl	%r8d, %edi
+	xchgq	%r8, %rdi
 	call	__pthread_disable_asynccancel
+.LcleanupEND:
+	movq	%r8, %rdi
 
-	testq	%r12, %r12
-	je	3b
-	cmpq	$-EWOULDBLOCK, %r12
-	je	3b
-	negq	%r12
+	testq	%rcx, %rcx
+	je	3f
+	cmpq	$-EWOULDBLOCK, %rcx
+	jne	4f
+
+3:
+#if VALUE == 0
+	movl	(%rdi), %eax
+#else
+	movl	VALUE(%rdi), %eax
+#endif
+5:	testl	%eax, %eax
+	je	6b
+
+	leal	-1(%rax), %edx
+	LOCK
+#if VALUE == 0
+	cmpxchgl %edx, (%rdi)
+#else
+	cmpxchgl %edx, VALUE(%rdi)
+#endif
+	jne	5b
+
+	xorl	%eax, %eax
+
+9:	LOCK
+	subq	$1, NWAITERS(%rdi)
+
+	leaq	8(%rsp), %rsp
+	cfi_adjust_cfa_offset(-8)
+
+	retq
+
+	cfi_adjust_cfa_offset(8)
+4:	negq	%rcx
 #if USE___THREAD
 	movq	errno@gottpoff(%rip), %rdx
-	movl	%r12d, %fs:(%rdx)
+	movl	%ecx, %fs:(%rdx)
 #else
+# error "not supported.  %rcx and %rdi must be preserved"
 	callq	__errno_location@plt
-	movl	%r12d, (%rax)
+	movl	%ecx, (%rax)
 #endif
 	orl	$-1, %eax
 
-	popq	%r13
-	cfi_adjust_cfa_offset(-8)
-	cfi_restore(13)
-	popq	%r12
-	cfi_adjust_cfa_offset(-8)
-	cfi_restore(12)
+	jmp 9b
+	.size	sem_wait,.-sem_wait
 
-	retq
 
-4:	/* Canceled.  */
-	movq	$0xffffffffffffffff, %fs:RESULT
+	.type	sem_wait_cleanup,@function
+sem_wait_cleanup:
+	movq	(%rsp), %rdi
 	LOCK
-	orl	$0x10, %fs:CANCELHANDLING
-	movq	%fs:CLEANUP_JMP_BUF, %rdi
-	jmp	HIDDEN_JUMPTARGET (__pthread_unwind)
+	subq	$1, NWAITERS(%rdi)
+	movq	%rax, %rdi
+.LcallUR:
+	call	_Unwind_Resume@PLT
+	hlt
+.LENDCODE:
 	cfi_endproc
-	.size	sem_wait,.-sem_wait
+	.size	sem_wait_cleanup,.-sem_wait_cleanup
+
+
+	.section .gcc_except_table,"a",@progbits
+.LexceptSTART:
+	.byte	DW_EH_PE_omit			# @LPStart format
+	.byte	DW_EH_PE_omit			# @TType format
+	.byte	DW_EH_PE_uleb128		# call-site format
+	.uleb128 .Lcstend-.Lcstbegin
+.Lcstbegin:
+	.uleb128 .LcleanupSTART-.LSTARTCODE
+	.uleb128 .LcleanupEND-.LcleanupSTART
+	.uleb128 sem_wait_cleanup-.LSTARTCODE
+	.uleb128  0
+	.uleb128 .LcallUR-.LSTARTCODE
+	.uleb128 .LENDCODE-.LcallUR
+	.uleb128 0
+	.uleb128  0
+.Lcstend:
+
+
+#ifdef SHARED
+	.hidden	DW.ref.__gcc_personality_v0
+	.weak	DW.ref.__gcc_personality_v0
+	.section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
+	.align	8
+	.type	DW.ref.__gcc_personality_v0, @object
+	.size	DW.ref.__gcc_personality_v0, 8
+DW.ref.__gcc_personality_v0:
+	.quad	__gcc_personality_v0
+#endif