path: root/main/musl/0002-d86af2a0-to-4992ace9.patch

diff --git a/arch/arm/bits/alltypes.h.in b/arch/arm/bits/alltypes.h.in
index bd23a6a..0d750cc 100644
--- a/arch/arm/bits/alltypes.h.in
+++ b/arch/arm/bits/alltypes.h.in
@@ -17,7 +17,7 @@ TYPEDEF long time_t;
 TYPEDEF long suseconds_t;
 
 TYPEDEF struct { union { int __i[9]; unsigned __s[9]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[6]; void *__p[6]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[6]; volatile void *volatile __p[6]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[12]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[8]; void *__p[8]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[5]; void *__p[5]; } __u; } pthread_barrier_t;
diff --git a/arch/i386/bits/alltypes.h.in b/arch/i386/bits/alltypes.h.in
index efd2c07..502c882 100644
--- a/arch/i386/bits/alltypes.h.in
+++ b/arch/i386/bits/alltypes.h.in
@@ -31,7 +31,7 @@ TYPEDEF long time_t;
 TYPEDEF long suseconds_t;
 
 TYPEDEF struct { union { int __i[9]; unsigned __s[9]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[6]; void *__p[6]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[6]; volatile void *volatile __p[6]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[12]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[8]; void *__p[8]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[5]; void *__p[5]; } __u; } pthread_barrier_t;
diff --git a/arch/microblaze/bits/alltypes.h.in b/arch/microblaze/bits/alltypes.h.in
index 6bd7942..4657d14 100644
--- a/arch/microblaze/bits/alltypes.h.in
+++ b/arch/microblaze/bits/alltypes.h.in
@@ -17,7 +17,7 @@ TYPEDEF long time_t;
 TYPEDEF long suseconds_t;
 
 TYPEDEF struct { union { int __i[9]; unsigned __s[9]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[6]; void *__p[6]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[6]; volatile void *volatile __p[6]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[12]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[8]; void *__p[8]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[5]; void *__p[5]; } __u; } pthread_barrier_t;
diff --git a/arch/mips/bits/alltypes.h.in b/arch/mips/bits/alltypes.h.in
index 6bd7942..4657d14 100644
--- a/arch/mips/bits/alltypes.h.in
+++ b/arch/mips/bits/alltypes.h.in
@@ -17,7 +17,7 @@ TYPEDEF long time_t;
 TYPEDEF long suseconds_t;
 
 TYPEDEF struct { union { int __i[9]; unsigned __s[9]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[6]; void *__p[6]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[6]; volatile void *volatile __p[6]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[12]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[8]; void *__p[8]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[5]; void *__p[5]; } __u; } pthread_barrier_t;
diff --git a/arch/or1k/bits/alltypes.h.in b/arch/or1k/bits/alltypes.h.in
index bd23a6a..0d750cc 100644
--- a/arch/or1k/bits/alltypes.h.in
+++ b/arch/or1k/bits/alltypes.h.in
@@ -17,7 +17,7 @@ TYPEDEF long time_t;
 TYPEDEF long suseconds_t;
 
 TYPEDEF struct { union { int __i[9]; unsigned __s[9]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[6]; void *__p[6]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[6]; volatile void *volatile __p[6]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[12]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[8]; void *__p[8]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[5]; void *__p[5]; } __u; } pthread_barrier_t;
diff --git a/arch/powerpc/bits/alltypes.h.in b/arch/powerpc/bits/alltypes.h.in
index e9d8dd8..378124c 100644
--- a/arch/powerpc/bits/alltypes.h.in
+++ b/arch/powerpc/bits/alltypes.h.in
@@ -17,7 +17,7 @@ TYPEDEF long time_t;
 TYPEDEF long suseconds_t;
 
 TYPEDEF struct { union { int __i[9]; unsigned __s[9]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[6]; void *__p[6]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[6]; volatile void *volatile __p[6]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[12]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[8]; void *__p[8]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[5]; void *__p[5]; } __u; } pthread_barrier_t;
diff --git a/arch/sh/bits/alltypes.h.in b/arch/sh/bits/alltypes.h.in
index e9d8dd8..378124c 100644
--- a/arch/sh/bits/alltypes.h.in
+++ b/arch/sh/bits/alltypes.h.in
@@ -17,7 +17,7 @@ TYPEDEF long time_t;
 TYPEDEF long suseconds_t;
 
 TYPEDEF struct { union { int __i[9]; unsigned __s[9]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[6]; void *__p[6]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[6]; volatile void *volatile __p[6]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[12]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[8]; void *__p[8]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[5]; void *__p[5]; } __u; } pthread_barrier_t;
diff --git a/arch/x32/bits/alltypes.h.in b/arch/x32/bits/alltypes.h.in
index b077fc9..8930efa 100644
--- a/arch/x32/bits/alltypes.h.in
+++ b/arch/x32/bits/alltypes.h.in
@@ -22,7 +22,7 @@ TYPEDEF long long time_t;
 TYPEDEF long long suseconds_t;
 
 TYPEDEF struct { union { int __i[14]; unsigned long __s[7]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[10]; void *__p[5]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[10]; volatile void *volatile __p[5]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[6]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[14]; void *__p[7]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[8]; void *__p[4]; } __u; } pthread_barrier_t;
diff --git a/arch/x86_64/bits/alltypes.h.in b/arch/x86_64/bits/alltypes.h.in
index 277e944..34b7d6a 100644
--- a/arch/x86_64/bits/alltypes.h.in
+++ b/arch/x86_64/bits/alltypes.h.in
@@ -22,7 +22,7 @@ TYPEDEF long time_t;
 TYPEDEF long suseconds_t;
 
 TYPEDEF struct { union { int __i[14]; unsigned long __s[7]; } __u; } pthread_attr_t;
-TYPEDEF struct { union { int __i[10]; void *__p[5]; } __u; } pthread_mutex_t;
+TYPEDEF struct { union { int __i[10]; volatile void *volatile __p[5]; } __u; } pthread_mutex_t;
 TYPEDEF struct { union { int __i[12]; void *__p[6]; } __u; } pthread_cond_t;
 TYPEDEF struct { union { int __i[14]; void *__p[7]; } __u; } pthread_rwlock_t;
 TYPEDEF struct { union { int __i[8]; void *__p[4]; } __u; } pthread_barrier_t;
diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h
index 650e811..2d090f8 100644
--- a/src/internal/pthread_impl.h
+++ b/src/internal/pthread_impl.h
@@ -33,9 +33,9 @@ struct pthread {
 	pthread_attr_t attr;
 	volatile int dead;
 	struct {
-		void **head;
+		volatile void *volatile head;
 		long off;
-		void *pending;
+		volatile void *volatile pending;
 	} robust_list;
 	int unblock_cancel;
 	int timer_id;
@@ -66,16 +66,16 @@ struct __timer {
 #define _m_prev __u.__p[3]
 #define _m_next __u.__p[4]
 #define _m_count __u.__i[5]
-#define _c_mutex __u.__p[0]
+#define _c_shared __u.__p[0]
 #define _c_seq __u.__i[2]
 #define _c_waiters __u.__i[3]
 #define _c_clock __u.__i[4]
-#define _c_lock __u.__i[5]
-#define _c_lockwait __u.__i[6]
-#define _c_waiters2 __u.__i[7]
-#define _c_destroy __u.__i[8]
+#define _c_lock __u.__i[8]
+#define _c_head __u.__p[1]
+#define _c_tail __u.__p[5]
 #define _rw_lock __u.__i[0]
 #define _rw_waiters __u.__i[1]
+#define _rw_shared __u.__i[2]
 #define _b_lock __u.__i[0]
 #define _b_waiters __u.__i[1]
 #define _b_limit __u.__i[2]
@@ -108,8 +108,13 @@ void __unmapself(void *, size_t);
 
 int __timedwait(volatile int *, int, clockid_t, const struct timespec *, void (*)(void *), void *, int);
 void __wait(volatile int *, volatile int *, int, int);
-#define __wake(addr, cnt, priv) \
-	__syscall(SYS_futex, addr, FUTEX_WAKE, (cnt)<0?INT_MAX:(cnt))
+static inline void __wake(volatile void *addr, int cnt, int priv)
+{
+	if (priv) priv = 128;
+	if (cnt<0) cnt = INT_MAX;
+	__syscall(SYS_futex, addr, FUTEX_WAKE|priv, cnt) != -EINVAL ||
+	__syscall(SYS_futex, addr, FUTEX_WAKE, cnt);
+}
 
 void __acquire_ptc();
 void __release_ptc();
diff --git a/src/mman/mmap.c b/src/mman/mmap.c
index 1917a54..56e39a7 100644
--- a/src/mman/mmap.c
+++ b/src/mman/mmap.c
@@ -16,8 +16,6 @@ weak_alias(dummy0, __vm_unlock);
 
 void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off)
 {
-	void *ret;
-
 	if (off & OFF_MASK) {
 		errno = EINVAL;
 		return MAP_FAILED;
@@ -26,14 +24,15 @@ void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off)
 		errno = ENOMEM;
 		return MAP_FAILED;
 	}
-	if (flags & MAP_FIXED) __vm_lock(-1);
+	if (flags & MAP_FIXED) {
+		__vm_lock(-1);
+		__vm_unlock();
+	}
 #ifdef SYS_mmap2
-	ret = (void *)syscall(SYS_mmap2, start, len, prot, flags, fd, off/UNIT);
+	return (void *)syscall(SYS_mmap2, start, len, prot, flags, fd, off/UNIT);
 #else
-	ret = (void *)syscall(SYS_mmap, start, len, prot, flags, fd, off);
+	return (void *)syscall(SYS_mmap, start, len, prot, flags, fd, off);
 #endif
-	if (flags & MAP_FIXED) __vm_unlock();
-	return ret;
 }
 
 weak_alias(__mmap, mmap);
diff --git a/src/mman/munmap.c b/src/mman/munmap.c
index 8488d75..359c691 100644
--- a/src/mman/munmap.c
+++ b/src/mman/munmap.c
@@ -11,8 +11,8 @@ int __munmap(void *start, size_t len)
 {
 	int ret;
 	__vm_lock(-1);
-	ret = syscall(SYS_munmap, start, len);
 	__vm_unlock();
+	ret = syscall(SYS_munmap, start, len);
 	return ret;
 }
 
diff --git a/src/thread/__timedwait.c b/src/thread/__timedwait.c
index 302273a..39eb996 100644
--- a/src/thread/__timedwait.c
+++ b/src/thread/__timedwait.c
@@ -4,12 +4,15 @@
 #include "futex.h"
 #include "syscall.h"
 
-static int do_wait(volatile int *addr, int val,
-	clockid_t clk, const struct timespec *at, int priv)
+int __timedwait(volatile int *addr, int val,
+	clockid_t clk, const struct timespec *at,
+	void (*cleanup)(void *), void *arg, int priv)
 {
-	int r;
+	int r, cs;
 	struct timespec to, *top=0;
 
+	if (priv) priv = 128;
+
 	if (at) {
 		if (at->tv_nsec >= 1000000000UL) return EINVAL;
 		if (clock_gettime(clk, &to)) return EINVAL;
@@ -22,21 +25,12 @@ static int do_wait(volatile int *addr, int val,
 		top = &to;
 	}
 
-	r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT, val, top);
-	if (r == EINTR || r == EINVAL || r == ETIMEDOUT) return r;
-	return 0;
-}
-
-int __timedwait(volatile int *addr, int val,
-	clockid_t clk, const struct timespec *at,
-	void (*cleanup)(void *), void *arg, int priv)
-{
-	int r, cs;
-
 	if (!cleanup) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs);
 	pthread_cleanup_push(cleanup, arg);
 
-	r = do_wait(addr, val, clk, at, priv);
+	r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT|priv, val, top);
+	if (r == EINVAL) r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT, val, top);
+	if (r != EINTR && r != ETIMEDOUT) r = 0;
 
 	pthread_cleanup_pop(0);
 	if (!cleanup) pthread_setcancelstate(cs, 0);
diff --git a/src/thread/__wait.c b/src/thread/__wait.c
index a1e4780..ec1e820 100644
--- a/src/thread/__wait.c
+++ b/src/thread/__wait.c
@@ -3,13 +3,15 @@
 void __wait(volatile int *addr, volatile int *waiters, int val, int priv)
 {
 	int spins=10000;
-	if (priv) priv = 128; priv=0;
+	if (priv) priv = 128;
 	while (spins--) {
 		if (*addr==val) a_spin();
 		else return;
 	}
 	if (waiters) a_inc(waiters);
-	while (*addr==val)
-		__syscall(SYS_futex, addr, FUTEX_WAIT|priv, val, 0);
+	while (*addr==val) {
+		__syscall(SYS_futex, addr, FUTEX_WAIT|priv, val, 0) != -EINVAL
+		|| __syscall(SYS_futex, addr, FUTEX_WAIT, val, 0);
+	}
 	if (waiters) a_dec(waiters);
 }
diff --git a/src/thread/pthread_attr_get.c b/src/thread/pthread_attr_get.c
index 03fc91e..3d296bf 100644
--- a/src/thread/pthread_attr_get.c
+++ b/src/thread/pthread_attr_get.c
@@ -75,7 +75,7 @@ int pthread_mutexattr_getprotocol(const pthread_mutexattr_t *restrict a, int *re
 }
 int pthread_mutexattr_getpshared(const pthread_mutexattr_t *restrict a, int *restrict pshared)
 {
-	*pshared = a->__attr>>31;
+	*pshared = a->__attr / 128U % 2;
 	return 0;
 }
 
diff --git a/src/thread/pthread_barrier_wait.c b/src/thread/pthread_barrier_wait.c
index 5e60338..6b329c9 100644
--- a/src/thread/pthread_barrier_wait.c
+++ b/src/thread/pthread_barrier_wait.c
@@ -87,7 +87,8 @@ int pthread_barrier_wait(pthread_barrier_t *b)
 			a_spin();
 		a_inc(&inst->finished);
 		while (inst->finished == 1)
-			__syscall(SYS_futex, &inst->finished, FUTEX_WAIT,1,0);
+			__syscall(SYS_futex,&inst->finished,FUTEX_WAIT|128,1,0) != -EINTR
+			|| __syscall(SYS_futex,&inst->finished,FUTEX_WAIT,1,0);
 		return PTHREAD_BARRIER_SERIAL_THREAD;
 	}
 
diff --git a/src/thread/pthread_cond_broadcast.c b/src/thread/pthread_cond_broadcast.c
index 0901daf..69f840f 100644
--- a/src/thread/pthread_cond_broadcast.c
+++ b/src/thread/pthread_cond_broadcast.c
@@ -1,39 +1,12 @@
 #include "pthread_impl.h"
 
+int __private_cond_signal(pthread_cond_t *, int);
+
 int pthread_cond_broadcast(pthread_cond_t *c)
 {
-	pthread_mutex_t *m;
-
+	if (!c->_c_shared) return __private_cond_signal(c, -1);
 	if (!c->_c_waiters) return 0;
-
 	a_inc(&c->_c_seq);
-
-	/* If cond var is process-shared, simply wake all waiters. */
-	if (c->_c_mutex == (void *)-1) {
-		__wake(&c->_c_seq, -1, 0);
-		return 0;
-	}
-
-	/* Block waiters from returning so we can use the mutex. */
-	while (a_swap(&c->_c_lock, 1))
-		__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
-	if (!c->_c_waiters)
-		goto out;
-	m = c->_c_mutex;
-
-	/* Move waiter count to the mutex */
-	a_fetch_add(&m->_m_waiters, c->_c_waiters2);
-	c->_c_waiters2 = 0;
-
-	/* Perform the futex requeue, waking one waiter unless we know
-	 * that the calling thread holds the mutex. */
-	__syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE,
-		!m->_m_type || (m->_m_lock&INT_MAX)!=__pthread_self()->tid,
-		INT_MAX, &m->_m_lock);
-
-out:
-	a_store(&c->_c_lock, 0);
-	if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0);
-
+	__wake(&c->_c_seq, -1, 0);
 	return 0;
 }
diff --git a/src/thread/pthread_cond_destroy.c b/src/thread/pthread_cond_destroy.c
index a096c55..8c55516 100644
--- a/src/thread/pthread_cond_destroy.c
+++ b/src/thread/pthread_cond_destroy.c
@@ -2,12 +2,13 @@
 
 int pthread_cond_destroy(pthread_cond_t *c)
 {
-	int priv = c->_c_mutex != (void *)-1;
-	int cnt;
-	c->_c_destroy = 1;
-	if (c->_c_waiters)
-		__wake(&c->_c_seq, -1, priv);
-	while ((cnt = c->_c_waiters))
-		__wait(&c->_c_waiters, 0, cnt, priv);
+	if (c->_c_shared && c->_c_waiters) {
+		int cnt;
+		a_or(&c->_c_waiters, 0x80000000);
+		a_inc(&c->_c_seq);
+		__wake(&c->_c_seq, -1, 0);
+		while ((cnt = c->_c_waiters) & 0x7fffffff)
+			__wait(&c->_c_waiters, 0, cnt, 0);
+	}
 	return 0;
 }
diff --git a/src/thread/pthread_cond_init.c b/src/thread/pthread_cond_init.c
index 357ecd5..8c484dd 100644
--- a/src/thread/pthread_cond_init.c
+++ b/src/thread/pthread_cond_init.c
@@ -5,7 +5,7 @@ int pthread_cond_init(pthread_cond_t *restrict c, const pthread_condattr_t *rest
 	*c = (pthread_cond_t){0};
 	if (a) {
 		c->_c_clock = a->__attr & 0x7fffffff;
-		if (a->__attr>>31) c->_c_mutex = (void *)-1;
+		if (a->__attr>>31) c->_c_shared = (void *)-1;
 	}
 	return 0;
 }
diff --git a/src/thread/pthread_cond_signal.c b/src/thread/pthread_cond_signal.c
index 71bcdcd..119c00a 100644
--- a/src/thread/pthread_cond_signal.c
+++ b/src/thread/pthread_cond_signal.c
@@ -1,9 +1,12 @@
 #include "pthread_impl.h"
 
+int __private_cond_signal(pthread_cond_t *, int);
+
 int pthread_cond_signal(pthread_cond_t *c)
 {
+	if (!c->_c_shared) return __private_cond_signal(c, 1);
 	if (!c->_c_waiters) return 0;
 	a_inc(&c->_c_seq);
-	if (c->_c_waiters) __wake(&c->_c_seq, 1, 0);
+	__wake(&c->_c_seq, 1, 0);
 	return 0;
 }
diff --git a/src/thread/pthread_cond_timedwait.c b/src/thread/pthread_cond_timedwait.c
index 99d62cc..c5cf66c 100644
--- a/src/thread/pthread_cond_timedwait.c
+++ b/src/thread/pthread_cond_timedwait.c
@@ -1,47 +1,130 @@
 #include "pthread_impl.h"
 
-struct cm {
-	pthread_cond_t *c;
-	pthread_mutex_t *m;
+/*
+ * struct waiter
+ *
+ * Waiter objects have automatic storage on the waiting thread, and
+ * are used in building a linked list representing waiters currently
+ * waiting on the condition variable or a group of waiters woken
+ * together by a broadcast or signal; in the case of signal, this is a
+ * degenerate list of one member.
+ *
+ * Waiter lists attached to the condition variable itself are
+ * protected by the lock on the cv. Detached waiter lists are never
+ * modified again, but can only be traversed in reverse order, and are
+ * protected by the "barrier" locks in each node, which are unlocked
+ * in turn to control wake order.
+ *
+ * Since process-shared cond var semantics do not necessarily allow
+ * one thread to see another's automatic storage (they may be in
+ * different processes), the waiter list is not used for the
+ * process-shared case, but the structure is still used to store data
+ * needed by the cancellation cleanup handler.
+ */
+
+struct waiter {
+	struct waiter *prev, *next;
+	int state, barrier, mutex_ret;
+	int *notify;
+	pthread_mutex_t *mutex;
+	pthread_cond_t *cond;
+	int shared;
 };
 
-static void unwait(pthread_cond_t *c, pthread_mutex_t *m)
+/* Self-synchronized-destruction-safe lock functions */
+
+static inline void lock(volatile int *l)
 {
-	/* Removing a waiter is non-trivial if we could be using requeue
-	 * based broadcast signals, due to mutex access issues, etc. */
+	if (a_cas(l, 0, 1)) {
+		a_cas(l, 1, 2);
+		do __wait(l, 0, 2, 1);
+		while (a_cas(l, 0, 2));
+	}
+}
 
-	if (c->_c_mutex == (void *)-1) {
-		a_dec(&c->_c_waiters);
-		if (c->_c_destroy) __wake(&c->_c_waiters, 1, 0);
+static inline void unlock(volatile int *l)
+{
+	if (a_swap(l, 0)==2)
+		__wake(l, 1, 1);
+}
+
+static inline void unlock_requeue(volatile int *l, volatile int *r, int w)
+{
+	a_store(l, 0);
+	if (w) __wake(l, 1, 1);
+	else __syscall(SYS_futex, l, FUTEX_REQUEUE|128, 0, 1, r) != -EINVAL
+		|| __syscall(SYS_futex, l, FUTEX_REQUEUE, 0, 1, r);
+}
+
+enum {
+	WAITING,
+	SIGNALED,
+	LEAVING,
+};
+
+static void unwait(void *arg)
+{
+	struct waiter *node = arg;
+
+	if (node->shared) {
+		pthread_cond_t *c = node->cond;
+		pthread_mutex_t *m = node->mutex;
+		if (a_fetch_add(&c->_c_waiters, -1) == -0x7fffffff)
+			__wake(&c->_c_waiters, 1, 0);
+		node->mutex_ret = pthread_mutex_lock(m);
 		return;
 	}
 
-	while (a_swap(&c->_c_lock, 1))
-		__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
+	int oldstate = a_cas(&node->state, WAITING, LEAVING);
+
+	if (oldstate == WAITING) {
+		/* Access to cv object is valid because this waiter was not
+		 * yet signaled and a new signal/broadcast cannot return
+		 * after seeing a LEAVING waiter without getting notified
+		 * via the futex notify below. */
+
+		pthread_cond_t *c = node->cond;
+		lock(&c->_c_lock);
+		
+		if (c->_c_head == node) c->_c_head = node->next;
+		else if (node->prev) node->prev->next = node->next;
+		if (c->_c_tail == node) c->_c_tail = node->prev;
+		else if (node->next) node->next->prev = node->prev;
+		
+		unlock(&c->_c_lock);
+
+		if (node->notify) {
+			if (a_fetch_add(node->notify, -1)==1)
+				__wake(node->notify, 1, 1);
+		}
+	} else {
+		/* Lock barrier first to control wake order. */
+		lock(&node->barrier);
+	}
 
-	if (c->_c_waiters2) c->_c_waiters2--;
-	else a_dec(&m->_m_waiters);
+	node->mutex_ret = pthread_mutex_lock(node->mutex);
 
-	a_store(&c->_c_lock, 0);
-	if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1);
+	if (oldstate == WAITING) return;
 
-	a_dec(&c->_c_waiters);
-	if (c->_c_destroy) __wake(&c->_c_waiters, 1, 1);
-}
+	if (!node->next) a_inc(&node->mutex->_m_waiters);
 
-static void cleanup(void *p)
-{
-	struct cm *cm = p;
-	unwait(cm->c, cm->m);
-	pthread_mutex_lock(cm->m);
+	/* Unlock the barrier that's holding back the next waiter, and
+	 * either wake it or requeue it to the mutex. */
+	if (node->prev) {
+		unlock_requeue(&node->prev->barrier,
+			&node->mutex->_m_lock,
+			node->mutex->_m_type & 128);
+	} else {
+		a_dec(&node->mutex->_m_waiters);
+	}
 }
 
 int pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restrict m, const struct timespec *restrict ts)
 {
-	struct cm cm = { .c=c, .m=m };
-	int r, e=0, seq;
+	struct waiter node = { .cond = c, .mutex = m };
+	int e, seq, *fut, clock = c->_c_clock;
 
-	if (m->_m_type && (m->_m_lock&INT_MAX) != __pthread_self()->tid)
+	if ((m->_m_type&15) && (m->_m_lock&INT_MAX) != __pthread_self()->tid)
 		return EPERM;
 
 	if (ts && ts->tv_nsec >= 1000000000UL)
@@ -49,28 +132,68 @@ int pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restrict
 
 	pthread_testcancel();
 
-	a_inc(&c->_c_waiters);
-
-	if (c->_c_mutex != (void *)-1) {
-		c->_c_mutex = m;
-		while (a_swap(&c->_c_lock, 1))
-			__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
-		c->_c_waiters2++;
-		a_store(&c->_c_lock, 0);
-		if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1);
+	if (c->_c_shared) {
+		node.shared = 1;
+		fut = &c->_c_seq;
+		seq = c->_c_seq;
+		a_inc(&c->_c_waiters);
+	} else {
+		lock(&c->_c_lock);
+
+		seq = node.barrier = 2;
+		fut = &node.barrier;
+		node.state = WAITING;
+		node.next = c->_c_head;
+		c->_c_head = &node;
+		if (!c->_c_tail) c->_c_tail = &node;
+		else node.next->prev = &node;
+
+		unlock(&c->_c_lock);
 	}
 
-	seq = c->_c_seq;
-
 	pthread_mutex_unlock(m);
 
-	do e = __timedwait(&c->_c_seq, seq, c->_c_clock, ts, cleanup, &cm, 0);
-	while (c->_c_seq == seq && (!e || e==EINTR));
+	do e = __timedwait(fut, seq, clock, ts, unwait, &node, !node.shared);
+	while (*fut==seq && (!e || e==EINTR));
 	if (e == EINTR) e = 0;
 
-	unwait(c, m);
+	unwait(&node);
+
+	return node.mutex_ret ? node.mutex_ret : e;
+}
+
+int __private_cond_signal(pthread_cond_t *c, int n)
+{
+	struct waiter *p, *first=0;
+	int ref = 0, cur;
+
+	lock(&c->_c_lock);
+	for (p=c->_c_tail; n && p; p=p->prev) {
+		if (a_cas(&p->state, WAITING, SIGNALED) != WAITING) {
+			ref++;
+			p->notify = &ref;
+		} else {
+			n--;
+			if (!first) first=p;
+		}
+	}
+	/* Split the list, leaving any remainder on the cv. */
+	if (p) {
+		if (p->next) p->next->prev = 0;
+		p->next = 0;
+	} else {
+		c->_c_head = 0;
+	}
+	c->_c_tail = p;
+	unlock(&c->_c_lock);
+
+	/* Wait for any waiters in the LEAVING state to remove
+	 * themselves from the list before returning or allowing
+	 * signaled threads to proceed. */
+	while ((cur = ref)) __wait(&ref, 0, cur, 1);
 
-	if ((r=pthread_mutex_lock(m))) return r;
+	/* Allow first signaled waiter, if any, to proceed. */
+	if (first) unlock(&first->barrier);
 
-	return e;
+	return 0;
 }
diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c
index e77e54a..c8c117b 100644
--- a/src/thread/pthread_create.c
+++ b/src/thread/pthread_create.c
@@ -10,6 +10,7 @@ static void dummy_0()
 weak_alias(dummy_0, __acquire_ptc);
 weak_alias(dummy_0, __release_ptc);
 weak_alias(dummy_0, __pthread_tsd_run_dtors);
+weak_alias(dummy_0, __do_private_robust_list);
 
 _Noreturn void pthread_exit(void *result)
 {
@@ -63,6 +64,8 @@ _Noreturn void pthread_exit(void *result)
 			a_dec(&libc.bytelocale_cnt_minus_1);
 	}
 
+	__do_private_robust_list();
+
 	if (self->detached && self->map_base) {
 		/* Detached threads must avoid the kernel clear_child_tid
 		 * feature, since the virtual address will have been
diff --git a/src/thread/pthread_mutex_consistent.c b/src/thread/pthread_mutex_consistent.c
index 65da29f..96b83b5 100644
--- a/src/thread/pthread_mutex_consistent.c
+++ b/src/thread/pthread_mutex_consistent.c
@@ -2,9 +2,9 @@
 
 int pthread_mutex_consistent(pthread_mutex_t *m)
 {
-	if (m->_m_type < 8) return EINVAL;
-	if ((m->_m_lock & 0x3fffffff) != __pthread_self()->tid)
+	if (!(m->_m_type & 8)) return EINVAL;
+	if ((m->_m_lock & 0x7fffffff) != __pthread_self()->tid)
 		return EPERM;
-	m->_m_type -= 8;
+	m->_m_type &= ~8U;
 	return 0;
 }
diff --git a/src/thread/pthread_mutex_init.c b/src/thread/pthread_mutex_init.c
index 9d85a35..acf45a7 100644
--- a/src/thread/pthread_mutex_init.c
+++ b/src/thread/pthread_mutex_init.c
@@ -3,6 +3,6 @@
 int pthread_mutex_init(pthread_mutex_t *restrict m, const pthread_mutexattr_t *restrict a)
 {
 	*m = (pthread_mutex_t){0};
-	if (a) m->_m_type = a->__attr & 7;
+	if (a) m->_m_type = a->__attr;
 	return 0;
 }
diff --git a/src/thread/pthread_mutex_lock.c b/src/thread/pthread_mutex_lock.c
index 42b5af6..2a9a3aa 100644
--- a/src/thread/pthread_mutex_lock.c
+++ b/src/thread/pthread_mutex_lock.c
@@ -2,7 +2,8 @@
 
 int pthread_mutex_lock(pthread_mutex_t *m)
 {
-	if (m->_m_type == PTHREAD_MUTEX_NORMAL && !a_cas(&m->_m_lock, 0, EBUSY))
+	if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL
+	    && !a_cas(&m->_m_lock, 0, EBUSY))
 		return 0;
 
 	return pthread_mutex_timedlock(m, 0);
diff --git a/src/thread/pthread_mutex_timedlock.c b/src/thread/pthread_mutex_timedlock.c
index 7b1afc0..2a959d2 100644
--- a/src/thread/pthread_mutex_timedlock.c
+++ b/src/thread/pthread_mutex_timedlock.c
@@ -2,21 +2,23 @@
 
 int pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec *restrict at)
 {
-	int r, t;
-
-	if (m->_m_type == PTHREAD_MUTEX_NORMAL && !a_cas(&m->_m_lock, 0, EBUSY))
+	if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL
+	    && !a_cas(&m->_m_lock, 0, EBUSY))
 		return 0;
 
+	int r, t, priv = (m->_m_type & 128) ^ 128;
+
 	while ((r=pthread_mutex_trylock(m)) == EBUSY) {
-		if (!(r=m->_m_lock) || (r&0x40000000)) continue;
+		if (!(r=m->_m_lock) || ((r&0x40000000) && (m->_m_type&4)))
+			continue;
 		if ((m->_m_type&3) == PTHREAD_MUTEX_ERRORCHECK
-		 && (r&0x1fffffff) == __pthread_self()->tid)
+		 && (r&0x7fffffff) == __pthread_self()->tid)
 			return EDEADLK;
 
 		a_inc(&m->_m_waiters);
 		t = r | 0x80000000;
 		a_cas(&m->_m_lock, r, t);
-		r = __timedwait(&m->_m_lock, t, CLOCK_REALTIME, at, 0, 0, 0);
+		r = __timedwait(&m->_m_lock, t, CLOCK_REALTIME, at, 0, 0, priv);
 		a_dec(&m->_m_waiters);
 		if (r && r != EINTR) break;
 	}
diff --git a/src/thread/pthread_mutex_trylock.c b/src/thread/pthread_mutex_trylock.c
index 00ad65d..e851517 100644
--- a/src/thread/pthread_mutex_trylock.c
+++ b/src/thread/pthread_mutex_trylock.c
@@ -1,52 +1,58 @@
 #include "pthread_impl.h"
 
-int pthread_mutex_trylock(pthread_mutex_t *m)
+int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
 {
-	int tid, old, own;
-	pthread_t self;
-
-	if (m->_m_type == PTHREAD_MUTEX_NORMAL)
-		return a_cas(&m->_m_lock, 0, EBUSY) & EBUSY;
-
-	self = __pthread_self();
-	tid = self->tid;
-
-	if (m->_m_type >= 4) {
-		if (!self->robust_list.off)
-			__syscall(SYS_set_robust_list,
-				&self->robust_list, 3*sizeof(long));
+	int old, own;
+	int type = m->_m_type & 15;
+	pthread_t self = __pthread_self();
+	int tid = self->tid;
+
+	if (!self->robust_list.off) {
+		__syscall(SYS_set_robust_list, &self->robust_list, 3*sizeof(long));
+		self->robust_list.head = &self->robust_list.head;
 		self->robust_list.off = (char*)&m->_m_lock-(char *)&m->_m_next;
-		self->robust_list.pending = &m->_m_next;
 	}
 
 	old = m->_m_lock;
 	own = old & 0x7fffffff;
-	if (own == tid && (m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE) {
+	if (own == tid && (type&3) == PTHREAD_MUTEX_RECURSIVE) {
 		if ((unsigned)m->_m_count >= INT_MAX) return EAGAIN;
 		m->_m_count++;
 		return 0;
 	}
+	if (own == 0x40000000) return ENOTRECOVERABLE;
 
-	if ((own && !(own & 0x40000000)) || a_cas(&m->_m_lock, old, tid)!=old)
-		return EBUSY;
-
-	if (m->_m_type < 4) return 0;
+	if (m->_m_type & 128) {
+		if (m->_m_waiters) tid |= 0x80000000;
+		self->robust_list.pending = &m->_m_next;
+	}
 
-	if (m->_m_type >= 8) {
-		m->_m_lock = 0;
-		return ENOTRECOVERABLE;
+	if ((own && (!(own & 0x40000000) || !(type & 4)))
+	    || a_cas(&m->_m_lock, old, tid) != old) {
+		self->robust_list.pending = 0;
+		return EBUSY;
 	}
-	m->_m_next = self->robust_list.head;
+
+	volatile void *next = self->robust_list.head;
+	m->_m_next = next;
 	m->_m_prev = &self->robust_list.head;
-	if (self->robust_list.head)
-		self->robust_list.head[-1] = &m->_m_next;
+	if (next != &self->robust_list.head) *(volatile void *volatile *)
+		((char *)next - sizeof(void *)) = &m->_m_next;
 	self->robust_list.head = &m->_m_next;
 	self->robust_list.pending = 0;
+
 	if (own) {
 		m->_m_count = 0;
-		m->_m_type += 8;
+		m->_m_type |= 8;
 		return EOWNERDEAD;
 	}
 
 	return 0;
 }
+
+int pthread_mutex_trylock(pthread_mutex_t *m)
+{
+	if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL)
+		return a_cas(&m->_m_lock, 0, EBUSY) & EBUSY;
+	return __pthread_mutex_trylock_owner(m);
+}
diff --git a/src/thread/pthread_mutex_unlock.c b/src/thread/pthread_mutex_unlock.c
index b4bd74b..46761d9 100644
--- a/src/thread/pthread_mutex_unlock.c
+++ b/src/thread/pthread_mutex_unlock.c
@@ -8,30 +8,31 @@ int pthread_mutex_unlock(pthread_mutex_t *m)
 	pthread_t self;
 	int waiters = m->_m_waiters;
 	int cont;
-	int robust = 0;
+	int type = m->_m_type & 15;
+	int priv = (m->_m_type & 128) ^ 128;
 
-	if (m->_m_type != PTHREAD_MUTEX_NORMAL) {
-		if (!m->_m_lock)
-			return EPERM;
+	if (type != PTHREAD_MUTEX_NORMAL) {
 		self = __pthread_self();
-		if ((m->_m_lock&0x1fffffff) != self->tid)
+		if ((m->_m_lock&0x7fffffff) != self->tid)
 			return EPERM;
-		if ((m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count)
+		if ((type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count)
 			return m->_m_count--, 0;
-		if (m->_m_type >= 4) {
-			robust = 1;
+		if (!priv) {
 			self->robust_list.pending = &m->_m_next;
-			*(void **)m->_m_prev = m->_m_next;
-			if (m->_m_next) ((void **)m->_m_next)[-1] = m->_m_prev;
 			__vm_lock_impl(+1);
 		}
+		volatile void *prev = m->_m_prev;
+		volatile void *next = m->_m_next;
+		*(volatile void *volatile *)prev = next;
+		if (next != &self->robust_list.head) *(volatile void *volatile *)
+			((char *)next - sizeof(void *)) = prev;
 	}
-	cont = a_swap(&m->_m_lock, 0);
-	if (robust) {
+	cont = a_swap(&m->_m_lock, (type & 8) ? 0x40000000 : 0);
+	if (type != PTHREAD_MUTEX_NORMAL && !priv) {
 		self->robust_list.pending = 0;
 		__vm_unlock_impl();
 	}
 	if (waiters || cont<0)
-		__wake(&m->_m_lock, 1, 0);
+		__wake(&m->_m_lock, 1, priv);
 	return 0;
 }
diff --git a/src/thread/pthread_mutexattr_setpshared.c b/src/thread/pthread_mutexattr_setpshared.c
index 8c7a1e2..100f6ff 100644
--- a/src/thread/pthread_mutexattr_setpshared.c
+++ b/src/thread/pthread_mutexattr_setpshared.c
@@ -3,7 +3,7 @@
 int pthread_mutexattr_setpshared(pthread_mutexattr_t *a, int pshared)
 {
 	if (pshared > 1U) return EINVAL;
-	a->__attr &= 0x7fffffff;
-	a->__attr |= pshared<<31;
+	a->__attr &= ~128U;
+	a->__attr |= pshared<<7;
 	return 0;
 }
diff --git a/src/thread/pthread_mutexattr_setrobust.c b/src/thread/pthread_mutexattr_setrobust.c
index dcfa4cf..d062788 100644
--- a/src/thread/pthread_mutexattr_setrobust.c
+++ b/src/thread/pthread_mutexattr_setrobust.c
@@ -1,4 +1,28 @@
 #include "pthread_impl.h"
+#include <stddef.h>
+
+void __do_private_robust_list()
+{
+	pthread_t self = __pthread_self();
+	volatile void *volatile *p;
+	volatile void *volatile *prev;
+	volatile void *volatile *next;
+	pthread_mutex_t *m;
+
+	prev = &self->robust_list.head;
+	for (p=self->robust_list.head; p&&p!=&self->robust_list.head; p=next) {
+		next = *p;
+		m = (void *)((char *)p - offsetof(pthread_mutex_t, _m_next));
+		if (!(m->_m_type & 128)) {
+			int waiters = m->_m_waiters;
+			*prev = next;
+			int cont = a_swap(&m->_m_lock, self->tid|0x40000000);
+			if (cont < 0 || waiters) __wake(&m->_m_lock, 1, 1);
+		} else {
+			prev = p;
+		}
+	}
+}
 
 int pthread_mutexattr_setrobust(pthread_mutexattr_t *a, int robust)
 {
diff --git a/src/thread/pthread_once.c b/src/thread/pthread_once.c
index e01f6d4..2eb0f93 100644
--- a/src/thread/pthread_once.c
+++ b/src/thread/pthread_once.c
@@ -3,7 +3,7 @@
 static void undo(void *control)
 {
 	a_store(control, 0);
-	__wake(control, 1, 0);
+	__wake(control, 1, 1);
 }
 
 int pthread_once(pthread_once_t *control, void (*init)(void))
@@ -25,10 +25,10 @@ int pthread_once(pthread_once_t *control, void (*init)(void))
 		pthread_cleanup_pop(0);
 
 		a_store(control, 2);
-		if (waiters) __wake(control, -1, 0);
+		if (waiters) __wake(control, -1, 1);
 		return 0;
 	case 1:
-		__wait(control, &waiters, 1, 0);
+		__wait(control, &waiters, 1, 1);
 		continue;
 	case 2:
 		return 0;
diff --git a/src/thread/pthread_rwlock_init.c b/src/thread/pthread_rwlock_init.c
index 82df52e..a2c0b47 100644
--- a/src/thread/pthread_rwlock_init.c
+++ b/src/thread/pthread_rwlock_init.c
@@ -3,7 +3,6 @@
 int pthread_rwlock_init(pthread_rwlock_t *restrict rw, const pthread_rwlockattr_t *restrict a)
 {
 	*rw = (pthread_rwlock_t){0};
-	if (a) {
-	}
+	if (a) rw->_rw_shared = a->__attr[0]*128;
 	return 0;
 }
diff --git a/src/thread/pthread_rwlock_timedrdlock.c b/src/thread/pthread_rwlock_timedrdlock.c
index c0c94c9..a2b4d44 100644
--- a/src/thread/pthread_rwlock_timedrdlock.c
+++ b/src/thread/pthread_rwlock_timedrdlock.c
@@ -8,7 +8,7 @@ int pthread_rwlock_timedrdlock(pthread_rwlock_t *restrict rw, const struct times
 		t = r | 0x80000000;
 		a_inc(&rw->_rw_waiters);
 		a_cas(&rw->_rw_lock, r, t);
-		r = __timedwait(&rw->_rw_lock, t, CLOCK_REALTIME, at, 0, 0, 0);
+		r = __timedwait(&rw->_rw_lock, t, CLOCK_REALTIME, at, 0, 0, rw->_rw_shared^128);
 		a_dec(&rw->_rw_waiters);
 		if (r && r != EINTR) return r;
 	}
diff --git a/src/thread/pthread_rwlock_timedwrlock.c b/src/thread/pthread_rwlock_timedwrlock.c
index 339a167..63a32ec 100644
--- a/src/thread/pthread_rwlock_timedwrlock.c
+++ b/src/thread/pthread_rwlock_timedwrlock.c
@@ -8,7 +8,7 @@ int pthread_rwlock_timedwrlock(pthread_rwlock_t *restrict rw, const struct times
 		t = r | 0x80000000;
 		a_inc(&rw->_rw_waiters);
 		a_cas(&rw->_rw_lock, r, t);
-		r = __timedwait(&rw->_rw_lock, t, CLOCK_REALTIME, at, 0, 0, 0);
+		r = __timedwait(&rw->_rw_lock, t, CLOCK_REALTIME, at, 0, 0, rw->_rw_shared^128);
 		a_dec(&rw->_rw_waiters);
 		if (r && r != EINTR) return r;
 	}
diff --git a/src/thread/pthread_rwlock_unlock.c b/src/thread/pthread_rwlock_unlock.c
index a6d2085..7b5eec8 100644
--- a/src/thread/pthread_rwlock_unlock.c
+++ b/src/thread/pthread_rwlock_unlock.c
@@ -2,7 +2,7 @@
 
 int pthread_rwlock_unlock(pthread_rwlock_t *rw)
 {
-	int val, cnt, waiters, new;
+	int val, cnt, waiters, new, priv = rw->_rw_shared^128;
 
 	do {
 		val = rw->_rw_lock;
@@ -12,7 +12,7 @@ int pthread_rwlock_unlock(pthread_rwlock_t *rw)
 	} while (a_cas(&rw->_rw_lock, val, new) != val);
 
 	if (!new && (waiters || val<0))
-		__wake(&rw->_rw_lock, cnt, 0);
+		__wake(&rw->_rw_lock, cnt, priv);
 
 	return 0;
 }
diff --git a/src/thread/sem_init.c b/src/thread/sem_init.c
index e8e419c..5509243 100644
--- a/src/thread/sem_init.c
+++ b/src/thread/sem_init.c
@@ -10,5 +10,6 @@ int sem_init(sem_t *sem, int pshared, unsigned value)
 	}
 	sem->__val[0] = value;
 	sem->__val[1] = 0;
+	sem->__val[2] = pshared ? 0 : 128;
 	return 0;
 }
diff --git a/src/thread/sem_post.c b/src/thread/sem_post.c
index 14a2dfe..31e3293 100644
--- a/src/thread/sem_post.c
+++ b/src/thread/sem_post.c
@@ -3,7 +3,7 @@
 
 int sem_post(sem_t *sem)
 {
-	int val, waiters;
+	int val, waiters, priv = sem->__val[2];
 	do {
 		val = sem->__val[0];
 		waiters = sem->__val[1];
@@ -12,6 +12,6 @@ int sem_post(sem_t *sem)
 			return -1;
 		}
 	} while (a_cas(sem->__val, val, val+1+(val<0)) != val);
-	if (val<0 || waiters) __wake(sem->__val, 1, 0);
+	if (val<0 || waiters) __wake(sem->__val, 1, priv);
 	return 0;
 }
diff --git a/src/thread/sem_timedwait.c b/src/thread/sem_timedwait.c
index 6d0d011..bfcb6dc 100644
--- a/src/thread/sem_timedwait.c
+++ b/src/thread/sem_timedwait.c
@@ -12,7 +12,7 @@ int sem_timedwait(sem_t *restrict sem, const struct timespec *restrict at)
 		int r;
 		a_inc(sem->__val+1);
 		a_cas(sem->__val, 0, -1);
-		r = __timedwait(sem->__val, -1, CLOCK_REALTIME, at, cleanup, sem->__val+1, 0);
+		r = __timedwait(sem->__val, -1, CLOCK_REALTIME, at, cleanup, sem->__val+1, sem->__val[2]);
 		a_dec(sem->__val+1);
 		if (r) {
 			errno = r;