diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h index 650e811..826191c 100644 --- a/src/internal/pthread_impl.h +++ b/src/internal/pthread_impl.h @@ -76,6 +76,7 @@ struct __timer { #define _c_destroy __u.__i[8] #define _rw_lock __u.__i[0] #define _rw_waiters __u.__i[1] +#define _rw_shared __u.__i[2] #define _b_lock __u.__i[0] #define _b_waiters __u.__i[1] #define _b_limit __u.__i[2] @@ -108,8 +109,13 @@ void __unmapself(void *, size_t); int __timedwait(volatile int *, int, clockid_t, const struct timespec *, void (*)(void *), void *, int); void __wait(volatile int *, volatile int *, int, int); -#define __wake(addr, cnt, priv) \ - __syscall(SYS_futex, addr, FUTEX_WAKE, (cnt)<0?INT_MAX:(cnt)) +static inline void __wake(volatile void *addr, int cnt, int priv) +{ + if (priv) priv = 128; + if (cnt<0) cnt = INT_MAX; + __syscall(SYS_futex, addr, FUTEX_WAKE|priv, cnt) != -EINVAL || + __syscall(SYS_futex, addr, FUTEX_WAKE, cnt); +} void __acquire_ptc(); void __release_ptc(); diff --git a/src/thread/__timedwait.c b/src/thread/__timedwait.c index 302273a..39eb996 100644 --- a/src/thread/__timedwait.c +++ b/src/thread/__timedwait.c @@ -4,12 +4,15 @@ #include "futex.h" #include "syscall.h" -static int do_wait(volatile int *addr, int val, - clockid_t clk, const struct timespec *at, int priv) +int __timedwait(volatile int *addr, int val, + clockid_t clk, const struct timespec *at, + void (*cleanup)(void *), void *arg, int priv) { - int r; + int r, cs; struct timespec to, *top=0; + if (priv) priv = 128; + if (at) { if (at->tv_nsec >= 1000000000UL) return EINVAL; if (clock_gettime(clk, &to)) return EINVAL; @@ -22,21 +25,12 @@ static int do_wait(volatile int *addr, int val, top = &to; } - r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT, val, top); - if (r == EINTR || r == EINVAL || r == ETIMEDOUT) return r; - return 0; -} - -int __timedwait(volatile int *addr, int val, - clockid_t clk, const struct timespec *at, - void (*cleanup)(void *), void *arg, int priv) -{ - int r, cs; - if (!cleanup) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); pthread_cleanup_push(cleanup, arg); - r = do_wait(addr, val, clk, at, priv); + r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT|priv, val, top); + if (r == EINVAL) r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT, val, top); + if (r != EINTR && r != ETIMEDOUT) r = 0; pthread_cleanup_pop(0); if (!cleanup) pthread_setcancelstate(cs, 0); diff --git a/src/thread/__wait.c b/src/thread/__wait.c index a1e4780..ec1e820 100644 --- a/src/thread/__wait.c +++ b/src/thread/__wait.c @@ -3,13 +3,15 @@ void __wait(volatile int *addr, volatile int *waiters, int val, int priv) { int spins=10000; - if (priv) priv = 128; priv=0; + if (priv) priv = 128; while (spins--) { if (*addr==val) a_spin(); else return; } if (waiters) a_inc(waiters); - while (*addr==val) - __syscall(SYS_futex, addr, FUTEX_WAIT|priv, val, 0); + while (*addr==val) { + __syscall(SYS_futex, addr, FUTEX_WAIT|priv, val, 0) != -EINVAL + || __syscall(SYS_futex, addr, FUTEX_WAIT, val, 0); + } if (waiters) a_dec(waiters); } diff --git a/src/thread/pthread_attr_get.c b/src/thread/pthread_attr_get.c index 03fc91e..3d296bf 100644 --- a/src/thread/pthread_attr_get.c +++ b/src/thread/pthread_attr_get.c @@ -75,7 +75,7 @@ int pthread_mutexattr_getprotocol(const pthread_mutexattr_t *restrict a, int *re } int pthread_mutexattr_getpshared(const pthread_mutexattr_t *restrict a, int *restrict pshared) { - *pshared = a->__attr>>31; + *pshared = a->__attr / 128U % 2; return 0; } diff --git a/src/thread/pthread_cond_broadcast.c b/src/thread/pthread_cond_broadcast.c index 0901daf..f445646 100644 --- a/src/thread/pthread_cond_broadcast.c +++ b/src/thread/pthread_cond_broadcast.c @@ -33,7 +33,7 @@ int pthread_cond_broadcast(pthread_cond_t *c) out: a_store(&c->_c_lock, 0); - if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0); + if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1); return 0; } diff --git a/src/thread/pthread_cond_signal.c b/src/thread/pthread_cond_signal.c index 71bcdcd..5fd72f9 100644 --- a/src/thread/pthread_cond_signal.c +++ b/src/thread/pthread_cond_signal.c @@ -4,6 +4,6 @@ int pthread_cond_signal(pthread_cond_t *c) { if (!c->_c_waiters) return 0; a_inc(&c->_c_seq); - if (c->_c_waiters) __wake(&c->_c_seq, 1, 0); + if (c->_c_waiters) __wake(&c->_c_seq, 1, c->_c_mutex!=(void*)-1); return 0; } diff --git a/src/thread/pthread_cond_timedwait.c b/src/thread/pthread_cond_timedwait.c index 99d62cc..a51a46c 100644 --- a/src/thread/pthread_cond_timedwait.c +++ b/src/thread/pthread_cond_timedwait.c @@ -41,7 +41,7 @@ int pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restrict struct cm cm = { .c=c, .m=m }; int r, e=0, seq; - if (m->_m_type && (m->_m_lock&INT_MAX) != __pthread_self()->tid) + if ((m->_m_type&15) && (m->_m_lock&INT_MAX) != __pthread_self()->tid) return EPERM; if (ts && ts->tv_nsec >= 1000000000UL) diff --git a/src/thread/pthread_mutex_init.c b/src/thread/pthread_mutex_init.c index 9d85a35..acf45a7 100644 --- a/src/thread/pthread_mutex_init.c +++ b/src/thread/pthread_mutex_init.c @@ -3,6 +3,6 @@ int pthread_mutex_init(pthread_mutex_t *restrict m, const pthread_mutexattr_t *restrict a) { *m = (pthread_mutex_t){0}; - if (a) m->_m_type = a->__attr & 7; + if (a) m->_m_type = a->__attr; return 0; } diff --git a/src/thread/pthread_mutex_timedlock.c b/src/thread/pthread_mutex_timedlock.c index 7b1afc0..849febb 100644 --- a/src/thread/pthread_mutex_timedlock.c +++ b/src/thread/pthread_mutex_timedlock.c @@ -2,11 +2,12 @@ int pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec *restrict at) { - int r, t; - - if (m->_m_type == PTHREAD_MUTEX_NORMAL && !a_cas(&m->_m_lock, 0, EBUSY)) + if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL + && !a_cas(&m->_m_lock, 0, EBUSY)) return 0; + int r, t, priv = (m->_m_type & 128) ^ 128; + while ((r=pthread_mutex_trylock(m)) == EBUSY) { if (!(r=m->_m_lock) || (r&0x40000000)) continue; if ((m->_m_type&3) == PTHREAD_MUTEX_ERRORCHECK @@ -16,7 +17,7 @@ int pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec * a_inc(&m->_m_waiters); t = r | 0x80000000; a_cas(&m->_m_lock, r, t); - r = __timedwait(&m->_m_lock, t, CLOCK_REALTIME, at, 0, 0, 0); + r = __timedwait(&m->_m_lock, t, CLOCK_REALTIME, at, 0, 0, priv); a_dec(&m->_m_waiters); if (r && r != EINTR) break; } diff --git a/src/thread/pthread_mutex_trylock.c b/src/thread/pthread_mutex_trylock.c index 00ad65d..850fcb9 100644 --- a/src/thread/pthread_mutex_trylock.c +++ b/src/thread/pthread_mutex_trylock.c @@ -1,17 +1,13 @@ #include "pthread_impl.h" -int pthread_mutex_trylock(pthread_mutex_t *m) +int __pthread_mutex_trylock_owner(pthread_mutex_t *m) { - int tid, old, own; - pthread_t self; - - if (m->_m_type == PTHREAD_MUTEX_NORMAL) - return a_cas(&m->_m_lock, 0, EBUSY) & EBUSY; + int old, own; + int type = m->_m_type & 15; + pthread_t self = __pthread_self(); + int tid = self->tid; - self = __pthread_self(); - tid = self->tid; - - if (m->_m_type >= 4) { + if (type >= 4) { if (!self->robust_list.off) __syscall(SYS_set_robust_list, &self->robust_list, 3*sizeof(long)); @@ -21,7 +17,7 @@ int pthread_mutex_trylock(pthread_mutex_t *m) old = m->_m_lock; own = old & 0x7fffffff; - if (own == tid && (m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE) { + if (own == tid && (type&3) == PTHREAD_MUTEX_RECURSIVE) { if ((unsigned)m->_m_count >= INT_MAX) return EAGAIN; m->_m_count++; return 0; @@ -30,9 +26,9 @@ int pthread_mutex_trylock(pthread_mutex_t *m) if ((own && !(own & 0x40000000)) || a_cas(&m->_m_lock, old, tid)!=old) return EBUSY; - if (m->_m_type < 4) return 0; + if (type < 4) return 0; - if (m->_m_type >= 8) { + if (type >= 8) { m->_m_lock = 0; return ENOTRECOVERABLE; } @@ -50,3 +46,10 @@ int pthread_mutex_trylock(pthread_mutex_t *m) return 0; } + +int pthread_mutex_trylock(pthread_mutex_t *m) +{ + if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL) + return a_cas(&m->_m_lock, 0, EBUSY) & EBUSY; + return __pthread_mutex_trylock_owner(m); +} diff --git a/src/thread/pthread_mutex_unlock.c b/src/thread/pthread_mutex_unlock.c index b4bd74b..769d6e5 100644 --- a/src/thread/pthread_mutex_unlock.c +++ b/src/thread/pthread_mutex_unlock.c @@ -9,16 +9,18 @@ int pthread_mutex_unlock(pthread_mutex_t *m) int waiters = m->_m_waiters; int cont; int robust = 0; + int type = m->_m_type & 15; + int priv = (m->_m_type & 128) ^ 128; - if (m->_m_type != PTHREAD_MUTEX_NORMAL) { + if (type != PTHREAD_MUTEX_NORMAL) { if (!m->_m_lock) return EPERM; self = __pthread_self(); if ((m->_m_lock&0x1fffffff) != self->tid) return EPERM; - if ((m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count) + if ((type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count) return m->_m_count--, 0; - if (m->_m_type >= 4) { + if (type >= 4) { robust = 1; self->robust_list.pending = &m->_m_next; *(void **)m->_m_prev = m->_m_next; @@ -32,6 +34,6 @@ int pthread_mutex_unlock(pthread_mutex_t *m) __vm_unlock_impl(); } if (waiters || cont<0) - __wake(&m->_m_lock, 1, 0); + __wake(&m->_m_lock, 1, priv); return 0; } diff --git a/src/thread/pthread_mutexattr_setpshared.c b/src/thread/pthread_mutexattr_setpshared.c index 8c7a1e2..100f6ff 100644 --- a/src/thread/pthread_mutexattr_setpshared.c +++ b/src/thread/pthread_mutexattr_setpshared.c @@ -3,7 +3,7 @@ int pthread_mutexattr_setpshared(pthread_mutexattr_t *a, int pshared) { if (pshared > 1U) return EINVAL; - a->__attr &= 0x7fffffff; - a->__attr |= pshared<<31; + a->__attr &= ~128U; + a->__attr |= pshared<<7; return 0; } diff --git a/src/thread/pthread_once.c b/src/thread/pthread_once.c index e01f6d4..2eb0f93 100644 --- a/src/thread/pthread_once.c +++ b/src/thread/pthread_once.c @@ -3,7 +3,7 @@ static void undo(void *control) { a_store(control, 0); - __wake(control, 1, 0); + __wake(control, 1, 1); } int pthread_once(pthread_once_t *control, void (*init)(void)) @@ -25,10 +25,10 @@ int pthread_once(pthread_once_t *control, void (*init)(void)) pthread_cleanup_pop(0); a_store(control, 2); - if (waiters) __wake(control, -1, 0); + if (waiters) __wake(control, -1, 1); return 0; case 1: - __wait(control, &waiters, 1, 0); + __wait(control, &waiters, 1, 1); continue; case 2: return 0; diff --git a/src/thread/pthread_rwlock_init.c b/src/thread/pthread_rwlock_init.c index 82df52e..a2c0b47 100644 --- a/src/thread/pthread_rwlock_init.c +++ b/src/thread/pthread_rwlock_init.c @@ -3,7 +3,6 @@ int pthread_rwlock_init(pthread_rwlock_t *restrict rw, const pthread_rwlockattr_t *restrict a) { *rw = (pthread_rwlock_t){0}; - if (a) { - } + if (a) rw->_rw_shared = a->__attr[0]*128; return 0; } diff --git a/src/thread/pthread_rwlock_timedrdlock.c b/src/thread/pthread_rwlock_timedrdlock.c index c0c94c9..a2b4d44 100644 --- a/src/thread/pthread_rwlock_timedrdlock.c +++ b/src/thread/pthread_rwlock_timedrdlock.c @@ -8,7 +8,7 @@ int pthread_rwlock_timedrdlock(pthread_rwlock_t *restrict rw, const struct times t = r | 0x80000000; a_inc(&rw->_rw_waiters); a_cas(&rw->_rw_lock, r, t); - r = __timedwait(&rw->_rw_lock, t, CLOCK_REALTIME, at, 0, 0, 0); + r = __timedwait(&rw->_rw_lock, t, CLOCK_REALTIME, at, 0, 0, rw->_rw_shared^128); a_dec(&rw->_rw_waiters); if (r && r != EINTR) return r; } diff --git a/src/thread/pthread_rwlock_timedwrlock.c b/src/thread/pthread_rwlock_timedwrlock.c index 339a167..63a32ec 100644 --- a/src/thread/pthread_rwlock_timedwrlock.c +++ b/src/thread/pthread_rwlock_timedwrlock.c @@ -8,7 +8,7 @@ int pthread_rwlock_timedwrlock(pthread_rwlock_t *restrict rw, const struct times t = r | 0x80000000; a_inc(&rw->_rw_waiters); a_cas(&rw->_rw_lock, r, t); - r = __timedwait(&rw->_rw_lock, t, CLOCK_REALTIME, at, 0, 0, 0); + r = __timedwait(&rw->_rw_lock, t, CLOCK_REALTIME, at, 0, 0, rw->_rw_shared^128); a_dec(&rw->_rw_waiters); if (r && r != EINTR) return r; } diff --git a/src/thread/pthread_rwlock_unlock.c b/src/thread/pthread_rwlock_unlock.c index a6d2085..7b5eec8 100644 --- a/src/thread/pthread_rwlock_unlock.c +++ b/src/thread/pthread_rwlock_unlock.c @@ -2,7 +2,7 @@ int pthread_rwlock_unlock(pthread_rwlock_t *rw) { - int val, cnt, waiters, new; + int val, cnt, waiters, new, priv = rw->_rw_shared^128; do { val = rw->_rw_lock; @@ -12,7 +12,7 @@ int pthread_rwlock_unlock(pthread_rwlock_t *rw) } while (a_cas(&rw->_rw_lock, val, new) != val); if (!new && (waiters || val<0)) - __wake(&rw->_rw_lock, cnt, 0); + __wake(&rw->_rw_lock, cnt, priv); return 0; } diff --git a/src/thread/sem_init.c b/src/thread/sem_init.c index e8e419c..5509243 100644 --- a/src/thread/sem_init.c +++ b/src/thread/sem_init.c @@ -10,5 +10,6 @@ int sem_init(sem_t *sem, int pshared, unsigned value) } sem->__val[0] = value; sem->__val[1] = 0; + sem->__val[2] = pshared ? 0 : 128; return 0; } diff --git a/src/thread/sem_post.c b/src/thread/sem_post.c index 14a2dfe..31e3293 100644 --- a/src/thread/sem_post.c +++ b/src/thread/sem_post.c @@ -3,7 +3,7 @@ int sem_post(sem_t *sem) { - int val, waiters; + int val, waiters, priv = sem->__val[2]; do { val = sem->__val[0]; waiters = sem->__val[1]; @@ -12,6 +12,6 @@ int sem_post(sem_t *sem) return -1; } } while (a_cas(sem->__val, val, val+1+(val<0)) != val); - if (val<0 || waiters) __wake(sem->__val, 1, 0); + if (val<0 || waiters) __wake(sem->__val, 1, priv); return 0; } diff --git a/src/thread/sem_timedwait.c b/src/thread/sem_timedwait.c index 6d0d011..bfcb6dc 100644 --- a/src/thread/sem_timedwait.c +++ b/src/thread/sem_timedwait.c @@ -12,7 +12,7 @@ int sem_timedwait(sem_t *restrict sem, const struct timespec *restrict at) int r; a_inc(sem->__val+1); a_cas(sem->__val, 0, -1); - r = __timedwait(sem->__val, -1, CLOCK_REALTIME, at, cleanup, sem->__val+1, 0); + r = __timedwait(sem->__val, -1, CLOCK_REALTIME, at, cleanup, sem->__val+1, sem->__val[2]); a_dec(sem->__val+1); if (r) { errno = r;