Make the new mutexes faster:
- Eliminate mutexattr_private and just set a bit in ptm_owner if the mutex is recursive. This forces the slow path to be taken for recursive mutexes. Overload an unused field in pthread_mutex_t to record whether or not it's an errorcheck mutex. - Streamline pthread_mutex_lock / pthread_mutex_unlock a bit more. As a side effect makes it possible to have assembly stubs for them.
This commit is contained in:
ad
parent
4042b7d22a
commit
b0efccf4cd
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: pthread_int.h,v 1.53 2007/09/10 11:34:05 skrll Exp $ */
|
||||
/* $NetBSD: pthread_int.h,v 1.54 2007/09/11 18:11:29 ad Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc.
|
||||
@ -257,6 +257,7 @@ void pthread__membar_producer(void);
|
||||
void pthread__membar_consumer(void);
|
||||
|
||||
int pthread__mutex_owned(pthread_t, pthread_mutex_t *);
|
||||
int pthread__mutex_catchup(pthread_mutex_t *);
|
||||
|
||||
#ifndef pthread__smt_pause
|
||||
#define pthread__smt_pause() /* nothing */
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: pthread_mutex2.c,v 1.5 2007/09/11 11:30:15 ad Exp $ */
|
||||
/* $NetBSD: pthread_mutex2.c,v 1.6 2007/09/11 18:11:29 ad Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc.
|
||||
@ -37,7 +37,7 @@
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__RCSID("$NetBSD: pthread_mutex2.c,v 1.5 2007/09/11 11:30:15 ad Exp $");
|
||||
__RCSID("$NetBSD: pthread_mutex2.c,v 1.6 2007/09/11 18:11:29 ad Exp $");
|
||||
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
@ -55,16 +55,31 @@ __RCSID("$NetBSD: pthread_mutex2.c,v 1.5 2007/09/11 11:30:15 ad Exp $");
|
||||
* the list head in order for the hint arguments to _lwp_park /
|
||||
* _lwp_unpark_all to match.
|
||||
*/
|
||||
#define pt_nextwaiter pt_sleep.ptqe_next
|
||||
#define ptm_waiters ptm_blocked.ptqh_first
|
||||
#define pt_nextwaiter pt_sleep.ptqe_next
|
||||
#define ptm_waiters ptm_blocked.ptqh_first
|
||||
#define ptm_errorcheck ptm_blocked.ptqh_last
|
||||
|
||||
#define MUTEX_WAITERS_BIT (1UL)
|
||||
#define MUTEX_WAITERS_BIT (0x01UL)
|
||||
#define MUTEX_RECURSIVE_BIT (0x02UL)
|
||||
#define MUTEX_THREAD (-16L)
|
||||
|
||||
#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT)
|
||||
#define MUTEX_OWNER(x) ((uintptr_t)(x) & ~MUTEX_WAITERS_BIT)
|
||||
#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT)
|
||||
#define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT)
|
||||
#define MUTEX_OWNER(x) ((uintptr_t)(x) & MUTEX_THREAD)
|
||||
#define MUTEX_GET_RECURSE(ptm) ((intptr_t)(ptm)->ptm_private)
|
||||
#define MUTEX_SET_RECURSE(ptm, delta) \
|
||||
((ptm)->ptm_private = (void *)((intptr_t)(ptm)->ptm_private + delta))
|
||||
|
||||
#if __GNUC_PREREQ__(3, 0)
|
||||
#define NOINLINE __attribute ((noinline))
|
||||
#else
|
||||
#define NOINLINE /* nothing */
|
||||
#endif
|
||||
|
||||
static void pthread__mutex_wakeup(pthread_t, pthread_mutex_t *);
|
||||
static int pthread__mutex_lock_slow(pthread_mutex_t *, void *);
|
||||
static int pthread__mutex_lock_slow(pthread_mutex_t *);
|
||||
static int pthread__mutex_unlock_slow(pthread_mutex_t *);
|
||||
static void pthread__mutex_pause(void);
|
||||
|
||||
__strong_alias(__libc_mutex_init,pthread_mutex_init)
|
||||
__strong_alias(__libc_mutex_lock,pthread_mutex_lock)
|
||||
@ -85,51 +100,34 @@ mutex_cas(volatile void *ptr, void **old, void *new)
|
||||
return pthread__atomic_cas_ptr(ptr, old, new);
|
||||
}
|
||||
|
||||
struct mutex_private {
|
||||
int type;
|
||||
int recursecount;
|
||||
};
|
||||
|
||||
static const struct mutex_private mutex_private_default = {
|
||||
PTHREAD_MUTEX_DEFAULT,
|
||||
0,
|
||||
};
|
||||
|
||||
struct mutexattr_private {
|
||||
int type;
|
||||
};
|
||||
|
||||
static const struct mutexattr_private mutexattr_private_default = {
|
||||
PTHREAD_MUTEX_DEFAULT,
|
||||
};
|
||||
|
||||
int
|
||||
pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
|
||||
{
|
||||
struct mutexattr_private *map;
|
||||
struct mutex_private *mp;
|
||||
intptr_t type;
|
||||
|
||||
pthread__error(EINVAL, "Invalid mutex attribute",
|
||||
(attr == NULL) || (attr->ptma_magic == _PT_MUTEXATTR_MAGIC));
|
||||
if (attr == NULL)
|
||||
type = PTHREAD_MUTEX_NORMAL;
|
||||
else
|
||||
type = (intptr_t)attr->ptma_private;
|
||||
|
||||
if (attr != NULL && (map = attr->ptma_private) != NULL &&
|
||||
memcmp(map, &mutexattr_private_default, sizeof(*map)) != 0) {
|
||||
mp = malloc(sizeof(*mp));
|
||||
if (mp == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
mp->type = map->type;
|
||||
mp->recursecount = 0;
|
||||
} else {
|
||||
/* LINTED cast away const */
|
||||
mp = (struct mutex_private *) &mutex_private_default;
|
||||
switch (type) {
|
||||
case PTHREAD_MUTEX_ERRORCHECK:
|
||||
ptm->ptm_errorcheck = (void *)1;
|
||||
ptm->ptm_owner = NULL;
|
||||
break;
|
||||
case PTHREAD_MUTEX_RECURSIVE:
|
||||
ptm->ptm_errorcheck = NULL;
|
||||
ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT;
|
||||
break;
|
||||
default:
|
||||
ptm->ptm_errorcheck = NULL;
|
||||
ptm->ptm_owner = NULL;
|
||||
break;
|
||||
}
|
||||
|
||||
ptm->ptm_magic = _PT_MUTEX_MAGIC;
|
||||
ptm->ptm_owner = NULL;
|
||||
ptm->ptm_waiters = NULL;
|
||||
pthread_lockinit(&ptm->ptm_lock);
|
||||
ptm->ptm_private = mp;
|
||||
ptm->ptm_private = NULL;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -142,13 +140,9 @@ pthread_mutex_destroy(pthread_mutex_t *ptm)
|
||||
pthread__error(EINVAL, "Invalid mutex",
|
||||
ptm->ptm_magic == _PT_MUTEX_MAGIC);
|
||||
pthread__error(EBUSY, "Destroying locked mutex",
|
||||
ptm->ptm_owner == NULL);
|
||||
MUTEX_OWNER(ptm->ptm_owner) == 0);
|
||||
|
||||
ptm->ptm_magic = _PT_MUTEX_DEAD;
|
||||
if (ptm->ptm_private != NULL &&
|
||||
ptm->ptm_private != (const void *)&mutex_private_default)
|
||||
free(ptm->ptm_private);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -173,72 +167,65 @@ pthread_mutex_lock(pthread_mutex_t *ptm)
|
||||
|
||||
owner = NULL;
|
||||
self = pthread__self();
|
||||
|
||||
if (__predict_true(mutex_cas(&ptm->ptm_owner, &owner, self)))
|
||||
return 0;
|
||||
return pthread__mutex_lock_slow(ptm, owner);
|
||||
|
||||
return pthread__mutex_lock_slow(ptm);
|
||||
}
|
||||
|
||||
#if __GNUC_PREREQ__(3, 0)
|
||||
__attribute ((noinline))
|
||||
#endif
|
||||
static int
|
||||
pthread__mutex_lock_slow(pthread_mutex_t *ptm, void *owner)
|
||||
/* We want function call overhead. */
|
||||
NOINLINE static void
|
||||
pthread__mutex_pause(void)
|
||||
{
|
||||
extern int pthread__started;
|
||||
struct mutex_private *mp;
|
||||
void *waiters, *new;
|
||||
|
||||
pthread__smt_pause();
|
||||
}
|
||||
|
||||
NOINLINE static int
|
||||
pthread__mutex_lock_slow(pthread_mutex_t *ptm)
|
||||
{
|
||||
void *waiters, *new, *owner;
|
||||
pthread_t self;
|
||||
sigset_t ss;
|
||||
int count;
|
||||
|
||||
pthread__error(EINVAL, "Invalid mutex",
|
||||
ptm->ptm_magic == _PT_MUTEX_MAGIC);
|
||||
|
||||
/* Recursive or errorcheck? */
|
||||
mp = ptm->ptm_private;
|
||||
owner = ptm->ptm_owner;
|
||||
self = pthread__self();
|
||||
if (MUTEX_OWNER(owner) == (uintptr_t)self && mp != NULL) {
|
||||
switch (mp->type) {
|
||||
case PTHREAD_MUTEX_ERRORCHECK:
|
||||
return EDEADLK;
|
||||
case PTHREAD_MUTEX_RECURSIVE:
|
||||
if (mp->recursecount == INT_MAX)
|
||||
|
||||
/* Recursive or errorcheck? */
|
||||
if (MUTEX_OWNER(owner) == (uintptr_t)self) {
|
||||
if (MUTEX_RECURSIVE(owner)) {
|
||||
if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
|
||||
return EAGAIN;
|
||||
mp->recursecount++;
|
||||
MUTEX_SET_RECURSE(ptm, +1);
|
||||
return 0;
|
||||
}
|
||||
if (ptm->ptm_errorcheck)
|
||||
return EDEADLK;
|
||||
}
|
||||
|
||||
/* Spin for a while. */
|
||||
count = pthread__nspins;
|
||||
while (owner != NULL && --count > 0) {
|
||||
pthread__smt_pause();
|
||||
while (MUTEX_OWNER(owner) == 0 && --count > 0) {
|
||||
pthread__mutex_pause();
|
||||
owner = ptm->ptm_owner;
|
||||
}
|
||||
|
||||
for (;; owner = ptm->ptm_owner) {
|
||||
/* If it has become free, try to acquire it again. */
|
||||
while (owner == NULL) {
|
||||
if (mutex_cas(&ptm->ptm_owner, &owner, self))
|
||||
while (MUTEX_OWNER(owner) == 0) {
|
||||
new = (void *)((uintptr_t)self | (uintptr_t)owner);
|
||||
if (mutex_cas(&ptm->ptm_owner, &owner, new))
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Nope, still held. */
|
||||
if (pthread__started == 0) {
|
||||
/* The spec says we must deadlock, so... */
|
||||
mp = ptm->ptm_private;
|
||||
pthread__assert(mp->type == PTHREAD_MUTEX_NORMAL);
|
||||
(void) sigprocmask(SIG_SETMASK, NULL, &ss);
|
||||
for (;;) {
|
||||
sigsuspend(&ss);
|
||||
}
|
||||
/*NOTREACHED*/
|
||||
}
|
||||
|
||||
/*
|
||||
* Add thread to the list of waiters. Issue a memory
|
||||
* barrier to ensure sleeponq/nextwaiter are visible
|
||||
* before we enter the waiters list.
|
||||
* Nope, still held. Add thread to the list of waiters.
|
||||
* Issue a memory barrier to ensure sleeponq/nextwaiter
|
||||
* are visible before we enter the waiters list.
|
||||
*/
|
||||
self->pt_sleeponq = 1;
|
||||
for (waiters = ptm->ptm_waiters;;) {
|
||||
@ -268,7 +255,7 @@ pthread__mutex_lock_slow(pthread_mutex_t *ptm, void *owner)
|
||||
for (owner = ptm->ptm_owner;;) {
|
||||
if (MUTEX_HAS_WAITERS(owner))
|
||||
break;
|
||||
if (owner == NULL) {
|
||||
if (MUTEX_OWNER(owner) == 0) {
|
||||
pthread__mutex_wakeup(self, ptm);
|
||||
break;
|
||||
}
|
||||
@ -293,109 +280,126 @@ pthread__mutex_lock_slow(pthread_mutex_t *ptm, void *owner)
|
||||
int
|
||||
pthread_mutex_trylock(pthread_mutex_t *ptm)
|
||||
{
|
||||
struct mutex_private *mp;
|
||||
pthread_t self;
|
||||
void *value;
|
||||
|
||||
pthread__error(EINVAL, "Invalid mutex",
|
||||
ptm->ptm_magic == _PT_MUTEX_MAGIC);
|
||||
|
||||
self = pthread__self();
|
||||
|
||||
value = NULL;
|
||||
|
||||
if (mutex_cas(&ptm->ptm_owner, &value, self))
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* These tests can be performed without holding the
|
||||
* interlock because these fields are only modified
|
||||
* if we know we own the mutex.
|
||||
*/
|
||||
mp = ptm->ptm_private;
|
||||
if (mp != NULL && mp->type == PTHREAD_MUTEX_RECURSIVE &&
|
||||
MUTEX_OWNER(value) == (uintptr_t)self) {
|
||||
if (mp->recursecount == INT_MAX)
|
||||
if (MUTEX_OWNER(value) == (uintptr_t)self && MUTEX_RECURSIVE(value)) {
|
||||
if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
|
||||
return EAGAIN;
|
||||
mp->recursecount++;
|
||||
MUTEX_SET_RECURSE(ptm, +1);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return EBUSY;
|
||||
}
|
||||
|
||||
NOINLINE int
|
||||
pthread__mutex_catchup(pthread_mutex_t *ptm)
|
||||
{
|
||||
pthread_t self;
|
||||
|
||||
self = pthread__self();
|
||||
|
||||
if (self->pt_nwaiters == 1) {
|
||||
/*
|
||||
* If the calling thread is about to block, defer
|
||||
* unparking the target until _lwp_park() is called.
|
||||
*/
|
||||
if (self->pt_willpark && self->pt_unpark == 0) {
|
||||
self->pt_unpark = self->pt_waiters[0];
|
||||
self->pt_unparkhint = &ptm->ptm_waiters;
|
||||
} else {
|
||||
(void)_lwp_unpark(self->pt_waiters[0],
|
||||
&ptm->ptm_waiters);
|
||||
}
|
||||
} else {
|
||||
(void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
|
||||
&ptm->ptm_waiters);
|
||||
}
|
||||
self->pt_nwaiters = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
pthread_mutex_unlock(pthread_mutex_t *ptm)
|
||||
{
|
||||
struct mutex_private *mp;
|
||||
pthread_t self, owner;
|
||||
void *owner;
|
||||
pthread_t self;
|
||||
|
||||
self = pthread__self();
|
||||
owner = self;
|
||||
|
||||
if (__predict_false(!mutex_cas(&ptm->ptm_owner, &owner, NULL)))
|
||||
return pthread__mutex_unlock_slow(ptm);
|
||||
|
||||
/*
|
||||
* There were no waiters, but we may have deferred waking
|
||||
* other threads until mutex unlock - we must wake them now.
|
||||
*/
|
||||
if (self->pt_nwaiters != 0)
|
||||
return pthread__mutex_catchup(ptm);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
NOINLINE static int
|
||||
pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
|
||||
{
|
||||
pthread_t self, owner, new;
|
||||
int weown;
|
||||
|
||||
pthread__error(EINVAL, "Invalid mutex",
|
||||
ptm->ptm_magic == _PT_MUTEX_MAGIC);
|
||||
|
||||
/*
|
||||
* These tests can be performed without holding the
|
||||
* interlock because these fields are only modified
|
||||
* if we know we own the mutex.
|
||||
*/
|
||||
self = pthread_self();
|
||||
owner = ptm->ptm_owner;
|
||||
weown = (MUTEX_OWNER(owner) == (uintptr_t)self);
|
||||
mp = ptm->ptm_private;
|
||||
|
||||
if (mp == NULL) {
|
||||
if (__predict_false(!weown)) {
|
||||
pthread__error(EPERM, "Unlocking unlocked mutex",
|
||||
(owner != NULL));
|
||||
pthread__error(EPERM,
|
||||
"Unlocking mutex owned by another thread", weown);
|
||||
}
|
||||
} else if (mp->type == PTHREAD_MUTEX_RECURSIVE) {
|
||||
if (ptm->ptm_errorcheck) {
|
||||
if (!weown)
|
||||
return EPERM;
|
||||
if (mp->recursecount != 0) {
|
||||
mp->recursecount--;
|
||||
/*
|
||||
* Wake any threads we deferred waking until
|
||||
* mutex unlock.
|
||||
*/
|
||||
if (self->pt_nwaiters != 0) {
|
||||
(void)_lwp_unpark_all(self->pt_waiters,
|
||||
self->pt_nwaiters, &ptm->ptm_waiters);
|
||||
self->pt_nwaiters = 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
} else if (mp->type == PTHREAD_MUTEX_ERRORCHECK) {
|
||||
new = NULL;
|
||||
} else if (MUTEX_RECURSIVE(owner)) {
|
||||
if (!weown)
|
||||
return EPERM;
|
||||
if (__predict_false(!weown)) {
|
||||
pthread__error(EPERM, "Unlocking unlocked mutex",
|
||||
(owner != NULL));
|
||||
pthread__error(EPERM,
|
||||
"Unlocking mutex owned by another thread", weown);
|
||||
}
|
||||
if (MUTEX_GET_RECURSE(ptm) != 0) {
|
||||
MUTEX_SET_RECURSE(ptm, -1);
|
||||
new = owner;
|
||||
} else
|
||||
new = (pthread_t)MUTEX_RECURSIVE_BIT;
|
||||
} else {
|
||||
pthread__error(EPERM,
|
||||
"Unlocking unlocked mutex", (owner != NULL));
|
||||
pthread__error(EPERM,
|
||||
"Unlocking mutex owned by another thread", weown);
|
||||
new = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Release the mutex. If there appear to be waiters, then
|
||||
* wake them up.
|
||||
*/
|
||||
owner = pthread__atomic_swap_ptr(&ptm->ptm_owner, NULL);
|
||||
if (MUTEX_HAS_WAITERS(owner) != 0) {
|
||||
pthread__mutex_wakeup(self, ptm);
|
||||
return 0;
|
||||
if (new != owner) {
|
||||
owner = pthread__atomic_swap_ptr(&ptm->ptm_owner, new);
|
||||
if (MUTEX_HAS_WAITERS(owner) != 0) {
|
||||
pthread__mutex_wakeup(self, ptm);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* There were no waiters, but we may have deferred waking
|
||||
* other threads until mutex unlock - we must wake them now.
|
||||
*/
|
||||
if (self->pt_nwaiters != 0) {
|
||||
(void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
|
||||
&ptm->ptm_waiters);
|
||||
self->pt_nwaiters = 0;
|
||||
}
|
||||
if (self->pt_nwaiters != 0)
|
||||
return pthread__mutex_catchup(ptm);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -462,25 +466,15 @@ pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
pthread_mutexattr_init(pthread_mutexattr_t *attr)
|
||||
{
|
||||
struct mutexattr_private *map;
|
||||
|
||||
map = malloc(sizeof(*map));
|
||||
if (map == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
*map = mutexattr_private_default;
|
||||
|
||||
attr->ptma_magic = _PT_MUTEXATTR_MAGIC;
|
||||
attr->ptma_private = map;
|
||||
|
||||
attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
|
||||
{
|
||||
@ -488,10 +482,6 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
|
||||
pthread__error(EINVAL, "Invalid mutex attribute",
|
||||
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
|
||||
|
||||
attr->ptma_magic = _PT_MUTEXATTR_DEAD;
|
||||
if (attr->ptma_private != NULL)
|
||||
free(attr->ptma_private);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -499,15 +489,11 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
|
||||
int
|
||||
pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
|
||||
{
|
||||
struct mutexattr_private *map;
|
||||
|
||||
pthread__error(EINVAL, "Invalid mutex attribute",
|
||||
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
|
||||
|
||||
map = attr->ptma_private;
|
||||
|
||||
*typep = map->type;
|
||||
|
||||
*typep = (int)(intptr_t)attr->ptma_private;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -515,25 +501,19 @@ pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
|
||||
int
|
||||
pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
|
||||
{
|
||||
struct mutexattr_private *map;
|
||||
|
||||
pthread__error(EINVAL, "Invalid mutex attribute",
|
||||
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
|
||||
|
||||
map = attr->ptma_private;
|
||||
|
||||
switch (type) {
|
||||
case PTHREAD_MUTEX_NORMAL:
|
||||
case PTHREAD_MUTEX_ERRORCHECK:
|
||||
case PTHREAD_MUTEX_RECURSIVE:
|
||||
map->type = type;
|
||||
break;
|
||||
|
||||
attr->ptma_private = (void *)(intptr_t)type;
|
||||
return 0;
|
||||
default:
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user