Aren/NetBSD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Remove libpthread's atomic ops.

Browse Source 
- Remove the old spinlock-based mutex and rwlock implementations.
- Use the atomic ops from libc.
This commit is contained in:
ad 2008-02-10 18:50:54 +00:00
parent 749400b704
commit a67e1e3475
16 changed files with 876 additions and 1905 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/libpthread/Makefile
View File

@ -1,4 +1,4 @@
# $NetBSD: Makefile,v 1.49 2008/01/15 03:37:14 rmind Exp $
# $NetBSD: Makefile,v 1.50 2008/02/10 18:50:54 ad Exp $
#
WARNS= 4
@ -54,9 +54,7 @@ SRCS+= pthread_cond.c
SRCS+= pthread_lock.c
SRCS+= pthread_misc.c
SRCS+= pthread_mutex.c
SRCS+= pthread_mutex2.c
SRCS+= pthread_rwlock.c
SRCS+= pthread_rwlock2.c
SRCS+= pthread_specific.c
SRCS+= pthread_spin.c
SRCS+= pthread_tsd.c
@ -82,10 +80,10 @@ pthread_lock.po: pthread_lock.o
${_MKTARGET_CREATE}
cp pthread_lock.o pthread_lock.po
COPTS.pthread_mutex2.c+= -fomit-frame-pointer -falign-functions=32
pthread_mutex2.po: pthread_mutex2.o
COPTS.pthread_mutex.c+= -fomit-frame-pointer -falign-functions=32
pthread_mutex.po: pthread_mutex.o
${_MKTARGET_CREATE}
cp pthread_mutex2.o pthread_mutex2.po
cp pthread_mutex.o pthread_mutex.po
COPTS.pthread.c += -Wno-stack-protector

5
lib/libpthread/arch/hppa/pthread_md.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_md.h,v 1.4 2007/09/24 12:19:39 skrll Exp $ */
/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@ -98,4 +98,7 @@ do { \
(uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \
} while (/*CONSTCOND*/0)
/* Don't need additional memory barriers. */
#define PTHREAD__ATOMIC_IS_MEMBAR
#endif /* !_LIB_PTHREAD_HPPA_MD_H */

30
lib/libpthread/arch/i386/_context_u.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: _context_u.S,v 1.7 2007/11/13 17:20:09 ad Exp $ */
/* $NetBSD: _context_u.S,v 1.8 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
@ -100,31 +100,3 @@ STUB(_swapcontext_u_xmm)
movl %eax, (UC_REGS + _REG_UESP * 4)(%ecx)
movl 8(%esp), %ecx
SETC
STUB(pthread__atomic_swap_ptr)
movl 4(%esp), %ecx
movl 8(%esp), %eax
xchgl %eax, (%ecx)
ret
STUB(pthread__atomic_or_ulong)
movl 4(%esp), %ecx
movl 8(%esp), %eax
lock
orl %eax, (%ecx)
ret
STUB(pthread__membar_full)
lock
addl $0, -4(%esp)
ret
STUB(pthread__membar_producer)
/* A store is enough */
addl $0, -4(%esp)
ret
STUB(pthread__membar_consumer)
lock
addl $0, -4(%esp)
ret

34
lib/libpthread/arch/i386/pthread_md.h
View File

@ -1,7 +1,7 @@
/* $NetBSD: pthread_md.h,v 1.11 2007/11/13 17:20:10 ad Exp $ */
/* $NetBSD: pthread_md.h,v 1.12 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
* Copyright (c) 2001, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@ -182,9 +182,11 @@ pthread__sp(void)
} while (/*CONSTCOND*/0)
#define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory")
#define PTHREAD__HAVE_ATOMIC
/* #define PTHREAD__HAVE_THREADREG */
/* Don't need additional memory barriers. */
#define PTHREAD__ATOMIC_IS_MEMBAR
void pthread__threadreg_set(pthread_t);
static inline pthread_t
@ -199,30 +201,4 @@ pthread__threadreg_get(void)
return self;
}
static inline void *
pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new)
{
volatile uintptr_t *cast = ptr;
void *ret;
__asm __volatile ("lock; cmpxchgl %2, %1"
: "=a" (ret), "=m" (*cast)
: "r" (new), "m" (*cast), "0" (old));
return ret;
}
static inline void *
pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new)
{
volatile uintptr_t *cast = ptr;
void *ret;
__asm __volatile ("cmpxchgl %2, %1"
: "=a" (ret), "=m" (*cast)
: "r" (new), "m" (*cast), "0" (old));
return ret;
}
#endif /* _LIB_PTHREAD_I386_MD_H */

5
lib/libpthread/arch/m68k/pthread_md.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:16 perry Exp $ */
/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@ -99,4 +99,7 @@ pthread__sp(void)
(uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \
} while (/*CONSTCOND*/0)
/* m68k will not go SMP */
#define PTHREAD__ATOMIC_IS_MEMBAR
#endif /* _LIB_PTHREAD_M68K_MD_H */

5
lib/libpthread/arch/sh3/pthread_md.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_md.h,v 1.3 2005/12/24 21:11:17 perry Exp $ */
/* $NetBSD: pthread_md.h,v 1.4 2008/02/10 18:50:55 ad Exp $ */
/*
* Copyright 2003 Wasabi Systems, Inc.
@ -90,4 +90,7 @@ pthread__sp(void)
#define PTHREAD_FPREG_TO_UCONTEXT(uc, freg)
#endif
/* sh3 will not go SMP */
#define PTHREAD__ATOMIC_IS_MEMBAR
#endif /* _LIB_PTHREAD_SH3_MD_H */

5
lib/libpthread/arch/sparc/pthread_md.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:17 perry Exp $ */
/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2002 The NetBSD Foundation, Inc.
@ -92,4 +92,7 @@ pthread__sp(void)
(uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \
} while (/*CONSTCOND*/0)
/* Don't need additional memory barriers. */
#define PTHREAD__ATOMIC_IS_MEMBAR
#endif /* _LIB_PTHREAD_SPARC_MD_H */

5
lib/libpthread/arch/vax/pthread_md.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:17 perry Exp $ */
/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@ -85,4 +85,7 @@ pthread__sp(void)
#define PTHREAD_FPREG_TO_UCONTEXT(uc, reg) do { \
} while (/*CONSTCOND*/0)
/* Don't need additional memory barriers. */
#define PTHREAD__ATOMIC_IS_MEMBAR
#endif /* _LIB_PTHREAD_VAX_MD_H */

28
lib/libpthread/arch/x86_64/_context_u.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: _context_u.S,v 1.9 2007/11/13 17:20:10 ad Exp $ */
/* $NetBSD: _context_u.S,v 1.10 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@ -137,29 +137,3 @@ STUB(_swapcontext_u)
movq %rax, (UC_REGS + _REG_URSP * 8)(%rdi)
movq %rsi, %rdi
SETC
STUB(pthread__atomic_swap_ptr)
movq %rsi, %rax
lock
xchgq %rax, (%rdi)
ret
STUB(pthread__atomic_or_ulong)
lock
orq %rsi, (%rdi)
ret
STUB(pthread__membar_full)
lock
addq $0, -8(%rsp)
ret
STUB(pthread__membar_producer)
/* A store is enough */
movq $0, -8(%rsp)
ret
STUB(pthread__membar_consumer)
lock
addq $0, -8(%rsp)
ret

32
lib/libpthread/arch/x86_64/pthread_md.h
View File

@ -1,7 +1,7 @@
/* $NetBSD: pthread_md.h,v 1.7 2007/11/13 17:20:10 ad Exp $ */
/* $NetBSD: pthread_md.h,v 1.8 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
* Copyright (c) 2001, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@ -104,32 +104,8 @@ pthread__sp(void)
} while (/*CONSTCOND*/0)
#define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory")
#define PTHREAD__HAVE_ATOMIC
static inline void *
pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new)
{
volatile uintptr_t *cast = ptr;
void *ret;
__asm __volatile ("lock; cmpxchgq %2, %1"
: "=a" (ret), "=m" (*cast)
: "r" (new), "m" (*cast), "0" (old));
return ret;
}
static inline void *
pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new)
{
volatile uintptr_t *cast = ptr;
void *ret;
__asm __volatile ("cmpxchgq %2, %1"
: "=a" (ret), "=m" (*cast)
: "r" (new), "m" (*cast), "0" (old));
return ret;
}
/* Don't need additional memory barriers. */
#define PTHREAD__ATOMIC_IS_MEMBAR
#endif /* _LIB_PTHREAD_X86_64_MD_H */

16
lib/libpthread/pthread_int.h
View File

@ -1,7 +1,7 @@
/* $NetBSD: pthread_int.h,v 1.65 2008/01/08 20:56:08 christos Exp $ */
/* $NetBSD: pthread_int.h,v 1.66 2008/02/10 18:50:54 ad Exp $ */
/*-
* Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc.
* Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@ -51,6 +51,10 @@
#include "pthread_queue.h"
#include "pthread_md.h"
/* Need to use libc-private names for atomic operations. */
#include "../../common/lib/libc/atomic/atomic_op_namespace.h"
#include <sys/atomic.h>
#include <sys/tree.h>
#include <lwp.h>
@ -289,14 +293,6 @@ void pthread__errorfunc(const char *, int, const char *, const char *)
PTHREAD_HIDE;
char *pthread__getenv(const char *) PTHREAD_HIDE;
void pthread__cancelled(void) PTHREAD_HIDE;
void *pthread__atomic_cas_ptr(volatile void *, const void *, const void *) PTHREAD_HIDE;
void *pthread__atomic_swap_ptr(volatile void *, const void *) PTHREAD_HIDE;
void pthread__atomic_or_ulong(volatile unsigned long *, unsigned long) PTHREAD_HIDE;
void pthread__membar_full(void) PTHREAD_HIDE;
void pthread__membar_producer(void) PTHREAD_HIDE;
void pthread__membar_consumer(void) PTHREAD_HIDE;
int pthread__mutex_deferwake(pthread_t, pthread_mutex_t *) PTHREAD_HIDE;
#ifndef pthread__smt_pause

9
lib/libpthread/pthread_misc.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_misc.c,v 1.6 2008/02/09 17:07:54 yamt Exp $ */
/* $NetBSD: pthread_misc.c,v 1.7 2008/02/10 18:50:54 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_misc.c,v 1.6 2008/02/09 17:07:54 yamt Exp $");
__RCSID("$NetBSD: pthread_misc.c,v 1.7 2008/02/10 18:50:54 ad Exp $");
#include <errno.h>
#include <string.h>
@ -173,13 +173,12 @@ pthread__sched_yield(void)
self = pthread__self();
#ifdef PTHREAD__HAVE_ATOMIC
/* Memory barrier for unlocked mutex release. */
pthread__membar_producer();
#endif
membar_producer();
self->pt_blocking++;
error = _sys_sched_yield();
self->pt_blocking--;
membar_sync();
return error;
}

652
lib/libpthread/pthread_mutex.c
View File

@ -1,7 +1,7 @@
/* $NetBSD: pthread_mutex.c,v 1.43 2008/01/25 02:12:10 rafal Exp $ */
/* $NetBSD: pthread_mutex.c,v 1.44 2008/02/10 18:50:54 ad Exp $ */
/*-
* Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc.
* Copyright (c) 2001, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@ -37,24 +37,51 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_mutex.c,v 1.43 2008/01/25 02:12:10 rafal Exp $");
__RCSID("$NetBSD: pthread_mutex.c,v 1.44 2008/02/10 18:50:54 ad Exp $");
#include <sys/types.h>
#include <machine/lock.h>
#include <sys/lwpctl.h>
#include <errno.h>
#include <limits.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "pthread.h"
#include "pthread_int.h"
#ifndef PTHREAD__HAVE_ATOMIC
/*
* Note that it's important to use the address of ptm_waiters as
* the list head in order for the hint arguments to _lwp_park /
* _lwp_unpark_all to match.
*/
#define pt_nextwaiter pt_sleep.ptqe_next
#define ptm_waiters ptm_blocked.ptqh_first
#define ptm_errorcheck ptm_lock
static int pthread_mutex_lock_slow(pthread_t, pthread_mutex_t *);
static void once_cleanup(void *);
#define MUTEX_WAITERS_BIT ((uintptr_t)0x01)
#define MUTEX_RECURSIVE_BIT ((uintptr_t)0x02)
#define MUTEX_DEFERRED_BIT ((uintptr_t)0x04)
#define MUTEX_THREAD ((uintptr_t)-16L)
#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT)
#define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT)
#define MUTEX_OWNER(x) ((uintptr_t)(x) & MUTEX_THREAD)
#define MUTEX_GET_RECURSE(ptm) ((intptr_t)(ptm)->ptm_private)
#define MUTEX_SET_RECURSE(ptm, delta) \
((ptm)->ptm_private = (void *)((intptr_t)(ptm)->ptm_private + delta))
#if __GNUC_PREREQ__(3, 0)
#define NOINLINE __attribute ((noinline))
#else
#define NOINLINE /* nothing */
#endif
static void pthread__mutex_wakeup(pthread_t, pthread_mutex_t *);
static int pthread__mutex_lock_slow(pthread_mutex_t *);
static int pthread__mutex_unlock_slow(pthread_mutex_t *);
static void pthread__mutex_pause(void);
int _pthread_mutex_held_np(pthread_mutex_t *);
pthread_t _pthread_mutex_owner_np(pthread_mutex_t *);
@ -74,309 +101,431 @@ __strong_alias(__libc_mutexattr_settype,pthread_mutexattr_settype)
__strong_alias(__libc_thr_once,pthread_once)
struct mutex_private {
int type;
int recursecount;
};
static const struct mutex_private mutex_private_default = {
PTHREAD_MUTEX_DEFAULT,
0,
};
struct mutexattr_private {
int type;
};
static const struct mutexattr_private mutexattr_private_default = {
PTHREAD_MUTEX_DEFAULT,
};
int
pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
{
struct mutexattr_private *map;
struct mutex_private *mp;
intptr_t type;
pthread__error(EINVAL, "Invalid mutex attribute",
(attr == NULL) || (attr->ptma_magic == _PT_MUTEXATTR_MAGIC));
if (attr == NULL)
type = PTHREAD_MUTEX_NORMAL;
else
type = (intptr_t)attr->ptma_private;
if (attr != NULL && (map = attr->ptma_private) != NULL &&
memcmp(map, &mutexattr_private_default, sizeof(*map)) != 0) {
mp = malloc(sizeof(*mp));
if (mp == NULL)
return ENOMEM;
mp->type = map->type;
mp->recursecount = 0;
} else {
/* LINTED cast away const */
mp = (struct mutex_private *) &mutex_private_default;
switch (type) {
case PTHREAD_MUTEX_ERRORCHECK:
ptm->ptm_errorcheck = 1;
ptm->ptm_owner = NULL;
break;
case PTHREAD_MUTEX_RECURSIVE:
ptm->ptm_errorcheck = 0;
ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT;
break;
default:
ptm->ptm_errorcheck = 0;
ptm->ptm_owner = NULL;
break;
}
mutex->ptm_magic = _PT_MUTEX_MAGIC;
mutex->ptm_owner = NULL;
pthread_lockinit(&mutex->ptm_lock);
pthread_lockinit(&mutex->ptm_interlock);
PTQ_INIT(&mutex->ptm_blocked);
mutex->ptm_private = mp;
ptm->ptm_magic = _PT_MUTEX_MAGIC;
ptm->ptm_waiters = NULL;
ptm->ptm_private = NULL;
return 0;
}
int
pthread_mutex_destroy(pthread_mutex_t *mutex)
pthread_mutex_destroy(pthread_mutex_t *ptm)
{
pthread__error(EINVAL, "Invalid mutex",
mutex->ptm_magic == _PT_MUTEX_MAGIC);
ptm->ptm_magic == _PT_MUTEX_MAGIC);
pthread__error(EBUSY, "Destroying locked mutex",
__SIMPLELOCK_UNLOCKED_P(&mutex->ptm_lock));
mutex->ptm_magic = _PT_MUTEX_DEAD;
if (mutex->ptm_private != NULL &&
mutex->ptm_private != (const void *)&mutex_private_default)
free(mutex->ptm_private);
MUTEX_OWNER(ptm->ptm_owner) == 0);
ptm->ptm_magic = _PT_MUTEX_DEAD;
return 0;
}
/*
* Note regarding memory visibility: Pthreads has rules about memory
* visibility and mutexes. Very roughly: Memory a thread can see when
* it unlocks a mutex can be seen by another thread that locks the
* same mutex.
*
* A memory barrier after a lock and before an unlock will provide
* this behavior. This code relies on pthread__spintrylock() to issue
* a barrier after obtaining a lock, and on pthread__spinunlock() to
* issue a barrier before releasing a lock.
*/
int
pthread_mutex_lock(pthread_mutex_t *mutex)
pthread_mutex_lock(pthread_mutex_t *ptm)
{
pthread_t self;
int error;
void *val;
self = pthread__self();
/*
* Note that if we get the lock, we don't have to deal with any
* non-default lock type handling.
*/
if (__predict_false(pthread__spintrylock(self, &mutex->ptm_lock) == 0)) {
error = pthread_mutex_lock_slow(self, mutex);
if (error)
return error;
val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
if (__predict_true(val == NULL)) {
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
/*
* We have the lock!
*/
mutex->ptm_owner = self;
return 0;
return pthread__mutex_lock_slow(ptm);
}
static int
pthread_mutex_lock_slow(pthread_t self, pthread_mutex_t *mutex)
/* We want function call overhead. */
NOINLINE static void
pthread__mutex_pause(void)
{
struct mutex_private *mp;
int count;
pthread__smt_pause();
}
/*
* Spin while the holder is running. 'lwpctl' gives us the true
* status of the thread. pt_blocking is set by libpthread in order
* to cut out system call and kernel spinlock overhead on remote CPUs
* (could represent many thousands of clock cycles). pt_blocking also
* makes this thread yield if the target is calling sched_yield().
*/
NOINLINE static void *
pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner)
{
pthread_t thread;
unsigned int count, i;
for (count = 2;; owner = ptm->ptm_owner) {
thread = (pthread_t)MUTEX_OWNER(owner);
if (thread == NULL)
break;
if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE ||
thread->pt_blocking)
break;
if (count < 128)
count += count;
for (i = count; i != 0; i--)
pthread__mutex_pause();
}
return owner;
}
NOINLINE static int
pthread__mutex_lock_slow(pthread_mutex_t *ptm)
{
void *waiters, *new, *owner, *next;
pthread_t self;
pthread__error(EINVAL, "Invalid mutex",
mutex->ptm_magic == _PT_MUTEX_MAGIC);
ptm->ptm_magic == _PT_MUTEX_MAGIC);
for (;;) {
/* Spin for a while. */
count = pthread__nspins;
while (__SIMPLELOCK_LOCKED_P(&mutex->ptm_lock) && --count > 0)
pthread__smt_pause();
if (count > 0) {
if (pthread__spintrylock(self, &mutex->ptm_lock) != 0)
break;
continue;
owner = ptm->ptm_owner;
self = pthread__self();
/* Recursive or errorcheck? */
if (MUTEX_OWNER(owner) == (uintptr_t)self) {
if (MUTEX_RECURSIVE(owner)) {
if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
return EAGAIN;
MUTEX_SET_RECURSE(ptm, +1);
return 0;
}
if (ptm->ptm_errorcheck)
return EDEADLK;
}
/* Okay, didn't look free. Get the interlock... */
pthread__spinlock(self, &mutex->ptm_interlock);
for (;; owner = ptm->ptm_owner) {
/* Spin while the owner is running. */
owner = pthread__mutex_spin(ptm, owner);
/* If it has become free, try to acquire it again. */
if (MUTEX_OWNER(owner) == 0) {
for (; MUTEX_OWNER(owner) == 0; owner = next) {
new = (void *)
((uintptr_t)self | (uintptr_t)owner);
next = atomic_cas_ptr(&ptm->ptm_owner, owner,
new);
if (next == owner) {
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
}
/*
* We have lost the race to acquire the mutex.
* The new owner could be running on another
* CPU, in which case we should spin and avoid
* the overhead of blocking.
*/
if (!MUTEX_HAS_WAITERS(owner))
continue;
}
/*
* The mutex_unlock routine will get the interlock
* before looking at the list of sleepers, so if the
* lock is held we can safely put ourselves on the
* sleep queue. If it's not held, we can try taking it
* again.
* Nope, still held. Add thread to the list of waiters.
* Issue a memory barrier to ensure sleeponq/nextwaiter
* are visible before we enter the waiters list.
*/
PTQ_INSERT_HEAD(&mutex->ptm_blocked, self, pt_sleep);
if (__SIMPLELOCK_UNLOCKED_P(&mutex->ptm_lock)) {
PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep);
pthread__spinunlock(self, &mutex->ptm_interlock);
continue;
self->pt_sleeponq = 1;
for (waiters = ptm->ptm_waiters;; waiters = next) {
self->pt_nextwaiter = waiters;
membar_producer();
next = atomic_cas_ptr(&ptm->ptm_waiters, waiters, self);
if (next == waiters)
break;
}
mp = mutex->ptm_private;
if (mutex->ptm_owner == self && mp != NULL) {
switch (mp->type) {
case PTHREAD_MUTEX_ERRORCHECK:
PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep);
pthread__spinunlock(self, &mutex->ptm_interlock);
return EDEADLK;
case PTHREAD_MUTEX_RECURSIVE:
/*
* Set the waiters bit and block.
*
* Note that the mutex can become unlocked before we set
* the waiters bit. If that happens it's not safe to sleep
* as we may never be awoken: we must remove the current
* thread from the waiters list and try again.
*
* Because we are doing this atomically, we can't remove
* one waiter: we must remove all waiters and awken them,
* then sleep in _lwp_park() until we have been awoken.
*
* Issue a memory barrier to ensure that we are reading
* the value of ptm_owner/pt_sleeponq after we have entered
* the waiters list (the CAS itself must be atomic).
*/
membar_consumer();
for (owner = ptm->ptm_owner;; owner = next) {
if (MUTEX_HAS_WAITERS(owner))
break;
if (MUTEX_OWNER(owner) == 0) {
pthread__mutex_wakeup(self, ptm);
break;
}
new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT);
next = atomic_cas_ptr(&ptm->ptm_owner, owner, new);
if (next == owner) {
/*
* It's safe to do this without
* holding the interlock, because
* we only modify it if we know we
* own the mutex.
* pthread_mutex_unlock() can do a
* non-interlocked CAS. We cannot
* know if our attempt to set the
* waiters bit has succeeded while
* the holding thread is running.
* There are many assumptions; see
* sys/kern/kern_mutex.c for details.
* In short, we must spin if we see
* that the holder is running again.
*/
PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep);
pthread__spinunlock(self, &mutex->ptm_interlock);
if (mp->recursecount == INT_MAX)
return EAGAIN;
mp->recursecount++;
return 0;
membar_sync();
next = pthread__mutex_spin(ptm, owner);
}
}
/*
* Locking a mutex is not a cancellation
* point, so we don't need to do the
* test-cancellation dance. We may get woken
* up spuriously by pthread_cancel or signals,
* but it's okay since we're just going to
* retry.
* We may have been awoken by the current thread above,
* or will be awoken by the current holder of the mutex.
* The key requirement is that we must not proceed until
* told that we are no longer waiting (via pt_sleeponq
* being set to zero). Otherwise it is unsafe to re-enter
* the thread onto the waiters list.
*/
self->pt_sleeponq = 1;
self->pt_sleepobj = &mutex->ptm_blocked;
pthread__spinunlock(self, &mutex->ptm_interlock);
(void)pthread__park(self, &mutex->ptm_interlock,
&mutex->ptm_blocked, NULL, 0, &mutex->ptm_blocked);
while (self->pt_sleeponq) {
self->pt_blocking++;
(void)_lwp_park(NULL, 0, &ptm->ptm_waiters, NULL);
self->pt_blocking--;
membar_sync();
}
}
return 0;
}
int
pthread_mutex_trylock(pthread_mutex_t *mutex)
pthread_mutex_trylock(pthread_mutex_t *ptm)
{
struct mutex_private *mp;
pthread_t self;
pthread__error(EINVAL, "Invalid mutex",
mutex->ptm_magic == _PT_MUTEX_MAGIC);
void *val;
self = pthread__self();
val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
if (__predict_true(val == NULL)) {
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
if (pthread__spintrylock(self, &mutex->ptm_lock) == 0) {
if (MUTEX_OWNER(val) == (uintptr_t)self && MUTEX_RECURSIVE(val)) {
if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
return EAGAIN;
MUTEX_SET_RECURSE(ptm, +1);
return 0;
}
return EBUSY;
}
int
pthread_mutex_unlock(pthread_mutex_t *ptm)
{
pthread_t self;
void *value;
/*
* Note this may be a non-interlocked CAS. See lock_slow()
* above and sys/kern/kern_mutex.c for details.
*/
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_exit();
#endif
self = pthread__self();
value = atomic_cas_ptr_ni(&ptm->ptm_owner, self, NULL);
if (__predict_true(value == self))
return 0;
return pthread__mutex_unlock_slow(ptm);
}
NOINLINE static int
pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
{
pthread_t self, owner, new;
int weown, error, deferred;
pthread__error(EINVAL, "Invalid mutex",
ptm->ptm_magic == _PT_MUTEX_MAGIC);
self = pthread__self();
owner = ptm->ptm_owner;
weown = (MUTEX_OWNER(owner) == (uintptr_t)self);
deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT);
error = 0;
if (ptm->ptm_errorcheck) {
if (!weown) {
error = EPERM;
new = owner;
} else {
new = NULL;
}
} else if (MUTEX_RECURSIVE(owner)) {
if (!weown) {
error = EPERM;
new = owner;
} else if (MUTEX_GET_RECURSE(ptm) != 0) {
MUTEX_SET_RECURSE(ptm, -1);
new = owner;
} else {
new = (pthread_t)MUTEX_RECURSIVE_BIT;
}
} else {
pthread__error(EPERM,
"Unlocking unlocked mutex", (owner != NULL));
pthread__error(EPERM,
"Unlocking mutex owned by another thread", weown);
new = NULL;
}
/*
* Release the mutex. If there appear to be waiters, then
* wake them up.
*/
if (new != owner) {
owner = atomic_swap_ptr(&ptm->ptm_owner, new);
if (MUTEX_HAS_WAITERS(owner) != 0) {
pthread__mutex_wakeup(self, ptm);
return 0;
}
}
/*
* There were no waiters, but we may have deferred waking
* other threads until mutex unlock - we must wake them now.
*/
if (!deferred)
return error;
if (self->pt_nwaiters == 1) {
/*
* These tests can be performed without holding the
* interlock because these fields are only modified
* if we know we own the mutex.
* If the calling thread is about to block, defer
* unparking the target until _lwp_park() is called.
*/
mp = mutex->ptm_private;
if (mp != NULL && mp->type == PTHREAD_MUTEX_RECURSIVE &&
mutex->ptm_owner == self) {
if (mp->recursecount == INT_MAX)
return EAGAIN;
mp->recursecount++;
return 0;
if (self->pt_willpark && self->pt_unpark == 0) {
self->pt_unpark = self->pt_waiters[0];
self->pt_unparkhint = &ptm->ptm_waiters;
} else {
(void)_lwp_unpark(self->pt_waiters[0],
&ptm->ptm_waiters);
}
return EBUSY;
} else {
(void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
&ptm->ptm_waiters);
}
self->pt_nwaiters = 0;
mutex->ptm_owner = self;
return 0;
return error;
}
int
pthread_mutex_unlock(pthread_mutex_t *mutex)
static void
pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm)
{
struct mutex_private *mp;
pthread_t self;
int weown;
pthread__error(EINVAL, "Invalid mutex",
mutex->ptm_magic == _PT_MUTEX_MAGIC);
pthread_t thread, next;
ssize_t n, rv;
/*
* These tests can be performed without holding the
* interlock because these fields are only modified
* if we know we own the mutex.
* Take ownership of the current set of waiters. No
* need for a memory barrier following this, all loads
* are dependent upon 'thread'.
*/
self = pthread__self();
weown = (mutex->ptm_owner == self);
mp = mutex->ptm_private;
thread = atomic_swap_ptr(&ptm->ptm_waiters, NULL);
if (mp == NULL) {
if (__predict_false(!weown)) {
pthread__error(EPERM, "Unlocking unlocked mutex",
(mutex->ptm_owner != 0));
pthread__error(EPERM,
"Unlocking mutex owned by another thread", weown);
for (;;) {
/*
* Pull waiters from the queue and add to our list.
* Use a memory barrier to ensure that we safely
* read the value of pt_nextwaiter before 'thread'
* sees pt_sleeponq being cleared.
*/
for (n = self->pt_nwaiters, self->pt_nwaiters = 0;
n < pthread__unpark_max && thread != NULL;
thread = next) {
next = thread->pt_nextwaiter;
if (thread != self) {
self->pt_waiters[n++] = thread->pt_lid;
membar_sync();
}
thread->pt_sleeponq = 0;
/* No longer safe to touch 'thread' */
}
} else if (mp->type == PTHREAD_MUTEX_RECURSIVE) {
if (!weown)
return EPERM;
if (mp->recursecount != 0) {
mp->recursecount--;
return 0;
}
} else if (mp->type == PTHREAD_MUTEX_ERRORCHECK) {
if (!weown)
return EPERM;
if (__predict_false(!weown)) {
pthread__error(EPERM, "Unlocking unlocked mutex",
(mutex->ptm_owner != 0));
pthread__error(EPERM,
"Unlocking mutex owned by another thread", weown);
switch (n) {
case 0:
return;
case 1:
/*
* If the calling thread is about to block,
* defer unparking the target until _lwp_park()
* is called.
*/
if (self->pt_willpark && self->pt_unpark == 0) {
self->pt_unpark = self->pt_waiters[0];
self->pt_unparkhint = &ptm->ptm_waiters;
return;
}
rv = (ssize_t)_lwp_unpark(self->pt_waiters[0],
&ptm->ptm_waiters);
if (rv != 0 && errno != EALREADY && errno != EINTR &&
errno != ESRCH) {
pthread__errorfunc(__FILE__, __LINE__,
__func__, "_lwp_unpark failed");
}
return;
default:
rv = _lwp_unpark_all(self->pt_waiters, (size_t)n,
&ptm->ptm_waiters);
if (rv != 0 && errno != EINTR) {
pthread__errorfunc(__FILE__, __LINE__,
__func__, "_lwp_unpark_all failed");
}
break;
}
}
mutex->ptm_owner = NULL;
pthread__spinunlock(self, &mutex->ptm_lock);
/*
* Do a double-checked locking dance to see if there are any
* waiters. If we don't see any waiters, we can exit, because
* we've already released the lock. If we do see waiters, they
* were probably waiting on us... there's a slight chance that
* they are waiting on a different thread's ownership of the
* lock that happened between the unlock above and this
* examination of the queue; if so, no harm is done, as the
* waiter will loop and see that the mutex is still locked.
*/
pthread__spinlock(self, &mutex->ptm_interlock);
pthread__unpark_all(self, &mutex->ptm_interlock, &mutex->ptm_blocked);
return 0;
}
int
pthread_mutexattr_init(pthread_mutexattr_t *attr)
{
struct mutexattr_private *map;
map = malloc(sizeof(*map));
if (map == NULL)
return ENOMEM;
*map = mutexattr_private_default;
attr->ptma_magic = _PT_MUTEXATTR_MAGIC;
attr->ptma_private = map;
attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT;
return 0;
}
int
pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
{
@ -384,10 +533,6 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
attr->ptma_magic = _PT_MUTEXATTR_DEAD;
if (attr->ptma_private != NULL)
free(attr->ptma_private);
return 0;
}
@ -395,15 +540,11 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
int
pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
{
struct mutexattr_private *map;
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
map = attr->ptma_private;
*typep = map->type;
*typep = (int)(intptr_t)attr->ptma_private;
return 0;
}
@ -411,25 +552,19 @@ pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
int
pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
{
struct mutexattr_private *map;
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
map = attr->ptma_private;
switch (type) {
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_RECURSIVE:
map->type = type;
break;
attr->ptma_private = (void *)(intptr_t)type;
return 0;
default:
return EINVAL;
}
return 0;
}
@ -459,24 +594,27 @@ pthread_once(pthread_once_t *once_control, void (*routine)(void))
}
int
pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *mutex)
pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *ptm)
{
return mutex->ptm_owner == thread;
if (MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)thread)
return 0;
atomic_or_ulong((volatile unsigned long *)
(uintptr_t)&ptm->ptm_owner,
(unsigned long)MUTEX_DEFERRED_BIT);
return 1;
}
int
_pthread_mutex_held_np(pthread_mutex_t *mutex)
_pthread_mutex_held_np(pthread_mutex_t *ptm)
{
return mutex->ptm_owner == pthread__self();
return MUTEX_OWNER(ptm->ptm_owner) == (uintptr_t)pthread__self();
}
pthread_t
_pthread_mutex_owner_np(pthread_mutex_t *mutex)
_pthread_mutex_owner_np(pthread_mutex_t *ptm)
{
return (pthread_t)mutex->ptm_owner;
return (pthread_t)MUTEX_OWNER(ptm->ptm_owner);
}
#endif /* !PTHREAD__HAVE_ATOMIC */

623
lib/libpthread/pthread_mutex2.c
View File

@ -1,623 +0,0 @@
/* $NetBSD: pthread_mutex2.c,v 1.17 2007/12/24 14:46:29 ad Exp $ */
/*-
* Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the NetBSD
* Foundation, Inc. and its contributors.
* 4. Neither the name of The NetBSD Foundation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_mutex2.c,v 1.17 2007/12/24 14:46:29 ad Exp $");
#include <sys/types.h>
#include <sys/lwpctl.h>
#include <errno.h>
#include <limits.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "pthread.h"
#include "pthread_int.h"
#ifdef PTHREAD__HAVE_ATOMIC
/*
* Note that it's important to use the address of ptm_waiters as
* the list head in order for the hint arguments to _lwp_park /
* _lwp_unpark_all to match.
*/
#define pt_nextwaiter pt_sleep.ptqe_next
#define ptm_waiters ptm_blocked.ptqh_first
#define ptm_errorcheck ptm_lock
#define MUTEX_WAITERS_BIT ((uintptr_t)0x01)
#define MUTEX_RECURSIVE_BIT ((uintptr_t)0x02)
#define MUTEX_DEFERRED_BIT ((uintptr_t)0x04)
#define MUTEX_THREAD ((uintptr_t)-16L)
#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT)
#define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT)
#define MUTEX_OWNER(x) ((uintptr_t)(x) & MUTEX_THREAD)
#define MUTEX_GET_RECURSE(ptm) ((intptr_t)(ptm)->ptm_private)
#define MUTEX_SET_RECURSE(ptm, delta) \
((ptm)->ptm_private = (void *)((intptr_t)(ptm)->ptm_private + delta))
#if __GNUC_PREREQ__(3, 0)
#define NOINLINE __attribute ((noinline))
#else
#define NOINLINE /* nothing */
#endif
static void pthread__mutex_wakeup(pthread_t, pthread_mutex_t *);
static int pthread__mutex_lock_slow(pthread_mutex_t *);
static int pthread__mutex_unlock_slow(pthread_mutex_t *);
static void pthread__mutex_pause(void);
int _pthread_mutex_held_np(pthread_mutex_t *);
pthread_t _pthread_mutex_owner_np(pthread_mutex_t *);
__weak_alias(pthread_mutex_held_np,_pthread_mutex_held_np)
__weak_alias(pthread_mutex_owner_np,_pthread_mutex_owner_np)
__strong_alias(__libc_mutex_init,pthread_mutex_init)
__strong_alias(__libc_mutex_lock,pthread_mutex_lock)
__strong_alias(__libc_mutex_trylock,pthread_mutex_trylock)
__strong_alias(__libc_mutex_unlock,pthread_mutex_unlock)
__strong_alias(__libc_mutex_destroy,pthread_mutex_destroy)
__strong_alias(__libc_mutexattr_init,pthread_mutexattr_init)
__strong_alias(__libc_mutexattr_destroy,pthread_mutexattr_destroy)
__strong_alias(__libc_mutexattr_settype,pthread_mutexattr_settype)
__strong_alias(__libc_thr_once,pthread_once)
static inline int
mutex_cas(volatile void *ptr, void **old, void *new)
{
void *oldv;
oldv = *old;
*old = pthread__atomic_cas_ptr(ptr, oldv, new);
return *old == oldv;
}
static inline int
mutex_cas_ni(volatile void *ptr, void **old, void *new)
{
void *oldv;
oldv = *old;
*old = pthread__atomic_cas_ptr_ni(ptr, oldv, new);
return *old == oldv;
}
int
pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
{
intptr_t type;
if (attr == NULL)
type = PTHREAD_MUTEX_NORMAL;
else
type = (intptr_t)attr->ptma_private;
switch (type) {
case PTHREAD_MUTEX_ERRORCHECK:
ptm->ptm_errorcheck = 1;
ptm->ptm_owner = NULL;
break;
case PTHREAD_MUTEX_RECURSIVE:
ptm->ptm_errorcheck = 0;
ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT;
break;
default:
ptm->ptm_errorcheck = 0;
ptm->ptm_owner = NULL;
break;
}
ptm->ptm_magic = _PT_MUTEX_MAGIC;
ptm->ptm_waiters = NULL;
ptm->ptm_private = NULL;
return 0;
}
int
pthread_mutex_destroy(pthread_mutex_t *ptm)
{
pthread__error(EINVAL, "Invalid mutex",
ptm->ptm_magic == _PT_MUTEX_MAGIC);
pthread__error(EBUSY, "Destroying locked mutex",
MUTEX_OWNER(ptm->ptm_owner) == 0);
ptm->ptm_magic = _PT_MUTEX_DEAD;
return 0;
}
int
pthread_mutex_lock(pthread_mutex_t *ptm)
{
void *owner;
pthread_t self;
owner = NULL;
self = pthread__self();
if (__predict_true(mutex_cas(&ptm->ptm_owner, &owner, self)))
return 0;
return pthread__mutex_lock_slow(ptm);
}
/* We want function call overhead. */
NOINLINE static void
pthread__mutex_pause(void)
{
pthread__smt_pause();
}
/*
* Spin while the holder is running. 'lwpctl' gives us the true
* status of the thread. pt_blocking is set by libpthread in order
* to cut out system call and kernel spinlock overhead on remote CPUs
* (could represent many thousands of clock cycles). pt_blocking also
* makes this thread yield if the target is calling sched_yield().
*/
NOINLINE static void *
pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner)
{
pthread_t thread;
for (;; owner = ptm->ptm_owner) {
thread = (pthread_t)MUTEX_OWNER(owner);
if (thread == NULL)
break;
if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE ||
thread->pt_blocking)
break;
pthread__mutex_pause();
pthread__mutex_pause();
pthread__mutex_pause();
pthread__mutex_pause();
}
return owner;
}
NOINLINE static int
pthread__mutex_lock_slow(pthread_mutex_t *ptm)
{
void *waiters, *new, *owner;
pthread_t self;
pthread__error(EINVAL, "Invalid mutex",
ptm->ptm_magic == _PT_MUTEX_MAGIC);
owner = ptm->ptm_owner;
self = pthread__self();
/* Recursive or errorcheck? */
if (MUTEX_OWNER(owner) == (uintptr_t)self) {
if (MUTEX_RECURSIVE(owner)) {
if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
return EAGAIN;
MUTEX_SET_RECURSE(ptm, +1);
return 0;
}
if (ptm->ptm_errorcheck)
return EDEADLK;
}
for (;; owner = ptm->ptm_owner) {
/* Spin while the owner is running. */
owner = pthread__mutex_spin(ptm, owner);
/* If it has become free, try to acquire it again. */
if (MUTEX_OWNER(owner) == 0) {
while (MUTEX_OWNER(owner) == 0) {
new = (void *)
((uintptr_t)self | (uintptr_t)owner);
if (mutex_cas(&ptm->ptm_owner, &owner, new))
return 0;
}
/*
* We have lost the race to acquire the mutex.
* The new owner could be running on another
* CPU, in which case we should spin and avoid
* the overhead of blocking.
*/
continue;
}
/*
* Nope, still held. Add thread to the list of waiters.
* Issue a memory barrier to ensure sleeponq/nextwaiter
* are visible before we enter the waiters list.
*/
self->pt_sleeponq = 1;
for (waiters = ptm->ptm_waiters;;) {
self->pt_nextwaiter = waiters;
pthread__membar_producer();
if (mutex_cas(&ptm->ptm_waiters, &waiters, self))
break;
}
/*
* Set the waiters bit and block.
*
* Note that the mutex can become unlocked before we set
* the waiters bit. If that happens it's not safe to sleep
* as we may never be awoken: we must remove the current
* thread from the waiters list and try again.
*
* Because we are doing this atomically, we can't remove
* one waiter: we must remove all waiters and awken them,
* then sleep in _lwp_park() until we have been awoken.
*
* Issue a memory barrier to ensure that we are reading
* the value of ptm_owner/pt_sleeponq after we have entered
* the waiters list (the CAS itself must be atomic).
*/
pthread__membar_consumer();
for (owner = ptm->ptm_owner;;) {
if (MUTEX_HAS_WAITERS(owner))
break;
if (MUTEX_OWNER(owner) == 0) {
pthread__mutex_wakeup(self, ptm);
break;
}
new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT);
if (mutex_cas(&ptm->ptm_owner, &owner, new)) {
/*
* pthread_mutex_unlock() can do a
* non-interlocked CAS. We cannot
* know if our attempt to set the
* waiters bit has succeeded while
* the holding thread is running.
* There are many assumptions; see
* sys/kern/kern_mutex.c for details.
* In short, we must spin if we see
* that the holder is running again.
*/
pthread__membar_full();
owner = pthread__mutex_spin(ptm, owner);
}
}
/*
* We may have been awoken by the current thread above,
* or will be awoken by the current holder of the mutex.
* The key requirement is that we must not proceed until
* told that we are no longer waiting (via pt_sleeponq
* being set to zero). Otherwise it is unsafe to re-enter
* the thread onto the waiters list.
*/
while (self->pt_sleeponq) {
self->pt_blocking++;
(void)_lwp_park(NULL, 0, &ptm->ptm_waiters, NULL);
self->pt_blocking--;
}
}
}
int
pthread_mutex_trylock(pthread_mutex_t *ptm)
{
pthread_t self;
void *value;
self = pthread__self();
value = NULL;
if (mutex_cas(&ptm->ptm_owner, &value, self))
return 0;
if (MUTEX_OWNER(value) == (uintptr_t)self && MUTEX_RECURSIVE(value)) {
if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
return EAGAIN;
MUTEX_SET_RECURSE(ptm, +1);
return 0;
}
return EBUSY;
}
int
pthread_mutex_unlock(pthread_mutex_t *ptm)
{
void *owner;
pthread_t self;
self = pthread__self();
owner = self;
/*
* Note this may be a non-interlocked CAS. See lock_slow()
* above and sys/kern/kern_mutex.c for details.
*/
if (__predict_true(mutex_cas_ni(&ptm->ptm_owner, &owner, NULL)))
return 0;
return pthread__mutex_unlock_slow(ptm);
}
NOINLINE static int
pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
{
pthread_t self, owner, new;
int weown, error, deferred;
pthread__error(EINVAL, "Invalid mutex",
ptm->ptm_magic == _PT_MUTEX_MAGIC);
self = pthread__self();
owner = ptm->ptm_owner;
weown = (MUTEX_OWNER(owner) == (uintptr_t)self);
deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT);
error = 0;
if (ptm->ptm_errorcheck) {
if (!weown) {
error = EPERM;
new = owner;
} else {
new = NULL;
}
} else if (MUTEX_RECURSIVE(owner)) {
if (!weown) {
error = EPERM;
new = owner;
} else if (MUTEX_GET_RECURSE(ptm) != 0) {
MUTEX_SET_RECURSE(ptm, -1);
new = owner;
} else {
new = (pthread_t)MUTEX_RECURSIVE_BIT;
}
} else {
pthread__error(EPERM,
"Unlocking unlocked mutex", (owner != NULL));
pthread__error(EPERM,
"Unlocking mutex owned by another thread", weown);
new = NULL;
}
/*
* Release the mutex. If there appear to be waiters, then
* wake them up.
*/
if (new != owner) {
owner = pthread__atomic_swap_ptr(&ptm->ptm_owner, new);
if (MUTEX_HAS_WAITERS(owner) != 0) {
pthread__mutex_wakeup(self, ptm);
return 0;
}
}
/*
* There were no waiters, but we may have deferred waking
* other threads until mutex unlock - we must wake them now.
*/
if (!deferred)
return error;
if (self->pt_nwaiters == 1) {
/*
* If the calling thread is about to block, defer
* unparking the target until _lwp_park() is called.
*/
if (self->pt_willpark && self->pt_unpark == 0) {
self->pt_unpark = self->pt_waiters[0];
self->pt_unparkhint = &ptm->ptm_waiters;
} else {
(void)_lwp_unpark(self->pt_waiters[0],
&ptm->ptm_waiters);
}
} else {
(void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
&ptm->ptm_waiters);
}
self->pt_nwaiters = 0;
return error;
}
static void
pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm)
{
pthread_t thread, next;
ssize_t n, rv;
/*
* Take ownership of the current set of waiters. No
* need for a memory barrier following this, all loads
* are dependent upon 'thread'.
*/
thread = pthread__atomic_swap_ptr(&ptm->ptm_waiters, NULL);
for (;;) {
/*
* Pull waiters from the queue and add to our list.
* Use a memory barrier to ensure that we safely
* read the value of pt_nextwaiter before 'thread'
* sees pt_sleeponq being cleared.
*/
for (n = self->pt_nwaiters, self->pt_nwaiters = 0;
n < pthread__unpark_max && thread != NULL;
thread = next) {
next = thread->pt_nextwaiter;
self->pt_waiters[n++] = thread->pt_lid;
pthread__membar_full();
thread->pt_sleeponq = 0;
/* No longer safe to touch 'thread' */
}
switch (n) {
case 0:
return;
case 1:
/*
* If the calling thread is about to block,
* defer unparking the target until _lwp_park()
* is called.
*/
if (self->pt_willpark && self->pt_unpark == 0) {
self->pt_unpark = self->pt_waiters[0];
self->pt_unparkhint = &ptm->ptm_waiters;
return;
}
rv = (ssize_t)_lwp_unpark(self->pt_waiters[0],
&ptm->ptm_waiters);
if (rv != 0 && errno != EALREADY && errno != EINTR &&
errno != ESRCH) {
pthread__errorfunc(__FILE__, __LINE__,
__func__, "_lwp_unpark failed");
}
return;
default:
rv = _lwp_unpark_all(self->pt_waiters, (size_t)n,
&ptm->ptm_waiters);
if (rv != 0 && errno != EINTR) {
pthread__errorfunc(__FILE__, __LINE__,
__func__, "_lwp_unpark_all failed");
}
break;
}
}
}
int
pthread_mutexattr_init(pthread_mutexattr_t *attr)
{
attr->ptma_magic = _PT_MUTEXATTR_MAGIC;
attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT;
return 0;
}
int
pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
{
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
return 0;
}
int
pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
{
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
*typep = (int)(intptr_t)attr->ptma_private;
return 0;
}
int
pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
{
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
switch (type) {
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_RECURSIVE:
attr->ptma_private = (void *)(intptr_t)type;
return 0;
default:
return EINVAL;
}
}
static void
once_cleanup(void *closure)
{
pthread_mutex_unlock((pthread_mutex_t *)closure);
}
int
pthread_once(pthread_once_t *once_control, void (*routine)(void))
{
if (once_control->pto_done == 0) {
pthread_mutex_lock(&once_control->pto_mutex);
pthread_cleanup_push(&once_cleanup, &once_control->pto_mutex);
if (once_control->pto_done == 0) {
routine();
once_control->pto_done = 1;
}
pthread_cleanup_pop(1);
}
return 0;
}
int
pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *ptm)
{
if (MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)thread)
return 0;
pthread__atomic_or_ulong((volatile unsigned long *)
(uintptr_t)&ptm->ptm_owner,
(unsigned long)MUTEX_DEFERRED_BIT);
return 1;
}
int
_pthread_mutex_held_np(pthread_mutex_t *ptm)
{
return MUTEX_OWNER(ptm->ptm_owner) == (uintptr_t)pthread__self();
}
pthread_t
_pthread_mutex_owner_np(pthread_mutex_t *ptm)
{
return (pthread_t)MUTEX_OWNER(ptm->ptm_owner);
}
#endif /* PTHREAD__HAVE_ATOMIC */

738
lib/libpthread/pthread_rwlock.c
View File

@ -1,11 +1,11 @@
/* $NetBSD: pthread_rwlock.c,v 1.26 2008/01/31 11:50:40 ad Exp $ */
/* $NetBSD: pthread_rwlock.c,v 1.27 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc.
* Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Nathan J. Williams and Andrew Doran.
* by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -37,23 +37,32 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_rwlock.c,v 1.26 2008/01/31 11:50:40 ad Exp $");
__RCSID("$NetBSD: pthread_rwlock.c,v 1.27 2008/02/10 18:50:55 ad Exp $");
#include <errno.h>
#include <stddef.h>
#include "pthread.h"
#include "pthread_int.h"
#ifndef PTHREAD__HAVE_ATOMIC
#define _RW_LOCKED 0
#define _RW_WANT_WRITE 1
#define _RW_WANT_READ 2
__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np)
__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np)
__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np)
static int pthread__rwlock_wrlock(pthread_rwlock_t *, const struct timespec *);
static int pthread__rwlock_rdlock(pthread_rwlock_t *, const struct timespec *);
static void pthread__rwlock_early(void *);
int _pthread_rwlock_held_np(pthread_rwlock_t *);
int _pthread_rwlock_rdheld_np(pthread_rwlock_t *);
int _pthread_rwlock_wrheld_np(pthread_rwlock_t *);
#ifndef lint
__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np);
__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np);
__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np);
#endif
__strong_alias(__libc_rwlock_init,pthread_rwlock_init)
__strong_alias(__libc_rwlock_rdlock,pthread_rwlock_rdlock)
__strong_alias(__libc_rwlock_wrlock,pthread_rwlock_wrlock)
@ -62,368 +71,517 @@ __strong_alias(__libc_rwlock_trywrlock,pthread_rwlock_trywrlock)
__strong_alias(__libc_rwlock_unlock,pthread_rwlock_unlock)
__strong_alias(__libc_rwlock_destroy,pthread_rwlock_destroy)
static inline uintptr_t
rw_cas(pthread_rwlock_t *ptr, uintptr_t o, uintptr_t n)
{
return (uintptr_t)atomic_cas_ptr(&ptr->ptr_owner, (void *)o,
(void *)n);
}
int
pthread_rwlock_init(pthread_rwlock_t *rwlock,
pthread_rwlock_init(pthread_rwlock_t *ptr,
const pthread_rwlockattr_t *attr)
{
#ifdef ERRORCHECK
if ((rwlock == NULL) ||
(attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC)))
if (attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
return EINVAL;
#endif
rwlock->ptr_magic = _PT_RWLOCK_MAGIC;
pthread_lockinit(&rwlock->ptr_interlock);
PTQ_INIT(&rwlock->ptr_rblocked);
PTQ_INIT(&rwlock->ptr_wblocked);
rwlock->ptr_nreaders = 0;
rwlock->ptr_writer = NULL;
ptr->ptr_magic = _PT_RWLOCK_MAGIC;
pthread_lockinit(&ptr->ptr_interlock);
PTQ_INIT(&ptr->ptr_rblocked);
PTQ_INIT(&ptr->ptr_wblocked);
ptr->ptr_nreaders = 0;
ptr->ptr_owner = NULL;
return 0;
}
int
pthread_rwlock_destroy(pthread_rwlock_t *rwlock)
pthread_rwlock_destroy(pthread_rwlock_t *ptr)
{
#ifdef ERRORCHECK
if ((rwlock == NULL) ||
(rwlock->ptr_magic != _PT_RWLOCK_MAGIC) ||
(!PTQ_EMPTY(&rwlock->ptr_rblocked)) ||
(!PTQ_EMPTY(&rwlock->ptr_wblocked)) ||
(rwlock->ptr_nreaders != 0) ||
(rwlock->ptr_writer != NULL))
if ((ptr->ptr_magic != _PT_RWLOCK_MAGIC) ||
(!PTQ_EMPTY(&ptr->ptr_rblocked)) ||
(!PTQ_EMPTY(&ptr->ptr_wblocked)) ||
(ptr->ptr_nreaders != 0) ||
(ptr->ptr_owner != NULL))
return EINVAL;
#endif
rwlock->ptr_magic = _PT_RWLOCK_DEAD;
ptr->ptr_magic = _PT_RWLOCK_DEAD;
return 0;
}
int
pthread_rwlock_rdlock(pthread_rwlock_t *rwlock)
static int
pthread__rwlock_rdlock(pthread_rwlock_t *ptr, const struct timespec *ts)
{
uintptr_t owner, next;
pthread_t self;
#ifdef ERRORCHECK
if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
return EINVAL;
#endif
self = pthread__self();
pthread__spinlock(self, &rwlock->ptr_interlock);
#ifdef ERRORCHECK
if (rwlock->ptr_writer == self) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EDEADLK;
}
#endif
/*
* Don't get a readlock if there is a writer or if there are waiting
* writers; i.e. prefer writers to readers. This strategy is dictated
* by SUSv3.
*/
while ((rwlock->ptr_writer != NULL) ||
(!PTQ_EMPTY(&rwlock->ptr_wblocked))) {
PTQ_INSERT_TAIL(&rwlock->ptr_rblocked, self, pt_sleep);
self->pt_sleeponq = 1;
self->pt_sleepobj = &rwlock->ptr_rblocked;
pthread__spinunlock(self, &rwlock->ptr_interlock);
(void)pthread__park(self, &rwlock->ptr_interlock,
&rwlock->ptr_rblocked, NULL, 0, &rwlock->ptr_rblocked);
pthread__spinlock(self, &rwlock->ptr_interlock);
}
rwlock->ptr_nreaders++;
pthread__spinunlock(self, &rwlock->ptr_interlock);
return 0;
}
int
pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
{
pthread_t self;
#ifdef ERRORCHECK
if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
return EINVAL;
#endif
int error;
self = pthread__self();
pthread__spinlock(self, &rwlock->ptr_interlock);
/*
* Don't get a readlock if there is a writer or if there are waiting
* writers; i.e. prefer writers to readers. This strategy is dictated
* by SUSv3.
*/
if ((rwlock->ptr_writer != NULL) ||
(!PTQ_EMPTY(&rwlock->ptr_wblocked))) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EBUSY;
}
rwlock->ptr_nreaders++;
pthread__spinunlock(self, &rwlock->ptr_interlock);
return 0;
}
int
pthread_rwlock_wrlock(pthread_rwlock_t *rwlock)
{
pthread_t self;
extern int pthread__started;
#ifdef ERRORCHECK
if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
self = pthread__self();
pthread__spinlock(self, &rwlock->ptr_interlock);
#ifdef ERRORCHECK
if (rwlock->ptr_writer == self) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EDEADLK;
}
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) == 0) {
next = rw_cas(ptr, owner, owner + RW_READ_INCR);
if (owner == next) {
/* Got it! */
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
/*
* Prefer writers to readers here; permit writers even if there are
* waiting readers.
*/
while ((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL)) {
#ifdef ERRORCHECK
if (pthread__started == 0) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EDEADLK;
return 0;
}
/*
* Didn't get it -- spin around again (we'll
* probably sleep on the next iteration).
*/
continue;
}
#endif
PTQ_INSERT_TAIL(&rwlock->ptr_wblocked, self, pt_sleep);
if ((owner & RW_THREAD) == (uintptr_t)self)
return EDEADLK;
/*
* Grab the interlock. Once we have that, we
* can adjust the waiter bits and sleep queue.
*/
pthread__spinlock(self, &ptr->ptr_interlock);
/*
* Mark the rwlock as having waiters. If the set fails,
* then we may not need to sleep and should spin again.
*/
next = rw_cas(ptr, owner, owner | RW_HAS_WAITERS);
if (owner != next) {
pthread__spinunlock(self, &ptr->ptr_interlock);
continue;
}
/* The waiters bit is set - it's safe to sleep. */
PTQ_INSERT_HEAD(&ptr->ptr_rblocked, self, pt_sleep);
ptr->ptr_nreaders++;
self->pt_rwlocked = _RW_WANT_READ;
self->pt_sleeponq = 1;
self->pt_sleepobj = &rwlock->ptr_wblocked;
pthread__spinunlock(self, &rwlock->ptr_interlock);
(void)pthread__park(self, &rwlock->ptr_interlock,
&rwlock->ptr_wblocked, NULL, 0, &rwlock->ptr_wblocked);
pthread__spinlock(self, &rwlock->ptr_interlock);
self->pt_sleepobj = &ptr->ptr_rblocked;
self->pt_early = pthread__rwlock_early;
pthread__spinunlock(self, &ptr->ptr_interlock);
error = pthread__park(self, &ptr->ptr_interlock,
&ptr->ptr_rblocked, ts, 0, &ptr->ptr_rblocked);
/* Did we get the lock? */
if (self->pt_rwlocked == _RW_LOCKED) {
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
if (error != 0)
return error;
pthread__errorfunc(__FILE__, __LINE__, __func__,
"direct handoff failure");
}
rwlock->ptr_writer = self;
pthread__spinunlock(self, &rwlock->ptr_interlock);
return 0;
}
int
pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
pthread_rwlock_tryrdlock(pthread_rwlock_t *ptr)
{
pthread_t self;
uintptr_t owner, next;
#ifdef ERRORCHECK
if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
self = pthread__self();
pthread__spinlock(self, &rwlock->ptr_interlock);
/*
* Prefer writers to readers here; permit writers even if there are
* waiting readers.
* Don't get a readlock if there is a writer or if there are waiting
* writers; i.e. prefer writers to readers. This strategy is dictated
* by SUSv3.
*/
if ((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL)) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EBUSY;
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) != 0)
return EBUSY;
next = rw_cas(ptr, owner, owner + RW_READ_INCR);
if (owner == next) {
/* Got it! */
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
}
}
rwlock->ptr_writer = self;
pthread__spinunlock(self, &rwlock->ptr_interlock);
static int
pthread__rwlock_wrlock(pthread_rwlock_t *ptr, const struct timespec *ts)
{
uintptr_t owner, next;
pthread_t self;
int error;
return 0;
self = pthread__self();
#ifdef ERRORCHECK
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & RW_THREAD) == 0) {
next = rw_cas(ptr, owner,
(uintptr_t)self | RW_WRITE_LOCKED);
if (owner == next) {
/* Got it! */
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
/*
* Didn't get it -- spin around again (we'll
* probably sleep on the next iteration).
*/
continue;
}
if ((owner & RW_THREAD) == (uintptr_t)self)
return EDEADLK;
/*
* Grab the interlock. Once we have that, we
* can adjust the waiter bits and sleep queue.
*/
pthread__spinlock(self, &ptr->ptr_interlock);
/*
* Mark the rwlock as having waiters. If the set fails,
* then we may not need to sleep and should spin again.
*/
next = rw_cas(ptr, owner,
owner | RW_HAS_WAITERS | RW_WRITE_WANTED);
if (owner != next) {
pthread__spinunlock(self, &ptr->ptr_interlock);
continue;
}
/* The waiters bit is set - it's safe to sleep. */
PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep);
self->pt_rwlocked = _RW_WANT_WRITE;
self->pt_sleeponq = 1;
self->pt_sleepobj = &ptr->ptr_wblocked;
self->pt_early = pthread__rwlock_early;
pthread__spinunlock(self, &ptr->ptr_interlock);
error = pthread__park(self, &ptr->ptr_interlock,
&ptr->ptr_wblocked, ts, 0, &ptr->ptr_wblocked);
/* Did we get the lock? */
if (self->pt_rwlocked == _RW_LOCKED) {
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
if (error != 0)
return error;
pthread__errorfunc(__FILE__, __LINE__, __func__,
"direct handoff failure");
}
}
int
pthread_rwlock_timedrdlock(pthread_rwlock_t *rwlock,
const struct timespec *abs_timeout)
pthread_rwlock_trywrlock(pthread_rwlock_t *ptr)
{
uintptr_t owner, next;
pthread_t self;
int retval;
#ifdef ERRORCHECK
if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
self = pthread__self();
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
if (owner != 0)
return EBUSY;
next = rw_cas(ptr, owner, (uintptr_t)self | RW_WRITE_LOCKED);
if (owner == next) {
/* Got it! */
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
}
}
int
pthread_rwlock_rdlock(pthread_rwlock_t *ptr)
{
return pthread__rwlock_rdlock(ptr, NULL);
}
int
pthread_rwlock_timedrdlock(pthread_rwlock_t *ptr,
const struct timespec *abs_timeout)
{
if (abs_timeout == NULL)
return EINVAL;
#endif
if ((abs_timeout->tv_nsec >= 1000000000) ||
(abs_timeout->tv_nsec < 0) ||
(abs_timeout->tv_sec < 0))
return EINVAL;
self = pthread__self();
pthread__spinlock(self, &rwlock->ptr_interlock);
#ifdef ERRORCHECK
if (rwlock->ptr_writer == self) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EDEADLK;
}
#endif
/*
* Don't get a readlock if there is a writer or if there are waiting
* writers; i.e. prefer writers to readers. This strategy is dictated
* by SUSv3.
*/
retval = 0;
while ((retval == 0) && ((rwlock->ptr_writer != NULL) ||
(!PTQ_EMPTY(&rwlock->ptr_wblocked)))) {
PTQ_INSERT_TAIL(&rwlock->ptr_rblocked, self, pt_sleep);
self->pt_sleeponq = 1;
self->pt_sleepobj = &rwlock->ptr_rblocked;
pthread__spinunlock(self, &rwlock->ptr_interlock);
retval = pthread__park(self, &rwlock->ptr_interlock,
&rwlock->ptr_rblocked, abs_timeout, 0,
&rwlock->ptr_rblocked);
pthread__spinlock(self, &rwlock->ptr_interlock);
}
/* One last chance to get the lock, in case it was released between
the alarm firing and when this thread got rescheduled, or in case
a signal handler kept it busy */
if ((rwlock->ptr_writer == NULL) &&
(PTQ_EMPTY(&rwlock->ptr_wblocked))) {
rwlock->ptr_nreaders++;
retval = 0;
}
pthread__spinunlock(self, &rwlock->ptr_interlock);
return retval;
return pthread__rwlock_rdlock(ptr, abs_timeout);
}
int
pthread_rwlock_wrlock(pthread_rwlock_t *ptr)
{
return pthread__rwlock_wrlock(ptr, NULL);
}
int
pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlock,
const struct timespec *abs_timeout)
pthread_rwlock_timedwrlock(pthread_rwlock_t *ptr,
const struct timespec *abs_timeout)
{
pthread_t self;
int retval;
extern int pthread__started;
#ifdef ERRORCHECK
if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
return EINVAL;
if (abs_timeout == NULL)
return EINVAL;
#endif
if ((abs_timeout->tv_nsec >= 1000000000) ||
(abs_timeout->tv_nsec < 0) ||
(abs_timeout->tv_sec < 0))
return EINVAL;
self = pthread__self();
pthread__spinlock(self, &rwlock->ptr_interlock);
#ifdef ERRORCHECK
if (rwlock->ptr_writer == self) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EDEADLK;
}
#endif
/*
* Prefer writers to readers here; permit writers even if there are
* waiting readers.
*/
retval = 0;
while (retval == 0 &&
((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL))) {
#ifdef ERRORCHECK
if (pthread__started == 0) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EDEADLK;
}
#endif
PTQ_INSERT_TAIL(&rwlock->ptr_wblocked, self, pt_sleep);
self->pt_sleeponq = 1;
self->pt_sleepobj = &rwlock->ptr_wblocked;
pthread__spinunlock(self, &rwlock->ptr_interlock);
retval = pthread__park(self, &rwlock->ptr_interlock,
&rwlock->ptr_wblocked, abs_timeout, 0,
&rwlock->ptr_wblocked);
pthread__spinlock(self, &rwlock->ptr_interlock);
}
if ((rwlock->ptr_nreaders == 0) && (rwlock->ptr_writer == NULL)) {
rwlock->ptr_writer = self;
retval = 0;
}
pthread__spinunlock(self, &rwlock->ptr_interlock);
return retval;
return pthread__rwlock_wrlock(ptr, abs_timeout);
}
int
pthread_rwlock_unlock(pthread_rwlock_t *rwlock)
pthread_rwlock_unlock(pthread_rwlock_t *ptr)
{
pthread_t self, writer;
uintptr_t owner, decr, new, next;
pthread_t self, thread;
#ifdef ERRORCHECK
if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
if ((ptr == NULL) || (ptr->ptr_magic != _PT_RWLOCK_MAGIC))
return EINVAL;
#endif
writer = NULL;
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_exit();
#endif
/*
* Since we used an add operation to set the required lock
* bits, we can use a subtract to clear them, which makes
* the read-release and write-release path similar.
*/
self = pthread__self();
pthread__spinlock(self, &rwlock->ptr_interlock);
if (rwlock->ptr_writer != NULL) {
/* Releasing a write lock. */
#ifdef ERRORCHECK
if (rwlock->ptr_writer != self) {
pthread__spinunlock(self, &rwlock->ptr_interlock);
owner = (uintptr_t)ptr->ptr_owner;
if ((owner & RW_WRITE_LOCKED) != 0) {
decr = (uintptr_t)self | RW_WRITE_LOCKED;
if ((owner & RW_THREAD) != (uintptr_t)self) {
return EPERM;
}
#endif
rwlock->ptr_writer = NULL;
writer = PTQ_FIRST(&rwlock->ptr_wblocked);
if (writer != NULL) {
PTQ_REMOVE(&rwlock->ptr_wblocked, writer, pt_sleep);
}
} else
#ifdef ERRORCHECK
if (rwlock->ptr_nreaders > 0)
#endif
{
/* Releasing a read lock. */
rwlock->ptr_nreaders--;
if (rwlock->ptr_nreaders == 0) {
writer = PTQ_FIRST(&rwlock->ptr_wblocked);
if (writer != NULL)
PTQ_REMOVE(&rwlock->ptr_wblocked, writer,
pt_sleep);
}
#ifdef ERRORCHECK
} else {
pthread__spinunlock(self, &rwlock->ptr_interlock);
return EPERM;
#endif
decr = RW_READ_INCR;
if (owner == 0) {
return EPERM;
}
}
if (writer != NULL)
pthread__unpark(self, &rwlock->ptr_interlock,
&rwlock->ptr_wblocked, writer);
else
pthread__unpark_all(self, &rwlock->ptr_interlock,
&rwlock->ptr_rblocked);
for (;; owner = next) {
/*
* Compute what we expect the new value of the lock to be.
* Only proceed to do direct handoff if there are waiters,
* and if the lock would become unowned.
*/
new = (owner - decr);
if ((new & (RW_THREAD | RW_HAS_WAITERS)) != RW_HAS_WAITERS) {
next = rw_cas(ptr, owner, new);
if (owner == next) {
/* Released! */
return 0;
}
continue;
}
return 0;
/*
* Grab the interlock. Once we have that, we can adjust
* the waiter bits. We must check to see if there are
* still waiters before proceeding.
*/
pthread__spinlock(self, &ptr->ptr_interlock);
owner = (uintptr_t)ptr->ptr_owner;
if ((owner & RW_HAS_WAITERS) == 0) {
pthread__spinunlock(self, &ptr->ptr_interlock);
next = owner;
continue;
}
/*
* Give the lock away. SUSv3 dictates that we must give
* preference to writers.
*/
if ((thread = PTQ_FIRST(&ptr->ptr_wblocked)) != NULL) {
new = (uintptr_t)thread | RW_WRITE_LOCKED;
if (PTQ_NEXT(thread, pt_sleep) != NULL)
new |= RW_HAS_WAITERS | RW_WRITE_WANTED;
else if (ptr->ptr_nreaders != 0)
new |= RW_HAS_WAITERS;
/*
* Set in the new value. The lock becomes owned
* by the writer that we are about to wake.
*/
(void)atomic_swap_ptr(&ptr->ptr_owner, (void *)new);
/* Wake the writer. */
PTQ_REMOVE(&ptr->ptr_wblocked, thread, pt_sleep);
thread->pt_rwlocked = _RW_LOCKED;
pthread__unpark(self, &ptr->ptr_interlock,
&ptr->ptr_wblocked, thread);
} else {
new = 0;
PTQ_FOREACH(thread, &ptr->ptr_rblocked, pt_sleep) {
/*
* May have already been handed the lock,
* since pthread__unpark_all() can release
* our interlock before awakening all
* threads.
*/
if (thread->pt_sleepobj == NULL)
continue;
new += RW_READ_INCR;
thread->pt_rwlocked = _RW_LOCKED;
}
/*
* Set in the new value. The lock becomes owned
* by the readers that we are about to wake.
*/
(void)atomic_swap_ptr(&ptr->ptr_owner, (void *)new);
/* Wake up all sleeping readers. */
ptr->ptr_nreaders = 0;
pthread__unpark_all(self, &ptr->ptr_interlock,
&ptr->ptr_rblocked);
}
return 0;
}
}
/*
* Called when a timedlock awakens early to adjust the waiter bits.
* The rwlock's interlock is held on entry, and the caller has been
* removed from the waiters lists.
*/
static void
pthread__rwlock_early(void *obj)
{
uintptr_t owner, set, new, next;
pthread_rwlock_t *ptr;
pthread_t self;
u_int off;
self = pthread__self();
switch (self->pt_rwlocked) {
case _RW_WANT_READ:
off = offsetof(pthread_rwlock_t, ptr_rblocked);
break;
case _RW_WANT_WRITE:
off = offsetof(pthread_rwlock_t, ptr_wblocked);
break;
default:
pthread__errorfunc(__FILE__, __LINE__, __func__,
"bad value of pt_rwlocked");
off = 0;
/* NOTREACHED */
break;
}
/* LINTED mind your own business */
ptr = (pthread_rwlock_t *)((uint8_t *)obj - off);
owner = (uintptr_t)ptr->ptr_owner;
if ((owner & RW_THREAD) == 0) {
pthread__errorfunc(__FILE__, __LINE__, __func__,
"lock not held");
}
if (!PTQ_EMPTY(&ptr->ptr_wblocked))
set = RW_HAS_WAITERS | RW_WRITE_WANTED;
else if (ptr->ptr_nreaders != 0)
set = RW_HAS_WAITERS;
else
set = 0;
for (;; owner = next) {
new = (owner & ~(RW_HAS_WAITERS | RW_WRITE_WANTED)) | set;
next = rw_cas(ptr, owner, new);
if (owner == next)
break;
}
}
int
_pthread_rwlock_held_np(pthread_rwlock_t *ptr)
{
uintptr_t owner = (uintptr_t)ptr->ptr_owner;
return (owner & RW_THREAD) != 0;
}
int
_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr)
{
uintptr_t owner = (uintptr_t)ptr->ptr_owner;
return (owner & RW_THREAD) != 0 && (owner & RW_WRITE_LOCKED) == 0;
}
int
_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr)
{
uintptr_t owner = (uintptr_t)ptr->ptr_owner;
return (owner & (RW_THREAD | RW_WRITE_LOCKED)) ==
((uintptr_t)pthread__self() | RW_WRITE_LOCKED);
}
int
pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
{
#ifdef ERRORCHECK
if (attr == NULL)
return EINVAL;
#endif
attr->ptra_magic = _PT_RWLOCKATTR_MAGIC;
return 0;
@ -433,35 +591,11 @@ pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
int
pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
{
#ifdef ERRORCHECK
if ((attr == NULL) ||
(attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
return EINVAL;
#endif
attr->ptra_magic = _PT_RWLOCKATTR_DEAD;
return 0;
}
int
_pthread_rwlock_held_np(pthread_rwlock_t *ptr)
{
return ptr->ptr_writer != NULL || ptr->ptr_nreaders != 0;
}
int
_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr)
{
return ptr->ptr_nreaders != 0;
}
int
_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr)
{
return ptr->ptr_writer == pthread__self();
}
#endif /* !PTHREAD__HAVE_ATOMIC */

584
lib/libpthread/pthread_rwlock2.c
View File

@ -1,584 +0,0 @@
/* $NetBSD: pthread_rwlock2.c,v 1.9 2008/01/31 11:50:40 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the NetBSD
* Foundation, Inc. and its contributors.
* 4. Neither the name of The NetBSD Foundation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_rwlock2.c,v 1.9 2008/01/31 11:50:40 ad Exp $");
#include <errno.h>
#include <stddef.h>
#include "pthread.h"
#include "pthread_int.h"
#ifdef PTHREAD__HAVE_ATOMIC
#define _RW_LOCKED 0
#define _RW_WANT_WRITE 1
#define _RW_WANT_READ 2
static int pthread__rwlock_wrlock(pthread_rwlock_t *, const struct timespec *);
static int pthread__rwlock_rdlock(pthread_rwlock_t *, const struct timespec *);
static void pthread__rwlock_early(void *);
int _pthread_rwlock_held_np(pthread_rwlock_t *);
int _pthread_rwlock_rdheld_np(pthread_rwlock_t *);
int _pthread_rwlock_wrheld_np(pthread_rwlock_t *);
#ifndef lint
__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np);
__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np);
__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np);
#endif
__strong_alias(__libc_rwlock_init,pthread_rwlock_init)
__strong_alias(__libc_rwlock_rdlock,pthread_rwlock_rdlock)
__strong_alias(__libc_rwlock_wrlock,pthread_rwlock_wrlock)
__strong_alias(__libc_rwlock_tryrdlock,pthread_rwlock_tryrdlock)
__strong_alias(__libc_rwlock_trywrlock,pthread_rwlock_trywrlock)
__strong_alias(__libc_rwlock_unlock,pthread_rwlock_unlock)
__strong_alias(__libc_rwlock_destroy,pthread_rwlock_destroy)
static inline uintptr_t
rw_cas(pthread_rwlock_t *ptr, uintptr_t o, uintptr_t n)
{
return (uintptr_t)pthread__atomic_cas_ptr(&ptr->ptr_owner, (void *)o,
(void *)n);
}
int
pthread_rwlock_init(pthread_rwlock_t *ptr,
const pthread_rwlockattr_t *attr)
{
if (attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
return EINVAL;
ptr->ptr_magic = _PT_RWLOCK_MAGIC;
pthread_lockinit(&ptr->ptr_interlock);
PTQ_INIT(&ptr->ptr_rblocked);
PTQ_INIT(&ptr->ptr_wblocked);
ptr->ptr_nreaders = 0;
ptr->ptr_owner = NULL;
return 0;
}
int
pthread_rwlock_destroy(pthread_rwlock_t *ptr)
{
if ((ptr->ptr_magic != _PT_RWLOCK_MAGIC) ||
(!PTQ_EMPTY(&ptr->ptr_rblocked)) ||
(!PTQ_EMPTY(&ptr->ptr_wblocked)) ||
(ptr->ptr_nreaders != 0) ||
(ptr->ptr_owner != NULL))
return EINVAL;
ptr->ptr_magic = _PT_RWLOCK_DEAD;
return 0;
}
static int
pthread__rwlock_rdlock(pthread_rwlock_t *ptr, const struct timespec *ts)
{
uintptr_t owner, next;
pthread_t self;
int error;
self = pthread__self();
#ifdef ERRORCHECK
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) == 0) {
next = rw_cas(ptr, owner, owner + RW_READ_INCR);
if (owner == next) {
/* Got it! */
return 0;
}
/*
* Didn't get it -- spin around again (we'll
* probably sleep on the next iteration).
*/
continue;
}
if ((owner & RW_THREAD) == (uintptr_t)self)
return EDEADLK;
/*
* Grab the interlock. Once we have that, we
* can adjust the waiter bits and sleep queue.
*/
pthread__spinlock(self, &ptr->ptr_interlock);
/*
* Mark the rwlock as having waiters. If the set fails,
* then we may not need to sleep and should spin again.
*/
next = rw_cas(ptr, owner, owner | RW_HAS_WAITERS);
if (owner != next) {
pthread__spinunlock(self, &ptr->ptr_interlock);
continue;
}
/* The waiters bit is set - it's safe to sleep. */
PTQ_INSERT_HEAD(&ptr->ptr_rblocked, self, pt_sleep);
ptr->ptr_nreaders++;
self->pt_rwlocked = _RW_WANT_READ;
self->pt_sleeponq = 1;
self->pt_sleepobj = &ptr->ptr_rblocked;
self->pt_early = pthread__rwlock_early;
pthread__spinunlock(self, &ptr->ptr_interlock);
error = pthread__park(self, &ptr->ptr_interlock,
&ptr->ptr_rblocked, ts, 0, &ptr->ptr_rblocked);
/* Did we get the lock? */
if (self->pt_rwlocked == _RW_LOCKED)
return 0;
if (error != 0)
return error;
pthread__errorfunc(__FILE__, __LINE__, __func__,
"direct handoff failure");
}
}
int
pthread_rwlock_tryrdlock(pthread_rwlock_t *ptr)
{
uintptr_t owner, next;
#ifdef ERRORCHECK
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
/*
* Don't get a readlock if there is a writer or if there are waiting
* writers; i.e. prefer writers to readers. This strategy is dictated
* by SUSv3.
*/
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) != 0)
return EBUSY;
next = rw_cas(ptr, owner, owner + RW_READ_INCR);
if (owner == next) {
/* Got it! */
return 0;
}
}
}
static int
pthread__rwlock_wrlock(pthread_rwlock_t *ptr, const struct timespec *ts)
{
uintptr_t owner, next;
pthread_t self;
int error;
self = pthread__self();
#ifdef ERRORCHECK
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & RW_THREAD) == 0) {
next = rw_cas(ptr, owner,
(uintptr_t)self | RW_WRITE_LOCKED);
if (owner == next) {
/* Got it! */
return 0;
}
/*
* Didn't get it -- spin around again (we'll
* probably sleep on the next iteration).
*/
continue;
}
if ((owner & RW_THREAD) == (uintptr_t)self)
return EDEADLK;
/*
* Grab the interlock. Once we have that, we
* can adjust the waiter bits and sleep queue.
*/
pthread__spinlock(self, &ptr->ptr_interlock);
/*
* Mark the rwlock as having waiters. If the set fails,
* then we may not need to sleep and should spin again.
*/
next = rw_cas(ptr, owner,
owner | RW_HAS_WAITERS | RW_WRITE_WANTED);
if (owner != next) {
pthread__spinunlock(self, &ptr->ptr_interlock);
continue;
}
/* The waiters bit is set - it's safe to sleep. */
PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep);
self->pt_rwlocked = _RW_WANT_WRITE;
self->pt_sleeponq = 1;
self->pt_sleepobj = &ptr->ptr_wblocked;
self->pt_early = pthread__rwlock_early;
pthread__spinunlock(self, &ptr->ptr_interlock);
error = pthread__park(self, &ptr->ptr_interlock,
&ptr->ptr_wblocked, ts, 0, &ptr->ptr_wblocked);
/* Did we get the lock? */
if (self->pt_rwlocked == _RW_LOCKED)
return 0;
if (error != 0)
return error;
pthread__errorfunc(__FILE__, __LINE__, __func__,
"direct handoff failure");
}
}
int
pthread_rwlock_trywrlock(pthread_rwlock_t *ptr)
{
uintptr_t owner, next;
pthread_t self;
#ifdef ERRORCHECK
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
self = pthread__self();
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
if (owner != 0)
return EBUSY;
next = rw_cas(ptr, owner, (uintptr_t)self | RW_WRITE_LOCKED);
if (owner == next) {
/* Got it! */
return 0;
}
}
}
int
pthread_rwlock_rdlock(pthread_rwlock_t *ptr)
{
return pthread__rwlock_rdlock(ptr, NULL);
}
int
pthread_rwlock_timedrdlock(pthread_rwlock_t *ptr,
const struct timespec *abs_timeout)
{
if (abs_timeout == NULL)
return EINVAL;
if ((abs_timeout->tv_nsec >= 1000000000) ||
(abs_timeout->tv_nsec < 0) ||
(abs_timeout->tv_sec < 0))
return EINVAL;
return pthread__rwlock_rdlock(ptr, abs_timeout);
}
int
pthread_rwlock_wrlock(pthread_rwlock_t *ptr)
{
return pthread__rwlock_wrlock(ptr, NULL);
}
int
pthread_rwlock_timedwrlock(pthread_rwlock_t *ptr,
const struct timespec *abs_timeout)
{
if (abs_timeout == NULL)
return EINVAL;
if ((abs_timeout->tv_nsec >= 1000000000) ||
(abs_timeout->tv_nsec < 0) ||
(abs_timeout->tv_sec < 0))
return EINVAL;
return pthread__rwlock_wrlock(ptr, abs_timeout);
}
int
pthread_rwlock_unlock(pthread_rwlock_t *ptr)
{
uintptr_t owner, decr, new, next;
pthread_t self, thread;
#ifdef ERRORCHECK
if ((ptr == NULL) || (ptr->ptr_magic != _PT_RWLOCK_MAGIC))
return EINVAL;
#endif
self = pthread__self();
/*
* Since we used an add operation to set the required lock
* bits, we can use a subtract to clear them, which makes
* the read-release and write-release path similar.
*/
owner = (uintptr_t)ptr->ptr_owner;
if ((owner & RW_WRITE_LOCKED) != 0) {
decr = (uintptr_t)self | RW_WRITE_LOCKED;
if ((owner & RW_THREAD) != (uintptr_t)self) {
return EPERM;
}
} else {
decr = RW_READ_INCR;
if (owner == 0) {
return EPERM;
}
}
for (;; owner = next) {
/*
* Compute what we expect the new value of the lock to be.
* Only proceed to do direct handoff if there are waiters,
* and if the lock would become unowned.
*/
new = (owner - decr);
if ((new & (RW_THREAD | RW_HAS_WAITERS)) != RW_HAS_WAITERS) {
next = rw_cas(ptr, owner, new);
if (owner == next) {
/* Released! */
return 0;
}
continue;
}
/*
* Grab the interlock. Once we have that, we can adjust
* the waiter bits. We must check to see if there are
* still waiters before proceeding.
*/
pthread__spinlock(self, &ptr->ptr_interlock);
owner = (uintptr_t)ptr->ptr_owner;
if ((owner & RW_HAS_WAITERS) == 0) {
pthread__spinunlock(self, &ptr->ptr_interlock);
next = owner;
continue;
}
/*
* Give the lock away. SUSv3 dictates that we must give
* preference to writers.
*/
if ((thread = PTQ_FIRST(&ptr->ptr_wblocked)) != NULL) {
new = (uintptr_t)thread | RW_WRITE_LOCKED;
if (PTQ_NEXT(thread, pt_sleep) != NULL)
new |= RW_HAS_WAITERS | RW_WRITE_WANTED;
else if (ptr->ptr_nreaders != 0)
new |= RW_HAS_WAITERS;
/*
* Set in the new value. The lock becomes owned
* by the writer that we are about to wake.
*/
(void)pthread__atomic_swap_ptr(&ptr->ptr_owner,
(void *)new);
/* Wake the writer. */
PTQ_REMOVE(&ptr->ptr_wblocked, thread, pt_sleep);
thread->pt_rwlocked = _RW_LOCKED;
pthread__unpark(self, &ptr->ptr_interlock,
&ptr->ptr_wblocked, thread);
} else {
new = 0;
PTQ_FOREACH(thread, &ptr->ptr_rblocked, pt_sleep) {
/*
* May have already been handed the lock,
* since pthread__unpark_all() can release
* our interlock before awakening all
* threads.
*/
if (thread->pt_sleepobj == NULL)
continue;
new += RW_READ_INCR;
thread->pt_rwlocked = _RW_LOCKED;
}
/*
* Set in the new value. The lock becomes owned
* by the readers that we are about to wake.
*/
(void)pthread__atomic_swap_ptr(&ptr->ptr_owner,
(void *)new);
/* Wake up all sleeping readers. */
ptr->ptr_nreaders = 0;
pthread__unpark_all(self, &ptr->ptr_interlock,
&ptr->ptr_rblocked);
}
return 0;
}
}
/*
* Called when a timedlock awakens early to adjust the waiter bits.
* The rwlock's interlock is held on entry, and the caller has been
* removed from the waiters lists.
*/
static void
pthread__rwlock_early(void *obj)
{
uintptr_t owner, set, new, next;
pthread_rwlock_t *ptr;
pthread_t self;
u_int off;
self = pthread__self();
switch (self->pt_rwlocked) {
case _RW_WANT_READ:
off = offsetof(pthread_rwlock_t, ptr_rblocked);
break;
case _RW_WANT_WRITE:
off = offsetof(pthread_rwlock_t, ptr_wblocked);
break;
default:
pthread__errorfunc(__FILE__, __LINE__, __func__,
"bad value of pt_rwlocked");
off = 0;
/* NOTREACHED */
break;
}
/* LINTED mind your own business */
ptr = (pthread_rwlock_t *)((uint8_t *)obj - off);
owner = (uintptr_t)ptr->ptr_owner;
if ((owner & RW_THREAD) == 0) {
pthread__errorfunc(__FILE__, __LINE__, __func__,
"lock not held");
}
if (!PTQ_EMPTY(&ptr->ptr_wblocked))
set = RW_HAS_WAITERS | RW_WRITE_WANTED;
else if (ptr->ptr_nreaders != 0)
set = RW_HAS_WAITERS;
else
set = 0;
for (;; owner = next) {
new = (owner & ~(RW_HAS_WAITERS | RW_WRITE_WANTED)) | set;
next = rw_cas(ptr, owner, new);
if (owner == next)
break;
}
}
int
_pthread_rwlock_held_np(pthread_rwlock_t *ptr)
{
uintptr_t owner = (uintptr_t)ptr->ptr_owner;
return (owner & RW_THREAD) != 0;
}
int
_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr)
{
uintptr_t owner = (uintptr_t)ptr->ptr_owner;
return (owner & RW_THREAD) != 0 && (owner & RW_WRITE_LOCKED) == 0;
}
int
_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr)
{
uintptr_t owner = (uintptr_t)ptr->ptr_owner;
return (owner & (RW_THREAD | RW_WRITE_LOCKED)) ==
((uintptr_t)pthread__self() | RW_WRITE_LOCKED);
}
int
pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
{
if (attr == NULL)
return EINVAL;
attr->ptra_magic = _PT_RWLOCKATTR_MAGIC;
return 0;
}
int
pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
{
if ((attr == NULL) ||
(attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
return EINVAL;
attr->ptra_magic = _PT_RWLOCKATTR_DEAD;
return 0;
}
#endif /* PTHREAD__HAVE_ATOMIC */