- Reinitialize the absolute minimum when recycling user thread state.

Chops another ~10% off create/join in a loop on i386.
- Disable low level debugging as this is stable. Improves benchmarks
  across the board by a small percentage. Uncontested mutex acquire
  and release in a loop becomes about 8% quicker.
- Minor cleanup.
This commit is contained in:
ad 2007-08-16 12:01:49 +00:00
parent 9247850afb
commit b8833ff53f
7 changed files with 129 additions and 120 deletions

View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread.c,v 1.76 2007/08/16 01:09:34 ad Exp $ */
/* $NetBSD: pthread.c,v 1.77 2007/08/16 12:01:49 ad Exp $ */
/*-
* Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc.
@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread.c,v 1.76 2007/08/16 01:09:34 ad Exp $");
__RCSID("$NetBSD: pthread.c,v 1.77 2007/08/16 12:01:49 ad Exp $");
#define __EXPOSE_STACK 1
@ -69,18 +69,17 @@ __RCSID("$NetBSD: pthread.c,v 1.76 2007/08/16 01:09:34 ad Exp $");
/* Maximum number of LWPs to unpark in one operation. */
#define PTHREAD__UNPARK_MAX 128
/* How many times to try acquiring spin locks on MP systems. */
#define PTHREAD__NSPINS 1024
static void pthread__create_tramp(void *(*)(void *), void *);
static void pthread__initthread(pthread_t);
static void pthread__scrubthread(pthread_t, char *, int);
static int pthread__stackid_setup(void *, size_t, pthread_t *);
static int pthread__stackalloc(pthread_t *);
static void pthread__initmain(pthread_t *);
int pthread__started;
pthread_spin_t pthread__queue_lock = __SIMPLELOCK_UNLOCKED;
pthread_spin_t pthread__allqueue_lock = __SIMPLELOCK_UNLOCKED;
pthread_spin_t pthread__deadqueue_lock = __SIMPLELOCK_UNLOCKED;
pthread_queue_t pthread__allqueue;
pthread_queue_t pthread__deadqueue;
@ -147,15 +146,15 @@ pthread_init(void)
{
pthread_t first;
char *p;
int i, mib[2], ncpu;
int i, mib[2];
size_t len;
extern int __isthreaded;
mib[0] = CTL_HW;
mib[1] = HW_NCPU;
len = sizeof(ncpu);
if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1)
len = sizeof(pthread__concurrency);
if (sysctl(mib, 2, &pthread__concurrency, &len, NULL, 0) == -1)
err(1, "sysctl(hw.ncpu");
mib[0] = CTL_KERN;
@ -166,18 +165,9 @@ pthread_init(void)
err(1, "sysctl(hw.osrevision");
/* Initialize locks first; they're needed elsewhere. */
pthread__lockprim_init(ncpu);
pthread__lockprim_init();
/*
* Get number of CPUs, and maximum number of LWPs that can be
* unparked at once.
*/
if ((pthread__concurrency = ncpu) > 1)
pthread__nspins = PTHREAD__NSPINS;
else
pthread__nspins = 1;
if ((p = getenv("PTHREAD_NSPINS")) != NULL)
pthread__nspins = atoi(p);
/* Fetch parameters. */
i = (int)_lwp_unpark_all(NULL, 0, NULL);
if (i == -1)
err(1, "_lwp_unpark_all");
@ -192,16 +182,14 @@ pthread_init(void)
/* Create the thread structure corresponding to main() */
pthread__initmain(&first);
pthread__initthread(first);
pthread__scrubthread(first, NULL, 0);
first->pt_state = PT_STATE_RUNNING;
first->pt_lid = _lwp_self();
PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
/* Start subsystems */
PTHREAD_MD_INIT
#ifdef PTHREAD__DEBUG
pthread__debug_init(ncpu);
#endif
pthread__debug_init();
for (p = getenv("PTHREAD_DIAGASSERT"); p && *p; p++) {
switch (*p) {
@ -250,9 +238,6 @@ pthread__child_callback(void)
static void
pthread__start(void)
{
pthread_t self;
self = pthread__self(); /* should be the "main()" thread */
/*
* Per-process timers are cleared by fork(); despite the
@ -260,7 +245,7 @@ pthread__start(void)
* fork() before creating any threads.
*/
pthread_atfork(NULL, NULL, pthread__child_callback);
SDPRINTF(("(pthread__start %p) Started.\n", self));
SDPRINTF(("(pthread__start %p) Started.\n", pthread__self()));
}
@ -271,19 +256,13 @@ pthread__initthread(pthread_t t)
{
t->pt_magic = PT_MAGIC;
t->pt_state = PT_STATE_RUNNING;
t->pt_spinlocks = 0;
t->pt_exitval = NULL;
t->pt_flags = 0;
t->pt_cancel = 0;
t->pt_errno = 0;
t->pt_name = NULL;
t->pt_willpark = 0;
t->pt_unpark = 0;
t->pt_sleeponq = 0;
t->pt_sleepobj = NULL;
t->pt_signalled = 0;
t->pt_sleepq = NULL;
t->pt_havespecific = 0;
pthread_lockinit(&t->pt_lock);
PTQ_INIT(&t->pt_cleanup_stack);
@ -291,6 +270,19 @@ pthread__initthread(pthread_t t)
memset(&t->pt_specific, 0, sizeof(int) * PTHREAD_KEYS_MAX);
}
static void
pthread__scrubthread(pthread_t t, char *name, int flags)
{
t->pt_state = PT_STATE_RUNNING;
t->pt_exitval = NULL;
t->pt_flags = flags;
t->pt_cancel = 0;
t->pt_errno = 0;
t->pt_name = name;
t->pt_lid = 0;
}
int
pthread_create(pthread_t *thread, const pthread_attr_t *attr,
@ -331,31 +323,37 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr,
self = pthread__self();
newthread = NULL;
/*
* Try to reclaim a dead thread.
*/
if (!PTQ_EMPTY(&pthread__deadqueue)) {
pthread_spinlock(self, &pthread__queue_lock);
pthread_spinlock(self, &pthread__deadqueue_lock);
newthread = PTQ_FIRST(&pthread__deadqueue);
if (newthread != NULL) {
PTQ_REMOVE(&pthread__deadqueue, newthread, pt_allq);
pthread_spinunlock(self, &pthread__queue_lock);
PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq);
pthread_spinunlock(self, &pthread__deadqueue_lock);
if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) {
/* Still running? */
if (_lwp_kill(newthread->pt_lid, 0) == 0 ||
errno != ESRCH) {
pthread_spinlock(self,
&pthread__queue_lock);
&pthread__deadqueue_lock);
PTQ_INSERT_TAIL(&pthread__deadqueue,
newthread, pt_allq);
newthread, pt_deadq);
pthread_spinunlock(self,
&pthread__queue_lock);
&pthread__deadqueue_lock);
newthread = NULL;
}
}
} else
pthread_spinunlock(self, &pthread__queue_lock);
pthread_spinunlock(self, &pthread__deadqueue_lock);
}
/*
* If necessary set up a stack, allocate space for a pthread_st,
* and initialize it.
*/
if (newthread == NULL) {
/* Set up a stack and allocate space for a pthread_st. */
ret = pthread__stackalloc(&newthread);
if (ret != 0) {
if (name)
@ -363,31 +361,30 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr,
return ret;
}
/*
* Set up state that will not change. This is used only
* when creating the thread.
*/
/* This is used only when creating the thread. */
_INITCONTEXT_U(&newthread->pt_uc);
#ifdef PTHREAD_MACHINE_HAS_ID_REGISTER
pthread__uc_id(&newthread->pt_uc) = newthread;
#endif
newthread->pt_uc.uc_stack = newthread->pt_stack;
newthread->pt_uc.uc_link = NULL;
/* Add to list of all threads. */
pthread_spinlock(self, &pthread__allqueue_lock);
PTQ_INSERT_HEAD(&pthread__allqueue, newthread, pt_allq);
pthread_spinunlock(self, &pthread__allqueue_lock);
/* Will be reset by the thread upon exit. */
pthread__initthread(newthread);
}
/* Set up state. */
pthread__initthread(newthread);
newthread->pt_flags = nattr.pta_flags;
newthread->pt_name = name;
/*
* Create the new LWP.
*/
pthread__scrubthread(newthread, name, nattr.pta_flags);
makecontext(&newthread->pt_uc, pthread__create_tramp, 2,
startfunc, arg);
/* Add to list of all threads. */
pthread_spinlock(self, &pthread__queue_lock);
PTQ_INSERT_HEAD(&pthread__allqueue, newthread, pt_allq);
pthread_spinunlock(self, &pthread__queue_lock);
/* Create the new LWP. */
flag = 0;
if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0)
flag |= LWP_SUSPENDED;
@ -398,10 +395,10 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr,
SDPRINTF(("(pthread_create %p) _lwp_create: %s\n",
strerror(errno)));
free(name);
pthread_spinlock(self, &pthread__queue_lock);
PTQ_REMOVE(&pthread__allqueue, newthread, pt_allq);
PTQ_INSERT_HEAD(&pthread__deadqueue, newthread, pt_allq);
pthread_spinunlock(self, &pthread__queue_lock);
newthread->pt_state = PT_STATE_DEAD;
pthread_spinlock(self, &pthread__deadqueue_lock);
PTQ_INSERT_HEAD(&pthread__deadqueue, newthread, pt_deadq);
pthread_spinunlock(self, &pthread__deadqueue_lock);
return ret;
}
@ -506,10 +503,9 @@ pthread_exit(void *retval)
self->pt_state = PT_STATE_DEAD;
name = self->pt_name;
self->pt_name = NULL;
pthread_spinlock(self, &pthread__queue_lock);
PTQ_REMOVE(&pthread__allqueue, self, pt_allq);
PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_allq);
pthread_spinunlock(self, &pthread__queue_lock);
pthread_spinlock(self, &pthread__deadqueue_lock);
PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_deadq);
pthread_spinunlock(self, &pthread__deadqueue_lock);
pthread_spinunlock(self, &self->pt_lock);
if (name != NULL)
free(name);
@ -572,10 +568,9 @@ pthread_join(pthread_t thread, void **valptr)
name = thread->pt_name;
thread->pt_name = NULL;
thread->pt_state = PT_STATE_DEAD;
pthread_spinlock(self, &pthread__queue_lock);
PTQ_REMOVE(&pthread__allqueue, thread, pt_allq);
PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_allq);
pthread_spinunlock(self, &pthread__queue_lock);
pthread_spinlock(self, &pthread__deadqueue_lock);
PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq);
pthread_spinunlock(self, &pthread__deadqueue_lock);
SDPRINTF(("(pthread_join %p) Joined %p.\n", self, thread));
if (name != NULL)
free(name);
@ -811,13 +806,13 @@ pthread__find(pthread_t self, pthread_t id)
{
pthread_t target;
pthread_spinlock(self, &pthread__queue_lock);
pthread_spinlock(self, &pthread__allqueue_lock);
PTQ_FOREACH(target, &pthread__allqueue, pt_allq)
if (target == id)
break;
pthread_spinunlock(self, &pthread__queue_lock);
pthread_spinunlock(self, &pthread__allqueue_lock);
if (target == NULL)
if (target == NULL || target->pt_state == PT_STATE_DEAD)
return ESRCH;
return 0;

View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_debug.c,v 1.12 2007/03/02 18:53:52 ad Exp $ */
/* $NetBSD: pthread_debug.c,v 1.13 2007/08/16 12:01:49 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_debug.c,v 1.12 2007/03/02 18:53:52 ad Exp $");
__RCSID("$NetBSD: pthread_debug.c,v 1.13 2007/08/16 12:01:49 ad Exp $");
#include <err.h>
#include <errno.h>
@ -70,12 +70,12 @@ static struct linebuf *linebuf;
static void pthread__debug_printcounters(void);
static const char *pthread__counternames[] = PTHREADD_INITCOUNTERNAMES;
extern int pthread__maxconcurrency, pthread__started;
extern int pthread__started;
void pthread__debug_init(int ncpu)
void
pthread__debug_init(void)
{
time_t t;
int i;
if (getenv("PTHREAD_DEBUGCOUNTERS") != NULL)
atexit(pthread__debug_printcounters);
@ -83,11 +83,9 @@ void pthread__debug_init(int ncpu)
if (getenv("PTHREAD_DEBUGLOG") != NULL) {
t = time(NULL);
debugbuf = pthread__debuglog_init(0);
linebuf = malloc(ncpu * sizeof(struct linebuf));
linebuf = calloc(1000, sizeof(struct linebuf));
if (linebuf == NULL)
err(1, "Couldn't allocate linebuf");
for (i = 0; i < ncpu; i++)
linebuf[i].len = 0;
DPRINTF(("Started debugging %s (pid %d) at %s\n",
getprogname(), getpid(), ctime(&t)));
@ -164,17 +162,8 @@ pthread__debuglog_printf(const char *fmt, ...)
tmpbuf = linebuf[vpid].buf;
len = linebuf[vpid].len;
#if defined(PTHREAD_PID_DEBUG) || defined(PTHREAD_VP_DEBUG)
if (len == 0) {
#ifdef PTHREAD_PID_DEBUG
len += sprintf(tmpbuf, "[%05d]", getpid());
#endif
#ifdef PTHREAD_VP_DEBUG
if (pthread__maxconcurrency > 1)
len += sprintf(tmpbuf + len, "[%d]", vpid);
#endif
}
#endif
if (len == 0)
len += sprintf(tmpbuf, "[%05d.%05d]", getpid(), vpid);
va_start(ap, fmt);
len += vsnprintf(tmpbuf + len, (unsigned int)(MAXLINELEN - len),
@ -232,5 +221,12 @@ pthread__debuglog_newline(void)
return (linebuf[vpid].len == 0);
}
#endif /* PTHREAD__DEBUG */
#else /* PTHREAD__DEBUG */
void
pthread__debug_init(void)
{
}
#endif /* PTHREAD__DEBUG */

View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_debug.h,v 1.10 2007/08/01 21:48:19 ad Exp $ */
/* $NetBSD: pthread_debug.h,v 1.11 2007/08/16 12:01:49 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@ -80,16 +80,15 @@ struct pthread_msgbuf {
char msg_bufc[1];
};
void pthread__debug_init(int ncpu);
struct pthread_msgbuf* pthread__debuglog_init(int force);
void pthread__debuglog_printf(const char *fmt, ...);
void pthread__debug_init(void);
struct pthread_msgbuf *pthread__debuglog_init(int);
void pthread__debuglog_printf(const char *, ...);
int pthread__debuglog_newline(void);
#ifdef PTHREAD__DEBUG
#undef PTHREAD_ALARM_DEBUG
#define PTHREAD_MAIN_DEBUG
#undef PTHREAD_PID_DEBUG
#define PTHREAD_SPIN_DEBUG
#undef PTHREAD_SPIN_DEBUG_PRINT

View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_int.h,v 1.47 2007/08/16 01:09:35 ad Exp $ */
/* $NetBSD: pthread_int.h,v 1.48 2007/08/16 12:01:49 ad Exp $ */
/*-
* Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc.
@ -39,7 +39,7 @@
#ifndef _LIB_PTHREAD_INT_H
#define _LIB_PTHREAD_INT_H
#define PTHREAD__DEBUG
/* #define PTHREAD__DEBUG */
#define ERRORCHECK
#include "pthread_types.h"
@ -90,9 +90,6 @@ struct __pthread_st {
/* Stack of cancellation cleanup handlers and their arguments */
PTQ_HEAD(, pt_clean_t) pt_cleanup_stack;
/* Thread-specific data */
void *pt_specific[PTHREAD_KEYS_MAX];
/* For debugger: LWPs waiting to join. */
pthread_queue_t pt_joiners;
PTQ_ENTRY(__pthread_st) pt_joinq;
@ -100,6 +97,7 @@ struct __pthread_st {
/* LWP ID and entry on the list of all threads. */
lwpid_t pt_lid;
PTQ_ENTRY(__pthread_st) pt_allq;
PTQ_ENTRY(__pthread_st) pt_deadq;
/*
* General synchronization data. We try to align, as threads
@ -109,10 +107,13 @@ struct __pthread_st {
int pt_sleeponq; /* on a sleep queue */
int pt_signalled; /* Received pthread_cond_signal() */
void *pt_sleepobj; /* object slept on */
pthread_queue_t *pt_sleepq; /* sleep queue */
PTQ_ENTRY(__pthread_st) pt_sleep;
int pt_dummy2 __aligned(128);
/* Thread-specific data. Large so it sits close to the end. */
int pt_havespecific;
void *pt_specific[PTHREAD_KEYS_MAX];
/*
* Context for thread creation. At the end as it's cached
* and then only ever passed to _lwp_create().
@ -141,9 +142,12 @@ struct __pthread_st {
#define PT_ATTR_MAGIC 0x22220002
#define PT_ATTR_DEAD 0xDEAD0002
extern int pthread__stacksize_lg;
extern size_t pthread__stacksize;
extern vaddr_t pthread__stackmask;
extern int pthread__stacksize_lg;
extern size_t pthread__stacksize;
extern vaddr_t pthread__stackmask;
extern int pthread__nspins;
extern int pthread__concurrency;
extern int pthread__osrev;
/* Flag to be used in a ucontext_t's uc_flags indicating that
* the saved register state is "user" state only, not full
@ -165,7 +169,7 @@ int pthread__park(pthread_t self, pthread_spin_t *lock,
int cancelpt, const void *hint);
/* Internal locking primitives */
void pthread__lockprim_init(int ncpu);
void pthread__lockprim_init(void);
void pthread_lockinit(pthread_spin_t *lock);
void pthread_spinlock(pthread_t thread, pthread_spin_t *lock);
int pthread_spintrylock(pthread_t thread, pthread_spin_t *lock);
@ -242,7 +246,4 @@ void pthread__errorfunc(const char *file, int line, const char *function,
#define pthread__smt_pause() /* nothing */
#endif
extern int pthread__nspins;
extern int pthread__osrev;
#endif /* _LIB_PTHREAD_INT_H */

View File

@ -1,4 +1,4 @@
/* $NetBSD: pthread_lock.c,v 1.21 2007/08/04 13:37:49 ad Exp $ */
/* $NetBSD: pthread_lock.c,v 1.22 2007/08/16 12:01:49 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_lock.c,v 1.21 2007/08/04 13:37:49 ad Exp $");
__RCSID("$NetBSD: pthread_lock.c,v 1.22 2007/08/16 12:01:49 ad Exp $");
#include <sys/types.h>
#include <sys/lock.h>
@ -46,10 +46,14 @@ __RCSID("$NetBSD: pthread_lock.c,v 1.21 2007/08/04 13:37:49 ad Exp $");
#include <errno.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include "pthread.h"
#include "pthread_int.h"
/* How many times to try acquiring spin locks on MP systems. */
#define PTHREAD__NSPINS 1024
#ifdef PTHREAD_SPIN_DEBUG_PRINT
#define SDPRINTF(x) DPRINTF(x)
#else
@ -106,10 +110,18 @@ pthread__simple_unlock(__cpu_simple_lock_t *alp)
* we fall back onto machine-dependent atomic lock primitives.
*/
void
pthread__lockprim_init(int ncpu)
pthread__lockprim_init(void)
{
char *p;
if (ncpu != 1) {
if ((p = getenv("PTHREAD_NSPINS")) != NULL)
pthread__nspins = atoi(p);
else if (pthread__concurrency != 1)
pthread__nspins = PTHREAD__NSPINS;
else
pthread__nspins = 1;
if (pthread__concurrency != 1) {
pthread__atomic = 1;
return;
}

View File

@ -1,7 +1,7 @@
/* $NetBSD: pthread_specific.c,v 1.10 2003/08/13 18:52:01 nathanw Exp $ */
/* $NetBSD: pthread_specific.c,v 1.11 2007/08/16 12:01:49 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
* Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_specific.c,v 1.10 2003/08/13 18:52:01 nathanw Exp $");
__RCSID("$NetBSD: pthread_specific.c,v 1.11 2007/08/16 12:01:49 ad Exp $");
/* Functions and structures dealing with thread-specific data */
@ -61,6 +61,7 @@ pthread_setspecific(pthread_key_t key, const void *value)
*/
/*LINTED const cast*/
self->pt_specific[key] = (void *) value;
self->pt_havespecific = 1;
return 0;
}

View File

@ -1,7 +1,7 @@
/* $NetBSD: pthread_tsd.c,v 1.2 2003/09/29 09:50:22 wiz Exp $ */
/* $NetBSD: pthread_tsd.c,v 1.3 2007/08/16 12:01:49 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
* Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread_tsd.c,v 1.2 2003/09/29 09:50:22 wiz Exp $");
__RCSID("$NetBSD: pthread_tsd.c,v 1.3 2007/08/16 12:01:49 ad Exp $");
/* Functions and structures dealing with thread-specific data */
#include <errno.h>
@ -180,6 +180,9 @@ pthread__destroy_tsd(pthread_t self)
void *val;
void (*destructor)(void *);
if (!self->pt_havespecific)
return;
/* Butenhof, section 5.4.2 (page 167):
*
* ``Also, Pthreads sets the thread-specific data value for a
@ -222,4 +225,6 @@ pthread__destroy_tsd(pthread_t self)
}
}
} while (!done && iterations--);
self->pt_havespecific = 0;
}