always initialize thread pointer at program start

this is the first step in an overhaul aimed at greatly simplifying and
optimizing everything dealing with thread-local state.

previously, the thread pointer was initialized lazily on first access,
or at program startup if stack protector was in use, or at certain
random places where inconsistent state could be reached if it were not
initialized early. while believed to be fully correct, the logic was
fragile and non-obvious.

in the first phase of the thread pointer overhaul, support is retained
(and in some cases improved) for systems/situation where loading the
thread pointer fails, e.g. old kernels.

some notes on specific changes:

- the confusing use of libc.main_thread as an indicator that the
  thread pointer is initialized is eliminated in favor of an explicit
  has_thread_pointer predicate.

- sigaction no longer needs to ensure that the thread pointer is
  initialized before installing a signal handler (this was needed to
  prevent a situation where the signal handler caused the thread
  pointer to be initialized and the subsequent sigreturn cleared it
  again) but it still needs to ensure that implementation-internal
  thread-related signals are not blocked.

- pthread tsd initialization for the main thread is deferred in a new
  manner to minimize bloat in the static-linked __init_tp code.

- pthread_setcancelstate no longer needs special handling for the
  situation before the thread pointer is initialized. it simply fails
  on systems that cannot support a thread pointer, which are
  non-conforming anyway.

- pthread_cleanup_push/pop now check for missing thread pointer and
  nop themselves out in this case, so stdio no longer needs to avoid
  the cancellable path when the thread pointer is not available.

a number of cases remain where certain interfaces may crash if the
system does not support a thread pointer. at this point, these should
be limited to pthread interfaces, and the number of such cases should
be fewer than before.

src/env/__init_tls.c

@@ -7,8 +7,28 @@
 #include "atomic.h"
 #include "syscall.h"
 
+int __init_tp(void *p)
+{
+	pthread_t td = p;
+	td->self = td;
+	if (__set_thread_area(TP_ADJ(p)) < 0)
+		return -1;
+	td->tid = td->pid = __syscall(SYS_set_tid_address, &td->tid);
+	td->errno_ptr = &td->errno_val;
+	/* Currently, both of these predicates depend in the same thing:
+	 * successful initialization of the thread pointer. However, in
+	 * the future, we may support setups where setting the thread
+	 * pointer is possible but threads other than the main thread
+	 * cannot work, so it's best to keep the predicates separate. */
+	libc.has_thread_pointer = 1;
+	libc.can_do_threads = 1;
+	return 0;
+}
+
 #ifndef SHARED
 
+static long long builtin_tls[(sizeof(struct pthread) + 64)/sizeof(long long)];
+
 struct tls_image {
 	void *image;
 	size_t len, size, align;
@@ -62,10 +82,11 @@ typedef Elf64_Phdr Phdr;
 
 void __init_tls(size_t *aux)
 {
-	unsigned char *p, *mem;
+	unsigned char *p;
 	size_t n;
 	Phdr *phdr, *tls_phdr=0;
 	size_t base = 0;
+	void *mem;
 
 	libc.tls_size = sizeof(struct pthread);
 
@@ -76,28 +97,38 @@ void __init_tls(size_t *aux)
 		if (phdr->p_type == PT_TLS)
 			tls_phdr = phdr;
 	}
-	if (!tls_phdr) return;
 
-	T.image = (void *)(base + tls_phdr->p_vaddr);
-	T.len = tls_phdr->p_filesz;
-	T.size = tls_phdr->p_memsz;
-	T.align = tls_phdr->p_align;
+	if (tls_phdr) {
+		T.image = (void *)(base + tls_phdr->p_vaddr);
+		T.len = tls_phdr->p_filesz;
+		T.size = tls_phdr->p_memsz;
+		T.align = tls_phdr->p_align;
+	}
 
 	T.size += (-T.size - (uintptr_t)T.image) & (T.align-1);
 	if (T.align < 4*sizeof(size_t)) T.align = 4*sizeof(size_t);
 
 	libc.tls_size = 2*sizeof(void *)+T.size+T.align+sizeof(struct pthread);
 
-	mem = (void *)__syscall(
+	if (libc.tls_size > sizeof builtin_tls) {
+		mem = (void *)__syscall(
 #ifdef SYS_mmap2
-		SYS_mmap2,
+			SYS_mmap2,
 #else
-		SYS_mmap,
+			SYS_mmap,
 #endif
-		0, libc.tls_size, PROT_READ|PROT_WRITE,
-		MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
+			0, libc.tls_size, PROT_READ|PROT_WRITE,
+			MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
+		/* -4095...-1 cast to void * will crash on dereference anyway,
+		 * so don't bloat the init code checking for error codes and
+		 * explicitly calling a_crash(). */
+	} else {
+		mem = builtin_tls;
+	}
 
-	if (!__install_initial_tls(__copy_tls(mem))) a_crash();
+	/* Failure to initialize thread pointer is fatal if TLS is used. */
+	if (__init_tp(__copy_tls(mem)) < 0 && tls_phdr)
+		a_crash();
 }
 #else
 void __init_tls(size_t *auxv) { }

src/env/__stack_chk_fail.c

@@ -7,7 +7,13 @@ uintptr_t __stack_chk_guard;
 
 void __init_ssp(void *entropy)
 {
-	pthread_t self = __pthread_self_init();
+	/* Here the thread pointer is used without checking whether
+	 * it is available; this will crash if it's not. However,
+	 * this function is only meant to be called if the program
+	 * being run uses stack protector, and in that case, it would
+	 * crash without a thread pointer anyway, so it's better to
+	 * crash early before there is state to be lost on crash. */
+	pthread_t self = __pthread_self();
 	uintptr_t canary;
 	if (entropy) memcpy(&canary, entropy, sizeof canary);
 	else canary = (uintptr_t)&canary * 1103515245;

src/errno/__errno_location.c

@@ -3,6 +3,6 @@
 int *__errno_location(void)
 {
 	static int e;
-	if (libc.main_thread) return __pthread_self()->errno_ptr;
+	if (libc.has_thread_pointer) return __pthread_self()->errno_ptr;
 	return &e;
 }

src/internal/libc.h

@@ -6,12 +6,12 @@
 #include <limits.h>
 
 struct __libc {
-	void *main_thread;
+	int has_thread_pointer;
+	int can_do_threads;
 	int threaded;
 	int secure;
 	size_t *auxv;
 	volatile int threads_minus_1;
-	int canceldisable;
 	FILE *ofl_head;
 	int ofl_lock[2];
 	size_t tls_size;

src/ldso/dynlink.c

@@ -90,7 +90,7 @@ struct symdef {
 #include "reloc.h"
 
 void __init_ssp(size_t *);
-void *__install_initial_tls(void *);
+int __init_tp(void *);
 void __init_libc(char **, char *);
 
 const char *__libc_get_version(void);
@@ -108,6 +108,7 @@ static pthread_rwlock_t lock;
 static struct debug debug;
 static size_t tls_cnt, tls_offset, tls_align = 4*sizeof(size_t);
 static pthread_mutex_t init_fini_lock = { ._m_type = PTHREAD_MUTEX_RECURSIVE };
+static long long builtin_tls[(sizeof(struct pthread) + 64)/sizeof(long long)];
 
 struct debug *_dl_debug_addr = &debug;
 
@@ -673,9 +674,10 @@ static struct dso *load_library(const char *name, struct dso *needed_by)
 	/* Add a shortname only if name arg was not an explicit pathname. */
 	if (pathname != name) p->shortname = strrchr(p->name, '/')+1;
 	if (p->tls_image) {
-		if (runtime && !__pthread_self_init()) {
+		if (runtime && !libc.has_thread_pointer) {
 			munmap(map, p->map_len);
 			free(p);
+			errno = ENOSYS;
 			return 0;
 		}
 		p->tls_id = ++tls_cnt;
@@ -866,10 +868,13 @@ void *__copy_tls(unsigned char *mem)
 	pthread_t td;
 	struct dso *p;
 
-	if (!tls_cnt) return mem;
-
 	void **dtv = (void *)mem;
 	dtv[0] = (void *)tls_cnt;
+	if (!tls_cnt) {
+		td = (void *)(dtv+1);
+		td->dtv = dtv;
+		return td;
+	}
 
 #ifdef TLS_ABOVE_TP
 	mem += sizeof(void *) * (tls_cnt+1);
@@ -962,6 +967,7 @@ void *__dynlink(int argc, char **argv)
 	size_t vdso_base;
 	size_t *auxv;
 	char **envp = argv+argc+1;
+	void *initial_tls;
 
 	/* Find aux vector just past environ[] */
 	for (i=argc+1; argv[i]; i++)
@@ -1144,15 +1150,19 @@ void *__dynlink(int argc, char **argv)
 	reloc_all(app);
 
 	update_tls_size();
-	if (tls_cnt) {
-		void *mem = mmap(0, libc.tls_size, PROT_READ|PROT_WRITE,
-			MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
-		if (mem==MAP_FAILED ||
-		    !__install_initial_tls(__copy_tls(mem))) {
+	if (libc.tls_size > sizeof builtin_tls) {
+		initial_tls = calloc(libc.tls_size, 1);
+		if (!initial_tls) {
 			dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
 				argv[0], libc.tls_size);
 			_exit(127);
 		}
+	} else {
+		initial_tls = builtin_tls;
+	}
+	if (__init_tp(__copy_tls(initial_tls)) < 0 && tls_cnt) {
+		dprintf(2, "%s: Thread-local storage not supported by kernel.\n", argv[0]);
+		_exit(127);
 	}
 
 	if (ldso_fail) _exit(127);

src/process/fork.c

@@ -17,12 +17,11 @@ pid_t fork(void)
 	__fork_handler(-1);
 	__block_all_sigs(&set);
 	ret = syscall(SYS_fork);
-	if (libc.main_thread && !ret) {
+	if (libc.has_thread_pointer && !ret) {
 		pthread_t self = __pthread_self();
-		self->tid = self->pid = syscall(SYS_getpid);
+		self->tid = self->pid = __syscall(SYS_getpid);
 		memset(&self->robust_list, 0, sizeof self->robust_list);
 		libc.threads_minus_1 = 0;
-		libc.main_thread = self;
 	}
 	__restore_sigs(&set);
 	__fork_handler(!ret);

src/signal/sigaction.c

@@ -8,9 +8,7 @@
 
 void __restore(), __restore_rt();
 
-static pthread_t dummy(void) { return 0; }
-weak_alias(dummy, __pthread_self_def);
-
+static int unmask_done;
 static unsigned long handler_set[_NSIG/(8*sizeof(long))];
 
 void __get_handler_set(sigset_t *set)
@@ -29,7 +27,20 @@ int __libc_sigaction(int sig, const struct sigaction *restrict sa, struct sigact
 		if ((uintptr_t)sa->sa_handler > 1UL) {
 			a_or_l(handler_set+(sig-1)/(8*sizeof(long)),
 				1UL<<(sig-1)%(8*sizeof(long)));
-			__pthread_self_def();
+
+			/* If pthread_create has not yet been called,
+			 * implementation-internal signals might not
+			 * yet have been unblocked. They must be
+			 * unblocked before any signal handler is
+			 * installed, so that an application cannot
+			 * receive an illegal sigset_t (with them
+			 * blocked) as part of the ucontext_t passed
+			 * to the signal handler. */
+			if (!libc.threaded && !unmask_done) {
+				__syscall(SYS_rt_sigprocmask, SIG_UNBLOCK,
+					SIGPT_SET, 0, _NSIG/8);
+				unmask_done = 1;
+			}
 		}
 		ksa.handler = sa->sa_handler;
 		ksa.flags = sa->sa_flags | SA_RESTORER;

src/stdio/__stdio_read.c

@@ -16,13 +16,9 @@ size_t __stdio_read(FILE *f, unsigned char *buf, size_t len)
 	};
 	ssize_t cnt;
 
-	if (libc.main_thread) {
-		pthread_cleanup_push(cleanup, f);
-		cnt = syscall_cp(SYS_readv, f->fd, iov, 2);
-		pthread_cleanup_pop(0);
-	} else {
-		cnt = syscall(SYS_readv, f->fd, iov, 2);
-	}
+	pthread_cleanup_push(cleanup, f);
+	cnt = syscall_cp(SYS_readv, f->fd, iov, 2);
+	pthread_cleanup_pop(0);
 	if (cnt <= 0) {
 		f->flags |= F_EOF ^ ((F_ERR^F_EOF) & cnt);
 		f->rpos = f->rend = 0;

src/stdio/__stdio_write.c

@@ -19,13 +19,9 @@ size_t __stdio_write(FILE *f, const unsigned char *buf, size_t len)
 	int iovcnt = 2;
 	ssize_t cnt;
 	for (;;) {
-		if (libc.main_thread) {
-			pthread_cleanup_push(cleanup, f);
-			cnt = syscall_cp(SYS_writev, f->fd, iov, iovcnt);
-			pthread_cleanup_pop(0);
-		} else {
-			cnt = syscall(SYS_writev, f->fd, iov, iovcnt);
-		}
+		pthread_cleanup_push(cleanup, f);
+		cnt = syscall_cp(SYS_writev, f->fd, iov, iovcnt);
+		pthread_cleanup_pop(0);
 		if (cnt == rem) {
 			f->wend = f->buf + f->buf_size;
 			f->wpos = f->wbase = f->buf;

src/thread/cancel_impl.c

@@ -20,7 +20,7 @@ long (__syscall_cp)(syscall_arg_t nr,
 	pthread_t self;
 	long r;
 
-	if (!libc.main_thread || (self = __pthread_self())->canceldisable)
+	if (!libc.has_thread_pointer || (self = __pthread_self())->canceldisable)
 		return __syscall(nr, u, v, w, x, y, z);
 
 	r = __syscall_cp_asm(&self->cancel, nr, u, v, w, x, y, z);
@@ -57,6 +57,7 @@ static void cancel_handler(int sig, siginfo_t *si, void *ctx)
 
 void __testcancel()
 {
+	if (!libc.has_thread_pointer) return;
 	pthread_t self = pthread_self();
 	if (self->cancel && !self->canceldisable)
 		__cancel();

src/thread/pthread_create.c

@@ -77,6 +77,7 @@ _Noreturn void pthread_exit(void *result)
 
 void __do_cleanup_push(struct __ptcb *cb)
 {
+	if (!libc.has_thread_pointer) return;
 	struct pthread *self = pthread_self();
 	cb->__next = self->cancelbuf;
 	self->cancelbuf = cb;
@@ -84,6 +85,7 @@ void __do_cleanup_push(struct __ptcb *cb)
 
 void __do_cleanup_pop(struct __ptcb *cb)
 {
+	if (!libc.has_thread_pointer) return;
 	__pthread_self()->cancelbuf = cb->__next;
 }
 
@@ -110,6 +112,8 @@ static int start(void *p)
 /* pthread_key_create.c overrides this */
 static const size_t dummy = 0;
 weak_alias(dummy, __pthread_tsd_size);
+static const void *dummy_tsd[1] = { 0 };
+weak_alias(dummy_tsd, __pthread_tsd_main);
 
 static FILE *const dummy_file = 0;
 weak_alias(dummy_file, __stdin_used);
@@ -127,7 +131,7 @@ int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attrp
 {
 	int ret;
 	size_t size, guard;
-	struct pthread *self = pthread_self(), *new;
+	struct pthread *self, *new;
 	unsigned char *map = 0, *stack = 0, *tsd = 0, *stack_limit;
 	unsigned flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND
 		| CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS
@@ -135,13 +139,16 @@ int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attrp
 	int do_sched = 0;
 	pthread_attr_t attr = {0};
 
-	if (!self) return ENOSYS;
+	if (!libc.can_do_threads) return ENOSYS;
+	self = __pthread_self();
 	if (!libc.threaded) {
 		for (FILE *f=libc.ofl_head; f; f=f->next)
 			init_file_lock(f);
 		init_file_lock(__stdin_used);
 		init_file_lock(__stdout_used);
 		init_file_lock(__stderr_used);
+		__syscall(SYS_rt_sigprocmask, SIG_UNBLOCK, SIGPT_SET, 0, _NSIG/8);
+		self->tsd = __pthread_tsd_main;
 		libc.threaded = 1;
 	}
 	if (attrp) attr = *attrp;

src/thread/pthread_key_create.c

@@ -14,7 +14,13 @@ int pthread_key_create(pthread_key_t *k, void (*dtor)(void *))
 	unsigned i = (uintptr_t)&k / 16 % PTHREAD_KEYS_MAX;
 	unsigned j = i;
 
-	__pthread_self_init();
+	if (libc.has_thread_pointer) {
+		pthread_t self = __pthread_self();
+		/* This can only happen in the main thread before
+		 * pthread_create has been called. */
+		if (!self->tsd) self->tsd = __pthread_tsd_main;
+	}
+
 	if (!dtor) dtor = nodtor;
 	do {
 		if (!a_cas_p(keys+j, 0, (void *)dtor)) {

src/thread/pthread_self.c

@@ -1,45 +1,6 @@
 #include "pthread_impl.h"
 
-static struct pthread *main_thread = &(struct pthread){0};
-
-/* pthread_key_create.c overrides this */
-static const void *dummy[1] = { 0 };
-weak_alias(dummy, __pthread_tsd_main);
-
-static int init_main_thread()
+pthread_t pthread_self()
 {
-	__syscall(SYS_rt_sigprocmask, SIG_UNBLOCK,
-		SIGPT_SET, 0, _NSIG/8);
-	if (__set_thread_area(TP_ADJ(main_thread)) < 0) return -1;
-	main_thread->canceldisable = libc.canceldisable;
-	main_thread->tsd = (void **)__pthread_tsd_main;
-	main_thread->errno_ptr = __errno_location();
-	main_thread->self = main_thread;
-	main_thread->tid = main_thread->pid =
-		__syscall(SYS_set_tid_address, &main_thread->tid);
-	if (!main_thread->dtv)
-		main_thread->dtv = (void *)dummy;
-	libc.main_thread = main_thread;
-	return 0;
-}
-
-pthread_t __pthread_self_def()
-{
-	static int init, failed;
-	if (!init) {
-		if (failed) return 0;
-		if (init_main_thread() < 0) failed = 1;
-		if (failed) return 0;
-		init = 1;
-	}
 	return __pthread_self();
 }
-
-weak_alias(__pthread_self_def, pthread_self);
-weak_alias(__pthread_self_def, __pthread_self_init);
-
-void *__install_initial_tls(void *p)
-{
-	main_thread = p;
-	return __pthread_self_def();
-}

src/thread/pthread_setcancelstate.c

@@ -3,13 +3,9 @@
 int pthread_setcancelstate(int new, int *old)
 {
 	if (new > 1U) return EINVAL;
-	if (libc.main_thread) {
-		struct pthread *self = __pthread_self();
-		if (old) *old = self->canceldisable;
-		self->canceldisable = new;
-	} else {
-		if (old) *old = libc.canceldisable;
-		libc.canceldisable = new;
-	}
+	if (!libc.has_thread_pointer) return ENOSYS;
+	struct pthread *self = __pthread_self();
+	if (old) *old = self->canceldisable;
+	self->canceldisable = new;
 	return 0;
 }