Make the machdep.svs_enabled sysctl writable, and add the kernel code
needed to disable SVS at runtime. We set 'svs_enabled' to false, and hotpatch the kernel entry/exit points to eliminate the context switch code. We need to make sure there is no remote CPU that is executing the code we are hotpatching. So we use two barriers: * After the first one each CPU is guaranteed to be executing in svs_disable_cpu with interrupts disabled (this way it can't leave this place). * After the second one it is guaranteed that SVS is disabled, so we flush the cache, enable interrupts and continue execution normally. Between the two barriers, cpu0 will disable SVS (svs_enabled=false and hotpatch), and each CPU will restore the generic syscall entry point. Three notes: * We should call svs_pgg_update(true) afterwards, to put back PG_G on the kernel pages (for better performance). This will be done in another commit. * The fact that we disable interrupts does not prevent us from receiving an NMI, and it would be problematic. So we need to add some code to verify that PMCs are disabled before hotpatching. This will be done in another commit. * In svs_disable() we expect each CPU to be online. We need to add a check to make sure they indeed are. The sysctl allows only a 1->0 transition. There is no point in doing 0->1 transitions anyway, and it would be complicated to implement because we need to re-synchronize the CPU user page tables with the current ones (we lost track of them in the last 1->0 transition).
This commit is contained in:
maxv
parent
f929364d1e
commit
5ab8a4e764
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: locore.S,v 1.152 2018/02/22 08:56:51 maxv Exp $ */
|
||||
/* $NetBSD: locore.S,v 1.153 2018/02/22 10:42:10 maxv Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright-o-rama!
|
||||
@ -1591,14 +1591,14 @@ END(intrfastexit)
|
||||
TEXT_USER_END
|
||||
|
||||
#ifdef SVS
|
||||
.globl svs_enter
|
||||
.globl svs_enter_end
|
||||
.globl svs_enter_altstack
|
||||
.globl svs_enter_altstack_end
|
||||
.globl svs_leave
|
||||
.globl svs_leave_end
|
||||
.globl svs_leave_altstack
|
||||
.globl svs_leave_altstack_end
|
||||
.globl svs_enter, svs_enter_end
|
||||
.globl svs_enter_altstack, svs_enter_altstack_end
|
||||
.globl svs_leave, svs_leave_end
|
||||
.globl svs_leave_altstack, svs_leave_altstack_end
|
||||
.globl nosvs_enter, nosvs_enter_end
|
||||
.globl nosvs_enter_altstack, nosvs_enter_altstack_end
|
||||
.globl nosvs_leave, nosvs_leave_end
|
||||
.globl nosvs_leave_altstack, nosvs_leave_altstack_end
|
||||
|
||||
LABEL(svs_enter)
|
||||
movq SVS_UTLS+UTLS_KPDIRPA,%rax
|
||||
@ -1630,4 +1630,20 @@ LABEL(svs_leave_altstack)
|
||||
movq %rax,%cr3
|
||||
1234:
|
||||
LABEL(svs_leave_altstack_end)
|
||||
|
||||
LABEL(nosvs_enter)
|
||||
NOSVS_ENTER
|
||||
LABEL(nosvs_enter_end)
|
||||
|
||||
LABEL(nosvs_enter_altstack)
|
||||
NOSVS_ENTER_ALTSTACK
|
||||
LABEL(nosvs_enter_altstack_end)
|
||||
|
||||
LABEL(nosvs_leave)
|
||||
NOSVS_LEAVE
|
||||
LABEL(nosvs_leave_end)
|
||||
|
||||
LABEL(nosvs_leave_altstack)
|
||||
NOSVS_LEAVE_ALTSTACK
|
||||
LABEL(nosvs_leave_altstack_end)
|
||||
#endif
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: frameasm.h,v 1.35 2018/02/22 08:56:51 maxv Exp $ */
|
||||
/* $NetBSD: frameasm.h,v 1.36 2018/02/22 10:42:11 maxv Exp $ */
|
||||
|
||||
#ifndef _AMD64_MACHINE_FRAMEASM_H
|
||||
#define _AMD64_MACHINE_FRAMEASM_H
|
||||
@ -112,28 +112,36 @@
|
||||
#define UTLS_RSP0 16
|
||||
|
||||
#define SVS_ENTER_BYTES 22
|
||||
#define SVS_ENTER \
|
||||
HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES) ; \
|
||||
#define NOSVS_ENTER \
|
||||
.byte 0xEB, (SVS_ENTER_BYTES-2) /* jmp */ ; \
|
||||
.fill (SVS_ENTER_BYTES-2),1,0xCC
|
||||
#define SVS_ENTER \
|
||||
HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES) ; \
|
||||
NOSVS_ENTER
|
||||
|
||||
#define SVS_LEAVE_BYTES 31
|
||||
#define SVS_LEAVE \
|
||||
HOTPATCH(HP_NAME_SVS_LEAVE, SVS_LEAVE_BYTES) ; \
|
||||
#define NOSVS_LEAVE \
|
||||
.byte 0xEB, (SVS_LEAVE_BYTES-2) /* jmp */ ; \
|
||||
.fill (SVS_LEAVE_BYTES-2),1,0xCC
|
||||
#define SVS_LEAVE \
|
||||
HOTPATCH(HP_NAME_SVS_LEAVE, SVS_LEAVE_BYTES) ; \
|
||||
NOSVS_LEAVE
|
||||
|
||||
#define SVS_ENTER_ALT_BYTES 23
|
||||
#define SVS_ENTER_ALTSTACK \
|
||||
HOTPATCH(HP_NAME_SVS_ENTER_ALT, SVS_ENTER_ALT_BYTES) ; \
|
||||
#define NOSVS_ENTER_ALTSTACK \
|
||||
.byte 0xEB, (SVS_ENTER_ALT_BYTES-2) /* jmp */ ; \
|
||||
.fill (SVS_ENTER_ALT_BYTES-2),1,0xCC
|
||||
#define SVS_ENTER_ALTSTACK \
|
||||
HOTPATCH(HP_NAME_SVS_ENTER_ALT, SVS_ENTER_ALT_BYTES) ; \
|
||||
NOSVS_ENTER_ALTSTACK
|
||||
|
||||
#define SVS_LEAVE_ALT_BYTES 22
|
||||
#define SVS_LEAVE_ALTSTACK \
|
||||
HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \
|
||||
#define NOSVS_LEAVE_ALTSTACK \
|
||||
.byte 0xEB, (SVS_LEAVE_ALT_BYTES-2) /* jmp */ ; \
|
||||
.fill (SVS_LEAVE_ALT_BYTES-2),1,0xCC
|
||||
#define SVS_LEAVE_ALTSTACK \
|
||||
HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \
|
||||
NOSVS_LEAVE_ALTSTACK
|
||||
|
||||
#else
|
||||
#define SVS_ENTER /* nothing */
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $ */
|
||||
/* $NetBSD: svs.c,v 1.6 2018/02/22 10:42:11 maxv Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2018 The NetBSD Foundation, Inc.
|
||||
@ -30,7 +30,7 @@
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.6 2018/02/22 10:42:11 maxv Exp $");
|
||||
|
||||
#include "opt_svs.h"
|
||||
|
||||
@ -38,6 +38,8 @@ __KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $");
|
||||
#include <sys/systm.h>
|
||||
#include <sys/proc.h>
|
||||
#include <sys/cpu.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/xcall.h>
|
||||
|
||||
#include <x86/cputypes.h>
|
||||
#include <machine/cpuvar.h>
|
||||
@ -541,6 +543,131 @@ svs_enable(void)
|
||||
x86_patch_window_close(psl, cr0);
|
||||
}
|
||||
|
||||
static void
|
||||
svs_disable_hotpatch(void)
|
||||
{
|
||||
extern uint8_t nosvs_enter, nosvs_enter_end;
|
||||
extern uint8_t nosvs_enter_altstack, nosvs_enter_altstack_end;
|
||||
extern uint8_t nosvs_leave, nosvs_leave_end;
|
||||
extern uint8_t nosvs_leave_altstack, nosvs_leave_altstack_end;
|
||||
u_long psl, cr0;
|
||||
uint8_t *bytes;
|
||||
size_t size;
|
||||
|
||||
x86_patch_window_open(&psl, &cr0);
|
||||
|
||||
bytes = &nosvs_enter;
|
||||
size = (size_t)&nosvs_enter_end - (size_t)&nosvs_enter;
|
||||
x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size);
|
||||
|
||||
bytes = &nosvs_enter_altstack;
|
||||
size = (size_t)&nosvs_enter_altstack_end -
|
||||
(size_t)&nosvs_enter_altstack;
|
||||
x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
|
||||
|
||||
bytes = &nosvs_leave;
|
||||
size = (size_t)&nosvs_leave_end - (size_t)&nosvs_leave;
|
||||
x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
|
||||
|
||||
bytes = &nosvs_leave_altstack;
|
||||
size = (size_t)&nosvs_leave_altstack_end -
|
||||
(size_t)&nosvs_leave_altstack;
|
||||
x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
|
||||
|
||||
x86_patch_window_close(psl, cr0);
|
||||
}
|
||||
|
||||
static volatile unsigned long svs_cpu_barrier1 __cacheline_aligned;
|
||||
static volatile unsigned long svs_cpu_barrier2 __cacheline_aligned;
|
||||
typedef void (vector)(void);
|
||||
|
||||
static void
|
||||
svs_disable_cpu(void *arg1, void *arg2)
|
||||
{
|
||||
struct cpu_info *ci = curcpu();
|
||||
extern vector Xsyscall;
|
||||
u_long psl;
|
||||
|
||||
psl = x86_read_psl();
|
||||
|
||||
atomic_dec_ulong(&svs_cpu_barrier1);
|
||||
while (atomic_cas_ulong(&svs_cpu_barrier1, 0, 0) != 0) {
|
||||
x86_pause();
|
||||
}
|
||||
|
||||
/* cpu0 is the one that does the hotpatch job */
|
||||
if (ci == &cpu_info_primary) {
|
||||
svs_enabled = false;
|
||||
svs_disable_hotpatch();
|
||||
}
|
||||
|
||||
/* put back the non-SVS syscall entry point */
|
||||
wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
|
||||
|
||||
atomic_dec_ulong(&svs_cpu_barrier2);
|
||||
while (atomic_cas_ulong(&svs_cpu_barrier2, 0, 0) != 0) {
|
||||
x86_pause();
|
||||
}
|
||||
|
||||
/* Write back and invalidate cache, flush pipelines. */
|
||||
wbinvd();
|
||||
x86_flush();
|
||||
|
||||
x86_write_psl(psl);
|
||||
}
|
||||
|
||||
static void
|
||||
svs_disable(void)
|
||||
{
|
||||
uint64_t xc;
|
||||
|
||||
/*
|
||||
* We expect all the CPUs to be online. XXX ensure they are.
|
||||
*/
|
||||
svs_cpu_barrier1 = ncpu;
|
||||
svs_cpu_barrier2 = ncpu;
|
||||
|
||||
printf("[+] Disabling SVS\n");
|
||||
xc = xc_broadcast(0, svs_disable_cpu, NULL, NULL);
|
||||
xc_wait(xc);
|
||||
|
||||
/*
|
||||
* XXX printf("[+] Installing PG_G\n");
|
||||
* XXX svs_pgg_update(true);
|
||||
*/
|
||||
|
||||
printf("[+] Done\n");
|
||||
}
|
||||
|
||||
int sysctl_machdep_svs_enabled(SYSCTLFN_ARGS);
|
||||
|
||||
int
|
||||
sysctl_machdep_svs_enabled(SYSCTLFN_ARGS)
|
||||
{
|
||||
struct sysctlnode node;
|
||||
int error, val;
|
||||
|
||||
val = *(int *)rnode->sysctl_data;
|
||||
|
||||
node = *rnode;
|
||||
node.sysctl_data = &val;
|
||||
|
||||
error = sysctl_lookup(SYSCTLFN_CALL(&node));
|
||||
if (error != 0 || newp == NULL)
|
||||
return error;
|
||||
|
||||
if (val == 1) {
|
||||
error = EINVAL;
|
||||
} else {
|
||||
if (svs_enabled) {
|
||||
svs_disable();
|
||||
}
|
||||
error = 0;
|
||||
}
|
||||
|
||||
return error;
|
||||
}
|
||||
|
||||
void
|
||||
svs_init(bool early)
|
||||
{
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $ */
|
||||
/* $NetBSD: x86_machdep.c,v 1.106 2018/02/22 10:42:11 maxv Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
|
||||
@ -31,7 +31,7 @@
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.106 2018/02/22 10:42:11 maxv Exp $");
|
||||
|
||||
#include "opt_modular.h"
|
||||
#include "opt_physmem.h"
|
||||
@ -1259,11 +1259,12 @@ SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup")
|
||||
CTL_MACHDEP, CTL_CREATE, CTL_EOL);
|
||||
#endif
|
||||
#ifdef SVS
|
||||
int sysctl_machdep_svs_enabled(SYSCTLFN_ARGS);
|
||||
sysctl_createv(clog, 0, NULL, NULL,
|
||||
CTLFLAG_PERMANENT,
|
||||
CTLFLAG_READWRITE,
|
||||
CTLTYPE_BOOL, "svs_enabled",
|
||||
SYSCTL_DESCR("Whether the kernel uses SVS"),
|
||||
NULL, 0, &svs_enabled, 0,
|
||||
sysctl_machdep_svs_enabled, 0, &svs_enabled, 0,
|
||||
CTL_MACHDEP, CTL_CREATE, CTL_EOL);
|
||||
#endif
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user