Add the IBRS mitigation for SpectreV2 on amd64.
Different operations are performed during context transitions: user->kernel: IBRS <- 1 kernel->user: IBRS <- 0 And during context switches: user->user: IBPB <- 0 kernel->user: IBPB <- 0 [user->kernel:IBPB <- 0 this one may not be needed] We use two macros, IBRS_ENTER and IBRS_LEAVE, to set the IBRS bit. The thing is hotpatched for better performance, like SVS. The idea is that IBRS is a "privileged" bit, which is set to 1 in kernel mode and 0 in user mode. To protect the branch predictor between user processes (which are of the same privilege), we use the IBPB barrier. The Intel manual also talks about (MWAIT/HLT)+HyperThreading, and says that when using either of the two instructions IBRS must be disabled for better performance on the core. I'm not totally sure about this part, so I'm not adding it now. IBRS is available only when the Intel microcode update is applied. The mitigation must be enabled manually with machdep.spectreV2.mitigated. Tested by msaitoh a week ago (but I adapted a few things since). Probably more changes to come.
This commit is contained in:
maxv
parent
b1146198e9
commit
0223f0c872
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: amd64_trap.S,v 1.39 2018/03/20 18:27:58 maxv Exp $ */
|
||||
/* $NetBSD: amd64_trap.S,v 1.40 2018/03/28 16:02:49 maxv Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
|
||||
|
|
@ -134,6 +134,7 @@ IDTVEC(trap02)
|
|||
ZTRAP_NJ(T_NMI)
|
||||
subq $TF_REGSIZE,%rsp
|
||||
INTR_SAVE_GPRS
|
||||
IBRS_ENTER
|
||||
SVS_ENTER_ALTSTACK
|
||||
cld
|
||||
SMAP_ENABLE
|
||||
|
|
@ -162,6 +163,7 @@ IDTVEC(trap02)
|
|||
SVS_LEAVE_ALTSTACK
|
||||
|
||||
.Lnmileave:
|
||||
IBRS_LEAVE
|
||||
INTR_RESTORE_GPRS
|
||||
addq $TF_REGSIZE+16,%rsp
|
||||
iretq
|
||||
|
|
@ -231,6 +233,7 @@ IDTVEC(trap08)
|
|||
TRAP_NJ(T_DOUBLEFLT)
|
||||
subq $TF_REGSIZE,%rsp
|
||||
INTR_SAVE_GPRS
|
||||
IBRS_ENTER
|
||||
SVS_ENTER_ALTSTACK
|
||||
testb $SEL_UPL,TF_CS(%rsp)
|
||||
jz 1f
|
||||
|
|
@ -248,6 +251,7 @@ IDTVEC(trap08)
|
|||
call _C_LABEL(doubletrap)
|
||||
|
||||
SVS_LEAVE_ALTSTACK
|
||||
IBRS_LEAVE
|
||||
INTR_RESTORE_GPRS
|
||||
|
||||
testb $SEL_UPL,TF_CS(%rsp)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: locore.S,v 1.159 2018/03/20 18:27:58 maxv Exp $ */
|
||||
/* $NetBSD: locore.S,v 1.160 2018/03/28 16:02:49 maxv Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright-o-rama!
|
||||
|
|
@ -1098,6 +1098,10 @@ ENTRY(cpu_switchto)
|
|||
popq %rdx
|
||||
#endif
|
||||
|
||||
pushq %rdx
|
||||
callq _C_LABEL(speculation_barrier)
|
||||
popq %rdx
|
||||
|
||||
/* Switch to newlwp's stack. */
|
||||
movq L_PCB(%r12),%r14
|
||||
movq PCB_RSP(%r14),%rsp
|
||||
|
|
@ -1413,6 +1417,7 @@ IDTVEC(\name)
|
|||
cld
|
||||
#endif
|
||||
INTR_SAVE_GPRS
|
||||
IBRS_ENTER
|
||||
movw $GSEL(GUDATA_SEL, SEL_UPL),TF_DS(%rsp)
|
||||
movw $GSEL(GUDATA_SEL, SEL_UPL),TF_ES(%rsp)
|
||||
movw $0,TF_FS(%rsp)
|
||||
|
|
@ -1464,6 +1469,7 @@ IDTVEC_END(osyscall)
|
|||
_ALIGN_TEXT
|
||||
LABEL(syscall_sysret)
|
||||
SVS_LEAVE
|
||||
IBRS_LEAVE
|
||||
INTR_RESTORE_GPRS
|
||||
SWAPGS
|
||||
#ifndef XEN
|
||||
|
|
@ -1554,6 +1560,7 @@ END(pagezero)
|
|||
LABEL(intrfastexit)
|
||||
NOT_XEN(cli;)
|
||||
SVS_LEAVE
|
||||
IBRS_LEAVE
|
||||
INTR_RESTORE_GPRS
|
||||
addq $(TF_REGSIZE+16),%rsp /* iret frame */
|
||||
|
||||
|
|
@ -1649,3 +1656,25 @@ LABEL(nosvs_leave_altstack)
|
|||
NOSVS_LEAVE_ALTSTACK
|
||||
LABEL(nosvs_leave_altstack_end)
|
||||
#endif
|
||||
|
||||
.globl ibrs_enter, ibrs_enter_end
|
||||
.globl ibrs_leave, ibrs_leave_end
|
||||
|
||||
/* IBRS <- 1 */
|
||||
LABEL(ibrs_enter)
|
||||
movl $MSR_IA32_SPEC_CTRL,%ecx
|
||||
movl $IA32_SPEC_CTRL_IBRS,%eax
|
||||
movl $(IA32_SPEC_CTRL_IBRS >> 32),%edx
|
||||
wrmsr
|
||||
LABEL(ibrs_enter_end)
|
||||
|
||||
/* IBRS <- 0 */
|
||||
LABEL(ibrs_leave)
|
||||
testb $SEL_UPL,TF_CS(%rsp)
|
||||
jz 1234f
|
||||
movl $MSR_IA32_SPEC_CTRL,%ecx
|
||||
xorl %eax,%eax
|
||||
xorl %edx,%edx
|
||||
wrmsr
|
||||
1234:
|
||||
LABEL(ibrs_leave_end)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: frameasm.h,v 1.37 2018/02/25 13:14:27 maxv Exp $ */
|
||||
/* $NetBSD: frameasm.h,v 1.38 2018/03/28 16:02:49 maxv Exp $ */
|
||||
|
||||
#ifndef _AMD64_MACHINE_FRAMEASM_H
|
||||
#define _AMD64_MACHINE_FRAMEASM_H
|
||||
|
|
@ -44,6 +44,8 @@
|
|||
#define HP_NAME_SVS_LEAVE 6
|
||||
#define HP_NAME_SVS_ENTER_ALT 7
|
||||
#define HP_NAME_SVS_LEAVE_ALT 8
|
||||
#define HP_NAME_IBRS_ENTER 9
|
||||
#define HP_NAME_IBRS_LEAVE 10
|
||||
|
||||
#define HOTPATCH(name, size) \
|
||||
123: ; \
|
||||
|
|
@ -61,6 +63,26 @@
|
|||
HOTPATCH(HP_NAME_STAC, 3) ; \
|
||||
.byte 0x0F, 0x1F, 0x00 ; \
|
||||
|
||||
/*
|
||||
* IBRS
|
||||
*/
|
||||
|
||||
#define IBRS_ENTER_BYTES 17
|
||||
#define IBRS_ENTER \
|
||||
HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES) ; \
|
||||
NOIBRS_ENTER
|
||||
#define NOIBRS_ENTER \
|
||||
.byte 0xEB, (IBRS_ENTER_BYTES-2) /* jmp */ ; \
|
||||
.fill (IBRS_ENTER_BYTES-2),1,0xCC
|
||||
|
||||
#define IBRS_LEAVE_BYTES 21
|
||||
#define IBRS_LEAVE \
|
||||
HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES) ; \
|
||||
NOIBRS_LEAVE
|
||||
#define NOIBRS_LEAVE \
|
||||
.byte 0xEB, (IBRS_LEAVE_BYTES-2) /* jmp */ ; \
|
||||
.fill (IBRS_LEAVE_BYTES-2),1,0xCC
|
||||
|
||||
#define SWAPGS NOT_XEN(swapgs)
|
||||
|
||||
/*
|
||||
|
|
@ -158,6 +180,7 @@
|
|||
testb $SEL_UPL,TF_CS(%rsp) ; \
|
||||
je 98f ; \
|
||||
SWAPGS ; \
|
||||
IBRS_ENTER ; \
|
||||
SVS_ENTER ; \
|
||||
movw %gs,TF_GS(%rsp) ; \
|
||||
movw %fs,TF_FS(%rsp) ; \
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $ */
|
||||
/* $NetBSD: spectre.c,v 1.2 2018/03/28 16:02:49 maxv Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2018 NetBSD Foundation, Inc.
|
||||
|
|
@ -34,7 +34,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.2 2018/03/28 16:02:49 maxv Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
|
|
@ -45,6 +45,7 @@ __KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $");
|
|||
#include <machine/cpufunc.h>
|
||||
#include <machine/cpuvar.h>
|
||||
#include <machine/specialreg.h>
|
||||
#include <machine/frameasm.h>
|
||||
|
||||
#include <x86/cputypes.h>
|
||||
|
||||
|
|
@ -57,13 +58,46 @@ enum spec_mitigation {
|
|||
bool spec_mitigation_enabled __read_mostly = false;
|
||||
static enum spec_mitigation mitigation_method = MITIGATION_NONE;
|
||||
|
||||
void speculation_barrier(struct lwp *, struct lwp *);
|
||||
|
||||
void
|
||||
speculation_barrier(struct lwp *oldlwp, struct lwp *newlwp)
|
||||
{
|
||||
if (!spec_mitigation_enabled)
|
||||
return;
|
||||
|
||||
/*
|
||||
* From kernel thread to kernel thread, no need for a barrier.
|
||||
*/
|
||||
if ((oldlwp->l_flag & LW_SYSTEM) &&
|
||||
(newlwp->l_flag & LW_SYSTEM))
|
||||
return;
|
||||
|
||||
switch (mitigation_method) {
|
||||
case MITIGATION_INTEL_IBRS:
|
||||
wrmsr(MSR_IA32_PRED_CMD, IA32_PRED_CMD_IBPB);
|
||||
break;
|
||||
default:
|
||||
/* nothing */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
speculation_detect_method(void)
|
||||
{
|
||||
struct cpu_info *ci = curcpu();
|
||||
u_int descs[4];
|
||||
|
||||
if (cpu_vendor == CPUVENDOR_INTEL) {
|
||||
/* TODO: detect MITIGATION_INTEL_IBRS */
|
||||
if (cpuid_level >= 7) {
|
||||
x86_cpuid(7, descs);
|
||||
if (descs[3] & CPUID_SEF_IBRS) {
|
||||
/* descs[3] = %edx */
|
||||
mitigation_method = MITIGATION_INTEL_IBRS;
|
||||
return;
|
||||
}
|
||||
}
|
||||
mitigation_method = MITIGATION_NONE;
|
||||
} else if (cpu_vendor == CPUVENDOR_AMD) {
|
||||
/*
|
||||
|
|
@ -88,6 +122,118 @@ speculation_detect_method(void)
|
|||
}
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
#ifdef __x86_64__
|
||||
static volatile unsigned long ibrs_cpu_barrier1 __cacheline_aligned;
|
||||
static volatile unsigned long ibrs_cpu_barrier2 __cacheline_aligned;
|
||||
|
||||
static void
|
||||
ibrs_enable_hotpatch(void)
|
||||
{
|
||||
extern uint8_t ibrs_enter, ibrs_enter_end;
|
||||
extern uint8_t ibrs_leave, ibrs_leave_end;
|
||||
u_long psl, cr0;
|
||||
uint8_t *bytes;
|
||||
size_t size;
|
||||
|
||||
x86_patch_window_open(&psl, &cr0);
|
||||
|
||||
bytes = &ibrs_enter;
|
||||
size = (size_t)&ibrs_enter_end - (size_t)&ibrs_enter;
|
||||
x86_hotpatch(HP_NAME_IBRS_ENTER, bytes, size);
|
||||
|
||||
bytes = &ibrs_leave;
|
||||
size = (size_t)&ibrs_leave_end - (size_t)&ibrs_leave;
|
||||
x86_hotpatch(HP_NAME_IBRS_LEAVE, bytes, size);
|
||||
|
||||
x86_patch_window_close(psl, cr0);
|
||||
}
|
||||
|
||||
static void
|
||||
ibrs_change_cpu(void *arg1, void *arg2)
|
||||
{
|
||||
struct cpu_info *ci = curcpu();
|
||||
bool enabled = (bool)arg1;
|
||||
u_long psl;
|
||||
|
||||
psl = x86_read_psl();
|
||||
x86_disable_intr();
|
||||
|
||||
atomic_dec_ulong(&ibrs_cpu_barrier1);
|
||||
while (atomic_cas_ulong(&ibrs_cpu_barrier1, 0, 0) != 0) {
|
||||
x86_pause();
|
||||
}
|
||||
|
||||
/* cpu0 is the one that does the hotpatch job */
|
||||
if (ci == &cpu_info_primary) {
|
||||
if (enabled) {
|
||||
ibrs_enable_hotpatch();
|
||||
} else {
|
||||
/* TODO */
|
||||
}
|
||||
}
|
||||
|
||||
atomic_dec_ulong(&ibrs_cpu_barrier2);
|
||||
while (atomic_cas_ulong(&ibrs_cpu_barrier2, 0, 0) != 0) {
|
||||
x86_pause();
|
||||
}
|
||||
|
||||
/* Write back and invalidate cache, flush pipelines. */
|
||||
wbinvd();
|
||||
x86_flush();
|
||||
|
||||
x86_write_psl(psl);
|
||||
}
|
||||
|
||||
static int
|
||||
ibrs_change(bool enabled)
|
||||
{
|
||||
struct cpu_info *ci = NULL;
|
||||
CPU_INFO_ITERATOR cii;
|
||||
uint64_t xc;
|
||||
|
||||
mutex_enter(&cpu_lock);
|
||||
|
||||
/*
|
||||
* We expect all the CPUs to be online.
|
||||
*/
|
||||
for (CPU_INFO_FOREACH(cii, ci)) {
|
||||
struct schedstate_percpu *spc = &ci->ci_schedstate;
|
||||
if (spc->spc_flags & SPCF_OFFLINE) {
|
||||
printf("[!] cpu%d offline, IBRS not changed\n",
|
||||
cpu_index(ci));
|
||||
mutex_exit(&cpu_lock);
|
||||
return EOPNOTSUPP;
|
||||
}
|
||||
}
|
||||
|
||||
ibrs_cpu_barrier1 = ncpu;
|
||||
ibrs_cpu_barrier2 = ncpu;
|
||||
|
||||
printf("[+] %s SpectreV2 Mitigation (IBRS)...",
|
||||
(enabled == true) ? "Enabling" : "Disabling");
|
||||
xc = xc_broadcast(0, ibrs_change_cpu, (void *)enabled, NULL);
|
||||
xc_wait(xc);
|
||||
printf(" done!\n");
|
||||
|
||||
mutex_exit(&cpu_lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
/*
|
||||
* TODO: i386
|
||||
*/
|
||||
static int
|
||||
ibrs_change(bool enabled)
|
||||
{
|
||||
panic("not supported");
|
||||
}
|
||||
#endif
|
||||
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
static void
|
||||
mitigation_disable_cpu(void *arg1, void *arg2)
|
||||
{
|
||||
|
|
@ -95,6 +241,7 @@ mitigation_disable_cpu(void *arg1, void *arg2)
|
|||
|
||||
switch (mitigation_method) {
|
||||
case MITIGATION_NONE:
|
||||
case MITIGATION_INTEL_IBRS:
|
||||
panic("impossible");
|
||||
break;
|
||||
case MITIGATION_AMD_DIS_IND:
|
||||
|
|
@ -102,9 +249,6 @@ mitigation_disable_cpu(void *arg1, void *arg2)
|
|||
msr &= ~IC_CFG_DIS_IND;
|
||||
wrmsr(MSR_IC_CFG, msr);
|
||||
break;
|
||||
case MITIGATION_INTEL_IBRS:
|
||||
/* ibrs_disable() TODO */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -115,6 +259,7 @@ mitigation_enable_cpu(void *arg1, void *arg2)
|
|||
|
||||
switch (mitigation_method) {
|
||||
case MITIGATION_NONE:
|
||||
case MITIGATION_INTEL_IBRS:
|
||||
panic("impossible");
|
||||
break;
|
||||
case MITIGATION_AMD_DIS_IND:
|
||||
|
|
@ -122,9 +267,6 @@ mitigation_enable_cpu(void *arg1, void *arg2)
|
|||
msr |= IC_CFG_DIS_IND;
|
||||
wrmsr(MSR_IC_CFG, msr);
|
||||
break;
|
||||
case MITIGATION_INTEL_IBRS:
|
||||
/* ibrs_enable() TODO */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -132,6 +274,7 @@ static int
|
|||
mitigation_disable(void)
|
||||
{
|
||||
uint64_t xc;
|
||||
int error;
|
||||
|
||||
speculation_detect_method();
|
||||
|
||||
|
|
@ -148,7 +291,10 @@ mitigation_disable(void)
|
|||
spec_mitigation_enabled = false;
|
||||
return 0;
|
||||
case MITIGATION_INTEL_IBRS:
|
||||
/* TODO */
|
||||
error = ibrs_change(false);
|
||||
if (error)
|
||||
return error;
|
||||
spec_mitigation_enabled = false;
|
||||
return 0;
|
||||
default:
|
||||
panic("impossible");
|
||||
|
|
@ -159,6 +305,7 @@ static int
|
|||
mitigation_enable(void)
|
||||
{
|
||||
uint64_t xc;
|
||||
int error;
|
||||
|
||||
speculation_detect_method();
|
||||
|
||||
|
|
@ -175,7 +322,10 @@ mitigation_enable(void)
|
|||
spec_mitigation_enabled = true;
|
||||
return 0;
|
||||
case MITIGATION_INTEL_IBRS:
|
||||
/* TODO */
|
||||
error = ibrs_change(true);
|
||||
if (error)
|
||||
return error;
|
||||
spec_mitigation_enabled = true;
|
||||
return 0;
|
||||
default:
|
||||
panic("impossible");
|
||||
|
|
|
|||
Loading…
Reference in New Issue