Use the hotpatch framework for LFENCE/MFENCE.
This commit is contained in:
maxv
parent
5bdfc4e5b8
commit
129e4c2b33
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: atomic.S,v 1.27 2020/04/26 13:59:44 maxv Exp $ */
|
||||
/* $NetBSD: atomic.S,v 1.28 2020/04/26 14:49:17 maxv Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2007 The NetBSD Foundation, Inc.
|
||||
|
|
@ -45,11 +45,15 @@
|
|||
#ifdef _HARDKERNEL
|
||||
#include "opt_xen.h"
|
||||
#include <machine/frameasm.h>
|
||||
#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
|
||||
#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
|
||||
#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
|
||||
#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 7);
|
||||
#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 7);
|
||||
#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
|
||||
#else
|
||||
#define LOCK lock
|
||||
#define ENDLABEL(a) /* nothing */
|
||||
#define LOCK lock
|
||||
#define HOTPATCH_SSE2_LFENCE /* nothing */
|
||||
#define HOTPATCH_SSE2_MFENCE /* nothing */
|
||||
#define ENDLABEL(a) /* nothing */
|
||||
#endif
|
||||
|
||||
.text
|
||||
|
|
@ -177,11 +181,12 @@ ENTRY(_atomic_cas_32_ni)
|
|||
END(_atomic_cas_32_ni)
|
||||
|
||||
ENTRY(_membar_consumer)
|
||||
HOTPATCH_SSE2_LFENCE
|
||||
/* 7 bytes of instructions */
|
||||
LOCK
|
||||
addl $0, -4(%esp)
|
||||
ret
|
||||
END(_membar_consumer)
|
||||
ENDLABEL(membar_consumer_end)
|
||||
|
||||
ENTRY(_membar_producer)
|
||||
/* A store is enough */
|
||||
|
|
@ -190,11 +195,12 @@ ENTRY(_membar_producer)
|
|||
END(_membar_producer)
|
||||
|
||||
ENTRY(_membar_sync)
|
||||
HOTPATCH_SSE2_MFENCE
|
||||
/* 7 bytes of instructions */
|
||||
LOCK
|
||||
addl $0, -4(%esp)
|
||||
ret
|
||||
END(_membar_sync)
|
||||
ENDLABEL(membar_sync_end)
|
||||
|
||||
#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL)
|
||||
#ifdef XENPV
|
||||
|
|
@ -257,20 +263,6 @@ END(_atomic_cas_cx8)
|
|||
ENDLABEL(_atomic_cas_cx8_end)
|
||||
#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */
|
||||
|
||||
#ifdef _HARDKERNEL
|
||||
ENTRY(sse2_lfence)
|
||||
lfence
|
||||
ret
|
||||
END(sse2_lfence)
|
||||
ENDLABEL(sse2_lfence_end)
|
||||
|
||||
ENTRY(sse2_mfence)
|
||||
mfence
|
||||
ret
|
||||
END(sse2_mfence)
|
||||
ENDLABEL(sse2_mfence_end)
|
||||
#endif /* _HARDKERNEL */
|
||||
|
||||
ALIAS(atomic_add_32,_atomic_add_32)
|
||||
ALIAS(atomic_add_int,_atomic_add_32)
|
||||
ALIAS(atomic_add_long,_atomic_add_32)
|
||||
|
|
@ -402,3 +394,19 @@ STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni)
|
|||
|
||||
STRONG_ALIAS(_membar_enter,_membar_consumer)
|
||||
STRONG_ALIAS(_membar_exit,_membar_producer)
|
||||
|
||||
#ifdef _HARDKERNEL
|
||||
.section .rodata
|
||||
|
||||
LABEL(sse2_lfence)
|
||||
lfence
|
||||
ret
|
||||
nop; nop; nop;
|
||||
LABEL(sse2_lfence_end)
|
||||
|
||||
LABEL(sse2_mfence)
|
||||
mfence
|
||||
ret
|
||||
nop; nop; nop;
|
||||
LABEL(sse2_mfence_end)
|
||||
#endif /* _HARDKERNEL */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: atomic.S,v 1.21 2020/04/26 13:59:44 maxv Exp $ */
|
||||
/* $NetBSD: atomic.S,v 1.22 2020/04/26 14:49:17 maxv Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2007 The NetBSD Foundation, Inc.
|
||||
|
|
@ -40,11 +40,13 @@
|
|||
|
||||
#ifdef _HARDKERNEL
|
||||
#include <machine/frameasm.h>
|
||||
#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
|
||||
#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
|
||||
#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
|
||||
#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 8);
|
||||
#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 8);
|
||||
#else
|
||||
#define LOCK lock
|
||||
#define ENDLABEL(a) /* nothing */
|
||||
#define LOCK lock
|
||||
#define HOTPATCH_SSE2_LFENCE /* nothing */
|
||||
#define HOTPATCH_SSE2_MFENCE /* nothing */
|
||||
#endif
|
||||
|
||||
.text
|
||||
|
|
@ -254,11 +256,12 @@ END(_atomic_cas_64_ni)
|
|||
/* memory barriers */
|
||||
|
||||
ENTRY(_membar_consumer)
|
||||
HOTPATCH_SSE2_LFENCE
|
||||
/* 8 bytes of instructions */
|
||||
LOCK
|
||||
addq $0, -8(%rsp)
|
||||
ret
|
||||
END(_membar_consumer)
|
||||
ENDLABEL(membar_consumer_end)
|
||||
|
||||
ENTRY(_membar_producer)
|
||||
/* A store is enough */
|
||||
|
|
@ -267,25 +270,12 @@ ENTRY(_membar_producer)
|
|||
END(_membar_producer)
|
||||
|
||||
ENTRY(_membar_sync)
|
||||
HOTPATCH_SSE2_MFENCE
|
||||
/* 8 bytes of instructions */
|
||||
LOCK
|
||||
addq $0, -8(%rsp)
|
||||
ret
|
||||
END(_membar_sync)
|
||||
ENDLABEL(membar_sync_end)
|
||||
|
||||
#ifdef _HARDKERNEL
|
||||
ENTRY(sse2_lfence)
|
||||
lfence
|
||||
ret
|
||||
END(sse2_lfence)
|
||||
ENDLABEL(sse2_lfence_end)
|
||||
|
||||
ENTRY(sse2_mfence)
|
||||
mfence
|
||||
ret
|
||||
END(sse2_mfence)
|
||||
ENDLABEL(sse2_mfence_end)
|
||||
#endif /* _HARDKERNEL */
|
||||
|
||||
ALIAS(atomic_add_32,_atomic_add_32)
|
||||
ALIAS(atomic_add_64,_atomic_add_64)
|
||||
|
|
@ -425,3 +415,19 @@ STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni)
|
|||
|
||||
STRONG_ALIAS(_membar_enter,_membar_consumer)
|
||||
STRONG_ALIAS(_membar_exit,_membar_producer)
|
||||
|
||||
#ifdef _HARDKERNEL
|
||||
.section .rodata
|
||||
|
||||
LABEL(sse2_lfence)
|
||||
lfence
|
||||
ret
|
||||
nop; nop; nop; nop;
|
||||
LABEL(sse2_lfence_end)
|
||||
|
||||
LABEL(sse2_mfence)
|
||||
mfence
|
||||
ret
|
||||
nop; nop; nop; nop;
|
||||
LABEL(sse2_mfence_end)
|
||||
#endif /* _HARDKERNEL */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: frameasm.h,v 1.48 2020/04/25 15:26:16 bouyer Exp $ */
|
||||
/* $NetBSD: frameasm.h,v 1.49 2020/04/26 14:49:17 maxv Exp $ */
|
||||
|
||||
#ifndef _AMD64_MACHINE_FRAMEASM_H
|
||||
#define _AMD64_MACHINE_FRAMEASM_H
|
||||
|
|
@ -63,6 +63,8 @@
|
|||
#define HP_NAME_SVS_ENTER_NMI 11
|
||||
#define HP_NAME_SVS_LEAVE_NMI 12
|
||||
#define HP_NAME_MDS_LEAVE 13
|
||||
#define HP_NAME_SSE2_LFENCE 14
|
||||
#define HP_NAME_SSE2_MFENCE 15
|
||||
|
||||
#define HOTPATCH(name, size) \
|
||||
123: ; \
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: frameasm.h,v 1.30 2020/04/25 15:26:17 bouyer Exp $ */
|
||||
/* $NetBSD: frameasm.h,v 1.31 2020/04/26 14:49:17 maxv Exp $ */
|
||||
|
||||
#ifndef _I386_FRAMEASM_H_
|
||||
#define _I386_FRAMEASM_H_
|
||||
|
|
@ -48,6 +48,8 @@
|
|||
#define HP_NAME_STAC 2
|
||||
#define HP_NAME_NOLOCK 3
|
||||
#define HP_NAME_RETFENCE 4
|
||||
#define HP_NAME_SSE2_LFENCE 5
|
||||
#define HP_NAME_SSE2_MFENCE 6
|
||||
|
||||
#define HOTPATCH(name, size) \
|
||||
123: ; \
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: patch.c,v 1.41 2020/04/26 13:37:14 maxv Exp $ */
|
||||
/* $NetBSD: patch.c,v 1.42 2020/04/26 14:49:17 maxv Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
|
||||
|
|
@ -34,7 +34,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.41 2020/04/26 13:37:14 maxv Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.42 2020/04/26 14:49:17 maxv Exp $");
|
||||
|
||||
#include "opt_lockdebug.h"
|
||||
#ifdef i386
|
||||
|
|
@ -69,15 +69,6 @@ void i686_mutex_spin_exit(int);
|
|||
void i686_mutex_spin_exit_end(void);
|
||||
void i686_mutex_spin_exit_patch(void);
|
||||
|
||||
void membar_consumer(void);
|
||||
void membar_consumer_end(void);
|
||||
void membar_sync(void);
|
||||
void membar_sync_end(void);
|
||||
void sse2_lfence(void);
|
||||
void sse2_lfence_end(void);
|
||||
void sse2_mfence(void);
|
||||
void sse2_mfence_end(void);
|
||||
|
||||
void _atomic_cas_64(void);
|
||||
void _atomic_cas_64_end(void);
|
||||
void _atomic_cas_cx8(void);
|
||||
|
|
@ -219,16 +210,18 @@ x86_patch(bool early)
|
|||
* ordinary non-temporal stores are always issued in
|
||||
* program order to main memory and to other CPUs.
|
||||
*/
|
||||
patchfunc(
|
||||
sse2_lfence, sse2_lfence_end,
|
||||
membar_consumer, membar_consumer_end,
|
||||
NULL
|
||||
);
|
||||
patchfunc(
|
||||
sse2_mfence, sse2_mfence_end,
|
||||
membar_sync, membar_sync_end,
|
||||
NULL
|
||||
);
|
||||
extern uint8_t sse2_lfence, sse2_lfence_end;
|
||||
extern uint8_t sse2_mfence, sse2_mfence_end;
|
||||
uint8_t *bytes;
|
||||
size_t size;
|
||||
|
||||
bytes = &sse2_lfence;
|
||||
size = (size_t)&sse2_lfence_end - (size_t)&sse2_lfence;
|
||||
x86_hotpatch(HP_NAME_SSE2_LFENCE, bytes, size);
|
||||
|
||||
bytes = &sse2_mfence;
|
||||
size = (size_t)&sse2_mfence_end - (size_t)&sse2_mfence;
|
||||
x86_hotpatch(HP_NAME_SSE2_MFENCE, bytes, size);
|
||||
}
|
||||
|
||||
#ifdef i386
|
||||
|
|
|
|||
Loading…
Reference in New Issue