Improve inline asm around dsb/dmb/isb:

- always use volatile and mark them as memory barrier
- use the common version from locore.h in all places not included from
  userland
This commit is contained in:
joerg 2015-02-25 13:52:42 +00:00
parent 7f6d2a73ac
commit 5aebfecd7c
17 changed files with 64 additions and 126 deletions

View File

@ -1,4 +1,4 @@
/* $NetBSD: awin_space.c,v 1.3 2014/02/20 21:45:49 matt Exp $ */
/* $NetBSD: awin_space.c,v 1.4 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
@ -31,7 +31,7 @@
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: awin_space.c,v 1.3 2014/02/20 21:45:49 matt Exp $");
__KERNEL_RCSID(0, "$NetBSD: awin_space.c,v 1.4 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -366,17 +366,8 @@ awin_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
{
flags &= BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE;
if (flags) {
/* Issue an ARM11 Data Syncronisation Barrier (DSB) */
#ifdef _ARM_ARCH_7
__asm __volatile("dsb");
#else
__asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" : : "r" (0)
: "memory");
#endif
return;
}
if (flags)
arm_dsb();
}
void *

View File

@ -1,4 +1,4 @@
/* $NetBSD: amlogic_space.c,v 1.1 2015/02/07 17:20:17 jmcneill Exp $ */
/* $NetBSD: amlogic_space.c,v 1.2 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
@ -31,7 +31,7 @@
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: amlogic_space.c,v 1.1 2015/02/07 17:20:17 jmcneill Exp $");
__KERNEL_RCSID(0, "$NetBSD: amlogic_space.c,v 1.2 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -366,17 +366,8 @@ amlogic_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
{
flags &= BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE;
if (flags) {
/* Issue an ARM11 Data Syncronisation Barrier (DSB) */
#ifdef _ARM_ARCH_7
__asm __volatile("dsb");
#else
__asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" : : "r" (0)
: "memory");
#endif
return;
}
if (flags)
arm_dsb();
}
void *

View File

@ -1,4 +1,4 @@
/* $NetBSD: cpufunc.c,v 1.150 2014/07/31 07:14:42 skrll Exp $ */
/* $NetBSD: cpufunc.c,v 1.151 2015/02/25 13:52:42 joerg Exp $ */
/*
* arm7tdmi support code Copyright (c) 2001 John Fremlin
@ -49,7 +49,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.150 2014/07/31 07:14:42 skrll Exp $");
__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.151 2015/02/25 13:52:42 joerg Exp $");
#include "opt_compat_netbsd.h"
#include "opt_cpuoptions.h"
@ -1490,9 +1490,9 @@ get_cachesize_cp15(int cssr)
#if defined(CPU_ARMV7)
__asm volatile(".arch\tarmv7a");
__asm volatile("mcr p15, 2, %0, c0, c0, 0" :: "r" (cssr));
__asm volatile("isb"); /* sync to the new cssr */
__asm volatile("isb" ::: "memory"); /* sync to the new cssr */
#else
__asm volatile("mcr p15, 1, %0, c0, c0, 2" :: "r" (cssr));
__asm volatile("mcr p15, 1, %0, c0, c0, 2" :: "r" (cssr) : "memory");
#endif
__asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r" (csid));
return csid;

View File

@ -1,4 +1,4 @@
/* $NetBSD: bus_dma.c,v 1.90 2015/02/12 10:23:48 joerg Exp $ */
/* $NetBSD: bus_dma.c,v 1.91 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
@ -35,7 +35,7 @@
#include "opt_arm_bus_space.h"
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.90 2015/02/12 10:23:48 joerg Exp $");
__KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.91 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -832,13 +832,13 @@ _bus_dmamap_sync_segment(vaddr_t va, paddr_t pa, vsize_t len, int ops, bool read
*/
case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE:
STAT_INCR(sync_postreadwrite);
__asm __volatile("dmb" ::: "memory");
arm_dmb();
cpu_dcache_inv_range(va, len);
cpu_sdcache_inv_range(va, pa, len);
break;
case BUS_DMASYNC_POSTREAD:
STAT_INCR(sync_postread);
__asm __volatile("dmb" ::: "memory");
arm_dmb();
cpu_dcache_inv_range(va, len);
cpu_sdcache_inv_range(va, pa, len);
break;

View File

@ -1,4 +1,4 @@
/* $NetBSD: db_machdep.c,v 1.22 2014/10/12 05:40:56 skrll Exp $ */
/* $NetBSD: db_machdep.c,v 1.23 2015/02/25 13:52:42 joerg Exp $ */
/*
* Copyright (c) 1996 Mark Brinicombe
@ -33,7 +33,7 @@
#endif
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: db_machdep.c,v 1.22 2014/10/12 05:40:56 skrll Exp $");
__KERNEL_RCSID(0, "$NetBSD: db_machdep.c,v 1.23 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@ -385,7 +385,7 @@ db_show_tlb_cmd(db_expr_t addr, bool have_addr, db_expr_t count, const char *mod
armreg_tlbdataop_write(
__SHIFTIN(va_index, dti->dti_index)
| __SHIFTIN(way, ARM_TLBDATAOP_WAY));
__asm("isb");
arm_isb();
const uint32_t d0 = armreg_tlbdata0_read();
const uint32_t d1 = armreg_tlbdata1_read();
if ((d0 & ARM_TLBDATA_VALID)
@ -406,7 +406,7 @@ db_show_tlb_cmd(db_expr_t addr, bool have_addr, db_expr_t count, const char *mod
armreg_tlbdataop_write(
__SHIFTIN(way, ARM_TLBDATAOP_WAY)
| __SHIFTIN(va_index, dti->dti_index));
__asm("isb");
arm_isb();
const uint32_t d0 = armreg_tlbdata0_read();
const uint32_t d1 = armreg_tlbdata1_read();
if (d0 & ARM_TLBDATA_VALID) {

View File

@ -1,4 +1,4 @@
/* $NetBSD: pmap.c,v 1.316 2014/11/10 15:46:33 skrll Exp $ */
/* $NetBSD: pmap.c,v 1.317 2015/02/25 13:52:42 joerg Exp $ */
/*
* Copyright 2003 Wasabi Systems, Inc.
@ -215,7 +215,7 @@
#include <arm/locore.h>
__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.316 2014/11/10 15:46:33 skrll Exp $");
__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.317 2015/02/25 13:52:42 joerg Exp $");
//#define PMAP_DEBUG
#ifdef PMAP_DEBUG
@ -952,9 +952,7 @@ pmap_pte_sync_current(pmap_t pm, pt_entry_t *ptep)
{
if (PMAP_NEEDS_PTE_SYNC && pmap_is_cached(pm))
PTE_SYNC(ptep);
#if ARM_MMU_V7 > 0
__asm("dsb":::"memory");
#endif
arm_dsb();
}
#ifdef PMAP_INCLUDE_PTE_SYNC

View File

@ -1,4 +1,4 @@
/* $NetBSD: bcm2835_space.c,v 1.6 2013/04/14 15:11:52 skrll Exp $ */
/* $NetBSD: bcm2835_space.c,v 1.7 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
@ -31,7 +31,7 @@
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: bcm2835_space.c,v 1.6 2013/04/14 15:11:52 skrll Exp $");
__KERNEL_RCSID(0, "$NetBSD: bcm2835_space.c,v 1.7 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -356,13 +356,8 @@ bcm2835_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
{
flags &= BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE;
if (flags) {
/* Issue an ARM11 Data Syncronisation Barrier (DSB) */
__asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" : : "r" (0)
: "memory");
return;
}
if (flags)
arm_dsb();
}
void *

View File

@ -34,7 +34,7 @@
#include <sys/cdefs.h>
__KERNEL_RCSID(1, "$NetBSD: bcm53xx_pax.c,v 1.13 2014/03/30 01:12:18 matt Exp $");
__KERNEL_RCSID(1, "$NetBSD: bcm53xx_pax.c,v 1.14 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/bus.h>
@ -419,12 +419,12 @@ bcmpax_conf_addr_write(struct bcmpax_softc *sc, pcitag_t tag)
bcmpax_write_4(sc, PCIE_CFG_IND_ADDR,
__SHIFTIN(func, CFG_IND_ADDR_FUNC)
| __SHIFTIN(reg, CFG_IND_ADDR_REG));
__asm __volatile("dsb");
arm_dsb();
return PCIE_CFG_IND_DATA;
}
if (sc->sc_linkup) {
bcmpax_write_4(sc, PCIE_CFG_ADDR, tag);
__asm __volatile("dsb");
arm_dsb();
return PCIE_CFG_DATA;
}
return 0;

View File

@ -1,4 +1,4 @@
/* $NetBSD: bcmgen_space.c,v 1.4 2013/10/28 22:51:16 matt Exp $ */
/* $NetBSD: bcmgen_space.c,v 1.5 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
@ -31,7 +31,7 @@
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: bcmgen_space.c,v 1.4 2013/10/28 22:51:16 matt Exp $");
__KERNEL_RCSID(0, "$NetBSD: bcmgen_space.c,v 1.5 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -241,17 +241,8 @@ bcmgen_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
{
flags &= BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE;
if (flags) {
/* Issue an ARM11 Data Syncronisation Barrier (DSB) */
#ifdef _ARM_ARCH_7
__asm __volatile("dsb");
#else
__asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" : : "r" (0)
: "memory");
#endif
return;
}
if (flags)
arm_dsb();
}
void *

View File

@ -1,4 +1,4 @@
/* $NetBSD: pmap.h,v 1.137 2014/11/08 17:18:22 skrll Exp $ */
/* $NetBSD: pmap.h,v 1.138 2015/02/25 13:52:42 joerg Exp $ */
/*
* Copyright (c) 2002, 2003 Wasabi Systems, Inc.
@ -78,6 +78,7 @@
#include "opt_multiprocessor.h"
#endif
#include <arm/cpufunc.h>
#include <arm/locore.h>
#include <uvm/uvm_object.h>
#endif
@ -511,9 +512,7 @@ pmap_ptesync(pt_entry_t *ptep, size_t cnt)
cnt * sizeof(pt_entry_t));
#endif
}
#if ARM_MMU_V7 > 0
__asm("dsb");
#endif
arm_dsb();
}
#define PDE_SYNC(pdep) pmap_ptesync((pdep), 1)

View File

@ -1,4 +1,4 @@
/* $NetBSD: lock.h,v 1.31 2015/02/23 20:49:53 joerg Exp $ */
/* $NetBSD: lock.h,v 1.32 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2000, 2001 The NetBSD Foundation, Inc.
@ -143,9 +143,9 @@ static __inline void
__arm_membar_producer(void)
{
#ifdef _ARM_ARCH_7
__asm __volatile("dsb");
__asm __volatile("dsb" ::: "memory");
#elif defined(_ARM_ARCH_6)
__asm __volatile("mcr\tp15,0,%0,c7,c10,4" :: "r"(0));
__asm __volatile("mcr\tp15,0,%0,c7,c10,4" :: "r"(0) : "memory");
#endif
}
@ -153,9 +153,9 @@ static __inline void
__arm_membar_consumer(void)
{
#ifdef _ARM_ARCH_7
__asm __volatile("dmb");
__asm __volatile("dmb" ::: "memory");
#elif defined(_ARM_ARCH_6)
__asm __volatile("mcr\tp15,0,%0,c7,c10,5" :: "r"(0));
__asm __volatile("mcr\tp15,0,%0,c7,c10,5" :: "r"(0) : "memory");
#endif
}

View File

@ -1,4 +1,4 @@
/* $NetBSD: locore.h,v 1.18 2014/11/07 20:48:41 martin Exp $ */
/* $NetBSD: locore.h,v 1.19 2015/02/25 13:52:42 joerg Exp $ */
/*
* Copyright (c) 1994-1996 Mark Brinicombe.
@ -244,7 +244,7 @@ arm_dmb(void)
if (CPU_IS_ARMV6_P())
armreg_dmb_write(0);
else if (CPU_IS_ARMV7_P())
__asm __volatile("dmb");
__asm __volatile("dmb" ::: "memory");
}
static inline void
@ -253,7 +253,7 @@ arm_dsb(void)
if (CPU_IS_ARMV6_P())
armreg_dsb_write(0);
else if (CPU_IS_ARMV7_P())
__asm __volatile("dsb");
__asm __volatile("dsb" ::: "memory");
}
static inline void
@ -262,7 +262,7 @@ arm_isb(void)
if (CPU_IS_ARMV6_P())
armreg_isb_write(0);
else if (CPU_IS_ARMV7_P())
__asm __volatile("isb");
__asm __volatile("isb" ::: "memory");
}
#endif

View File

@ -1,4 +1,4 @@
/* $NetBSD: mutex.h,v 1.19 2014/11/08 17:18:54 skrll Exp $ */
/* $NetBSD: mutex.h,v 1.20 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2002, 2007 The NetBSD Foundation, Inc.
@ -90,7 +90,7 @@ struct kmutex {
*/
#ifdef MULTIPROCESSOR
#ifdef _ARM_ARCH_7
#define MUTEX_RECEIVE(mtx) __asm __volatile("dmb")
#define MUTEX_RECEIVE(mtx) __asm __volatile("dmb" ::: "memory")
#else
#define MUTEX_RECEIVE(mtx) membar_consumer()
#endif
@ -100,7 +100,7 @@ struct kmutex {
#ifdef MULTIPROCESSOR
#ifdef _ARM_ARCH_7
#define MUTEX_GIVE(mtx) __asm __volatile("dsb")
#define MUTEX_GIVE(mtx) __asm __volatile("dsb" ::: "memory")
#else
#define MUTEX_GIVE(mtx) membar_producer()
#endif

View File

@ -1,4 +1,4 @@
/* $NetBSD: rwlock.h,v 1.8 2014/06/12 08:50:52 ozaki-r Exp $ */
/* $NetBSD: rwlock.h,v 1.9 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2002, 2006 The NetBSD Foundation, Inc.
@ -42,8 +42,8 @@ struct krwlock {
#ifdef MULTIPROCESSOR
#ifdef _ARM_ARCH_7
#define RW_RECEIVE(rw) __asm __volatile("dmb")
#define RW_GIVE(rw) __asm __volatile("dsb")
#define RW_RECEIVE(rw) __asm __volatile("dmb" ::: "memory")
#define RW_GIVE(rw) __asm __volatile("dsb" ::: "memory")
#else
#define RW_RECEIVE(rw) membar_consumer()
#define RW_GIVE(rw) membar_producer()

View File

@ -1,4 +1,4 @@
/* $NetBSD: rockchip_space.c,v 1.1 2014/12/26 16:53:33 jmcneill Exp $ */
/* $NetBSD: rockchip_space.c,v 1.2 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
@ -31,7 +31,7 @@
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: rockchip_space.c,v 1.1 2014/12/26 16:53:33 jmcneill Exp $");
__KERNEL_RCSID(0, "$NetBSD: rockchip_space.c,v 1.2 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -366,17 +366,8 @@ rockchip_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
{
flags &= BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE;
if (flags) {
/* Issue an ARM11 Data Syncronisation Barrier (DSB) */
#ifdef _ARM_ARCH_7
__asm __volatile("dsb");
#else
__asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" : : "r" (0)
: "memory");
#endif
return;
}
if (flags)
arm_dsb();
}
void *

View File

@ -1,4 +1,4 @@
/* $NetBSD: exynos_space.c,v 1.1 2014/04/13 02:26:26 matt Exp $ */
/* $NetBSD: exynos_space.c,v 1.2 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
@ -31,7 +31,7 @@
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: exynos_space.c,v 1.1 2014/04/13 02:26:26 matt Exp $");
__KERNEL_RCSID(0, "$NetBSD: exynos_space.c,v 1.2 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -366,17 +366,8 @@ exynos_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
{
flags &= BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE;
if (flags) {
/* Issue an ARM11 Data Syncronisation Barrier (DSB) */
#ifdef _ARM_ARCH_7
__asm __volatile("dsb");
#else
__asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" : : "r" (0)
: "memory");
#endif
return;
}
if (flags)
arm_dsb();
}
void *

View File

@ -1,4 +1,4 @@
/* $NetBSD: zynq_space.c,v 1.1 2015/01/23 12:34:09 hkenken Exp $ */
/* $NetBSD: zynq_space.c,v 1.2 2015/02/25 13:52:42 joerg Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
* All rights reserved.
@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: zynq_space.c,v 1.1 2015/01/23 12:34:09 hkenken Exp $");
__KERNEL_RCSID(0, "$NetBSD: zynq_space.c,v 1.2 2015/02/25 13:52:42 joerg Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -229,17 +229,8 @@ zynq_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
{
flags &= BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE;
if (flags) {
/* Issue an ARM11 Data Syncronisation Barrier (DSB) */
#ifdef _ARM_ARCH_7
__asm __volatile("dsb");
#else
__asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" : : "r" (0)
: "memory");
#endif
return;
}
if (flags)
arm_dsb();
}
void *