barriers: Move __DMB() to the new API

Remove the arch-specific ARM-centric __DMB() macro and use the new
barrier API instead.

Signed-off-by: Carlo Caione <ccaione@baylibre.com>

arch/arm/core/aarch32/mpu/arm_mpu.c

@@ -7,6 +7,7 @@
 #include <zephyr/device.h>
 #include <zephyr/init.h>
 #include <zephyr/kernel.h>
+#include <zephyr/sys/barrier.h>
 #include "arm_core_mpu_dev.h"
 #include <zephyr/linker/linker-defs.h>
 #include <kernel_arch_data.h>
@@ -199,7 +200,7 @@ void arm_core_mpu_enable(void)
 void arm_core_mpu_disable(void)
 {
 	/* Force any outstanding transfers to complete before disabling MPU */
-	__DMB();
+	barrier_dmem_fence_full();
 
 	/* Disable MPU */
 	MPU->CTRL = 0;

arch/arm/core/aarch32/mpu/nxp_mpu.c

@@ -11,6 +11,7 @@
 #include "arm_core_mpu_dev.h"
 #include <zephyr/sys/__assert.h>
 #include <zephyr/sys/math_extras.h>
+#include <zephyr/sys/barrier.h>
 #include <zephyr/linker/linker-defs.h>
 
 #define LOG_LEVEL CONFIG_MPU_LOG_LEVEL
@@ -408,7 +409,7 @@ void arm_core_mpu_enable(void)
 void arm_core_mpu_disable(void)
 {
 	/* Force any outstanding transfers to complete before disabling MPU */
-	__DMB();
+	barrier_dmem_fence_full();
 
 	/* Disable MPU */
 	SYSMPU->CESR &= ~SYSMPU_CESR_VLD_MASK;

arch/arm64/core/cortex_r/arm_mpu.c

@@ -13,6 +13,7 @@
 #include <zephyr/linker/linker-defs.h>
 #include <zephyr/logging/log.h>
 #include <zephyr/sys/check.h>
+#include <zephyr/sys/barrier.h>
 
 LOG_MODULE_REGISTER(mpu, CONFIG_MPU_LOG_LEVEL);
 
@@ -87,7 +88,7 @@ void arm_core_mpu_disable(void)
 	uint64_t val;
 
 	/* Force any outstanding transfers to complete before disabling MPU */
-	dmb();
+	barrier_dmem_fence_full();
 
 	val = read_sctlr_el1();
 	val &= ~SCTLR_M_BIT;

drivers/ethernet/eth_dwmac.c

@@ -16,6 +16,7 @@ LOG_MODULE_REGISTER(LOG_MODULE_NAME);
 #include <zephyr/kernel.h>
 #include <zephyr/cache.h>
 #include <zephyr/net/ethernet.h>
+#include <zephyr/sys/barrier.h>
 #include <ethernet/eth_stats.h>
 
 #include "eth_dwmac_priv.h"
@@ -188,7 +189,7 @@ static int dwmac_send(const struct device *dev, struct net_pkt *pkt)
 	} while (frag);
 
 	/* make sure all the above made it to memory */
-	__DMB();
+	barrier_dmem_fence_full();
 
 	/* update the descriptor index head */
 	p->tx_desc_head = d_idx;
@@ -380,7 +381,7 @@ static void dwmac_rx_refill_thread(void *arg1, void *unused1, void *unused2)
 		d->des3 = RDES3_BUF1V | RDES3_IOC | RDES3_OWN;
 
 		/* commit the above to memory */
-		__DMB();
+		barrier_dmem_fence_full();
 
 		/* advance to the next descriptor */
 		p->rx_desc_head = INC_WRAP(d_idx, NB_RX_DESCS);

drivers/ethernet/eth_sam_gmac.c

@@ -34,6 +34,7 @@ LOG_MODULE_REGISTER(LOG_MODULE_NAME);
 #include <zephyr/kernel.h>
 #include <zephyr/device.h>
 #include <zephyr/sys/__assert.h>
+#include <zephyr/sys/barrier.h>
 #include <zephyr/sys/util.h>
 #include <errno.h>
 #include <stdbool.h>
@@ -1338,7 +1339,7 @@ static struct net_pkt *frame_get(struct gmac_queue *queue)
 		/* Guarantee that status word is written before the address
 		 * word to avoid race condition.
 		 */
-		__DMB();  /* data memory barrier */
+		barrier_dmem_fence_full();
 		/* Update buffer descriptor address word */
 		wrap = (tail == rx_desc_list->len-1U ? GMAC_RXW0_WRAP : 0);
 		rx_desc->w0 = ((uint32_t)frag->data & GMAC_RXW0_ADDR) | wrap;
@@ -1583,7 +1584,7 @@ static int eth_tx(const struct device *dev, struct net_pkt *pkt)
 	/* Guarantee that all the fragments have been written before removing
 	 * the used bit to avoid race condition.
 	 */
-	__DMB();  /* data memory barrier */
+	barrier_dmem_fence_full();
 
 	/* Remove the used bit of the first fragment to allow the controller
 	 * to process it and the following fragments.
@@ -1605,7 +1606,7 @@ static int eth_tx(const struct device *dev, struct net_pkt *pkt)
 	/* Guarantee that the first fragment got its bit removed before starting
 	 * sending packets to avoid packets getting stuck.
 	 */
-	__DMB();  /* data memory barrier */
+	barrier_dmem_fence_full();
 
 	/* Start transmission */
 	gmac->GMAC_NCR |= GMAC_NCR_TSTART;

drivers/serial/uart_pl011.c

@@ -14,6 +14,7 @@
 #include <zephyr/device.h>
 #include <zephyr/drivers/uart.h>
 #include <zephyr/sys/device_mmio.h>
+#include <zephyr/sys/barrier.h>
 #include <zephyr/irq.h>
 #if defined(CONFIG_PINCTRL)
 #include <zephyr/drivers/pinctrl.h>
@@ -212,7 +213,7 @@ static int pl011_set_baudrate(const struct device *dev,
 	get_uart(dev)->ibrd = bauddiv >> PL011_FBRD_WIDTH;
 	get_uart(dev)->fbrd = bauddiv & ((1u << PL011_FBRD_WIDTH) - 1u);
 
-	__DMB();
+	barrier_dmem_fence_full();
 
 	/* In order to internally update the contents of ibrd or fbrd, a
 	 * lcr_h write must always be performed at the end

drivers/timer/nrf_rtc_timer.c

@@ -13,6 +13,7 @@
 #include <zephyr/drivers/timer/nrf_rtc_timer.h>
 #include <zephyr/sys/util.h>
 #include <zephyr/sys_clock.h>
+#include <zephyr/sys/barrier.h>
 #include <hal/nrf_rtc.h>
 #include <zephyr/irq.h>
 
@@ -152,7 +153,7 @@ static bool compare_int_lock(int32_t chan)
 
 	nrf_rtc_int_disable(RTC, NRF_RTC_CHANNEL_INT_MASK(chan));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	__ISB();
 
 	return prev & BIT(chan);
@@ -387,7 +388,7 @@ uint64_t z_nrf_rtc_timer_read(void)
 {
 	uint64_t val = ((uint64_t)overflow_cnt) << COUNTER_BIT_WIDTH;
 
-	__DMB();
+	barrier_dmem_fence_full();
 
 	uint32_t cntr = counter();
 

drivers/usb/device/usb_dc_sam_usbc.c

@@ -14,6 +14,7 @@ LOG_MODULE_REGISTER(usb_dc_sam_usbc, CONFIG_USB_DRIVER_LOG_LEVEL);
 #include <soc.h>
 #include <string.h>
 #include <zephyr/sys/byteorder.h>
+#include <zephyr/sys/barrier.h>
 #include <zephyr/drivers/pinctrl.h>
 #include <zephyr/irq.h>
 
@@ -654,7 +655,7 @@ static void usb_dc_sam_usbc_isr(void)
 	}
 
 usb_dc_sam_usbc_isr_barrier:
-	__DMB();
+	barrier_dmem_fence_full();
 }
 
 int usb_dc_attach(void)

drivers/xen/events.c

@@ -9,6 +9,7 @@
 #include <zephyr/xen/public/xen.h>
 #include <zephyr/xen/public/event_channel.h>
 #include <zephyr/xen/events.h>
+#include <zephyr/sys/barrier.h>
 
 #include <errno.h>
 #include <zephyr/kernel.h>
@@ -219,7 +220,7 @@ static void events_isr(void *data)
 	 */
 	vcpu->evtchn_upcall_pending = 0;
 
-	dmb();
+	barrier_dmem_fence_full();
 
 	/* Can not use system atomic_t/atomic_set() due to 32-bit casting */
 	pos_selector = __atomic_exchange_n(&vcpu->evtchn_pending_sel,

drivers/xen/gnttab.c

@@ -23,6 +23,7 @@
 #include <zephyr/xen/public/grant_table.h>
 #include <zephyr/xen/public/memory.h>
 #include <zephyr/xen/public/xen.h>
+#include <zephyr/sys/barrier.h>
 
 #include <zephyr/init.h>
 #include <zephyr/kernel.h>
@@ -86,7 +87,7 @@ static void gnttab_grant_permit_access(grant_ref_t gref, domid_t domid,
 	gnttab.table[gref].frame = gfn;
 	gnttab.table[gref].domid = domid;
 	/* Need to be sure that gfn and domid will be set before flags */
-	__DMB();
+	barrier_dmem_fence_full();
 
 	gnttab.table[gref].flags = flags;
 }

include/zephyr/arch/arm/aarch32/cortex_a_r/sys_io.h

@@ -17,6 +17,7 @@
 #include <zephyr/types.h>
 #include <zephyr/sys/sys_io.h>
 #include <zephyr/arch/arm/aarch32/cortex_a_r/cmsis.h>
+#include <zephyr/sys/barrier.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -30,13 +31,13 @@ static ALWAYS_INLINE uint8_t sys_read8(mem_addr_t addr)
 
 	__asm__ volatile("ldrb %0, [%1]" : "=r" (val) : "r" (addr));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	return val;
 }
 
 static ALWAYS_INLINE void sys_write8(uint8_t data, mem_addr_t addr)
 {
-	__DMB();
+	barrier_dmem_fence_full();
 	__asm__ volatile("strb %0, [%1]" : : "r" (data), "r" (addr));
 }
 
@@ -46,13 +47,13 @@ static ALWAYS_INLINE uint16_t sys_read16(mem_addr_t addr)
 
 	__asm__ volatile("ldrh %0, [%1]" : "=r" (val) : "r" (addr));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	return val;
 }
 
 static ALWAYS_INLINE void sys_write16(uint16_t data, mem_addr_t addr)
 {
-	__DMB();
+	barrier_dmem_fence_full();
 	__asm__ volatile("strh %0, [%1]" : : "r" (data), "r" (addr));
 }
 
@@ -62,13 +63,13 @@ static ALWAYS_INLINE uint32_t sys_read32(mem_addr_t addr)
 
 	__asm__ volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	return val;
 }
 
 static ALWAYS_INLINE void sys_write32(uint32_t data, mem_addr_t addr)
 {
-	__DMB();
+	barrier_dmem_fence_full();
 	__asm__ volatile("str %0, [%1]" : : "r" (data), "r" (addr));
 }
 
@@ -78,7 +79,7 @@ static ALWAYS_INLINE uint64_t sys_read64(mem_addr_t addr)
 
 	__asm__ volatile("ldrd %Q0, %R0, [%1]" : "=r" (val) : "r" (addr));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	return val;
 }
 

include/zephyr/arch/arm64/lib_helpers.h

@@ -155,12 +155,10 @@ static ALWAYS_INLINE void disable_fiq(void)
 #define wfi()	__asm__ volatile("wfi" : : : "memory")
 
 #define dsb()	__asm__ volatile ("dsb sy" ::: "memory")
-#define dmb()	__asm__ volatile ("dmb sy" ::: "memory")
 #define isb()	__asm__ volatile ("isb" ::: "memory")
 
 /* Zephyr needs these as well */
 #define __ISB() isb()
-#define __DMB() dmb()
 #define __DSB() dsb()
 
 static inline bool is_el_implemented(unsigned int el)

include/zephyr/arch/arm64/sys_io.h

@@ -16,6 +16,7 @@
 
 #include <zephyr/types.h>
 #include <zephyr/sys/sys_io.h>
+#include <zephyr/sys/barrier.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -39,13 +40,13 @@ static ALWAYS_INLINE uint8_t sys_read8(mem_addr_t addr)
 
 	__asm__ volatile("ldrb %w0, [%1]" : "=r" (val) : "r" (addr));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	return val;
 }
 
 static ALWAYS_INLINE void sys_write8(uint8_t data, mem_addr_t addr)
 {
-	__DMB();
+	barrier_dmem_fence_full();
 	__asm__ volatile("strb %w0, [%1]" : : "r" (data), "r" (addr));
 }
 
@@ -55,13 +56,13 @@ static ALWAYS_INLINE uint16_t sys_read16(mem_addr_t addr)
 
 	__asm__ volatile("ldrh %w0, [%1]" : "=r" (val) : "r" (addr));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	return val;
 }
 
 static ALWAYS_INLINE void sys_write16(uint16_t data, mem_addr_t addr)
 {
-	__DMB();
+	barrier_dmem_fence_full();
 	__asm__ volatile("strh %w0, [%1]" : : "r" (data), "r" (addr));
 }
 
@@ -71,13 +72,13 @@ static ALWAYS_INLINE uint32_t sys_read32(mem_addr_t addr)
 
 	__asm__ volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	return val;
 }
 
 static ALWAYS_INLINE void sys_write32(uint32_t data, mem_addr_t addr)
 {
-	__DMB();
+	barrier_dmem_fence_full();
 	__asm__ volatile("str %w0, [%1]" : : "r" (data), "r" (addr));
 }
 
@@ -87,13 +88,13 @@ static ALWAYS_INLINE uint64_t sys_read64(mem_addr_t addr)
 
 	__asm__ volatile("ldr %x0, [%1]" : "=r" (val) : "r" (addr));
 
-	__DMB();
+	barrier_dmem_fence_full();
 	return val;
 }
 
 static ALWAYS_INLINE void sys_write64(uint64_t data, mem_addr_t addr)
 {
-	__DMB();
+	barrier_dmem_fence_full();
 	__asm__ volatile("str %x0, [%1]" : : "r" (data), "r" (addr));
 }
 

tests/arch/arm/arm_thread_swap/src/arm_thread_arch.c

@@ -8,6 +8,7 @@
 #include <zephyr/arch/cpu.h>
 #include <zephyr/arch/arm/aarch32/cortex_m/cmsis.h>
 #include <zephyr/kernel_structs.h>
+#include <zephyr/sys/barrier.h>
 #include <offsets_short_arch.h>
 #include <ksched.h>
 
@@ -392,7 +393,7 @@ static void alt_thread_entry(void)
 	/* Manually trigger a context-switch, to swap-out
 	 * the alternative test thread.
 	 */
-	__DMB();
+	barrier_dmem_fence_full();
 	SCB->ICSR |= SCB_ICSR_PENDSVSET_Msk;
 	irq_unlock(0);
 
@@ -593,7 +594,7 @@ ZTEST(arm_thread_swap, test_arm_thread_swap)
 	/* Manually trigger a context-switch to swap-out the current thread.
 	 * Request a return to a different interrupt lock state.
 	 */
-	__DMB();
+	barrier_dmem_fence_full();
 
 #if defined(CONFIG_NO_OPTIMIZATIONS)
 	SCB->ICSR |= SCB_ICSR_PENDSVSET_Msk;