diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 92d87b441d2..5673d9aa42d 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -340,7 +340,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h \ arch/default/odp/api/abi/sync_inlines.h \ - arch/default/odp/api/abi/time_inlines.h + arch/default/odp/api/abi/time_inlines.h \ + arch/default/odp/api/abi/ticketlock_generic.h \ + arch/default/odp/api/abi/ticketlock_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ @@ -370,7 +372,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/cpu.h \ arch/aarch64/odp/api/abi/sync_inlines.h \ arch/common/odp/api/abi/time_cpu_inlines.h \ - arch/aarch64/odp/api/abi/time_inlines.h + arch/aarch64/odp/api/abi/time_inlines.h \ + arch/default/odp/api/abi/ticketlock_generic.h \ + arch/aarch64/odp/api/abi/ticketlock_inlines.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -394,7 +398,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h \ arch/default/odp/api/abi/sync_inlines.h \ - arch/default/odp/api/abi/time_inlines.h + arch/default/odp/api/abi/time_inlines.h \ + arch/default/odp/api/abi/ticketlock_generic.h \ + arch/default/odp/api/abi/ticketlock_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -416,7 +422,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h \ arch/default/odp/api/abi/sync_inlines.h \ - arch/default/odp/api/abi/time_inlines.h + arch/default/odp/api/abi/time_inlines.h \ + arch/default/odp/api/abi/ticketlock_generic.h \ + arch/default/odp/api/abi/ticketlock_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -442,7 +450,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/x86/odp/api/abi/cpu.h \ arch/x86/odp/api/abi/sync_inlines.h \ arch/common/odp/api/abi/time_cpu_inlines.h \ - arch/x86/odp/api/abi/time_inlines.h + arch/x86/odp/api/abi/time_inlines.h \ + arch/default/odp/api/abi/ticketlock_generic.h \ + arch/default/odp/api/abi/ticketlock_inlines.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/ticketlock_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/ticketlock_inlines.h new file mode 100644 index 00000000000..c0255ccd9e5 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/ticketlock_inlines.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP Ticketlock inline functions + */ +#ifndef ODP_API_ABI_TICKETLOCK_INLINES_H_ +#define ODP_API_ABI_TICKETLOCK_INLINES_H_ + +static inline uint32_t monitorll32(uint32_t *var, int mm) +{ + uint32_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +static inline void +_odp_wait_until_equal_32(uint32_t expected, uint32_t *addr, int memorder) +{ + uint32_t value; + + __asm__ volatile("sevl" : : : ); + do { + __asm__ volatile("wfe" : : : ); + value = monitorll32(addr, memorder); + } while (expected != value); +} + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/ticketlock_generic.h b/platform/linux-generic/arch/default/odp/api/abi/ticketlock_generic.h new file mode 100644 index 00000000000..7d9b99ad913 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/ticketlock_generic.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2021, ARM Limited + * Copyright (c) 2021-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_TICKETLOCK_GENERIC_H_ +#define ODP_API_ABI_TICKETLOCK_GENERIC_H_ +#include + +static inline void +_odp_wait_until_equal_32(uint32_t expected, uint32_t *addr, int memorder) +{ + while (expected != __atomic_load_n(addr, memorder)) + odp_cpu_pause(); +} + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/ticketlock_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/ticketlock_inlines.h new file mode 100644 index 00000000000..34a9d5230aa --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/ticketlock_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h b/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h index eef052f8d99..91ae432cae5 100644 --- a/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h @@ -11,7 +11,7 @@ #include #include - +#include /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -47,8 +47,8 @@ _ODP_INLINE void odp_ticketlock_lock(odp_ticketlock_t *ticketlock) /* Spin waiting for our turn. Use load-acquire so that we acquire * all stores from the previous lock owner */ - while (ticket != odp_atomic_load_acq_u32(&ticketlock->cur_ticket)) - odp_cpu_pause(); + _odp_wait_until_equal_32(ticket, (uint32_t *)&ticketlock->cur_ticket, + __ATOMIC_ACQUIRE); } _ODP_INLINE int odp_ticketlock_trylock(odp_ticketlock_t *tklock)