From 0385534d2d257469595cb2e1307d3b211505b21f Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 23 Aug 2023 13:54:07 +0300 Subject: [PATCH] linux-gen: ring: fix ring size detection Previously, _RING_MPMC_ENQ_MULTI() and _RING_MPMC_ENQ_BATCH() could return zero even though the ring was not actually full. Read w_head before r_tail, which guarantees that w_head - r_tail <= size. Any additional delay in reading r_tail makes the subtraction result only smaller. This avoids returning zero when the ring is not actually full. Signed-off-by: Matias Elo Reported-by: Fan Hong --- .../include/odp_ring_mpmc_internal.h | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h index e3517926734..c03a8e9a0d4 100644 --- a/platform/linux-generic/include/odp_ring_mpmc_internal.h +++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h @@ -212,13 +212,17 @@ static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring, uint32_t old_head, new_head, r_tail, num_free, i; uint32_t size = ring_mask + 1; - /* Load acquires ensure that w_head load happens after r_tail load, - * and thus r_tail value is always behind or equal to w_head value. - * When CAS operation succeeds, this thread owns data between old - * and new w_head. */ + /* The CAS operation guarantees that w_head value is up to date. Load + * acquire is used to ensure that r_tail is read after w_head. This + * guarantees that w_head - r_tail <= size. Any additional delay in + * reading r_tail makes the subtraction result only smaller. This + * avoids returning zero when the ring is not actually full. + * + * When CAS operation succeeds, this thread owns data between old and + * new w_head. */ do { - r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); old_head = odp_atomic_load_acq_u32(&ring->r.w_head); + r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); num_free = size - (old_head - r_tail); @@ -259,13 +263,17 @@ static inline uint32_t _RING_MPMC_ENQ_BATCH(_ring_mpmc_gen_t *ring, uint32_t old_head, new_head, r_tail, num_free, i; uint32_t size = ring_mask + 1; - /* Load acquires ensure that w_head load happens after r_tail load, - * and thus r_tail value is always behind or equal to w_head value. - * When CAS operation succeeds, this thread owns data between old - * and new w_head. */ + /* The CAS operation guarantees that w_head value is up to date. Load + * acquire is used to ensure that r_tail is read after w_head. This + * guarantees that w_head - r_tail <= size. Any additional delay in + * reading r_tail makes the subtraction result only smaller. This + * avoids returning zero when the ring is not actually full. + * + * When CAS operation succeeds, this thread owns data between old and + * new w_head. */ do { - r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); old_head = odp_atomic_load_acq_u32(&ring->r.w_head); + r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); num_free = size - (old_head - r_tail);