From 026b6912d93eb6dd0ea39b26524878bceb1567c1 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Tue, 12 Sep 2023 12:23:50 +0000 Subject: [PATCH 1/8] test: dmafwd: configure cache sizes for transfer and completion pools Configure cache sizes for transfer buffer and completion event pools based on the cache size command line option. These are utilized in fast path and caching may improve performance in DMA copy case. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 20fac97900..2890db260e 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -45,7 +45,8 @@ enum { #define MAX_WORKERS (ODP_THREAD_COUNT_MAX - 1) #define MAX_PKTIO_INDEXES 1024U -#define MIN(a, b) (((a) <= (b)) ? (a) : (b)) +#define MIN(a, b) (((a) <= (b)) ? (a) : (b)) +#define MAX(a, b) (((a) >= (b)) ? (a) : (b)) #define DIV_IF(a, b) ((b) > 0U ? ((a) / (b)) : 0U) ODP_STATIC_ASSERT(MAX_IFS < UINT8_MAX, "Too large maximum interface count"); @@ -122,6 +123,8 @@ typedef struct prog_config_s { uint32_t num_pkts; uint32_t pkt_len; uint32_t cache_size; + uint32_t trs_cache_size; + uint32_t compl_cache_size; uint32_t time_sec; int num_thrs; uint8_t num_ifs; @@ -224,7 +227,7 @@ static void print_usage(dynamic_defs_t *dyn_defs) " -l, --pkt_len Maximum size of packet buffers in packet I/O pool. %u by\n" " default.\n" " -c, --worker_count Amount of workers. %u by default.\n" - " -C, --cache_size Packet pool cache size. %u by default.\n" + " -C, --cache_size Maximum cache size for pools. %u by default.\n" " -T, --time_sec Time in seconds to run. 0 means infinite. %u by default.\n" " -h, --help This help.\n" "\n", DEF_CPY_TYPE, dyn_defs->burst_size, dyn_defs->num_pkts, dyn_defs->pkt_len, @@ -345,6 +348,11 @@ static parse_result_t check_options(prog_config_t *config) return PRS_NOK; } + config->trs_cache_size = MIN(MAX(config->cache_size, pool_capa.buf.min_cache_size), + pool_capa.buf.max_cache_size); + config->compl_cache_size = MIN(MAX(config->cache_size, dma_capa.pool.min_cache_size), + dma_capa.pool.max_cache_size); + return PRS_OK; } @@ -637,6 +645,7 @@ static odp_bool_t setup_copy(prog_config_t *config) pool_param.buf.num = config->num_pkts; pool_param.buf.size = sizeof(transfer_t); + pool_param.buf.cache_size = config->trs_cache_size; pool_param.type = ODP_POOL_BUFFER; config->trs_pool = odp_pool_create(PROG_NAME "_dma_trs", &pool_param); @@ -656,6 +665,7 @@ static odp_bool_t setup_copy(prog_config_t *config) odp_dma_pool_param_init(&compl_pool_param); compl_pool_param.num = config->num_pkts; + compl_pool_param.cache_size = config->compl_cache_size; thr->compl_pool = odp_dma_pool_create(PROG_NAME "_dma_compl", &compl_pool_param); if (thr->compl_pool == ODP_POOL_INVALID) { @@ -998,7 +1008,7 @@ static void print_stats(const prog_config_t *config) " copy mode: %s\n" " burst size: %u\n" " packet length: %u\n" - " pool cache size: %u\n", config->copy_type == SW_COPY ? "SW" : "DMA", + " max cache size: %u\n", config->copy_type == SW_COPY ? "SW" : "DMA", config->burst_size, config->pkt_len, config->cache_size); for (int i = 0; i < config->num_thrs; ++i) { From 39c2ec448a92cbd232181580c6d75edc9e15da35 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Fri, 15 Sep 2023 08:33:50 +0000 Subject: [PATCH 2/8] test: dmafwd: unify ODP init/term error logs Remove trailing periods to match existing tester logging style. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 2890db260e..c9718498da 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -1064,12 +1064,12 @@ int main(int argc, char **argv) init_param.mem_model = odph_opts.mem_model; if (odp_init_global(&odp_instance, NULL, NULL)) { - ODPH_ERR("ODP global init failed, exiting.\n"); + ODPH_ERR("ODP global init failed, exiting\n"); exit(EXIT_FAILURE); } if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) { - ODPH_ERR("ODP local init failed, exiting.\n"); + ODPH_ERR("ODP local init failed, exiting\n"); exit(EXIT_FAILURE); } @@ -1140,12 +1140,12 @@ int main(int argc, char **argv) (void)odp_shm_free(shm_cfg); if (odp_term_local()) { - ODPH_ERR("ODP local terminate failed, exiting.\n"); + ODPH_ERR("ODP local terminate failed, exiting\n"); exit(EXIT_FAILURE); } if (odp_term_global(odp_instance)) { - ODPH_ERR("ODP global terminate failed, exiting.\n"); + ODPH_ERR("ODP global terminate failed, exiting\n"); exit(EXIT_FAILURE); } From 9496d8d97cc3f098603f79ced073500d6fe6b194 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Tue, 12 Sep 2023 12:44:02 +0000 Subject: [PATCH 3/8] test: dmafwd: unify capability check error logs Use suitable log wording in capability checks depending on the error condition (not supported vs. failure). Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index c9718498da..54494030f4 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -290,7 +290,7 @@ static parse_result_t check_options(prog_config_t *config) } if ((uint32_t)config->num_thrs > dma_capa.max_sessions) { - ODPH_ERR("Not enough DMA sessions supported: %d (max: %u)\n", config->num_thrs, + ODPH_ERR("Unsupported DMA session count: %d (max: %u)\n", config->num_thrs, dma_capa.max_sessions); return PRS_NOT_SUP; } @@ -299,13 +299,13 @@ static parse_result_t check_options(prog_config_t *config) burst_size = MIN(burst_size, MAX_BURST); if (config->burst_size == 0U || config->burst_size > burst_size) { - ODPH_ERR("Unsupported segment count for DMA: %u (min: 1, max: %u)\n", + ODPH_ERR("Invalid segment count for DMA: %u (min: 1, max: %u)\n", config->burst_size, burst_size); return PRS_NOK; } if (config->pkt_len > dma_capa.max_seg_len) { - ODPH_ERR("Unsupported packet length for DMA: %u (max: %u)\n", config->pkt_len, + ODPH_ERR("Invalid packet length for DMA: %u (max: %u)\n", config->pkt_len, dma_capa.max_seg_len); return PRS_NOK; } @@ -317,7 +317,7 @@ static parse_result_t check_options(prog_config_t *config) } if ((uint32_t)config->num_thrs > dma_capa.pool.max_pools) { - ODPH_ERR("Unsupported amount of completion pools: %d (max: %u)\n", + ODPH_ERR("Invalid amount of completion pools: %d (max: %u)\n", config->num_thrs, dma_capa.pool.max_pools); return PRS_NOK; } From 645baa583a7ed707ea76ca28e0d4889f079aac3b Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Wed, 13 Sep 2023 07:40:20 +0000 Subject: [PATCH 4/8] test: dmafwd: add DMA poll mode parsing support In preparation for DMA poll mode support, add parsing skeleton for the new mode. The mode itself is not yet supported and will fail with "not yet implemented" error if chosen. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd.c | 123 +++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 47 deletions(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 54494030f4..34e5e4016d 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -30,7 +30,8 @@ enum { SW_COPY = 0U, - DMA_COPY + DMA_COPY_EV, + DMA_COPY_POLL }; #define DEF_CPY_TYPE SW_COPY @@ -219,7 +220,8 @@ static void print_usage(dynamic_defs_t *dyn_defs) "\n" " -t, --copy_type Type of copy. %u by default.\n" " 0: SW\n" - " 1: DMA\n" + " 1: DMA with event completion\n" + " 2: DMA with poll completion\n" " -b, --burst_size Copy burst size. This many packets are accumulated before\n" " copy. %u by default.\n" " -n, --num_pkts Number of packet buffers allocated for packet I/O pool.\n" @@ -273,7 +275,8 @@ static parse_result_t check_options(prog_config_t *config) MAX_PKTIO_INDEXES); } - if (config->copy_type != SW_COPY && config->copy_type != DMA_COPY) { + if (config->copy_type != SW_COPY && config->copy_type != DMA_COPY_EV && + config->copy_type != DMA_COPY_POLL) { ODPH_ERR("Invalid copy type: %u\n", config->copy_type); return PRS_NOK; } @@ -310,16 +313,26 @@ static parse_result_t check_options(prog_config_t *config) return PRS_NOK; } - if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U || !dma_capa.queue_type_sched) { - ODPH_ERR("Unsupported completion mode (mode support: %x, scheduled queue " - "support: %u\n", dma_capa.compl_mode_mask, dma_capa.queue_type_sched); - return PRS_NOT_SUP; - } + if (config->copy_type == DMA_COPY_EV) { + if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U || + !dma_capa.queue_type_sched) { + ODPH_ERR("Unsupported DMA completion mode: event (mode support: %x, " + "scheduled queue support: %u)\n", dma_capa.compl_mode_mask, + dma_capa.queue_type_sched); + return PRS_NOT_SUP; + } - if ((uint32_t)config->num_thrs > dma_capa.pool.max_pools) { - ODPH_ERR("Invalid amount of completion pools: %d (max: %u)\n", - config->num_thrs, dma_capa.pool.max_pools); - return PRS_NOK; + if ((uint32_t)config->num_thrs > dma_capa.pool.max_pools) { + ODPH_ERR("Invalid amount of DMA completion pools: %d (max: %u)\n", + config->num_thrs, dma_capa.pool.max_pools); + return PRS_NOK; + } + } else if (config->copy_type == DMA_COPY_POLL) { + if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_POLL) == 0U) { + ODPH_ERR("Unsupported DMA completion mode: poll (mode support: %x)\n", + dma_capa.compl_mode_mask); + return PRS_NOT_SUP; + } } if (odp_pool_capability(&pool_capa) < 0) { @@ -655,34 +668,40 @@ static odp_bool_t setup_copy(prog_config_t *config) } for (int i = 0; i < config->num_thrs; ++i) { - thr = &config->thread_config[i]; - thr->dma_handle = odp_dma_create(PROG_NAME "_dma", &dma_params); - - if (thr->dma_handle == ODP_DMA_INVALID) { - ODPH_ERR("Error creating DMA session\n"); - return false; - } + if (config->copy_type == DMA_COPY_EV) { + thr = &config->thread_config[i]; + thr->dma_handle = odp_dma_create(PROG_NAME "_dma", &dma_params); - odp_dma_pool_param_init(&compl_pool_param); - compl_pool_param.num = config->num_pkts; - compl_pool_param.cache_size = config->compl_cache_size; - thr->compl_pool = odp_dma_pool_create(PROG_NAME "_dma_compl", &compl_pool_param); + if (thr->dma_handle == ODP_DMA_INVALID) { + ODPH_ERR("Error creating DMA session\n"); + return false; + } - if (thr->compl_pool == ODP_POOL_INVALID) { - ODPH_ERR("Error creating DMA event completion pool\n"); - return false; - } + odp_dma_pool_param_init(&compl_pool_param); + compl_pool_param.num = config->num_pkts; + compl_pool_param.cache_size = config->compl_cache_size; + thr->compl_pool = odp_dma_pool_create(PROG_NAME "_dma_compl", + &compl_pool_param); - thr->copy_pool = config->copy_pool; - thr->trs_pool = config->trs_pool; - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; - queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; - queue_param.sched.prio = odp_schedule_max_prio(); - thr->compl_q = odp_queue_create(PROG_NAME "_dma_compl", &queue_param); + if (thr->compl_pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating DMA event completion pool\n"); + return false; + } - if (thr->compl_q == ODP_QUEUE_INVALID) { - ODPH_ERR("Error creating DMA completion queue\n"); + thr->copy_pool = config->copy_pool; + thr->trs_pool = config->trs_pool; + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; + queue_param.sched.prio = odp_schedule_max_prio(); + thr->compl_q = odp_queue_create(PROG_NAME "_dma_compl", &queue_param); + + if (thr->compl_q == ODP_QUEUE_INVALID) { + ODPH_ERR("Error creating DMA completion queue\n"); + return false; + } + } else { + ODPH_ERR("DMA poll completion support not yet implemented\n"); return false; } } @@ -1001,14 +1020,18 @@ static void teardown(prog_config_t *config) static void print_stats(const prog_config_t *config) { const stats_t *stats; - const char *align = config->copy_type == SW_COPY ? " " : " "; + const char *align1 = config->copy_type == DMA_COPY_EV ? " " : ""; + const char *align2 = config->copy_type == SW_COPY ? " " : + config->copy_type == DMA_COPY_EV ? " " : + " "; printf("\n==================\n\n" "DMA forwarder done\n\n" " copy mode: %s\n" " burst size: %u\n" " packet length: %u\n" - " max cache size: %u\n", config->copy_type == SW_COPY ? "SW" : "DMA", + " max cache size: %u\n", config->copy_type == SW_COPY ? "SW" : + config->copy_type == DMA_COPY_EV ? "DMA-event" : "DMA-poll", config->burst_size, config->pkt_len, config->cache_size); for (int i = 0; i < config->num_thrs; ++i) { @@ -1020,15 +1043,21 @@ static void print_stats(const prog_config_t *config) printf(" packet copy errors: %" PRIu64 "\n", stats->copy_errs); } else { - printf(" successful DMA transfers: %" PRIu64 "\n" - " DMA transfer start errors: %" PRIu64 "\n" - " DMA transfer errors: %" PRIu64 "\n" - " transfer buffer allocation errors: %" PRIu64 "\n" - " completion event allocation errors: %" PRIu64 "\n" - " copy packet allocation errors: %" PRIu64 "\n", - stats->trs, stats->start_errs, stats->trs_errs, - stats->buf_alloc_errs, stats->compl_alloc_errs, + printf(" successful DMA transfers: %s%" PRIu64 "\n" + " DMA transfer start errors: %s%" PRIu64 "\n" + " DMA transfer errors: %s%" PRIu64 "\n" + " transfer buffer allocation errors: %s%" PRIu64 "\n" + " copy packet allocation errors: %s%" PRIu64 "\n", + align1, stats->trs, align1, stats->start_errs, align1, + stats->trs_errs, align1, stats->buf_alloc_errs, align1, stats->pkt_alloc_errs); + + if (config->copy_type == DMA_COPY_EV) + printf(" completion event allocation errors: %" PRIu64 "\n", + stats->compl_alloc_errs); + else + printf(" transfer ID allocation errors: %" PRIu64 "\n", + stats->compl_alloc_errs); } printf(" packets forwarded:%s%" PRIu64 "\n" @@ -1036,7 +1065,7 @@ static void print_stats(const prog_config_t *config) " call cycles per schedule round:\n" " total: %" PRIu64 "\n" " schedule: %" PRIu64 "\n" - " rounds: %" PRIu64 "\n", align, stats->fwd_pkts, align, + " rounds: %" PRIu64 "\n", align2, stats->fwd_pkts, align2, stats->discards, DIV_IF(stats->tot_cc, stats->sched_rounds), DIV_IF(stats->sched_cc, stats->sched_rounds), stats->sched_rounds); } From f164934b1cd5ce096718fc9154d657492326fb02 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Wed, 13 Sep 2023 11:06:38 +0000 Subject: [PATCH 5/8] test: dmafwd: refactor DMA copy In preparation for DMA poll mode support, restructure how transfers are initialized and sent in DMA copy case. This makes it easier to plug-in poll mode (and potentially others in the future) handling logic. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd.c | 209 ++++++++++++++++++++-------------- 1 file changed, 122 insertions(+), 87 deletions(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 34e5e4016d..7035266792 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -102,8 +102,25 @@ typedef struct pktio_s { uint8_t num_out_qs; } pktio_t; -typedef void (*ev_fn_t)(odp_dma_compl_t compl_ev, thread_config_t *config); -typedef void (*pkt_fn_t)(odp_packet_t pkts[], int num, pktio_t *pktio, thread_config_t *config); +typedef struct { + odp_packet_t src_pkts[MAX_BURST]; + odp_packet_t dst_pkts[MAX_BURST]; + pktio_t *pktio; + int num; +} transfer_t; + +/* Function for initializing transfer structures */ +typedef transfer_t *(*init_fn_t)(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, odp_dma_seg_t *src_segs, + odp_dma_seg_t *dst_segs, pktio_t *pktio, thread_config_t *config); +/* Function for starting transfers */ +typedef odp_bool_t (*start_fn_t)(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, thread_config_t *config); +/* Function for setting up packets for copy */ +typedef void (*pkt_fn_t)(odp_packet_t pkts[], int num, pktio_t *pktio, init_fn_t init_fn, + start_fn_t start_fn, thread_config_t *config); +/* Function for draining and tearing down inflight operations */ +typedef void (*drain_fn_t)(void); typedef struct prog_config_s { uint8_t pktio_idx_map[MAX_PKTIO_INDEXES]; @@ -118,8 +135,14 @@ typedef struct prog_config_s { odp_pool_t pktio_pool; odp_pool_t copy_pool; odp_pool_t trs_pool; - ev_fn_t ev_fn; - pkt_fn_t pkt_fn; + + struct { + init_fn_t init_fn; + start_fn_t start_fn; + pkt_fn_t pkt_fn; + drain_fn_t drain_fn; + }; + uint32_t burst_size; uint32_t num_pkts; uint32_t pkt_len; @@ -138,13 +161,6 @@ typedef struct { int num; } pkt_vec_t; -typedef struct { - odp_packet_t src_pkts[MAX_BURST]; - odp_packet_t dst_pkts[MAX_BURST]; - pktio_t *pktio; - int num; -} transfer_t; - static prog_config_t *prog_conf; static void terminate(int signal ODP_UNUSED) @@ -462,6 +478,7 @@ static inline int send_packets(odp_pktout_queue_t queue, odp_packet_t pkts[], in } static void sw_copy_and_send_packets(odp_packet_t pkts[], int num, pktio_t *pktio, + init_fn_t init_fn ODP_UNUSED, start_fn_t start_fn ODP_UNUSED, thread_config_t *config) { odp_packet_t old_pkt, new_pkt; @@ -490,41 +507,10 @@ static void sw_copy_and_send_packets(odp_packet_t pkts[], int num, pktio_t *pkti } } -static inline void send_dma_trs_packets(odp_dma_compl_t compl_ev, thread_config_t *config) -{ - odp_dma_result_t res; - odp_buffer_t buf; - transfer_t *trs; - pktio_t *pktio; - int num_sent; - stats_t *stats = &config->stats; - - memset(&res, 0, sizeof(res)); - odp_dma_compl_result(compl_ev, &res); - buf = (odp_buffer_t)res.user_ptr; - trs = (transfer_t *)odp_buffer_addr(buf); - pktio = trs->pktio; - - if (res.success) { - num_sent = send_packets(pktio->out_qs[config->thr_idx % pktio->num_out_qs], - trs->dst_pkts, trs->num); - ++stats->trs; - stats->fwd_pkts += num_sent; - stats->discards += trs->num - num_sent; - } else { - odp_packet_free_multi(trs->dst_pkts, trs->num); - ++stats->trs_errs; - } - - odp_packet_free_multi(trs->src_pkts, trs->num); - odp_buffer_free(buf); - odp_dma_compl_free(compl_ev); -} - -static inline transfer_t *init_dma_trs(odp_dma_transfer_param_t *trs_param, - odp_dma_compl_param_t *compl_param, odp_dma_seg_t *src_segs, - odp_dma_seg_t *dst_segs, pktio_t *pktio, - thread_config_t *config) +static transfer_t *init_dma_ev_trs(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, odp_dma_seg_t *src_segs, + odp_dma_seg_t *dst_segs, pktio_t *pktio, + thread_config_t *config) { odp_buffer_t buf; stats_t *stats = &config->stats; @@ -565,7 +551,22 @@ static inline transfer_t *init_dma_trs(odp_dma_transfer_param_t *trs_param, return trs; } -static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, thread_config_t *config) +static odp_bool_t start_dma_ev_trs(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, thread_config_t *config) +{ + const int ret = odp_dma_transfer_start(config->dma_handle, trs_param, compl_param); + + if (odp_unlikely(ret <= 0)) { + odp_buffer_free(compl_param->user_ptr); + odp_event_free(compl_param->event); + return false; + } + + return true; +} + +static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, init_fn_t init_fn, + start_fn_t start_fn, thread_config_t *config) { odp_dma_transfer_param_t trs_param; odp_dma_compl_param_t compl_param; @@ -583,8 +584,7 @@ static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, thread_config pkt = pkts[i]; if (odp_unlikely(trs == NULL)) { - trs = init_dma_trs(&trs_param, &compl_param, src_segs, dst_segs, pktio, - config); + trs = init_fn(&trs_param, &compl_param, src_segs, dst_segs, pktio, config); if (trs == NULL) { odp_packet_free(pkt); @@ -613,13 +613,43 @@ static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, thread_config } if (num_segs > 0U) - if (odp_dma_transfer_start(config->dma_handle, &trs_param, &compl_param) <= 0) { + if (odp_unlikely(!start_fn(&trs_param, &compl_param, config))) { odp_packet_free_multi(trs->src_pkts, trs->num); odp_packet_free_multi(trs->dst_pkts, trs->num); ++stats->start_errs; } } +static void drain_events(void) +{ + odp_event_t ev; + odp_event_type_t type; + odp_dma_result_t res; + odp_buffer_t buf; + transfer_t *trs; + + while (true) { + ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS * 2U)); + + if (ev == ODP_EVENT_INVALID) + break; + + type = odp_event_type(ev); + + if (type == ODP_EVENT_DMA_COMPL) { + memset(&res, 0, sizeof(res)); + odp_dma_compl_result(odp_dma_compl_from_event(ev), &res); + buf = (odp_buffer_t)res.user_ptr; + trs = (transfer_t *)odp_buffer_addr(buf); + odp_packet_free_multi(trs->src_pkts, trs->num); + odp_packet_free_multi(trs->dst_pkts, trs->num); + odp_buffer_free(buf); + } + + odp_event_free(ev); + } +} + static odp_bool_t setup_copy(prog_config_t *config) { odp_pool_param_t pool_param; @@ -647,7 +677,6 @@ static odp_bool_t setup_copy(prog_config_t *config) } if (config->copy_type == SW_COPY) { - config->ev_fn = NULL; config->pkt_fn = sw_copy_and_send_packets; for (int i = 0; i < config->num_thrs; ++i) @@ -706,8 +735,10 @@ static odp_bool_t setup_copy(prog_config_t *config) } } - config->ev_fn = send_dma_trs_packets; + config->init_fn = init_dma_ev_trs; + config->start_fn = start_dma_ev_trs; config->pkt_fn = dma_copy; + config->drain_fn = drain_events; return true; } @@ -800,6 +831,37 @@ static odp_bool_t setup_pktios(prog_config_t *config) return true; } +static inline void send_dma_trs_packets(odp_dma_compl_t compl_ev, thread_config_t *config) +{ + odp_dma_result_t res; + odp_buffer_t buf; + transfer_t *trs; + pktio_t *pktio; + int num_sent; + stats_t *stats = &config->stats; + + memset(&res, 0, sizeof(res)); + odp_dma_compl_result(compl_ev, &res); + buf = (odp_buffer_t)res.user_ptr; + trs = (transfer_t *)odp_buffer_addr(buf); + + if (res.success) { + pktio = trs->pktio; + num_sent = send_packets(pktio->out_qs[config->thr_idx % pktio->num_out_qs], + trs->dst_pkts, trs->num); + ++stats->trs; + stats->fwd_pkts += num_sent; + stats->discards += trs->num - num_sent; + } else { + odp_packet_free_multi(trs->dst_pkts, trs->num); + ++stats->trs_errs; + } + + odp_packet_free_multi(trs->src_pkts, trs->num); + odp_buffer_free(buf); + odp_dma_compl_free(compl_ev); +} + static inline void push_packet(odp_packet_t pkt, pkt_vec_t pkt_vecs[], uint8_t *pktio_idx_map) { uint8_t idx = pktio_idx_map[odp_packet_input_index(pkt)]; @@ -822,36 +884,6 @@ static void free_pending_packets(pkt_vec_t pkt_vecs[], uint32_t num_ifs) odp_packet_free_multi(pkt_vecs[i].pkts, pkt_vecs[i].num); } -static void drain(void) -{ - odp_event_t ev; - odp_event_type_t type; - odp_dma_result_t res; - odp_buffer_t buf; - transfer_t *trs; - - while (true) { - ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS * 2U)); - - if (ev == ODP_EVENT_INVALID) - break; - - type = odp_event_type(ev); - - if (type == ODP_EVENT_DMA_COMPL) { - memset(&res, 0, sizeof(res)); - odp_dma_compl_result(odp_dma_compl_from_event(ev), &res); - buf = (odp_buffer_t)res.user_ptr; - trs = (transfer_t *)odp_buffer_addr(buf); - odp_packet_free_multi(trs->src_pkts, trs->num); - odp_packet_free_multi(trs->dst_pkts, trs->num); - odp_buffer_free(buf); - } - - odp_event_free(ev); - } -} - static int process_packets(void *args) { thread_config_t *config = args; @@ -864,9 +896,10 @@ static int process_packets(void *args) int num_evs; odp_event_t ev; odp_event_type_t type; - ev_fn_t ev_fn = config->prog_config->ev_fn; uint8_t *pktio_map = config->prog_config->pktio_idx_map; stats_t *stats = &config->stats; + init_fn_t init_fn = config->prog_config->init_fn; + start_fn_t start_fn = config->prog_config->start_fn; pkt_fn_t pkt_fn = config->prog_config->pkt_fn; for (uint32_t i = 0U; i < num_ifs; ++i) { @@ -893,8 +926,7 @@ static int process_packets(void *args) type = odp_event_type(ev); if (type == ODP_EVENT_DMA_COMPL) { - if (ev_fn) - ev_fn(odp_dma_compl_from_event(ev), config); + send_dma_trs_packets(odp_dma_compl_from_event(ev), config); } else if (type == ODP_EVENT_PACKET) { push_packet(odp_packet_from_event(ev), pkt_vecs, pktio_map); } else { @@ -907,7 +939,8 @@ static int process_packets(void *args) pkt_vec = &pkt_vecs[i]; if (pkt_vec->num >= burst_size) { - pkt_fn(pkt_vec->pkts, burst_size, pkt_vec->pktio, config); + pkt_fn(pkt_vec->pkts, burst_size, pkt_vec->pktio, init_fn, + start_fn, config); pop_packets(pkt_vec, burst_size); } } @@ -919,7 +952,9 @@ static int process_packets(void *args) stats->sched_rounds = rounds; free_pending_packets(pkt_vecs, num_ifs); odp_barrier_wait(&config->prog_config->term_barrier); - drain(); + + if (config->prog_config->drain_fn) + config->prog_config->drain_fn(); return 0; } From f194a89f76c461aae92ac2197ecaf062bab8f009 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Fri, 15 Sep 2023 08:25:41 +0000 Subject: [PATCH 6/8] test: dmafwd: implement DMA poll mode In addition to event completion mode, DMA copy performance can now also be benchmarked with poll completion mode. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd.c | 292 +++++++++++++++++++++++++++++++--- 1 file changed, 268 insertions(+), 24 deletions(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 7035266792..987313f1f3 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -77,6 +77,8 @@ typedef struct { uint64_t buf_alloc_errs; uint64_t compl_alloc_errs; uint64_t pkt_alloc_errs; + uint64_t trs_poll_errs; + uint64_t trs_polled; uint64_t fwd_pkts; uint64_t discards; uint64_t sched_cc; @@ -91,6 +93,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_pool_t copy_pool; odp_pool_t trs_pool; odp_queue_t compl_q; + odp_stash_t inflight_stash; stats_t stats; int thr_idx; } thread_config_t; @@ -120,7 +123,7 @@ typedef odp_bool_t (*start_fn_t)(odp_dma_transfer_param_t *trs_param, typedef void (*pkt_fn_t)(odp_packet_t pkts[], int num, pktio_t *pktio, init_fn_t init_fn, start_fn_t start_fn, thread_config_t *config); /* Function for draining and tearing down inflight operations */ -typedef void (*drain_fn_t)(void); +typedef void (*drain_fn_t)(thread_config_t *config); typedef struct prog_config_s { uint8_t pktio_idx_map[MAX_PKTIO_INDEXES]; @@ -143,13 +146,17 @@ typedef struct prog_config_s { drain_fn_t drain_fn; }; + uint64_t inflight_obj_size; uint32_t burst_size; uint32_t num_pkts; uint32_t pkt_len; uint32_t cache_size; + uint32_t num_inflight; uint32_t trs_cache_size; uint32_t compl_cache_size; + uint32_t stash_cache_size; uint32_t time_sec; + odp_stash_type_t stash_type; int num_thrs; uint8_t num_ifs; uint8_t copy_type; @@ -209,6 +216,7 @@ static void init_config(prog_config_t *config) thr->dma_handle = ODP_DMA_INVALID; thr->compl_pool = ODP_POOL_INVALID; thr->compl_q = ODP_QUEUE_INVALID; + thr->inflight_stash = ODP_STASH_INVALID; } for (uint32_t i = 0U; i < MAX_IFS; ++i) @@ -273,11 +281,29 @@ static void parse_interfaces(prog_config_t *config, const char *optarg) free(tmp_str); } +static odp_bool_t get_stash_capa(odp_stash_capability_t *stash_capa, odp_stash_type_t *stash_type) +{ + if (odp_stash_capability(stash_capa, ODP_STASH_TYPE_FIFO) == 0) { + *stash_type = ODP_STASH_TYPE_FIFO; + return true; + } + + if (odp_stash_capability(stash_capa, ODP_STASH_TYPE_DEFAULT) == 0) { + *stash_type = ODP_STASH_TYPE_DEFAULT; + return true; + } + + return false; +} + static parse_result_t check_options(prog_config_t *config) { - unsigned int idx = odp_pktio_max_index(); + const unsigned int idx = odp_pktio_max_index(); odp_dma_capability_t dma_capa; uint32_t burst_size; + odp_stash_capability_t stash_capa; + const uint64_t obj_size = sizeof(odp_dma_transfer_id_t); + uint64_t max_num; odp_pool_capability_t pool_capa; if (config->num_ifs == 0U) { @@ -329,6 +355,8 @@ static parse_result_t check_options(prog_config_t *config) return PRS_NOK; } + config->num_inflight = dma_capa.max_transfers; + if (config->copy_type == DMA_COPY_EV) { if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U || !dma_capa.queue_type_sched) { @@ -349,6 +377,40 @@ static parse_result_t check_options(prog_config_t *config) dma_capa.compl_mode_mask); return PRS_NOT_SUP; } + + if (!get_stash_capa(&stash_capa, &config->stash_type)) { + ODPH_ERR("Error querying stash capabilities\n"); + return PRS_NOK; + } + + if ((uint32_t)config->num_thrs > stash_capa.max_stashes) { + ODPH_ERR("Invalid amount of stashes: %d (max: %u)\n", config->num_thrs, + stash_capa.max_stashes); + return PRS_NOK; + } + + if (obj_size == sizeof(uint8_t)) { + max_num = stash_capa.max_num.u8; + } else if (obj_size == sizeof(uint16_t)) { + max_num = stash_capa.max_num.u16; + } else if (obj_size <= sizeof(uint32_t)) { + max_num = stash_capa.max_num.u32; + } else if (obj_size <= sizeof(uint64_t)) { + max_num = stash_capa.max_num.u64; + } else if (obj_size <= sizeof(odp_u128_t)) { + max_num = stash_capa.max_num.u128; + } else { + ODPH_ERR("Invalid stash object size: %" PRIu64 "\n", obj_size); + return PRS_NOK; + } + + if (config->num_inflight > max_num) { + ODPH_ERR("Invalid stash size: %u (max: %" PRIu64 ")\n", + config->num_inflight, max_num); + return PRS_NOK; + } + + config->inflight_obj_size = obj_size; } if (odp_pool_capability(&pool_capa) < 0) { @@ -381,6 +443,7 @@ static parse_result_t check_options(prog_config_t *config) pool_capa.buf.max_cache_size); config->compl_cache_size = MIN(MAX(config->cache_size, dma_capa.pool.min_cache_size), dma_capa.pool.max_cache_size); + config->stash_cache_size = MIN(config->cache_size, stash_capa.max_cache_size); return PRS_OK; } @@ -551,6 +614,47 @@ static transfer_t *init_dma_ev_trs(odp_dma_transfer_param_t *trs_param, return trs; } +static transfer_t *init_dma_poll_trs(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, odp_dma_seg_t *src_segs, + odp_dma_seg_t *dst_segs, pktio_t *pktio, + thread_config_t *config) +{ + odp_buffer_t buf; + stats_t *stats = &config->stats; + transfer_t *trs; + + buf = odp_buffer_alloc(config->trs_pool); + + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) { + ++stats->buf_alloc_errs; + return NULL; + } + + trs = (transfer_t *)odp_buffer_addr(buf); + trs->num = 0; + trs->pktio = pktio; + trs_param->src_format = ODP_DMA_FORMAT_PACKET; + trs_param->dst_format = ODP_DMA_FORMAT_PACKET; + trs_param->num_src = 0U; + trs_param->num_dst = 0U; + trs_param->src_seg = src_segs; + trs_param->dst_seg = dst_segs; + compl_param->compl_mode = ODP_DMA_COMPL_POLL; + compl_param->transfer_id = odp_dma_transfer_id_alloc(config->dma_handle); + + if (odp_unlikely(compl_param->transfer_id == ODP_DMA_TRANSFER_ID_INVALID)) { + odp_buffer_free(buf); + ++stats->compl_alloc_errs; + return NULL; + } + + compl_param->user_ptr = buf; + memset(src_segs, 0, sizeof(*src_segs) * MAX_BURST); + memset(dst_segs, 0, sizeof(*dst_segs) * MAX_BURST); + + return trs; +} + static odp_bool_t start_dma_ev_trs(odp_dma_transfer_param_t *trs_param, odp_dma_compl_param_t *compl_param, thread_config_t *config) { @@ -565,6 +669,24 @@ static odp_bool_t start_dma_ev_trs(odp_dma_transfer_param_t *trs_param, return true; } +static odp_bool_t start_dma_poll_trs(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, thread_config_t *config) +{ + const int ret = odp_dma_transfer_start(config->dma_handle, trs_param, compl_param); + + if (odp_unlikely(ret <= 0)) { + odp_buffer_free(compl_param->user_ptr); + odp_dma_transfer_id_free(config->dma_handle, compl_param->transfer_id); + return false; + } + + if (odp_unlikely(odp_stash_put(config->inflight_stash, &compl_param->transfer_id, 1) != 1)) + /* Should not happen, but make it visible if it somehow does */ + ODPH_ABORT("DMA inflight transfer stash overflow, aborting"); + + return true; +} + static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, init_fn_t init_fn, start_fn_t start_fn, thread_config_t *config) { @@ -620,7 +742,7 @@ static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, init_fn_t ini } } -static void drain_events(void) +static void drain_events(thread_config_t *config ODP_UNUSED) { odp_event_t ev; odp_event_type_t type; @@ -650,18 +772,50 @@ static void drain_events(void) } } +static void drain_polled(thread_config_t *config) +{ + odp_dma_transfer_id_t id; + odp_dma_result_t res; + int ret; + odp_buffer_t buf; + transfer_t *trs; + + while (true) { + if (odp_stash_get(config->inflight_stash, &id, 1) != 1) + break; + + memset(&res, 0, sizeof(res)); + + do { + ret = odp_dma_transfer_done(config->dma_handle, id, &res); + } while (ret == 0); + + odp_dma_transfer_id_free(config->dma_handle, id); + + if (ret < 0) + continue; + + buf = (odp_buffer_t)res.user_ptr; + trs = (transfer_t *)odp_buffer_addr(buf); + odp_packet_free_multi(trs->src_pkts, trs->num); + odp_packet_free_multi(trs->dst_pkts, trs->num); + odp_buffer_free(buf); + } +} + static odp_bool_t setup_copy(prog_config_t *config) { odp_pool_param_t pool_param; thread_config_t *thr; - const odp_dma_param_t dma_params = { + const odp_dma_param_t dma_param = { .direction = ODP_DMA_MAIN_TO_MAIN, .type = ODP_DMA_TYPE_COPY, - .compl_mode_mask = ODP_DMA_COMPL_EVENT, + .compl_mode_mask = ODP_DMA_COMPL_EVENT | ODP_DMA_COMPL_POLL, .mt_mode = ODP_DMA_MT_SERIAL, .order = ODP_DMA_ORDER_NONE }; odp_dma_pool_param_t compl_pool_param; odp_queue_param_t queue_param; + odp_stash_param_t stash_param; odp_pool_param_init(&pool_param); pool_param.pkt.seg_len = config->pkt_len; @@ -697,15 +851,17 @@ static odp_bool_t setup_copy(prog_config_t *config) } for (int i = 0; i < config->num_thrs; ++i) { - if (config->copy_type == DMA_COPY_EV) { - thr = &config->thread_config[i]; - thr->dma_handle = odp_dma_create(PROG_NAME "_dma", &dma_params); + thr = &config->thread_config[i]; + thr->copy_pool = config->copy_pool; + thr->trs_pool = config->trs_pool; + thr->dma_handle = odp_dma_create(PROG_NAME "_dma", &dma_param); - if (thr->dma_handle == ODP_DMA_INVALID) { - ODPH_ERR("Error creating DMA session\n"); - return false; - } + if (thr->dma_handle == ODP_DMA_INVALID) { + ODPH_ERR("Error creating DMA session\n"); + return false; + } + if (config->copy_type == DMA_COPY_EV) { odp_dma_pool_param_init(&compl_pool_param); compl_pool_param.num = config->num_pkts; compl_pool_param.cache_size = config->compl_cache_size; @@ -717,8 +873,6 @@ static odp_bool_t setup_copy(prog_config_t *config) return false; } - thr->copy_pool = config->copy_pool; - thr->trs_pool = config->trs_pool; odp_queue_param_init(&queue_param); queue_param.type = ODP_QUEUE_TYPE_SCHED; queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; @@ -729,16 +883,32 @@ static odp_bool_t setup_copy(prog_config_t *config) ODPH_ERR("Error creating DMA completion queue\n"); return false; } + + config->init_fn = init_dma_ev_trs; + config->start_fn = start_dma_ev_trs; + config->drain_fn = drain_events; } else { - ODPH_ERR("DMA poll completion support not yet implemented\n"); - return false; + odp_stash_param_init(&stash_param); + stash_param.type = config->stash_type; + stash_param.put_mode = ODP_STASH_OP_LOCAL; + stash_param.get_mode = ODP_STASH_OP_LOCAL; + stash_param.num_obj = config->num_inflight; + stash_param.obj_size = config->inflight_obj_size; + stash_param.cache_size = config->stash_cache_size; + thr->inflight_stash = odp_stash_create("_dma_inflight", &stash_param); + + if (thr->inflight_stash == ODP_STASH_INVALID) { + ODPH_ERR("Error creating DMA inflight transfer stash\n"); + return false; + } + + config->init_fn = init_dma_poll_trs; + config->start_fn = start_dma_poll_trs; + config->drain_fn = drain_polled; } } - config->init_fn = init_dma_ev_trs; - config->start_fn = start_dma_ev_trs; config->pkt_fn = dma_copy; - config->drain_fn = drain_events; return true; } @@ -831,7 +1001,71 @@ static odp_bool_t setup_pktios(prog_config_t *config) return true; } -static inline void send_dma_trs_packets(odp_dma_compl_t compl_ev, thread_config_t *config) +static inline void send_dma_poll_trs_pkts(int burst_size, thread_config_t *config) +{ + odp_stash_t stash_handle = config->inflight_stash; + odp_dma_transfer_id_t ids[burst_size], id; + int32_t num; + odp_dma_t dma_handle = config->dma_handle; + odp_dma_result_t res; + int ret; + odp_buffer_t buf; + transfer_t *trs; + pktio_t *pktio; + int num_sent; + stats_t *stats = &config->stats; + + while (true) { + num = odp_stash_get(stash_handle, &ids, burst_size); + + if (num <= 0) + break; + + for (int32_t i = 0; i < num; ++i) { + id = ids[i]; + ret = odp_dma_transfer_done(dma_handle, id, &res); + + if (ret == 0) { + if (odp_unlikely(odp_stash_put(stash_handle, &id, 1) != 1)) + /* Should not happen, but make it visible if it somehow + * does */ + ODPH_ABORT("DMA inflight transfer stash overflow," + " aborting"); + + ++stats->trs_polled; + continue; + } + + odp_dma_transfer_id_free(dma_handle, id); + + if (ret < 0) { + ++stats->trs_poll_errs; + continue; + } + + buf = (odp_buffer_t)res.user_ptr; + trs = (transfer_t *)odp_buffer_addr(buf); + + if (res.success) { + pktio = trs->pktio; + num_sent = send_packets(pktio->out_qs[config->thr_idx % + pktio->num_out_qs], + trs->dst_pkts, trs->num); + ++stats->trs; + stats->fwd_pkts += num_sent; + stats->discards += trs->num - num_sent; + } else { + odp_packet_free_multi(trs->dst_pkts, trs->num); + ++stats->trs_errs; + } + + odp_packet_free_multi(trs->src_pkts, trs->num); + odp_buffer_free(buf); + } + } +} + +static inline void send_dma_ev_trs_pkts(odp_dma_compl_t compl_ev, thread_config_t *config) { odp_dma_result_t res; odp_buffer_t buf; @@ -891,6 +1125,7 @@ static int process_packets(void *args) pkt_vec_t pkt_vecs[num_ifs], *pkt_vec; odp_atomic_u32_t *is_running = &config->prog_config->is_running; uint64_t c1, c2, c3, c4, cdiff = 0U, rounds = 0U; + const uint8_t copy_type = config->prog_config->copy_type; const int burst_size = config->prog_config->burst_size; odp_event_t evs[burst_size]; int num_evs; @@ -918,6 +1153,9 @@ static int process_packets(void *args) cdiff += odp_cpu_cycles_diff(c4, c3); ++rounds; + if (copy_type == DMA_COPY_POLL) + send_dma_poll_trs_pkts(burst_size, config); + if (num_evs == 0) continue; @@ -926,7 +1164,7 @@ static int process_packets(void *args) type = odp_event_type(ev); if (type == ODP_EVENT_DMA_COMPL) { - send_dma_trs_packets(odp_dma_compl_from_event(ev), config); + send_dma_ev_trs_pkts(odp_dma_compl_from_event(ev), config); } else if (type == ODP_EVENT_PACKET) { push_packet(odp_packet_from_event(ev), pkt_vecs, pktio_map); } else { @@ -954,7 +1192,7 @@ static int process_packets(void *args) odp_barrier_wait(&config->prog_config->term_barrier); if (config->prog_config->drain_fn) - config->prog_config->drain_fn(); + config->prog_config->drain_fn(config); return 0; } @@ -1035,6 +1273,9 @@ static void teardown(prog_config_t *config) for (int i = 0; i < config->num_thrs; ++i) { thr = &config->thread_config[i]; + if (thr->inflight_stash != ODP_STASH_INVALID) + (void)odp_stash_destroy(thr->inflight_stash); + if (thr->compl_q != ODP_QUEUE_INVALID) (void)odp_queue_destroy(thr->compl_q); @@ -1091,8 +1332,11 @@ static void print_stats(const prog_config_t *config) printf(" completion event allocation errors: %" PRIu64 "\n", stats->compl_alloc_errs); else - printf(" transfer ID allocation errors: %" PRIu64 "\n", - stats->compl_alloc_errs); + printf(" transfer ID allocation errors: %" PRIu64 "\n" + " transfer poll errors: %" PRIu64 "\n" + " transfers polled: %" PRIu64 "\n", + stats->compl_alloc_errs, stats->trs_poll_errs, + stats->trs_polled); } printf(" packets forwarded:%s%" PRIu64 "\n" From 544b2dbe8cd6c9a06217044ce1d61a69985a1d47 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Fri, 15 Sep 2023 08:51:12 +0000 Subject: [PATCH 7/8] test: dmafwd: tune DMA transfer buffer and completion event pool counts Transfer buffer and DMA completion event pools should be able to accommodate maximum inflight DMA transfer count of buffers/events. Change relevant pool creations accordingly. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 987313f1f3..b36b81b6ac 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -371,6 +371,12 @@ static parse_result_t check_options(prog_config_t *config) config->num_thrs, dma_capa.pool.max_pools); return PRS_NOK; } + + if (config->num_inflight > dma_capa.pool.max_num) { + ODPH_ERR("Invalid amount of DMA completion events: %u (max: %u)\n", + config->num_inflight, dma_capa.pool.max_num); + return PRS_NOK; + } } else if (config->copy_type == DMA_COPY_POLL) { if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_POLL) == 0U) { ODPH_ERR("Unsupported DMA completion mode: poll (mode support: %x)\n", @@ -439,6 +445,12 @@ static parse_result_t check_options(prog_config_t *config) return PRS_NOK; } + if (config->num_inflight > pool_capa.buf.max_num) { + ODPH_ERR("Invalid pool buffer count: %u (max: %u)\n", config->num_inflight, + pool_capa.buf.max_num); + return PRS_NOK; + } + config->trs_cache_size = MIN(MAX(config->cache_size, pool_capa.buf.min_cache_size), pool_capa.buf.max_cache_size); config->compl_cache_size = MIN(MAX(config->cache_size, dma_capa.pool.min_cache_size), @@ -839,7 +851,7 @@ static odp_bool_t setup_copy(prog_config_t *config) return true; } - pool_param.buf.num = config->num_pkts; + pool_param.buf.num = config->num_inflight; pool_param.buf.size = sizeof(transfer_t); pool_param.buf.cache_size = config->trs_cache_size; pool_param.type = ODP_POOL_BUFFER; @@ -863,7 +875,7 @@ static odp_bool_t setup_copy(prog_config_t *config) if (config->copy_type == DMA_COPY_EV) { odp_dma_pool_param_init(&compl_pool_param); - compl_pool_param.num = config->num_pkts; + compl_pool_param.num = config->num_inflight; compl_pool_param.cache_size = config->compl_cache_size; thr->compl_pool = odp_dma_pool_create(PROG_NAME "_dma_compl", &compl_pool_param); From dbfc5d6887d9cf3956e916d1bc8ee3e8679c6386 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Fri, 15 Sep 2023 09:22:08 +0000 Subject: [PATCH 8/8] test: dmafwd: update runner script Add test run for new DMA poll mode. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd_run.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/performance/odp_dmafwd_run.sh b/test/performance/odp_dmafwd_run.sh index 85a2c20f1c..fa629bd0c0 100755 --- a/test/performance/odp_dmafwd_run.sh +++ b/test/performance/odp_dmafwd_run.sh @@ -60,9 +60,13 @@ echo "${BIN_NAME}: SW copy" echo "===================" ./${BIN_NAME}${EXEEXT} -i ${IF0} -b ${BATCH} -T ${TIME} -t 0 check_result $? -echo "${BIN_NAME}: DMA copy" +echo "${BIN_NAME}: DMA copy event" echo "====================" ./${BIN_NAME}${EXEEXT} -i ${IF0} -b ${BATCH} -T ${TIME} -t 1 check_result $? +echo "${BIN_NAME}: DMA copy poll" +echo "====================" +./${BIN_NAME}${EXEEXT} -i ${IF0} -b ${BATCH} -T ${TIME} -t 2 +check_result $? cleanup_interfaces check_exit