diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 20fac97900..b36b81b6ac 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -30,7 +30,8 @@ enum { SW_COPY = 0U, - DMA_COPY + DMA_COPY_EV, + DMA_COPY_POLL }; #define DEF_CPY_TYPE SW_COPY @@ -45,7 +46,8 @@ enum { #define MAX_WORKERS (ODP_THREAD_COUNT_MAX - 1) #define MAX_PKTIO_INDEXES 1024U -#define MIN(a, b) (((a) <= (b)) ? (a) : (b)) +#define MIN(a, b) (((a) <= (b)) ? (a) : (b)) +#define MAX(a, b) (((a) >= (b)) ? (a) : (b)) #define DIV_IF(a, b) ((b) > 0U ? ((a) / (b)) : 0U) ODP_STATIC_ASSERT(MAX_IFS < UINT8_MAX, "Too large maximum interface count"); @@ -75,6 +77,8 @@ typedef struct { uint64_t buf_alloc_errs; uint64_t compl_alloc_errs; uint64_t pkt_alloc_errs; + uint64_t trs_poll_errs; + uint64_t trs_polled; uint64_t fwd_pkts; uint64_t discards; uint64_t sched_cc; @@ -89,6 +93,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_pool_t copy_pool; odp_pool_t trs_pool; odp_queue_t compl_q; + odp_stash_t inflight_stash; stats_t stats; int thr_idx; } thread_config_t; @@ -100,8 +105,25 @@ typedef struct pktio_s { uint8_t num_out_qs; } pktio_t; -typedef void (*ev_fn_t)(odp_dma_compl_t compl_ev, thread_config_t *config); -typedef void (*pkt_fn_t)(odp_packet_t pkts[], int num, pktio_t *pktio, thread_config_t *config); +typedef struct { + odp_packet_t src_pkts[MAX_BURST]; + odp_packet_t dst_pkts[MAX_BURST]; + pktio_t *pktio; + int num; +} transfer_t; + +/* Function for initializing transfer structures */ +typedef transfer_t *(*init_fn_t)(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, odp_dma_seg_t *src_segs, + odp_dma_seg_t *dst_segs, pktio_t *pktio, thread_config_t *config); +/* Function for starting transfers */ +typedef odp_bool_t (*start_fn_t)(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, thread_config_t *config); +/* Function for setting up packets for copy */ +typedef void (*pkt_fn_t)(odp_packet_t pkts[], int num, pktio_t *pktio, init_fn_t init_fn, + start_fn_t start_fn, thread_config_t *config); +/* Function for draining and tearing down inflight operations */ +typedef void (*drain_fn_t)(thread_config_t *config); typedef struct prog_config_s { uint8_t pktio_idx_map[MAX_PKTIO_INDEXES]; @@ -116,13 +138,25 @@ typedef struct prog_config_s { odp_pool_t pktio_pool; odp_pool_t copy_pool; odp_pool_t trs_pool; - ev_fn_t ev_fn; - pkt_fn_t pkt_fn; + + struct { + init_fn_t init_fn; + start_fn_t start_fn; + pkt_fn_t pkt_fn; + drain_fn_t drain_fn; + }; + + uint64_t inflight_obj_size; uint32_t burst_size; uint32_t num_pkts; uint32_t pkt_len; uint32_t cache_size; + uint32_t num_inflight; + uint32_t trs_cache_size; + uint32_t compl_cache_size; + uint32_t stash_cache_size; uint32_t time_sec; + odp_stash_type_t stash_type; int num_thrs; uint8_t num_ifs; uint8_t copy_type; @@ -134,13 +168,6 @@ typedef struct { int num; } pkt_vec_t; -typedef struct { - odp_packet_t src_pkts[MAX_BURST]; - odp_packet_t dst_pkts[MAX_BURST]; - pktio_t *pktio; - int num; -} transfer_t; - static prog_config_t *prog_conf; static void terminate(int signal ODP_UNUSED) @@ -189,6 +216,7 @@ static void init_config(prog_config_t *config) thr->dma_handle = ODP_DMA_INVALID; thr->compl_pool = ODP_POOL_INVALID; thr->compl_q = ODP_QUEUE_INVALID; + thr->inflight_stash = ODP_STASH_INVALID; } for (uint32_t i = 0U; i < MAX_IFS; ++i) @@ -216,7 +244,8 @@ static void print_usage(dynamic_defs_t *dyn_defs) "\n" " -t, --copy_type Type of copy. %u by default.\n" " 0: SW\n" - " 1: DMA\n" + " 1: DMA with event completion\n" + " 2: DMA with poll completion\n" " -b, --burst_size Copy burst size. This many packets are accumulated before\n" " copy. %u by default.\n" " -n, --num_pkts Number of packet buffers allocated for packet I/O pool.\n" @@ -224,7 +253,7 @@ static void print_usage(dynamic_defs_t *dyn_defs) " -l, --pkt_len Maximum size of packet buffers in packet I/O pool. %u by\n" " default.\n" " -c, --worker_count Amount of workers. %u by default.\n" - " -C, --cache_size Packet pool cache size. %u by default.\n" + " -C, --cache_size Maximum cache size for pools. %u by default.\n" " -T, --time_sec Time in seconds to run. 0 means infinite. %u by default.\n" " -h, --help This help.\n" "\n", DEF_CPY_TYPE, dyn_defs->burst_size, dyn_defs->num_pkts, dyn_defs->pkt_len, @@ -252,11 +281,29 @@ static void parse_interfaces(prog_config_t *config, const char *optarg) free(tmp_str); } +static odp_bool_t get_stash_capa(odp_stash_capability_t *stash_capa, odp_stash_type_t *stash_type) +{ + if (odp_stash_capability(stash_capa, ODP_STASH_TYPE_FIFO) == 0) { + *stash_type = ODP_STASH_TYPE_FIFO; + return true; + } + + if (odp_stash_capability(stash_capa, ODP_STASH_TYPE_DEFAULT) == 0) { + *stash_type = ODP_STASH_TYPE_DEFAULT; + return true; + } + + return false; +} + static parse_result_t check_options(prog_config_t *config) { - unsigned int idx = odp_pktio_max_index(); + const unsigned int idx = odp_pktio_max_index(); odp_dma_capability_t dma_capa; uint32_t burst_size; + odp_stash_capability_t stash_capa; + const uint64_t obj_size = sizeof(odp_dma_transfer_id_t); + uint64_t max_num; odp_pool_capability_t pool_capa; if (config->num_ifs == 0U) { @@ -270,7 +317,8 @@ static parse_result_t check_options(prog_config_t *config) MAX_PKTIO_INDEXES); } - if (config->copy_type != SW_COPY && config->copy_type != DMA_COPY) { + if (config->copy_type != SW_COPY && config->copy_type != DMA_COPY_EV && + config->copy_type != DMA_COPY_POLL) { ODPH_ERR("Invalid copy type: %u\n", config->copy_type); return PRS_NOK; } @@ -287,7 +335,7 @@ static parse_result_t check_options(prog_config_t *config) } if ((uint32_t)config->num_thrs > dma_capa.max_sessions) { - ODPH_ERR("Not enough DMA sessions supported: %d (max: %u)\n", config->num_thrs, + ODPH_ERR("Unsupported DMA session count: %d (max: %u)\n", config->num_thrs, dma_capa.max_sessions); return PRS_NOT_SUP; } @@ -296,27 +344,79 @@ static parse_result_t check_options(prog_config_t *config) burst_size = MIN(burst_size, MAX_BURST); if (config->burst_size == 0U || config->burst_size > burst_size) { - ODPH_ERR("Unsupported segment count for DMA: %u (min: 1, max: %u)\n", + ODPH_ERR("Invalid segment count for DMA: %u (min: 1, max: %u)\n", config->burst_size, burst_size); return PRS_NOK; } if (config->pkt_len > dma_capa.max_seg_len) { - ODPH_ERR("Unsupported packet length for DMA: %u (max: %u)\n", config->pkt_len, + ODPH_ERR("Invalid packet length for DMA: %u (max: %u)\n", config->pkt_len, dma_capa.max_seg_len); return PRS_NOK; } - if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U || !dma_capa.queue_type_sched) { - ODPH_ERR("Unsupported completion mode (mode support: %x, scheduled queue " - "support: %u\n", dma_capa.compl_mode_mask, dma_capa.queue_type_sched); - return PRS_NOT_SUP; - } + config->num_inflight = dma_capa.max_transfers; - if ((uint32_t)config->num_thrs > dma_capa.pool.max_pools) { - ODPH_ERR("Unsupported amount of completion pools: %d (max: %u)\n", - config->num_thrs, dma_capa.pool.max_pools); - return PRS_NOK; + if (config->copy_type == DMA_COPY_EV) { + if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U || + !dma_capa.queue_type_sched) { + ODPH_ERR("Unsupported DMA completion mode: event (mode support: %x, " + "scheduled queue support: %u)\n", dma_capa.compl_mode_mask, + dma_capa.queue_type_sched); + return PRS_NOT_SUP; + } + + if ((uint32_t)config->num_thrs > dma_capa.pool.max_pools) { + ODPH_ERR("Invalid amount of DMA completion pools: %d (max: %u)\n", + config->num_thrs, dma_capa.pool.max_pools); + return PRS_NOK; + } + + if (config->num_inflight > dma_capa.pool.max_num) { + ODPH_ERR("Invalid amount of DMA completion events: %u (max: %u)\n", + config->num_inflight, dma_capa.pool.max_num); + return PRS_NOK; + } + } else if (config->copy_type == DMA_COPY_POLL) { + if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_POLL) == 0U) { + ODPH_ERR("Unsupported DMA completion mode: poll (mode support: %x)\n", + dma_capa.compl_mode_mask); + return PRS_NOT_SUP; + } + + if (!get_stash_capa(&stash_capa, &config->stash_type)) { + ODPH_ERR("Error querying stash capabilities\n"); + return PRS_NOK; + } + + if ((uint32_t)config->num_thrs > stash_capa.max_stashes) { + ODPH_ERR("Invalid amount of stashes: %d (max: %u)\n", config->num_thrs, + stash_capa.max_stashes); + return PRS_NOK; + } + + if (obj_size == sizeof(uint8_t)) { + max_num = stash_capa.max_num.u8; + } else if (obj_size == sizeof(uint16_t)) { + max_num = stash_capa.max_num.u16; + } else if (obj_size <= sizeof(uint32_t)) { + max_num = stash_capa.max_num.u32; + } else if (obj_size <= sizeof(uint64_t)) { + max_num = stash_capa.max_num.u64; + } else if (obj_size <= sizeof(odp_u128_t)) { + max_num = stash_capa.max_num.u128; + } else { + ODPH_ERR("Invalid stash object size: %" PRIu64 "\n", obj_size); + return PRS_NOK; + } + + if (config->num_inflight > max_num) { + ODPH_ERR("Invalid stash size: %u (max: %" PRIu64 ")\n", + config->num_inflight, max_num); + return PRS_NOK; + } + + config->inflight_obj_size = obj_size; } if (odp_pool_capability(&pool_capa) < 0) { @@ -345,6 +445,18 @@ static parse_result_t check_options(prog_config_t *config) return PRS_NOK; } + if (config->num_inflight > pool_capa.buf.max_num) { + ODPH_ERR("Invalid pool buffer count: %u (max: %u)\n", config->num_inflight, + pool_capa.buf.max_num); + return PRS_NOK; + } + + config->trs_cache_size = MIN(MAX(config->cache_size, pool_capa.buf.min_cache_size), + pool_capa.buf.max_cache_size); + config->compl_cache_size = MIN(MAX(config->cache_size, dma_capa.pool.min_cache_size), + dma_capa.pool.max_cache_size); + config->stash_cache_size = MIN(config->cache_size, stash_capa.max_cache_size); + return PRS_OK; } @@ -441,6 +553,7 @@ static inline int send_packets(odp_pktout_queue_t queue, odp_packet_t pkts[], in } static void sw_copy_and_send_packets(odp_packet_t pkts[], int num, pktio_t *pktio, + init_fn_t init_fn ODP_UNUSED, start_fn_t start_fn ODP_UNUSED, thread_config_t *config) { odp_packet_t old_pkt, new_pkt; @@ -469,46 +582,58 @@ static void sw_copy_and_send_packets(odp_packet_t pkts[], int num, pktio_t *pkti } } -static inline void send_dma_trs_packets(odp_dma_compl_t compl_ev, thread_config_t *config) +static transfer_t *init_dma_ev_trs(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, odp_dma_seg_t *src_segs, + odp_dma_seg_t *dst_segs, pktio_t *pktio, + thread_config_t *config) { - odp_dma_result_t res; odp_buffer_t buf; - transfer_t *trs; - pktio_t *pktio; - int num_sent; stats_t *stats = &config->stats; + transfer_t *trs; + odp_dma_compl_t c_ev; + + buf = odp_buffer_alloc(config->trs_pool); + + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) { + ++stats->buf_alloc_errs; + return NULL; + } - memset(&res, 0, sizeof(res)); - odp_dma_compl_result(compl_ev, &res); - buf = (odp_buffer_t)res.user_ptr; trs = (transfer_t *)odp_buffer_addr(buf); - pktio = trs->pktio; + trs->num = 0; + trs->pktio = pktio; + trs_param->src_format = ODP_DMA_FORMAT_PACKET; + trs_param->dst_format = ODP_DMA_FORMAT_PACKET; + trs_param->num_src = 0U; + trs_param->num_dst = 0U; + trs_param->src_seg = src_segs; + trs_param->dst_seg = dst_segs; + compl_param->compl_mode = ODP_DMA_COMPL_EVENT; + c_ev = odp_dma_compl_alloc(config->compl_pool); - if (res.success) { - num_sent = send_packets(pktio->out_qs[config->thr_idx % pktio->num_out_qs], - trs->dst_pkts, trs->num); - ++stats->trs; - stats->fwd_pkts += num_sent; - stats->discards += trs->num - num_sent; - } else { - odp_packet_free_multi(trs->dst_pkts, trs->num); - ++stats->trs_errs; + if (odp_unlikely(c_ev == ODP_DMA_COMPL_INVALID)) { + odp_buffer_free(buf); + ++stats->compl_alloc_errs; + return NULL; } - odp_packet_free_multi(trs->src_pkts, trs->num); - odp_buffer_free(buf); - odp_dma_compl_free(compl_ev); + compl_param->event = odp_dma_compl_to_event(c_ev); + compl_param->queue = config->compl_q; + compl_param->user_ptr = buf; + memset(src_segs, 0, sizeof(*src_segs) * MAX_BURST); + memset(dst_segs, 0, sizeof(*dst_segs) * MAX_BURST); + + return trs; } -static inline transfer_t *init_dma_trs(odp_dma_transfer_param_t *trs_param, - odp_dma_compl_param_t *compl_param, odp_dma_seg_t *src_segs, - odp_dma_seg_t *dst_segs, pktio_t *pktio, - thread_config_t *config) +static transfer_t *init_dma_poll_trs(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, odp_dma_seg_t *src_segs, + odp_dma_seg_t *dst_segs, pktio_t *pktio, + thread_config_t *config) { odp_buffer_t buf; stats_t *stats = &config->stats; transfer_t *trs; - odp_dma_compl_t c_ev; buf = odp_buffer_alloc(config->trs_pool); @@ -526,17 +651,15 @@ static inline transfer_t *init_dma_trs(odp_dma_transfer_param_t *trs_param, trs_param->num_dst = 0U; trs_param->src_seg = src_segs; trs_param->dst_seg = dst_segs; - compl_param->compl_mode = ODP_DMA_COMPL_EVENT; - c_ev = odp_dma_compl_alloc(config->compl_pool); + compl_param->compl_mode = ODP_DMA_COMPL_POLL; + compl_param->transfer_id = odp_dma_transfer_id_alloc(config->dma_handle); - if (odp_unlikely(c_ev == ODP_DMA_COMPL_INVALID)) { + if (odp_unlikely(compl_param->transfer_id == ODP_DMA_TRANSFER_ID_INVALID)) { odp_buffer_free(buf); ++stats->compl_alloc_errs; return NULL; } - compl_param->event = odp_dma_compl_to_event(c_ev); - compl_param->queue = config->compl_q; compl_param->user_ptr = buf; memset(src_segs, 0, sizeof(*src_segs) * MAX_BURST); memset(dst_segs, 0, sizeof(*dst_segs) * MAX_BURST); @@ -544,7 +667,40 @@ static inline transfer_t *init_dma_trs(odp_dma_transfer_param_t *trs_param, return trs; } -static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, thread_config_t *config) +static odp_bool_t start_dma_ev_trs(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, thread_config_t *config) +{ + const int ret = odp_dma_transfer_start(config->dma_handle, trs_param, compl_param); + + if (odp_unlikely(ret <= 0)) { + odp_buffer_free(compl_param->user_ptr); + odp_event_free(compl_param->event); + return false; + } + + return true; +} + +static odp_bool_t start_dma_poll_trs(odp_dma_transfer_param_t *trs_param, + odp_dma_compl_param_t *compl_param, thread_config_t *config) +{ + const int ret = odp_dma_transfer_start(config->dma_handle, trs_param, compl_param); + + if (odp_unlikely(ret <= 0)) { + odp_buffer_free(compl_param->user_ptr); + odp_dma_transfer_id_free(config->dma_handle, compl_param->transfer_id); + return false; + } + + if (odp_unlikely(odp_stash_put(config->inflight_stash, &compl_param->transfer_id, 1) != 1)) + /* Should not happen, but make it visible if it somehow does */ + ODPH_ABORT("DMA inflight transfer stash overflow, aborting"); + + return true; +} + +static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, init_fn_t init_fn, + start_fn_t start_fn, thread_config_t *config) { odp_dma_transfer_param_t trs_param; odp_dma_compl_param_t compl_param; @@ -562,8 +718,7 @@ static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, thread_config pkt = pkts[i]; if (odp_unlikely(trs == NULL)) { - trs = init_dma_trs(&trs_param, &compl_param, src_segs, dst_segs, pktio, - config); + trs = init_fn(&trs_param, &compl_param, src_segs, dst_segs, pktio, config); if (trs == NULL) { odp_packet_free(pkt); @@ -592,25 +747,87 @@ static void dma_copy(odp_packet_t pkts[], int num, pktio_t *pktio, thread_config } if (num_segs > 0U) - if (odp_dma_transfer_start(config->dma_handle, &trs_param, &compl_param) <= 0) { + if (odp_unlikely(!start_fn(&trs_param, &compl_param, config))) { odp_packet_free_multi(trs->src_pkts, trs->num); odp_packet_free_multi(trs->dst_pkts, trs->num); ++stats->start_errs; } } +static void drain_events(thread_config_t *config ODP_UNUSED) +{ + odp_event_t ev; + odp_event_type_t type; + odp_dma_result_t res; + odp_buffer_t buf; + transfer_t *trs; + + while (true) { + ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS * 2U)); + + if (ev == ODP_EVENT_INVALID) + break; + + type = odp_event_type(ev); + + if (type == ODP_EVENT_DMA_COMPL) { + memset(&res, 0, sizeof(res)); + odp_dma_compl_result(odp_dma_compl_from_event(ev), &res); + buf = (odp_buffer_t)res.user_ptr; + trs = (transfer_t *)odp_buffer_addr(buf); + odp_packet_free_multi(trs->src_pkts, trs->num); + odp_packet_free_multi(trs->dst_pkts, trs->num); + odp_buffer_free(buf); + } + + odp_event_free(ev); + } +} + +static void drain_polled(thread_config_t *config) +{ + odp_dma_transfer_id_t id; + odp_dma_result_t res; + int ret; + odp_buffer_t buf; + transfer_t *trs; + + while (true) { + if (odp_stash_get(config->inflight_stash, &id, 1) != 1) + break; + + memset(&res, 0, sizeof(res)); + + do { + ret = odp_dma_transfer_done(config->dma_handle, id, &res); + } while (ret == 0); + + odp_dma_transfer_id_free(config->dma_handle, id); + + if (ret < 0) + continue; + + buf = (odp_buffer_t)res.user_ptr; + trs = (transfer_t *)odp_buffer_addr(buf); + odp_packet_free_multi(trs->src_pkts, trs->num); + odp_packet_free_multi(trs->dst_pkts, trs->num); + odp_buffer_free(buf); + } +} + static odp_bool_t setup_copy(prog_config_t *config) { odp_pool_param_t pool_param; thread_config_t *thr; - const odp_dma_param_t dma_params = { + const odp_dma_param_t dma_param = { .direction = ODP_DMA_MAIN_TO_MAIN, .type = ODP_DMA_TYPE_COPY, - .compl_mode_mask = ODP_DMA_COMPL_EVENT, + .compl_mode_mask = ODP_DMA_COMPL_EVENT | ODP_DMA_COMPL_POLL, .mt_mode = ODP_DMA_MT_SERIAL, .order = ODP_DMA_ORDER_NONE }; odp_dma_pool_param_t compl_pool_param; odp_queue_param_t queue_param; + odp_stash_param_t stash_param; odp_pool_param_init(&pool_param); pool_param.pkt.seg_len = config->pkt_len; @@ -626,7 +843,6 @@ static odp_bool_t setup_copy(prog_config_t *config) } if (config->copy_type == SW_COPY) { - config->ev_fn = NULL; config->pkt_fn = sw_copy_and_send_packets; for (int i = 0; i < config->num_thrs; ++i) @@ -635,8 +851,9 @@ static odp_bool_t setup_copy(prog_config_t *config) return true; } - pool_param.buf.num = config->num_pkts; + pool_param.buf.num = config->num_inflight; pool_param.buf.size = sizeof(transfer_t); + pool_param.buf.cache_size = config->trs_cache_size; pool_param.type = ODP_POOL_BUFFER; config->trs_pool = odp_pool_create(PROG_NAME "_dma_trs", &pool_param); @@ -647,37 +864,62 @@ static odp_bool_t setup_copy(prog_config_t *config) for (int i = 0; i < config->num_thrs; ++i) { thr = &config->thread_config[i]; - thr->dma_handle = odp_dma_create(PROG_NAME "_dma", &dma_params); + thr->copy_pool = config->copy_pool; + thr->trs_pool = config->trs_pool; + thr->dma_handle = odp_dma_create(PROG_NAME "_dma", &dma_param); if (thr->dma_handle == ODP_DMA_INVALID) { ODPH_ERR("Error creating DMA session\n"); return false; } - odp_dma_pool_param_init(&compl_pool_param); - compl_pool_param.num = config->num_pkts; - thr->compl_pool = odp_dma_pool_create(PROG_NAME "_dma_compl", &compl_pool_param); + if (config->copy_type == DMA_COPY_EV) { + odp_dma_pool_param_init(&compl_pool_param); + compl_pool_param.num = config->num_inflight; + compl_pool_param.cache_size = config->compl_cache_size; + thr->compl_pool = odp_dma_pool_create(PROG_NAME "_dma_compl", + &compl_pool_param); - if (thr->compl_pool == ODP_POOL_INVALID) { - ODPH_ERR("Error creating DMA event completion pool\n"); - return false; - } + if (thr->compl_pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating DMA event completion pool\n"); + return false; + } - thr->copy_pool = config->copy_pool; - thr->trs_pool = config->trs_pool; - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; - queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; - queue_param.sched.prio = odp_schedule_max_prio(); - thr->compl_q = odp_queue_create(PROG_NAME "_dma_compl", &queue_param); - - if (thr->compl_q == ODP_QUEUE_INVALID) { - ODPH_ERR("Error creating DMA completion queue\n"); - return false; + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; + queue_param.sched.prio = odp_schedule_max_prio(); + thr->compl_q = odp_queue_create(PROG_NAME "_dma_compl", &queue_param); + + if (thr->compl_q == ODP_QUEUE_INVALID) { + ODPH_ERR("Error creating DMA completion queue\n"); + return false; + } + + config->init_fn = init_dma_ev_trs; + config->start_fn = start_dma_ev_trs; + config->drain_fn = drain_events; + } else { + odp_stash_param_init(&stash_param); + stash_param.type = config->stash_type; + stash_param.put_mode = ODP_STASH_OP_LOCAL; + stash_param.get_mode = ODP_STASH_OP_LOCAL; + stash_param.num_obj = config->num_inflight; + stash_param.obj_size = config->inflight_obj_size; + stash_param.cache_size = config->stash_cache_size; + thr->inflight_stash = odp_stash_create("_dma_inflight", &stash_param); + + if (thr->inflight_stash == ODP_STASH_INVALID) { + ODPH_ERR("Error creating DMA inflight transfer stash\n"); + return false; + } + + config->init_fn = init_dma_poll_trs; + config->start_fn = start_dma_poll_trs; + config->drain_fn = drain_polled; } } - config->ev_fn = send_dma_trs_packets; config->pkt_fn = dma_copy; return true; @@ -771,6 +1013,101 @@ static odp_bool_t setup_pktios(prog_config_t *config) return true; } +static inline void send_dma_poll_trs_pkts(int burst_size, thread_config_t *config) +{ + odp_stash_t stash_handle = config->inflight_stash; + odp_dma_transfer_id_t ids[burst_size], id; + int32_t num; + odp_dma_t dma_handle = config->dma_handle; + odp_dma_result_t res; + int ret; + odp_buffer_t buf; + transfer_t *trs; + pktio_t *pktio; + int num_sent; + stats_t *stats = &config->stats; + + while (true) { + num = odp_stash_get(stash_handle, &ids, burst_size); + + if (num <= 0) + break; + + for (int32_t i = 0; i < num; ++i) { + id = ids[i]; + ret = odp_dma_transfer_done(dma_handle, id, &res); + + if (ret == 0) { + if (odp_unlikely(odp_stash_put(stash_handle, &id, 1) != 1)) + /* Should not happen, but make it visible if it somehow + * does */ + ODPH_ABORT("DMA inflight transfer stash overflow," + " aborting"); + + ++stats->trs_polled; + continue; + } + + odp_dma_transfer_id_free(dma_handle, id); + + if (ret < 0) { + ++stats->trs_poll_errs; + continue; + } + + buf = (odp_buffer_t)res.user_ptr; + trs = (transfer_t *)odp_buffer_addr(buf); + + if (res.success) { + pktio = trs->pktio; + num_sent = send_packets(pktio->out_qs[config->thr_idx % + pktio->num_out_qs], + trs->dst_pkts, trs->num); + ++stats->trs; + stats->fwd_pkts += num_sent; + stats->discards += trs->num - num_sent; + } else { + odp_packet_free_multi(trs->dst_pkts, trs->num); + ++stats->trs_errs; + } + + odp_packet_free_multi(trs->src_pkts, trs->num); + odp_buffer_free(buf); + } + } +} + +static inline void send_dma_ev_trs_pkts(odp_dma_compl_t compl_ev, thread_config_t *config) +{ + odp_dma_result_t res; + odp_buffer_t buf; + transfer_t *trs; + pktio_t *pktio; + int num_sent; + stats_t *stats = &config->stats; + + memset(&res, 0, sizeof(res)); + odp_dma_compl_result(compl_ev, &res); + buf = (odp_buffer_t)res.user_ptr; + trs = (transfer_t *)odp_buffer_addr(buf); + + if (res.success) { + pktio = trs->pktio; + num_sent = send_packets(pktio->out_qs[config->thr_idx % pktio->num_out_qs], + trs->dst_pkts, trs->num); + ++stats->trs; + stats->fwd_pkts += num_sent; + stats->discards += trs->num - num_sent; + } else { + odp_packet_free_multi(trs->dst_pkts, trs->num); + ++stats->trs_errs; + } + + odp_packet_free_multi(trs->src_pkts, trs->num); + odp_buffer_free(buf); + odp_dma_compl_free(compl_ev); +} + static inline void push_packet(odp_packet_t pkt, pkt_vec_t pkt_vecs[], uint8_t *pktio_idx_map) { uint8_t idx = pktio_idx_map[odp_packet_input_index(pkt)]; @@ -793,36 +1130,6 @@ static void free_pending_packets(pkt_vec_t pkt_vecs[], uint32_t num_ifs) odp_packet_free_multi(pkt_vecs[i].pkts, pkt_vecs[i].num); } -static void drain(void) -{ - odp_event_t ev; - odp_event_type_t type; - odp_dma_result_t res; - odp_buffer_t buf; - transfer_t *trs; - - while (true) { - ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS * 2U)); - - if (ev == ODP_EVENT_INVALID) - break; - - type = odp_event_type(ev); - - if (type == ODP_EVENT_DMA_COMPL) { - memset(&res, 0, sizeof(res)); - odp_dma_compl_result(odp_dma_compl_from_event(ev), &res); - buf = (odp_buffer_t)res.user_ptr; - trs = (transfer_t *)odp_buffer_addr(buf); - odp_packet_free_multi(trs->src_pkts, trs->num); - odp_packet_free_multi(trs->dst_pkts, trs->num); - odp_buffer_free(buf); - } - - odp_event_free(ev); - } -} - static int process_packets(void *args) { thread_config_t *config = args; @@ -830,14 +1137,16 @@ static int process_packets(void *args) pkt_vec_t pkt_vecs[num_ifs], *pkt_vec; odp_atomic_u32_t *is_running = &config->prog_config->is_running; uint64_t c1, c2, c3, c4, cdiff = 0U, rounds = 0U; + const uint8_t copy_type = config->prog_config->copy_type; const int burst_size = config->prog_config->burst_size; odp_event_t evs[burst_size]; int num_evs; odp_event_t ev; odp_event_type_t type; - ev_fn_t ev_fn = config->prog_config->ev_fn; uint8_t *pktio_map = config->prog_config->pktio_idx_map; stats_t *stats = &config->stats; + init_fn_t init_fn = config->prog_config->init_fn; + start_fn_t start_fn = config->prog_config->start_fn; pkt_fn_t pkt_fn = config->prog_config->pkt_fn; for (uint32_t i = 0U; i < num_ifs; ++i) { @@ -856,6 +1165,9 @@ static int process_packets(void *args) cdiff += odp_cpu_cycles_diff(c4, c3); ++rounds; + if (copy_type == DMA_COPY_POLL) + send_dma_poll_trs_pkts(burst_size, config); + if (num_evs == 0) continue; @@ -864,8 +1176,7 @@ static int process_packets(void *args) type = odp_event_type(ev); if (type == ODP_EVENT_DMA_COMPL) { - if (ev_fn) - ev_fn(odp_dma_compl_from_event(ev), config); + send_dma_ev_trs_pkts(odp_dma_compl_from_event(ev), config); } else if (type == ODP_EVENT_PACKET) { push_packet(odp_packet_from_event(ev), pkt_vecs, pktio_map); } else { @@ -878,7 +1189,8 @@ static int process_packets(void *args) pkt_vec = &pkt_vecs[i]; if (pkt_vec->num >= burst_size) { - pkt_fn(pkt_vec->pkts, burst_size, pkt_vec->pktio, config); + pkt_fn(pkt_vec->pkts, burst_size, pkt_vec->pktio, init_fn, + start_fn, config); pop_packets(pkt_vec, burst_size); } } @@ -890,7 +1202,9 @@ static int process_packets(void *args) stats->sched_rounds = rounds; free_pending_packets(pkt_vecs, num_ifs); odp_barrier_wait(&config->prog_config->term_barrier); - drain(); + + if (config->prog_config->drain_fn) + config->prog_config->drain_fn(config); return 0; } @@ -971,6 +1285,9 @@ static void teardown(prog_config_t *config) for (int i = 0; i < config->num_thrs; ++i) { thr = &config->thread_config[i]; + if (thr->inflight_stash != ODP_STASH_INVALID) + (void)odp_stash_destroy(thr->inflight_stash); + if (thr->compl_q != ODP_QUEUE_INVALID) (void)odp_queue_destroy(thr->compl_q); @@ -991,14 +1308,18 @@ static void teardown(prog_config_t *config) static void print_stats(const prog_config_t *config) { const stats_t *stats; - const char *align = config->copy_type == SW_COPY ? " " : " "; + const char *align1 = config->copy_type == DMA_COPY_EV ? " " : ""; + const char *align2 = config->copy_type == SW_COPY ? " " : + config->copy_type == DMA_COPY_EV ? " " : + " "; printf("\n==================\n\n" "DMA forwarder done\n\n" " copy mode: %s\n" " burst size: %u\n" " packet length: %u\n" - " pool cache size: %u\n", config->copy_type == SW_COPY ? "SW" : "DMA", + " max cache size: %u\n", config->copy_type == SW_COPY ? "SW" : + config->copy_type == DMA_COPY_EV ? "DMA-event" : "DMA-poll", config->burst_size, config->pkt_len, config->cache_size); for (int i = 0; i < config->num_thrs; ++i) { @@ -1010,15 +1331,24 @@ static void print_stats(const prog_config_t *config) printf(" packet copy errors: %" PRIu64 "\n", stats->copy_errs); } else { - printf(" successful DMA transfers: %" PRIu64 "\n" - " DMA transfer start errors: %" PRIu64 "\n" - " DMA transfer errors: %" PRIu64 "\n" - " transfer buffer allocation errors: %" PRIu64 "\n" - " completion event allocation errors: %" PRIu64 "\n" - " copy packet allocation errors: %" PRIu64 "\n", - stats->trs, stats->start_errs, stats->trs_errs, - stats->buf_alloc_errs, stats->compl_alloc_errs, + printf(" successful DMA transfers: %s%" PRIu64 "\n" + " DMA transfer start errors: %s%" PRIu64 "\n" + " DMA transfer errors: %s%" PRIu64 "\n" + " transfer buffer allocation errors: %s%" PRIu64 "\n" + " copy packet allocation errors: %s%" PRIu64 "\n", + align1, stats->trs, align1, stats->start_errs, align1, + stats->trs_errs, align1, stats->buf_alloc_errs, align1, stats->pkt_alloc_errs); + + if (config->copy_type == DMA_COPY_EV) + printf(" completion event allocation errors: %" PRIu64 "\n", + stats->compl_alloc_errs); + else + printf(" transfer ID allocation errors: %" PRIu64 "\n" + " transfer poll errors: %" PRIu64 "\n" + " transfers polled: %" PRIu64 "\n", + stats->compl_alloc_errs, stats->trs_poll_errs, + stats->trs_polled); } printf(" packets forwarded:%s%" PRIu64 "\n" @@ -1026,7 +1356,7 @@ static void print_stats(const prog_config_t *config) " call cycles per schedule round:\n" " total: %" PRIu64 "\n" " schedule: %" PRIu64 "\n" - " rounds: %" PRIu64 "\n", align, stats->fwd_pkts, align, + " rounds: %" PRIu64 "\n", align2, stats->fwd_pkts, align2, stats->discards, DIV_IF(stats->tot_cc, stats->sched_rounds), DIV_IF(stats->sched_cc, stats->sched_rounds), stats->sched_rounds); } @@ -1054,12 +1384,12 @@ int main(int argc, char **argv) init_param.mem_model = odph_opts.mem_model; if (odp_init_global(&odp_instance, NULL, NULL)) { - ODPH_ERR("ODP global init failed, exiting.\n"); + ODPH_ERR("ODP global init failed, exiting\n"); exit(EXIT_FAILURE); } if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) { - ODPH_ERR("ODP local init failed, exiting.\n"); + ODPH_ERR("ODP local init failed, exiting\n"); exit(EXIT_FAILURE); } @@ -1130,12 +1460,12 @@ int main(int argc, char **argv) (void)odp_shm_free(shm_cfg); if (odp_term_local()) { - ODPH_ERR("ODP local terminate failed, exiting.\n"); + ODPH_ERR("ODP local terminate failed, exiting\n"); exit(EXIT_FAILURE); } if (odp_term_global(odp_instance)) { - ODPH_ERR("ODP global terminate failed, exiting.\n"); + ODPH_ERR("ODP global terminate failed, exiting\n"); exit(EXIT_FAILURE); } diff --git a/test/performance/odp_dmafwd_run.sh b/test/performance/odp_dmafwd_run.sh index 85a2c20f1c..fa629bd0c0 100755 --- a/test/performance/odp_dmafwd_run.sh +++ b/test/performance/odp_dmafwd_run.sh @@ -60,9 +60,13 @@ echo "${BIN_NAME}: SW copy" echo "===================" ./${BIN_NAME}${EXEEXT} -i ${IF0} -b ${BATCH} -T ${TIME} -t 0 check_result $? -echo "${BIN_NAME}: DMA copy" +echo "${BIN_NAME}: DMA copy event" echo "====================" ./${BIN_NAME}${EXEEXT} -i ${IF0} -b ${BATCH} -T ${TIME} -t 1 check_result $? +echo "${BIN_NAME}: DMA copy poll" +echo "====================" +./${BIN_NAME}${EXEEXT} -i ${IF0} -b ${BATCH} -T ${TIME} -t 2 +check_result $? cleanup_interfaces check_exit