From 2dc6500b2b2927b4c5a373a92b3d4d9516ec5c06 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Tue, 5 Nov 2024 17:12:10 -0600 Subject: [PATCH] Switch from `multisession` parallelization to `multicore` in `evaluate` stage (#53) * Switch from `multisession` parallelization to `multicore` in `evaluate` stage * Only use `multicore` parallelization if available in the `evaluate` stage --- pipeline/03-evaluate.R | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pipeline/03-evaluate.R b/pipeline/03-evaluate.R index 6828aaa..be1168b 100644 --- a/pipeline/03-evaluate.R +++ b/pipeline/03-evaluate.R @@ -12,8 +12,14 @@ tictoc::tic("Evaluate") # Load libraries, helpers, and recipes from files purrr::walk(list.files("R/", "\\.R$", full.names = TRUE), source) -# Enable parallel backend for generating stats more quickly -plan(multisession, workers = num_threads) +# Enable parallel backend for generating stats faster. +if (supportsMulticore()) { + # Limit to half the available cores to avoid hogging resources + plan(multicore, workers = ceiling(num_threads / 2)) +} else { + # Multisession performance begins to degrade beyond 5 workers + plan(multisession, workers = 5) +} # Renaming dictionary for input columns. We want the actual value of the column # to become geography_id and the NAME of the column to become geography_name