Merge pull request #73 from siemdejong/streamlit-pbar

fix: show progress bar on streamlit instead of spinner

README.md

@@ -72,7 +72,6 @@ Options:
   -e, --expansion_logo_path FILE  Expansion logo path.
   -f, --format [pdf|png]          Output format.
   -w, --workers INTEGER           Number of workers.
-  -c, --chunks INTEGER            Number of chunks for the workers to process
   --help                          Show this message and exit.
 ```
 

src/app/main.py

@@ -15,7 +15,15 @@
 
 
 def create_cards(
-    name, input_file, output_dir, expansion_logo, merge, side, format, workers, chunks
+    name,
+    input_file,
+    output_dir,
+    expansion_logo,
+    merge,
+    side,
+    format,
+    workers,
+    callbacks=None,
 ):
     main(
         name=name,
@@ -26,20 +34,22 @@ def create_cards(
         side=side,
         format=format,
         workers=workers,
-        chunks=chunks,
+        callbacks=callbacks,
     )
 
+
 def zip_cards(target_file: Path, source_dir: Path):
     with zipfile.ZipFile(target_file, "w", zipfile.ZIP_DEFLATED) as zip_file:
         for entry in source_dir.rglob("*"):
             if entry.suffix == ".zip":
                 continue
             zip_file.write(entry, entry.relative_to(tmp_dir))
 
+
 st.set_page_config(
     page_title="It Happens",
     page_icon=":material/sprint:",
-    initial_sidebar_state="expanded",    
+    initial_sidebar_state="expanded",
 )
 
 with st.sidebar:
@@ -117,28 +127,47 @@ def zip_cards(target_file: Path, source_dir: Path):
             options=np.arange(1, os.cpu_count() + 1),
             value=os.cpu_count(),
         )
-        chunks = st.select_slider(
-            "Number of chunks", options=np.arange(1, len(df) + 1), value=len(df)
-        )
 
     if st.button(":material/play_arrow: Create cards", use_container_width=True):
         with tempfile.TemporaryDirectory() as tmp_dir:
             tmp_dir = Path(tmp_dir)
-            with st.spinner("Creating your It Happens playing cards..."):
-                create_cards(
-                    name=expansion_name,
-                    input_file=input_file,
-                    output_dir=tmp_dir,
-                    expansion_logo=expansion_logo,
-                    merge=merge,
-                    side=side,
-                    format=format,
-                    workers=workers,
-                    chunks=chunks,
-                )
 
-                archive = tmp_dir / "ithappens-output.zip"
-                zip_cards(archive, tmp_dir)
+            pbar_text = "Creating your It Happens playing cards..."
+
+            class PbarCallback:
+                def __init__(self, total: int, text: str):
+                    self.pbar = st.progress(0, text)
+                    self.value = 0
+                    self.stepsize = 1 / total
+                    self.text = text
+
+                def __call__(self):
+                    self.advance()
+
+                def advance(self):
+                    self.value += self.stepsize
+                    self.pbar.progress(self.value, self.text)
+
+                def empty(self):
+                    self.pbar.empty()
+
+            pbar_callback = PbarCallback(len(df), pbar_text)
+            callbacks = (pbar_callback,)
+            create_cards(
+                name=expansion_name,
+                input_file=input_file,
+                output_dir=tmp_dir,
+                expansion_logo=expansion_logo,
+                merge=merge,
+                side=side,
+                format=format,
+                workers=workers,
+                callbacks=callbacks,
+            )
+            pbar_callback.empty()
+
+            archive = tmp_dir / "ithappens-output.zip"
+            zip_cards(archive, tmp_dir)
 
             with open(archive, "rb") as zip_file_buf:
                 st.download_button(

src/ithappens/cli/__init__.py

@@ -48,14 +48,6 @@ def cli():
 @click.option(
     "-w", "--workers", "workers", type=int, default=4, help="Number of workers."
 )
-@click.option(
-    "-c",
-    "--chunks",
-    "chunks",
-    type=int,
-    default=30,
-    help="Number of chunks for the workers to process",
-)
 def create(**kwargs):
     create_cli(**kwargs)
 

src/ithappens/create_cards.py

@@ -1,8 +1,8 @@
 import argparse
 import textwrap
 import io
+from collections.abc import Callable, Sequence
 from functools import partial
-from multiprocessing import Pool
 from pathlib import Path
 from typing import Literal, Optional, cast
 
@@ -424,10 +424,9 @@ def create_cards(
     side: Literal["front", "back", "both"],
     ext: Literal["pdf", "png"],
     workers: int,
-    chunks: int,
+    callbacks: Sequence[Callable] | None = None,
 ) -> None:
     nmax = df.shape[0]
-    chunksize = max(nmax // chunks, 1)
     create_card_par = partial(
         create_card,
         expansion_name=expansion_name,
@@ -437,14 +436,16 @@ def create_cards(
         ext=ext,
     )
     desc = "Plotting cards"
-    with Pool(workers) as p:
-        cards: list[Card] = list(
-            tqdm(
-                p.imap_unordered(create_card_par, df.iterrows(), chunksize),
-                total=nmax,
-                desc=desc,
-            )
-        )
+    from concurrent.futures import ProcessPoolExecutor, as_completed
+
+    with ProcessPoolExecutor(max_workers=workers) as executor:
+        futures = [executor.submit(create_card_par, row) for row in df.iterrows()]
+        cards = []
+        for future in tqdm(as_completed(futures), total=nmax, desc=desc):
+            card = future.result()
+            cards.append(card)
+            for callback in callbacks:
+                callback()
 
     if merge:
         with PdfPages(output_dir / "front" / "merged.pdf") as pdf:
@@ -477,6 +478,8 @@ def main(**args) -> None:
 
     df = parse_input_file(input_file)
 
+    callbacks = args.get("callbacks", None)
+
     create_cards(
         df,
         expansion_name,
@@ -486,7 +489,7 @@ def main(**args) -> None:
         args["side"],
         args["format"],
         args["workers"],
-        args["chunks"],
+        callbacks,
     )