From c6900ec1b03d132affa83f22916f2e3f29a4dcf5 Mon Sep 17 00:00:00 2001 From: siemdejong <28396796+siemdejong@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:33:14 +0200 Subject: [PATCH] fix: show progress bar on streamlit instead of spinner fixes #67 --- README.md | 1 - src/app/main.py | 69 +++++++++++++++++++++++++---------- src/ithappens/cli/__init__.py | 8 ---- src/ithappens/create_cards.py | 27 ++++++++------ 4 files changed, 64 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index f1a6bf7..27008b0 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,6 @@ Options: -e, --expansion_logo_path FILE Expansion logo path. -f, --format [pdf|png] Output format. -w, --workers INTEGER Number of workers. - -c, --chunks INTEGER Number of chunks for the workers to process --help Show this message and exit. ``` diff --git a/src/app/main.py b/src/app/main.py index 373d608..c4b06ad 100644 --- a/src/app/main.py +++ b/src/app/main.py @@ -15,7 +15,15 @@ def create_cards( - name, input_file, output_dir, expansion_logo, merge, side, format, workers, chunks + name, + input_file, + output_dir, + expansion_logo, + merge, + side, + format, + workers, + callbacks=None, ): main( name=name, @@ -26,9 +34,10 @@ def create_cards( side=side, format=format, workers=workers, - chunks=chunks, + callbacks=callbacks, ) + def zip_cards(target_file: Path, source_dir: Path): with zipfile.ZipFile(target_file, "w", zipfile.ZIP_DEFLATED) as zip_file: for entry in source_dir.rglob("*"): @@ -36,10 +45,11 @@ def zip_cards(target_file: Path, source_dir: Path): continue zip_file.write(entry, entry.relative_to(tmp_dir)) + st.set_page_config( page_title="It Happens", page_icon=":material/sprint:", - initial_sidebar_state="expanded", + initial_sidebar_state="expanded", ) with st.sidebar: @@ -117,28 +127,47 @@ def zip_cards(target_file: Path, source_dir: Path): options=np.arange(1, os.cpu_count() + 1), value=os.cpu_count(), ) - chunks = st.select_slider( - "Number of chunks", options=np.arange(1, len(df) + 1), value=len(df) - ) if st.button(":material/play_arrow: Create cards", use_container_width=True): with tempfile.TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) - with st.spinner("Creating your It Happens playing cards..."): - create_cards( - name=expansion_name, - input_file=input_file, - output_dir=tmp_dir, - expansion_logo=expansion_logo, - merge=merge, - side=side, - format=format, - workers=workers, - chunks=chunks, - ) - archive = tmp_dir / "ithappens-output.zip" - zip_cards(archive, tmp_dir) + pbar_text = "Creating your It Happens playing cards..." + + class PbarCallback: + def __init__(self, total: int, text: str): + self.pbar = st.progress(0, text) + self.value = 0 + self.stepsize = 1 / total + self.text = text + + def __call__(self): + self.advance() + + def advance(self): + self.value += self.stepsize + self.pbar.progress(self.value, self.text) + + def empty(self): + self.pbar.empty() + + pbar_callback = PbarCallback(len(df), pbar_text) + callbacks = (pbar_callback,) + create_cards( + name=expansion_name, + input_file=input_file, + output_dir=tmp_dir, + expansion_logo=expansion_logo, + merge=merge, + side=side, + format=format, + workers=workers, + callbacks=callbacks, + ) + pbar_callback.empty() + + archive = tmp_dir / "ithappens-output.zip" + zip_cards(archive, tmp_dir) with open(archive, "rb") as zip_file_buf: st.download_button( diff --git a/src/ithappens/cli/__init__.py b/src/ithappens/cli/__init__.py index a60b530..6805ef8 100644 --- a/src/ithappens/cli/__init__.py +++ b/src/ithappens/cli/__init__.py @@ -48,14 +48,6 @@ def cli(): @click.option( "-w", "--workers", "workers", type=int, default=4, help="Number of workers." ) -@click.option( - "-c", - "--chunks", - "chunks", - type=int, - default=30, - help="Number of chunks for the workers to process", -) def create(**kwargs): create_cli(**kwargs) diff --git a/src/ithappens/create_cards.py b/src/ithappens/create_cards.py index 7f66f0a..9162f7a 100644 --- a/src/ithappens/create_cards.py +++ b/src/ithappens/create_cards.py @@ -1,8 +1,8 @@ import argparse import textwrap import io +from collections.abc import Callable, Sequence from functools import partial -from multiprocessing import Pool from pathlib import Path from typing import Literal, Optional, cast @@ -424,10 +424,9 @@ def create_cards( side: Literal["front", "back", "both"], ext: Literal["pdf", "png"], workers: int, - chunks: int, + callbacks: Sequence[Callable] | None = None, ) -> None: nmax = df.shape[0] - chunksize = max(nmax // chunks, 1) create_card_par = partial( create_card, expansion_name=expansion_name, @@ -437,14 +436,16 @@ def create_cards( ext=ext, ) desc = "Plotting cards" - with Pool(workers) as p: - cards: list[Card] = list( - tqdm( - p.imap_unordered(create_card_par, df.iterrows(), chunksize), - total=nmax, - desc=desc, - ) - ) + from concurrent.futures import ProcessPoolExecutor, as_completed + + with ProcessPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(create_card_par, row) for row in df.iterrows()] + cards = [] + for future in tqdm(as_completed(futures), total=nmax, desc=desc): + card = future.result() + cards.append(card) + for callback in callbacks: + callback() if merge: with PdfPages(output_dir / "front" / "merged.pdf") as pdf: @@ -477,6 +478,8 @@ def main(**args) -> None: df = parse_input_file(input_file) + callbacks = args.get("callbacks", None) + create_cards( df, expansion_name, @@ -486,7 +489,7 @@ def main(**args) -> None: args["side"], args["format"], args["workers"], - args["chunks"], + callbacks, )