From 4d590405e7ab8fbebb2f2e938aebbeaca34ca63e Mon Sep 17 00:00:00 2001 From: Dan Snow Date: Sat, 30 Nov 2024 19:32:48 -0600 Subject: [PATCH] Use pandas iloc values for chunks and splitting --- data/src/utils/times.py | 2 +- data/src/utils/utils.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/src/utils/times.py b/data/src/utils/times.py index 3a04e0a..a2d5dbd 100644 --- a/data/src/utils/times.py +++ b/data/src/utils/times.py @@ -277,7 +277,7 @@ def _set_chunk_attributes(self) -> None: if self.chunk: chunk_start_idx, chunk_end_idx = self.chunk.split("-") self.chunk_start_idx = int(chunk_start_idx) - self.chunk_end_idx = int(chunk_end_idx) + 1 + self.chunk_end_idx = int(chunk_end_idx) self.chunk_size = self.chunk_end_idx - self.chunk_start_idx def _subset_origins(self) -> None: diff --git a/data/src/utils/utils.py b/data/src/utils/utils.py index 8a1602f..9f7752b 100644 --- a/data/src/utils/utils.py +++ b/data/src/utils/utils.py @@ -78,18 +78,18 @@ def split_range( chunk_size = math.ceil(n / n_chunks) for i in range(n // chunk_size): start = i * chunk_size - end = ((i + 1) * chunk_size) - 1 + end = (i + 1) * chunk_size chunk_ranges.append((start, end)) else: n_chunks_small = max(1, math.ceil(n / min_chunk_size)) for i in range(n_chunks_small): start = i * min_chunk_size - end = min((i + 1) * min_chunk_size - 1, n - 1) + end = min((i + 1) * min_chunk_size, n) chunk_ranges.append((start, end)) if chunk_ranges[-1][1] < n: start, _ = chunk_ranges[-1] - chunk_ranges[-1] = (start, n - 1) + chunk_ranges[-1] = (start, n) return chunk_ranges