Array refactor with new {store,retrieve}_chunks methods

- Reexport `safe_transmute::TriviallyTransmutable` as `array::TriviallyTransmutable` - Add `Array::chunks_subset{_bounded}` - Add `store_chunks`, `retrieve_chunks`, `erase_chunks` and variants to `Array` - Use macros to reduce common code patterns in `Array` - Separate `Array` methods into separate files for each storage trait - **Breaking**: Remove `_opt` and `par_` variants of `async_retrieve_array_subset` and `async_store_array_subset` (including `_elements` and `_ndarray` variants)
LDeakin · Jan 2, 2024 · 45ac10c · 45ac10c
1 parent 8295f28
commit 45ac10c
Show file tree

Hide file tree

Showing 11 changed files with 2,876 additions and 2,532 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+ - Reexport `safe_transmute::TriviallyTransmutable` as `array::TriviallyTransmutable`
+ - Add `Array::chunks_subset{_bounded}`
+ - Add `store_chunks`, `retrieve_chunks`, `erase_chunks` and variants to `Array`
+
+### Changed
+ - Use macros to reduce common code patterns in `Array`
+ - Separate `Array` methods into separate files for each storage trait
+ - **Breaking**: Remove `_opt` and `par_` variants of `async_retrieve_array_subset` and `async_store_array_subset` (including `_elements` and `_ndarray` variants)
+
 ## [0.8.0] - 2023-12-26
 
 ### Highlights

diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "zarrs"
-version = "0.8.0"
+version = "0.9.0"
 authors = ["Lachlan Deakin <ljdgit@gmail.com>"]
 edition = "2021"
 rust-version = "1.71"

diff --git a/src/array.rs b/src/array.rs
@@ -6,13 +6,10 @@
 //! Use [`ArrayBuilder`] to setup a new array, or use [`Array::new`] for an existing array.
 //! The documentation for [`Array`] details how to interact with arrays.
 
-#[cfg(feature = "async")]
-mod array_async;
 mod array_builder;
 mod array_errors;
 mod array_metadata;
 mod array_representation;
-mod array_sync;
 mod bytes_representation;
 pub mod chunk_grid;
 pub mod chunk_key_encoding;
@@ -42,7 +39,8 @@ pub use self::{
     nan_representations::{ZARR_NAN_BF16, ZARR_NAN_F16, ZARR_NAN_F32, ZARR_NAN_F64},
 };
 
-use safe_transmute::TriviallyTransmutable;
+/// Re-export of [`safe_transmute::TriviallyTransmutable`].
+pub use safe_transmute::TriviallyTransmutable;
 use serde::Serialize;
 
 use crate::{
@@ -100,20 +98,26 @@ pub type MaybeBytes = Option<Vec<u8>>;
 /// #### Sync API
 /// Array operations are divided into several categories based on the traits implemented for the backing [storage](crate::storage). In summary:
 ///  - [`ReadableStorageTraits`](crate::storage::ReadableStorageTraits): read array data and metadata
+///    - [`new`](Array::new)
 ///    - [`retrieve_chunk`](Array::retrieve_chunk)
+///    - [`retrieve_chunks`](Array::retrieve_chunks)
 ///    - [`retrieve_chunk_subset`](Array::retrieve_chunk_subset)
-///    - [`retrieve_array_subset`](Array::retrieve_array_subset) / [`par_retrieve_array_subset`](Array::par_retrieve_array_subset)
+///    - [`retrieve_array_subset`](Array::retrieve_array_subset)
 ///  - [`WritableStorageTraits`](crate::storage::WritableStorageTraits): write array data and metadata
+///    - [`store_metadata`](Array::store_metadata)
 ///    - [`store_chunk`](Array::store_chunk)
+///    - [`store_chunks`](Array::store_chunks)
 ///    - [`erase_chunk`](Array::erase_chunk)
+///    - [`erase_chunks`](Array::erase_chunks)
 ///  - [`ReadableWritableStorageTraits`](crate::storage::ReadableWritableStorageTraits): perform operations requiring both reading and writing
 ///    - [`store_chunk_subset`](Array::store_chunk_subset)
-///    - [`store_array_subset`](Array::store_array_subset) / [`par_store_array_subset`](Array::par_store_array_subset)
+///    - [`store_array_subset`](Array::store_array_subset)
 ///
-/// These `retrieve` and `store` methods have multiple variants:
-///   - The above variants store or retrieve data represented as bytes.
-///   - Variants with an `_elements` suffix can read and write array elements with a known type.
-///   - With the `ndarray` feature, method variants with an `_ndarray` suffix can be used to store or retrieve [`ndarray::Array`]s.
+/// Most `retrieve` and `store` methods have multiple variants:
+///   - Standard variants store or retrieve data represented as bytes.
+///   - `_elements` suffix variants can store or retrieve chunks with a known type.
+///   - `_ndarray` suffix variants can store or retrieve [`ndarray::Array`]s (requires `ndarray` feature).
+///   - Some variants support internal parallelisation, they have `par_` prefix and `_opt` suffix variants.
 ///
 /// #### Async API
 /// With the `async` feature and an async store, there are equivalent methods to the sync API with an `async_` prefix.
@@ -417,7 +421,7 @@ impl<TStorage: ?Sized> Array<TStorage> {
         .into()
     }
 
-    /// Create an array builder matching the parameters of this array
+    /// Create an array builder matching the parameters of this array.
     #[must_use]
     pub fn builder(&self) -> ArrayBuilder {
         ArrayBuilder::from_array(self)
@@ -460,6 +464,28 @@ impl<TStorage: ?Sized> Array<TStorage> {
         Ok(unsafe { chunk_subset.bound_unchecked(self.shape()) })
     }
 
+    /// Return the array subset of `chunks`.
+    ///
+    /// # Errors
+    /// Returns [`ArrayError::InvalidChunkGridIndicesError`] if a chunk in `chunks` is incompatible with the chunk grid.
+    #[allow(clippy::similar_names)]
+    pub fn chunks_subset(&self, chunks: &ArraySubset) -> Result<ArraySubset, ArrayError> {
+        let chunk0 = self.chunk_subset(chunks.start())?;
+        let chunk1 = self.chunk_subset(&chunks.end_inc())?;
+        let start = chunk0.start();
+        let end = chunk1.end_exc();
+        Ok(unsafe { ArraySubset::new_with_start_end_exc_unchecked(start.to_vec(), end) })
+    }
+
+    /// Return the array subset of `chunks` bounded by the array shape.
+    ///
+    /// # Errors
+    /// Returns [`ArrayError::InvalidChunkGridIndicesError`] if the `chunk_indices` are incompatible with the chunk grid.
+    pub fn chunks_subset_bounded(&self, chunks: &ArraySubset) -> Result<ArraySubset, ArrayError> {
+        let chunks_subset = self.chunks_subset(chunks)?;
+        Ok(unsafe { chunks_subset.bound_unchecked(self.shape()) })
+    }
+
     /// Get the chunk array representation at `chunk_index`.
     ///
     /// # Errors
@@ -517,6 +543,81 @@ impl<TStorage: ?Sized> Array<TStorage> {
     }
 }
 
+macro_rules! array_store_elements {
+    ( $self:expr, $elements:ident, $func:ident($($arg:tt)*) ) => {
+        if $self.data_type.size() != std::mem::size_of::<T>() {
+            Err(ArrayError::IncompatibleElementSize(
+                $self.data_type.size(),
+                std::mem::size_of::<T>(),
+            ))
+        } else {
+            let $elements = safe_transmute_to_bytes_vec($elements);
+            $self.$func($($arg)*)
+        }
+    };
+}
+
+#[cfg(feature = "ndarray")]
+macro_rules! array_store_ndarray {
+    ( $self:expr, $array:ident, $func:ident($($arg:tt)*) ) => {
+        if $self.data_type.size() != std::mem::size_of::<T>() {
+            Err(ArrayError::IncompatibleElementSize(
+                $self.data_type.size(),
+                std::mem::size_of::<T>(),
+            ))
+        } else {
+            let $array = $array.as_standard_layout().into_owned().into_raw_vec();
+            $self.$func($($arg)*)
+        }
+    };
+}
+
+#[cfg(feature = "async")]
+macro_rules! array_async_store_elements {
+    ( $self:expr, $elements:ident, $func:ident($($arg:tt)*) ) => {
+        if $self.data_type.size() != std::mem::size_of::<T>() {
+            Err(ArrayError::IncompatibleElementSize(
+                $self.data_type.size(),
+                std::mem::size_of::<T>(),
+            ))
+        } else {
+            let $elements = safe_transmute_to_bytes_vec($elements);
+            $self.$func($($arg)*).await
+        }
+    };
+}
+
+#[cfg(feature = "async")]
+#[cfg(feature = "ndarray")]
+macro_rules! array_async_store_ndarray {
+    ( $self:expr, $array:ident, $func:ident($($arg:tt)*) ) => {
+        if $self.data_type.size() != std::mem::size_of::<T>() {
+            Err(ArrayError::IncompatibleElementSize(
+                $self.data_type.size(),
+                std::mem::size_of::<T>(),
+            ))
+        } else {
+            let $array = $array.as_standard_layout().into_owned().into_raw_vec();
+            $self.$func($($arg)*).await
+        }
+    };
+}
+
+mod array_sync_readable;
+
+mod array_sync_writable;
+
+mod array_sync_readable_writable;
+
+#[cfg(feature = "async")]
+mod array_async_readable;
+
+#[cfg(feature = "async")]
+mod array_async_writable;
+
+#[cfg(feature = "async")]
+mod array_async_readable_writable;
+
 // Safe transmute, avoiding an allocation where possible
 //
 // A relevant discussion about this can be found here: https://github.com/nabijaczleweli/safe-transmute-rs/issues/16#issuecomment-471066699
@@ -623,9 +724,12 @@ mod tests {
     use itertools::Itertools;
     use rayon::prelude::{IntoParallelIterator, ParallelIterator};
 
-    use crate::storage::{
-        store::MemoryStore,
-        store_lock::{DefaultStoreLocks, StoreLocks},
+    use crate::{
+        array_subset::ArraySubset,
+        storage::{
+            store::MemoryStore,
+            store_lock::{DefaultStoreLocks, StoreLocks},
+        },
     };
 
     use super::*;