From a3472f4b05b31f978faa4ff63011dfd1c623ecd6 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Sun, 24 Dec 2023 23:45:52 +1100 Subject: [PATCH] Add `ArraySubset::new_with_ranges` --- CHANGELOG.md | 1 + examples/array_write_read.rs | 10 ++--- examples/async_array_write_read.rs | 10 ++--- examples/async_http_array_read.rs | 4 +- examples/http_array_read.rs | 4 +- examples/rectangular_array_write_read.rs | 8 ++-- examples/sharded_array_write_read.rs | 6 +-- examples/zip_array_write_read.rs | 10 ++--- src/array.rs | 10 ++--- src/array/codec.rs | 11 +++-- src/array/codec/array_to_array/bitround.rs | 4 +- src/array/codec/array_to_bytes/codec_chain.rs | 3 +- src/array/codec/array_to_bytes/sharding.rs | 7 ++-- src/array/codec/array_to_bytes/zfp.rs | 6 +-- src/array/codec/bytes_to_bytes/blosc.rs | 14 +++---- src/array_subset.rs | 28 +++++++------ src/array_subset/array_subset_iterators.rs | 42 +++++++++---------- src/storage/store/store_async/http_store.rs | 6 +-- src/storage/store/store_sync/http_store.rs | 4 +- 19 files changed, 91 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf3ca19c..82e5c392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Adds generic `StoreLocks`, `StoreKeyMutex`, and `StoreKeyMutexGuard` with associated traits and async variants - Includes `DefaultStoreLocks` and `DisabledStoreLocks` implementations - Readable and writable stores include a `new_with_locks` method to choose the store lock implementation + - Added `ArraySubset::new_with_ranges` ### Changed - **Breaking** `ReadableStorageTraits` is no longer a supertrait of `WritableStorageTraits` diff --git a/examples/array_write_read.rs b/examples/array_write_read.rs index 142318ad..e9d87336 100644 --- a/examples/array_write_read.rs +++ b/examples/array_write_read.rs @@ -75,13 +75,13 @@ fn array_write_read() -> Result<(), Box> { // Write a subset spanning multiple chunks, including updating chunks already written array.store_array_subset_elements::( - &ArraySubset::new_with_start_shape(vec![3, 3], vec![3, 3]).unwrap(), + &ArraySubset::new_with_ranges(&[3..6, 3..6]), vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], )?; // Store elements directly, in this case set the 7th column to 123.0 array.store_array_subset_elements::( - &ArraySubset::new_with_start_shape(vec![0, 6], vec![8, 1])?, + &ArraySubset::new_with_ranges(&[0..8, 6..7]), vec![123.0; 8], )?; @@ -90,7 +90,7 @@ fn array_write_read() -> Result<(), Box> { // chunk indices &[1, 1], // subset within chunk - &ArraySubset::new_with_start_shape(vec![3, 0], vec![1, 4])?, + &ArraySubset::new_with_ranges(&[3..4, 0..4]), vec![-4.0; 4], )?; @@ -98,7 +98,7 @@ fn array_write_read() -> Result<(), Box> { array.erase_chunk(&[0, 1])?; // Read the whole array - let subset_all = ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec())?; + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); let data_all = array.retrieve_array_subset_ndarray::(&subset_all)?; println!("The whole array is:\n{:?}\n", data_all); @@ -108,7 +108,7 @@ fn array_write_read() -> Result<(), Box> { println!("Chunk [1,0] is:\n{data_chunk:?}\n"); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2])?; // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array.retrieve_array_subset_ndarray::(&subset_4x2)?; println!("The middle 4x2 subset is:\n{:?}\n", data_4x2); diff --git a/examples/async_array_write_read.rs b/examples/async_array_write_read.rs index 65ad71a5..5a650bdf 100644 --- a/examples/async_array_write_read.rs +++ b/examples/async_array_write_read.rs @@ -83,7 +83,7 @@ async fn async_array_write_read() -> Result<(), Box> { // Write a subset spanning multiple chunks, including updating chunks already written array .async_store_array_subset_elements::( - &ArraySubset::new_with_start_shape(vec![3, 3], vec![3, 3]).unwrap(), + &ArraySubset::new_with_ranges(&[3..6, 3..6]), vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], ) .await?; @@ -91,7 +91,7 @@ async fn async_array_write_read() -> Result<(), Box> { // Store elements directly, in this case set the 7th column to 123.0 array .async_store_array_subset_elements::( - &ArraySubset::new_with_start_shape(vec![0, 6], vec![8, 1])?, + &ArraySubset::new_with_ranges(&[0..8, 6..7]), vec![123.0; 8], ) .await?; @@ -102,7 +102,7 @@ async fn async_array_write_read() -> Result<(), Box> { // chunk indices &[1, 1], // subset within chunk - &ArraySubset::new_with_start_shape(vec![3, 0], vec![1, 4])?, + &ArraySubset::new_with_ranges(&[3..4, 0..4]), vec![-4.0; 4], ) .await?; @@ -111,7 +111,7 @@ async fn async_array_write_read() -> Result<(), Box> { array.async_erase_chunk(&[0, 1]).await?; // Read the whole array - let subset_all = ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec())?; + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); let data_all = array .async_retrieve_array_subset_ndarray::(&subset_all) .await?; @@ -125,7 +125,7 @@ async fn async_array_write_read() -> Result<(), Box> { println!("Chunk [1,0] is:\n{data_chunk:?}\n"); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2])?; // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array .async_retrieve_array_subset_ndarray::(&subset_4x2) .await?; diff --git a/examples/async_http_array_read.rs b/examples/async_http_array_read.rs index 44645673..d8f2e54e 100644 --- a/examples/async_http_array_read.rs +++ b/examples/async_http_array_read.rs @@ -34,7 +34,7 @@ async fn http_array_read() -> Result<(), Box> { ); // Read the whole array - let subset_all = ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec())?; + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); let data_all = array .async_retrieve_array_subset_ndarray::(&subset_all) .await?; @@ -48,7 +48,7 @@ async fn http_array_read() -> Result<(), Box> { println!("Chunk [1,0] is:\n{data_chunk:?}\n"); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2])?; // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array .async_retrieve_array_subset_ndarray::(&subset_4x2) .await?; diff --git a/examples/http_array_read.rs b/examples/http_array_read.rs index e75484b5..a52e67f9 100644 --- a/examples/http_array_read.rs +++ b/examples/http_array_read.rs @@ -34,7 +34,7 @@ fn http_array_read() -> Result<(), Box> { ); // Read the whole array - let subset_all = ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec())?; + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); let data_all = array.retrieve_array_subset_ndarray::(&subset_all)?; println!("The whole array is:\n{:?}\n", data_all); @@ -44,7 +44,7 @@ fn http_array_read() -> Result<(), Box> { println!("Chunk [1,0] is:\n{data_chunk:?}\n"); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2])?; // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array.retrieve_array_subset_ndarray::(&subset_4x2)?; println!("The middle 4x2 subset is:\n{:?}\n", data_4x2); diff --git a/examples/rectangular_array_write_read.rs b/examples/rectangular_array_write_read.rs index 98c5deba..8f40233a 100644 --- a/examples/rectangular_array_write_read.rs +++ b/examples/rectangular_array_write_read.rs @@ -86,7 +86,7 @@ fn rectangular_array_write_read() -> Result<(), Box> { // Store elements directly, in this case set the 7th column to 123.0 array.store_array_subset_elements::( - &ArraySubset::new_with_start_shape(vec![0, 6], vec![8, 1])?, + &ArraySubset::new_with_ranges(&[0..8, 6..7]), vec![123.0; 8], )?; @@ -95,12 +95,12 @@ fn rectangular_array_write_read() -> Result<(), Box> { // chunk indices &[3, 1], // subset within chunk - &ArraySubset::new_with_start_shape(vec![1, 0], vec![1, 4])?, + &ArraySubset::new_with_ranges(&[1..2, 0..4]), vec![-4.0; 4], )?; // Read the whole array - let subset_all = ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec())?; + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); let data_all = array.retrieve_array_subset_ndarray::(&subset_all)?; println!("The whole array is:\n{:?}\n", data_all); @@ -110,7 +110,7 @@ fn rectangular_array_write_read() -> Result<(), Box> { println!("Chunk [1,0] is:\n{data_chunk:?}\n"); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2])?; // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array.retrieve_array_subset_ndarray::(&subset_4x2)?; println!("The middle 4x2 subset is:\n{:?}\n", data_4x2); diff --git a/examples/sharded_array_write_read.rs b/examples/sharded_array_write_read.rs index 206c617b..9cf2ea7d 100644 --- a/examples/sharded_array_write_read.rs +++ b/examples/sharded_array_write_read.rs @@ -94,7 +94,7 @@ fn sharded_array_write_read() -> Result<(), Box> { })?; // Read the whole array - let subset_all = ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec())?; // the center 4x2 region + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); // the center 4x2 region let data_all = array.retrieve_array_subset_ndarray::(&subset_all)?; println!("The whole array is:\n{:?}\n", data_all); @@ -104,12 +104,12 @@ fn sharded_array_write_read() -> Result<(), Box> { println!("Shard [1,0] is:\n{data_shard:?}\n"); // Read an inner chunk from the store - let subset_chunk_1_0 = ArraySubset::new_with_start_shape(vec![4, 0], vec![4, 4])?; + let subset_chunk_1_0 = ArraySubset::new_with_ranges(&[4..8, 0..4]); let data_chunk = array.retrieve_array_subset_ndarray::(&subset_chunk_1_0)?; println!("Chunk [1,0] is:\n{data_chunk:?}\n"); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2])?; // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array.retrieve_array_subset_ndarray::(&subset_4x2)?; println!("The middle 4x2 subset is:\n{:?}\n", data_4x2); diff --git a/examples/zip_array_write_read.rs b/examples/zip_array_write_read.rs index 2dea68db..64acaee8 100644 --- a/examples/zip_array_write_read.rs +++ b/examples/zip_array_write_read.rs @@ -65,13 +65,13 @@ fn write_array_to_storage( // Write a subset spanning multiple chunks, including updating chunks already written array.store_array_subset_elements::( - &ArraySubset::new_with_start_shape(vec![3, 3], vec![3, 3]).unwrap(), + &ArraySubset::new_with_ranges(&[3..6, 3..6]), vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], )?; // Store elements directly, in this case set the 7th column to 123.0 array.store_array_subset_elements::( - &ArraySubset::new_with_start_shape(vec![0, 6], vec![8, 1])?, + &ArraySubset::new_with_ranges(&[0..8, 6..7]), vec![123.0; 8], )?; @@ -80,7 +80,7 @@ fn write_array_to_storage( // chunk indices &[1, 1], // subset within chunk - &ArraySubset::new_with_start_shape(vec![3, 0], vec![1, 4])?, + &ArraySubset::new_with_ranges(&[3..4, 0..4]), vec![-4.0; 4], )?; @@ -91,7 +91,7 @@ fn read_array_from_store( array: Array, ) -> Result<(), Box> { // Read the whole array - let subset_all = ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec())?; + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); let data_all = array.retrieve_array_subset_ndarray::(&subset_all)?; println!("The whole array is:\n{:?}\n", data_all); @@ -101,7 +101,7 @@ fn read_array_from_store( println!("Chunk [1,0] is:\n{data_chunk:?}\n"); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2])?; // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array.retrieve_array_subset_ndarray::(&subset_4x2)?; println!("The middle 4x2 subset is:\n{:?}\n", data_4x2); diff --git a/src/array.rs b/src/array.rs index 3de47d97..6af45044 100644 --- a/src/array.rs +++ b/src/array.rs @@ -706,13 +706,12 @@ mod tests { array .store_array_subset_elements::( - &ArraySubset::new_with_start_shape(vec![3, 3], vec![3, 3]).unwrap(), + &ArraySubset::new_with_ranges(&[3..6, 3..6]), vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], ) .unwrap(); - let subset_all = - ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec()).unwrap(); + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); let data_all = array .retrieve_array_subset_elements::(&subset_all) .unwrap(); @@ -750,11 +749,10 @@ mod tests { let mut any_not_equal = false; for j in 1..10 { (0..100).into_par_iter().for_each(|i| { - let subset = ArraySubset::new_with_start_shape(vec![i, 0], vec![1, 4]).unwrap(); + let subset = ArraySubset::new_with_ranges(&[i..i + 1, 0..4]); array.store_array_subset(&subset, vec![j; 4]).unwrap(); }); - let subset_all = - ArraySubset::new_with_start_shape(vec![0, 0], array.shape().to_vec()).unwrap(); + let subset_all = ArraySubset::new_with_shape(array.shape().to_vec()); let data_all = array.retrieve_array_subset(&subset_all).unwrap(); let all_equal = data_all.iter().all_equal_value() == Ok(&j); if expect_equal { diff --git a/src/array/codec.rs b/src/array/codec.rs index f5e8d6bf..b0ae11aa 100644 --- a/src/array/codec.rs +++ b/src/array/codec.rs @@ -1091,7 +1091,7 @@ mod tests { #[test] fn test_array_subset_iterator1() { let array_shape = vec![2, 2]; - let array_subset = ArraySubset::new_with_start_shape(vec![0, 0], vec![2, 1]).unwrap(); + let array_subset = ArraySubset::new_with_shape(vec![2, 1]); let mut iter = array_subset.iter_contiguous_indices(&array_shape).unwrap(); assert_eq!(iter.next().unwrap(), (vec![0, 0], 1)); @@ -1102,7 +1102,7 @@ mod tests { #[test] fn test_array_subset_iterator2() { let array_shape = vec![2, 2]; - let array_subset = ArraySubset::new_with_start_shape(vec![1, 0], vec![1, 2]).unwrap(); + let array_subset = ArraySubset::new_with_ranges(&[1..2, 0..2]); let mut iter = array_subset.iter_contiguous_indices(&array_shape).unwrap(); assert_eq!(iter.next().unwrap(), (vec![1, 0], 2)); @@ -1112,7 +1112,7 @@ mod tests { #[test] fn test_array_subset_iterator3() { let array_shape = vec![2, 2]; - let array_subset = ArraySubset::new_with_start_shape(vec![0, 0], vec![2, 2]).unwrap(); + let array_subset = ArraySubset::new_with_shape(vec![2, 2]); let mut iter = array_subset.iter_contiguous_indices(&array_shape).unwrap(); assert_eq!(iter.next().unwrap(), (vec![0, 0], 4)); @@ -1122,8 +1122,7 @@ mod tests { #[test] fn test_array_subset_iterator4() { let array_shape = vec![2, 2, 2, 3]; - let array_subset = - ArraySubset::new_with_start_shape(vec![0, 0, 0, 0], vec![2, 1, 2, 3]).unwrap(); + let array_subset = ArraySubset::new_with_shape(vec![2, 1, 2, 3]); let mut iter = array_subset.iter_contiguous_indices(&array_shape).unwrap(); assert_eq!(iter.next().unwrap(), (vec![0, 0, 0, 0], 6)); @@ -1134,7 +1133,7 @@ mod tests { #[test] fn test_array_subset_iterator5() { let array_shape = vec![2, 2, 3]; - let array_subset = ArraySubset::new_with_start_shape(vec![0, 0, 1], vec![2, 2, 2]).unwrap(); + let array_subset = ArraySubset::new_with_ranges(&[0..2, 0..2, 1..3]); let mut iter = array_subset.iter_contiguous_indices(&array_shape).unwrap(); assert_eq!(iter.next().unwrap(), (vec![0, 0, 1], 2)); diff --git a/src/array/codec/array_to_array/bitround.rs b/src/array/codec/array_to_array/bitround.rs index 758b1302..70be5b63 100644 --- a/src/array/codec/array_to_array/bitround.rs +++ b/src/array/codec/array_to_array/bitround.rs @@ -248,8 +248,8 @@ mod tests { let encoded = codec.encode(bytes.clone(), &array_representation).unwrap(); let decoded_regions = [ - ArraySubset::new_with_start_shape(vec![3], vec![2]).unwrap(), - ArraySubset::new_with_start_shape(vec![17], vec![4]).unwrap(), + ArraySubset::new_with_ranges(&[3..5]), + ArraySubset::new_with_ranges(&[17..21]), ]; let input_handle = Box::new(std::io::Cursor::new(encoded)); let bytes_codec = BytesCodec::default(); diff --git a/src/array/codec/array_to_bytes/codec_chain.rs b/src/array/codec/array_to_bytes/codec_chain.rs index 0c5801ee..fcefcfaa 100644 --- a/src/array/codec/array_to_bytes/codec_chain.rs +++ b/src/array/codec/array_to_bytes/codec_chain.rs @@ -707,8 +707,7 @@ mod tests { let codec = CodecChain::from_metadata(&codec_configurations).unwrap(); let encoded = codec.encode(bytes, &array_representation).unwrap(); - let decoded_regions = - [ArraySubset::new_with_start_shape(vec![0, 1, 0], vec![2, 1, 1]).unwrap()]; + let decoded_regions = [ArraySubset::new_with_ranges(&[0..2, 1..2, 0..1])]; let input_handle = Box::new(std::io::Cursor::new(encoded)); let partial_decoder = codec .partial_decoder(input_handle, &array_representation) diff --git a/src/array/codec/array_to_bytes/sharding.rs b/src/array/codec/array_to_bytes/sharding.rs index 865e39f6..110f7740 100644 --- a/src/array/codec/array_to_bytes/sharding.rs +++ b/src/array/codec/array_to_bytes/sharding.rs @@ -270,7 +270,7 @@ mod tests { let codec = ShardingCodec::new_with_configuration(&codec_configuration).unwrap(); let encoded = codec.encode(bytes, &array_representation).unwrap(); - let decoded_regions = [ArraySubset::new_with_start_shape(vec![1, 0], vec![2, 1]).unwrap()]; + let decoded_regions = [ArraySubset::new_with_ranges(&[1..3, 0..1])]; let input_handle = Box::new(std::io::Cursor::new(encoded)); let partial_decoder = codec .partial_decoder(input_handle, &array_representation) @@ -305,8 +305,7 @@ mod tests { let codec = ShardingCodec::new_with_configuration(&codec_configuration).unwrap(); let encoded = codec.encode(bytes, &array_representation).unwrap(); - let decoded_regions = - [ArraySubset::new_with_start_shape(vec![1, 0, 0], vec![1, 2, 3]).unwrap()]; + let decoded_regions = [ArraySubset::new_with_ranges(&[1..2, 0..2, 0..3])]; let input_handle = Box::new(std::io::Cursor::new(encoded)); let partial_decoder = codec .partial_decoder(input_handle, &array_representation) @@ -337,7 +336,7 @@ mod tests { let codec = ShardingCodec::new_with_configuration(&codec_configuration).unwrap(); let encoded = codec.encode(bytes, &array_representation).unwrap(); - let decoded_regions = [ArraySubset::new_with_start_shape(vec![1, 0], vec![2, 1]).unwrap()]; + let decoded_regions = [ArraySubset::new_with_ranges(&[1..3, 0..1])]; let input_handle = Box::new(std::io::Cursor::new(encoded)); let partial_decoder = codec .partial_decoder(input_handle, &array_representation) diff --git a/src/array/codec/array_to_bytes/zfp.rs b/src/array/codec/array_to_bytes/zfp.rs index 2777364b..07bbdcc8 100644 --- a/src/array/codec/array_to_bytes/zfp.rs +++ b/src/array/codec/array_to_bytes/zfp.rs @@ -179,10 +179,8 @@ mod tests { let encoded = codec.encode(bytes.clone(), &array_representation).unwrap(); let decoded_regions = [ - ArraySubset::new_with_start_shape((&[0, 0, 0]).to_vec(), (&[1, 2, 3]).to_vec()) - .unwrap(), - ArraySubset::new_with_start_shape((&[0, 1, 2]).to_vec(), (&[3, 2, 1]).to_vec()) - .unwrap(), + ArraySubset::new_with_shape(vec![1, 2, 3]), + ArraySubset::new_with_ranges(&[0..3, 1..3, 2..3]), ]; let input_handle = Box::new(std::io::Cursor::new(encoded)); diff --git a/src/array/codec/bytes_to_bytes/blosc.rs b/src/array/codec/bytes_to_bytes/blosc.rs index 90052ade..9bf58e8d 100644 --- a/src/array/codec/bytes_to_bytes/blosc.rs +++ b/src/array/codec/bytes_to_bytes/blosc.rs @@ -351,14 +351,12 @@ mod tests { let codec = BloscCodec::new_with_configuration(&codec_configuration).unwrap(); let encoded = codec.encode(bytes).unwrap(); - let decoded_regions: Vec = - ArraySubset::new_with_start_shape(vec![0, 1, 0], vec![2, 1, 1]) - .unwrap() - .byte_ranges( - array_representation.shape(), - array_representation.element_size(), - ) - .unwrap(); + let decoded_regions: Vec = ArraySubset::new_with_ranges(&[0..2, 1..2, 0..1]) + .byte_ranges( + array_representation.shape(), + array_representation.element_size(), + ) + .unwrap(); let input_handle = Box::new(std::io::Cursor::new(encoded)); let partial_decoder = codec .partial_decoder(input_handle, &bytes_representation) diff --git a/src/array_subset.rs b/src/array_subset.rs index 9ab049fc..8400e37b 100644 --- a/src/array_subset.rs +++ b/src/array_subset.rs @@ -9,6 +9,8 @@ mod array_subset_iterators; +use std::ops::Range; + pub use array_subset_iterators::{ ChunksIterator, ContiguousIndicesIterator, ContiguousLinearisedIndicesIterator, IndicesIterator, LinearisedIndicesIterator, @@ -58,6 +60,14 @@ pub enum ArrayStoreBytesError { } impl ArraySubset { + /// Create a new array subset from a `ranges`. + #[must_use] + pub fn new_with_ranges(ranges: &[Range]) -> Self { + let start = ranges.iter().map(|range| range.start).collect(); + let shape = ranges.iter().map(|range| range.end - range.start).collect(); + Self { start, shape } + } + /// Create a new array subset with `size` starting at the origin. #[must_use] pub fn new_with_shape(shape: ArrayShape) -> Self { @@ -687,8 +697,7 @@ impl ArraySubset { #[must_use] pub unsafe fn in_subset_unchecked(&self, subset_other: &Self) -> Self { debug_assert_eq!(subset_other.dimensionality(), self.dimensionality()); - let mut starts = Vec::with_capacity(self.start.len()); - let mut shapes = Vec::with_capacity(self.start.len()); + let mut ranges = Vec::with_capacity(self.dimensionality()); for (start, size, other_start, other_size) in izip!( &self.start, &self.shape, @@ -698,11 +707,9 @@ impl ArraySubset { let output_start = start.saturating_sub(*other_start); let output_end = std::cmp::min((start + size).saturating_sub(*other_start), *other_size); - let output_size = output_end - output_start; - starts.push(output_start); - shapes.push(output_size); + ranges.push(output_start..output_end); } - unsafe { Self::new_with_start_shape_unchecked(starts, shapes) } + Self::new_with_ranges(&ranges) } /// Return the overlapping subset between this array subset and `subset_other`. @@ -730,8 +737,7 @@ impl ArraySubset { #[must_use] pub unsafe fn overlap_unchecked(&self, subset_other: &Self) -> Self { debug_assert_eq!(subset_other.dimensionality(), self.dimensionality()); - let mut starts = Vec::with_capacity(self.start.len()); - let mut shapes = Vec::with_capacity(self.start.len()); + let mut ranges = Vec::with_capacity(self.dimensionality()); for (start, size, other_start, other_size) in izip!( &self.start, &self.shape, @@ -740,11 +746,9 @@ impl ArraySubset { ) { let overlap_start = *std::cmp::max(start, other_start); let overlap_end = std::cmp::min(start + size, other_start + other_size); - let overlap_size = overlap_end - overlap_start; - starts.push(overlap_start); - shapes.push(overlap_size); + ranges.push(overlap_start..overlap_end); } - unsafe { Self::new_with_start_shape_unchecked(starts, shapes) } + Self::new_with_ranges(&ranges) } /// Return the subset relative to `start`. diff --git a/src/array_subset/array_subset_iterators.rs b/src/array_subset/array_subset_iterators.rs index 9dab994e..549d73c3 100644 --- a/src/array_subset/array_subset_iterators.rs +++ b/src/array_subset/array_subset_iterators.rs @@ -364,7 +364,7 @@ mod tests { #[test] fn array_subset_iter_indices() { - let subset = ArraySubset::new_with_start_shape(vec![1, 1], vec![2, 2]).unwrap(); + let subset = ArraySubset::new_with_ranges(&[1..3, 1..3]); let mut iter = subset.iter_indices(); assert_eq!(iter.next(), Some(vec![1, 1])); assert_eq!(iter.next(), Some(vec![1, 2])); @@ -375,7 +375,7 @@ mod tests { #[test] fn array_subset_iter_linearised_indices() { - let subset = ArraySubset::new_with_start_shape(vec![1, 1], vec![2, 2]).unwrap(); + let subset = ArraySubset::new_with_ranges(&[1..3, 1..3]); let mut iter = subset.iter_linearised_indices(&[4, 4]).unwrap(); // 0 1 2 3 // 4 5 6 7 @@ -390,7 +390,7 @@ mod tests { #[test] fn array_subset_iter_contiguous_indices1() { - let subset = ArraySubset::new_with_start_shape(vec![0, 0], vec![2, 2]).unwrap(); + let subset = ArraySubset::new_with_shape(vec![2, 2]); let mut iter = subset.iter_contiguous_indices(&[2, 2]).unwrap(); assert_eq!(iter.next(), Some((vec![0, 0], 4))); assert_eq!(iter.next(), None); @@ -398,7 +398,7 @@ mod tests { #[test] fn array_subset_iter_contiguous_indices2() { - let subset = ArraySubset::new_with_start_shape(vec![1, 1], vec![2, 2]).unwrap(); + let subset = ArraySubset::new_with_ranges(&[1..3, 1..3]); let mut iter = subset.iter_contiguous_indices(&[4, 4]).unwrap(); assert_eq!(iter.next(), Some((vec![1, 1], 2))); assert_eq!(iter.next(), Some((vec![2, 1], 2))); @@ -407,7 +407,7 @@ mod tests { #[test] fn array_subset_iter_contiguous_indices3() { - let subset = ArraySubset::new_with_start_shape(vec![1, 0, 0, 0], vec![2, 1, 2, 2]).unwrap(); + let subset = ArraySubset::new_with_ranges(&[1..3, 0..1, 0..2, 0..2]); let mut iter = subset.iter_contiguous_indices(&[3, 1, 2, 2]).unwrap(); assert_eq!(iter.next(), Some((vec![1, 0, 0, 0], 8))); assert_eq!(iter.next(), None); @@ -415,7 +415,7 @@ mod tests { #[test] fn array_subset_iter_continuous_linearised_indices() { - let subset = ArraySubset::new_with_start_shape(vec![1, 1], vec![2, 2]).unwrap(); + let subset = ArraySubset::new_with_ranges(&[1..3, 1..3]); let mut iter = subset.iter_contiguous_linearised_indices(&[4, 4]).unwrap(); // 0 1 2 3 // 4 5 6 7 @@ -429,29 +429,29 @@ mod tests { #[test] #[rustfmt::skip] fn array_subset_iter_chunks1() { - let subset = ArraySubset::new_with_start_shape(vec![1, 1], vec![4, 4]).unwrap(); + let subset = ArraySubset::new_with_ranges(&[1..5, 1..5]); let mut iter = subset.iter_chunks(&[2, 2]).unwrap(); - assert_eq!(iter.next(), Some((vec![0, 0], ArraySubset::new_with_start_shape(vec![0, 0], vec![2, 2]).unwrap()))); - assert_eq!(iter.next(), Some((vec![0, 1], ArraySubset::new_with_start_shape(vec![0, 2], vec![2, 2]).unwrap()))); - assert_eq!(iter.next(), Some((vec![0, 2], ArraySubset::new_with_start_shape(vec![0, 4], vec![2, 2]).unwrap()))); - assert_eq!(iter.next(), Some((vec![1, 0], ArraySubset::new_with_start_shape(vec![2, 0], vec![2, 2]).unwrap()))); - assert_eq!(iter.next(), Some((vec![1, 1], ArraySubset::new_with_start_shape(vec![2, 2], vec![2, 2]).unwrap()))); - assert_eq!(iter.next(), Some((vec![1, 2], ArraySubset::new_with_start_shape(vec![2, 4], vec![2, 2]).unwrap()))); - assert_eq!(iter.next(), Some((vec![2, 0], ArraySubset::new_with_start_shape(vec![4, 0], vec![2, 2]).unwrap()))); - assert_eq!(iter.next(), Some((vec![2, 1], ArraySubset::new_with_start_shape(vec![4, 2], vec![2, 2]).unwrap()))); - assert_eq!(iter.next(), Some((vec![2, 2], ArraySubset::new_with_start_shape(vec![4, 4], vec![2, 2]).unwrap()))); + assert_eq!(iter.next(), Some((vec![0, 0], ArraySubset::new_with_ranges(&[0..2, 0..2])))); + assert_eq!(iter.next(), Some((vec![0, 1], ArraySubset::new_with_ranges(&[0..2, 2..4])))); + assert_eq!(iter.next(), Some((vec![0, 2], ArraySubset::new_with_ranges(&[0..2, 4..6])))); + assert_eq!(iter.next(), Some((vec![1, 0], ArraySubset::new_with_ranges(&[2..4, 0..2])))); + assert_eq!(iter.next(), Some((vec![1, 1], ArraySubset::new_with_ranges(&[2..4, 2..4])))); + assert_eq!(iter.next(), Some((vec![1, 2], ArraySubset::new_with_ranges(&[2..4, 4..6])))); + assert_eq!(iter.next(), Some((vec![2, 0], ArraySubset::new_with_ranges(&[4..6, 0..2])))); + assert_eq!(iter.next(), Some((vec![2, 1], ArraySubset::new_with_ranges(&[4..6, 2..4])))); + assert_eq!(iter.next(), Some((vec![2, 2], ArraySubset::new_with_ranges(&[4..6, 4..6])))); assert_eq!(iter.next(), None); } #[test] #[rustfmt::skip] fn array_subset_iter_chunks2() { - let subset = ArraySubset::new_with_start_shape(vec![2, 2], vec![3, 4]).unwrap(); + let subset = ArraySubset::new_with_ranges(&[2..5, 2..6]); let mut iter = subset.iter_chunks(&[2, 3]).unwrap(); - assert_eq!(iter.next(), Some((vec![1, 0], ArraySubset::new_with_start_shape(vec![2, 0], vec![2, 3]).unwrap()))); - assert_eq!(iter.next(), Some((vec![1, 1], ArraySubset::new_with_start_shape(vec![2, 3], vec![2, 3]).unwrap()))); - assert_eq!(iter.next(), Some((vec![2, 0], ArraySubset::new_with_start_shape(vec![4, 0], vec![2, 3]).unwrap()))); - assert_eq!(iter.next(), Some((vec![2, 1], ArraySubset::new_with_start_shape(vec![4, 3], vec![2, 3]).unwrap()))); + assert_eq!(iter.next(), Some((vec![1, 0], ArraySubset::new_with_ranges(&[2..4, 0..3])))); + assert_eq!(iter.next(), Some((vec![1, 1], ArraySubset::new_with_ranges(&[2..4, 3..6])))); + assert_eq!(iter.next(), Some((vec![2, 0], ArraySubset::new_with_ranges(&[4..6, 0..3])))); + assert_eq!(iter.next(), Some((vec![2, 1], ArraySubset::new_with_ranges(&[4..6, 3..6])))); assert_eq!(iter.next(), None); } } diff --git a/src/storage/store/store_async/http_store.rs b/src/storage/store/store_async/http_store.rs index ea007647..81cca782 100644 --- a/src/storage/store/store_async/http_store.rs +++ b/src/storage/store/store_async/http_store.rs @@ -105,8 +105,7 @@ mod tests { assert_eq!(array.data_type(), &DataType::Float32); // Read the central 4x2 subset of the array - let subset_4x2 = - crate::array_subset::ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2]).unwrap(); // the center 4x2 region + let subset_4x2 = crate::array_subset::ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array .async_retrieve_array_subset_elements::(&subset_4x2) .await @@ -139,8 +138,7 @@ mod tests { assert_eq!(array.data_type(), &DataType::UInt16); // Read the central 4x2 subset of the array - let subset_4x2 = - crate::array_subset::ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2]).unwrap(); // the center 4x2 region + let subset_4x2 = crate::array_subset::ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array .async_retrieve_array_subset_elements::(&subset_4x2) .await diff --git a/src/storage/store/store_sync/http_store.rs b/src/storage/store/store_sync/http_store.rs index 67b0b000..a74ad10f 100644 --- a/src/storage/store/store_sync/http_store.rs +++ b/src/storage/store/store_sync/http_store.rs @@ -248,7 +248,7 @@ mod tests { assert_eq!(array.data_type(), &DataType::Float32); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2]).unwrap(); // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array .retrieve_array_subset_elements::(&subset_4x2) .unwrap(); @@ -278,7 +278,7 @@ mod tests { assert_eq!(array.data_type(), &DataType::UInt16); // Read the central 4x2 subset of the array - let subset_4x2 = ArraySubset::new_with_start_shape(vec![2, 3], vec![4, 2]).unwrap(); // the center 4x2 region + let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region let data_4x2 = array .retrieve_array_subset_elements::(&subset_4x2) .unwrap();