refactor(datatypes)!: major revisions

Data type extensions removed, r*/float16/bfloat16 now core and
the half crate is a required dependency.
Fixes for the bytes codec.
See changelog.

CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+ - **Breaking**: Added `UnsupportedDataTypeError`
+
+### Changed
+ - **Breaking**: `array::data_type::DataType` is now marked `#[non_exhaustive]`
+ - **Breaking**: Promote the `r*` (raw bits), `float16` and `bfloat16` data types to standard data types in `array::data_type::DataType`, rather than extension data types
+   - **Breaking**: Remove the crate features: `raw_bits`, `float16`, `bfloat16`
+   - **Breaking**: Removes `array::data_type::RawBitsDataType/Bfloat16DataType/Float16DataType`
+   - **Breaking**: `half` is now a required dependency
+
+### Fixed
+ - Bytes codec handling of complex and raw bits data types
+
+### Removed
+ - **Breaking**: Disabled data type extensions `array::data_type::DataType::Extension`.
+
 ## [0.5.1] - 2023-10-10
 
 ### Added

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "zarrs"
 description = "A library for the Zarr V3 storage format for multidimensional arrays and metadata"
-version = "0.5.1"
+version = "0.6.0"
 authors = ["Lachlan Deakin <ljdgit@gmail.com>"]
 edition = "2021"
 license = "MIT OR Apache-2.0"
@@ -11,18 +11,14 @@ categories = ["encoding"]
 exclude = [".dockerignore", ".github", ".editorconfig", "Dockerfile", "coverage.sh", "TODO.md"]
 
 [features]
-default = ["transpose", "blosc", "gzip", "sharding", "crc32c", "zstd", "raw_bits", "float16", "bfloat16", "http", "zip", "ndarray"]
+default = ["transpose", "blosc", "gzip", "sharding", "crc32c", "zstd", "http", "zip", "ndarray"]
 # Codecs
 transpose = ["dep:ndarray"]
 blosc = ["dep:blosc-sys"]
 gzip = ["dep:flate2"]
 sharding = []
 crc32c = ["dep:crc32fast"]
 zstd = ["dep:zstd"]
-# Optional/extension data types
-raw_bits = []
-float16 = ["dep:half"]
-bfloat16 = ["dep:half"]
 # Stores
 http = ["dep:reqwest", "dep:url"]
 zip = ["dep:zip"]
@@ -38,7 +34,7 @@ crc32fast = { version = "1.3", optional = true }
 derive_more = "0.99"
 dyn-clone = "1"
 flate2 = { version = "1", optional = true }
-half = { version = "2", optional = true }
+half = "2"
 inventory = "0.3"
 itertools = "0.11"
 ndarray = { version = "0.15", optional = true }

src/array/array_errors.rs

@@ -11,7 +11,7 @@ use crate::{
 use super::{
     chunk_grid::{InvalidArrayIndicesError, InvalidChunkGridIndicesError},
     codec::CodecError,
-    data_type::IncompatibleFillValueErrorMetadataError,
+    data_type::{IncompatibleFillValueErrorMetadataError, UnsupportedDataTypeError},
     ArrayIndices, ArrayShape,
 };
 
@@ -32,7 +32,7 @@ pub enum ArrayCreateError {
     UnsupportedAdditionalFieldError(#[from] UnsupportedAdditionalFieldError),
     /// Unsupported data type.
     #[error(transparent)]
-    DataTypeCreateError(PluginCreateError),
+    DataTypeCreateError(UnsupportedDataTypeError),
     /// Invalid fill value.
     #[error(transparent)]
     InvalidFillValue(#[from] IncompatibleFillValueErrorMetadataError),

src/array/codec/array_to_bytes/bytes.rs

@@ -14,6 +14,8 @@ pub use bytes_codec::BytesCodec;
 
 use derive_more::Display;
 
+use crate::array::DataType;
+
 /// The endianness of each element in an array, either `big` or `little`.
 #[derive(Copy, Clone, Eq, PartialEq, Debug, Display)]
 pub enum Endianness {
@@ -60,10 +62,26 @@ const NATIVE_ENDIAN: Endianness = Endianness::Big;
 #[cfg(target_endian = "little")]
 const NATIVE_ENDIAN: Endianness = Endianness::Little;
 
-fn reverse_endianness(v: &mut [u8], bytes_per_element: usize) {
-    if bytes_per_element > 1 {
-        v.chunks_exact_mut(bytes_per_element)
-            .for_each(<[u8]>::reverse);
+fn reverse_endianness(v: &mut [u8], data_type: &DataType) {
+    match data_type {
+        DataType::Bool | DataType::Int8 | DataType::UInt8 | DataType::RawBits(_) => {}
+        DataType::Int16
+        | DataType::Int32
+        | DataType::Int64
+        | DataType::UInt16
+        | DataType::UInt32
+        | DataType::UInt64
+        | DataType::Float16
+        | DataType::Float32
+        | DataType::Float64
+        | DataType::BFloat16 => {
+            v.chunks_exact_mut(data_type.size())
+                .for_each(<[u8]>::reverse);
+        }
+        DataType::Complex64 | DataType::Complex128 => {
+            v.chunks_exact_mut(data_type.size() / 2)
+                .for_each(<[u8]>::reverse);
+        }
     }
 }
 

src/array/codec/array_to_bytes/bytes/bytes_codec.rs

@@ -112,7 +112,7 @@ impl ArrayCodecTraits for BytesCodec {
 
         if let Some(endian) = &self.endian {
             if !endian.is_native() {
-                reverse_endianness(&mut decoded_value, decoded_representation.element_size());
+                reverse_endianness(&mut decoded_value, decoded_representation.data_type());
             }
         }
         Ok(decoded_value)
@@ -125,7 +125,7 @@ impl ArrayCodecTraits for BytesCodec {
     ) -> Result<Vec<u8>, CodecError> {
         if let Some(endian) = &self.endian {
             if !endian.is_native() {
-                reverse_endianness(&mut encoded_value, decoded_representation.element_size());
+                reverse_endianness(&mut encoded_value, decoded_representation.data_type());
             }
         }
         Ok(encoded_value)

src/array/codec/array_to_bytes/bytes/bytes_partial_decoder.rs

@@ -61,7 +61,7 @@ impl ArrayPartialDecoderTraits for BytesPartialDecoder<'_> {
                         if !endian.is_native() {
                             reverse_endianness(
                                 &mut bytes_subset,
-                                decoded_representation.element_size(),
+                                decoded_representation.data_type(),
                             );
                         }
                     }