Add preliminary support for files format prior to 2.5

Signed-off-by: Alexis Jeandet <alexis.jeandet@member.fsf.org>

include/cdfpp/cdf-io/common.hpp

@@ -57,6 +57,11 @@ inline bool is_v3x(const magic_numbers_t& magic)
     return cdf_version(magic).first >= 3;
 }
 
+inline bool is_v2_5_or_more(const magic_numbers_t& magic)
+{
+    return cdf_version(magic).first == 2 && cdf_version(magic).second >= 5;
+}
+
 inline bool is_cdf(const magic_numbers_t& magic_numbers) noexcept
 {
     return (((magic_numbers.first & 0xfff00000) == 0xCDF00000)
@@ -139,7 +144,8 @@ void add_var_attribute(cdf_repr& repr, const std::vector<uint32_t>& variable_ind
     {
         for (auto& [attr_name, attr_data] : attr)
         {
-            repr.var_attributes[v_index][attr_name] = VariableAttribute { attr_name, std::move(attr_data) };
+            repr.var_attributes[v_index][attr_name]
+                = VariableAttribute { attr_name, std::move(attr_data) };
         }
     }
 }

include/cdfpp/cdf-io/desc-records.hpp

@@ -33,13 +33,20 @@ struct v3x_tag
 {
 };
 
-struct v2x_tag
+struct v2_4_or_less_tag
+{
+};
+
+struct v2_5_or_more_tag
 {
 };
 
 template <typename version_t>
 inline constexpr bool is_v3_v = std::is_same_v<version_t, v3x_tag>;
 
+template <typename version_t>
+inline constexpr bool is_v2_4_or_less_v = std::is_same_v<version_t, v2_4_or_less_tag>;
+
 template <typename version_t>
 using cdf_offset_field_t = std::conditional_t<is_v3_v<version_t>, uint64_t, uint32_t>;
 
@@ -64,7 +71,8 @@ struct is_cdf_DR_header : std::false_type
 template <typename T>
 struct is_cdf_DR_header<T,
     decltype(std::is_same_v<cdf_DR_header<typename T::cdf_version_t, T::expected_record_type>, T>,
-        void())> : std::is_same<cdf_DR_header<typename T::cdf_version_t, T::expected_record_type>, T>
+        void())>
+        : std::is_same<cdf_DR_header<typename T::cdf_version_t, T::expected_record_type>, T>
 {
 };
 
@@ -84,12 +92,13 @@ struct cdf_CDR_t
     uint32_t Release;
     cdf_encoding Encoding;
     uint32_t Flags;
-    uint32_t rfuA;
-    uint32_t rfuB;
+    unused_field<int32_t> rfuA;
+    unused_field<int32_t> rfuB;
     uint32_t Increment;
     uint32_t Identifier;
-    uint32_t rfuE;
-    string_field<256> copyright; // ignore format < 2.6
+    unused_field<int32_t> rfuE;
+    std::conditional_t<is_v2_4_or_less_v<version_t>, string_field<1945>, string_field<256>>
+        copyright;
 };
 
 template <typename version_t>
@@ -108,9 +117,9 @@ struct cdf_GDR_t
     uint32_t rNumDims;
     uint32_t NzVars;
     cdf_offset_field_t<version_t> UIRhead;
-    uint32_t rfuC;
+    unused_field<int32_t> rfuC;
     uint32_t LeapSecondLastUpdated;
-    uint32_t rfuE;
+    unused_field<int32_t> rfuE;
     table_field<uint32_t, 0> rDimSizes;
 
     std::size_t size(const table_field<uint32_t, 0>&) const
@@ -131,11 +140,11 @@ struct cdf_ADR_t
     int32_t num;
     int32_t NgrEntries;
     int32_t MAXgrEntries;
-    int32_t rfuA;
+    unused_field<int32_t> rfuA;
     cdf_offset_field_t<version_t> AzEDRhead;
     int32_t NzEntries;
     int32_t MAXzEntries;
-    int32_t rfuE;
+    unused_field<int32_t> rfuE;
     cdf_string_field_t<version_t, 256, 64> Name;
 };
 
@@ -151,10 +160,10 @@ struct cdf_AgrEDR_t
     int32_t Num;
     int32_t NumElements;
     int32_t NumStrings;
-    int32_t rfB;
-    int32_t rfC;
-    int32_t rfD;
-    int32_t rfE;
+    unused_field<int32_t> rfB;
+    unused_field<int32_t> rfC;
+    unused_field<int32_t> rfD;
+    unused_field<int32_t> rfE;
     // table_field<uint32_t> Values;
 
     /*std::size_t size(const table_field<uint32_t, 0>&) const
@@ -175,10 +184,10 @@ struct cdf_AzEDR_t
     int32_t Num;
     int32_t NumElements;
     int32_t NumStrings;
-    int32_t rfB;
-    int32_t rfC;
-    int32_t rfD;
-    int32_t rfE;
+    unused_field<int32_t> rfB;
+    unused_field<int32_t> rfC;
+    unused_field<int32_t> rfD;
+    unused_field<int32_t> rfE;
     // table_field<uint32_t> Values;
 
     /*std::size_t size(const table_field<uint32_t, 0>&) const
@@ -233,9 +242,10 @@ struct cdf_rVDR_t
     cdf_offset_field_t<version_t> VXRtail;
     int32_t Flags;
     int32_t SRecords;
-    int32_t rfuB;
-    int32_t rfuC;
-    int32_t rfuF;
+    unused_field<int32_t> rfuB;
+    unused_field<int32_t> rfuC;
+    unused_field<std::conditional_t<is_v2_4_or_less_v<version_t>, table_field<char, 2>, int32_t>>
+        rfuF;
     int32_t NumElems;
     int32_t Num;
     cdf_offset_field_t<version_t> CPRorSPRoffset;
@@ -250,7 +260,8 @@ struct cdf_rVDR_t
         return rNumDims * sizeof(int32_t);
     }
 
-    std::size_t size(const table_field<int32_t, 1>&) const { return 0; }
+    constexpr std::size_t size(const table_field<int32_t, 1>&) const { return 0; }
+    constexpr std::size_t size(const table_field<char, 2>&) const { return 132; }
 };
 
 template <typename version_t>
@@ -266,9 +277,10 @@ struct cdf_zVDR_t
     cdf_offset_field_t<version_t> VXRtail;
     int32_t Flags;
     int32_t SRecords;
-    int32_t rfuB;
-    int32_t rfuC;
-    int32_t rfuF;
+    unused_field<int32_t> rfuB;
+    unused_field<int32_t> rfuC;
+    unused_field<std::conditional_t<is_v2_4_or_less_v<version_t>, table_field<char, 2>, int32_t>>
+        rfuF;
     int32_t NumElems;
     int32_t Num;
     cdf_offset_field_t<version_t> CPRorSPRoffset;
@@ -288,7 +300,8 @@ struct cdf_zVDR_t
         return this->zNumDims * sizeof(int32_t);
     }
 
-    std::size_t size(const table_field<int32_t, 2>&) const { return 0; }
+    constexpr std::size_t size(const table_field<int32_t, 2>&) const { return 0; }
+    constexpr std::size_t size(const table_field<char, 2>&) const { return 132; }
 };
 
 template <cdf_r_z type, typename version_t>
@@ -341,7 +354,7 @@ struct cdf_CVVR_t
     using cdf_version_t = version_t;
     inline static constexpr bool v3 = is_v3_v<version_t>;
     cdf_DR_header<version_t, cdf_record_type::CVVR> header;
-    uint32_t rfuA;
+    unused_field<uint32_t> rfuA;
     cdf_offset_field_t<version_t> cSize;
     table_field<char, 0> data;
     std::size_t size(const table_field<char, 0>&) const { return this->cSize; }
@@ -371,7 +384,7 @@ struct cdf_CPR_t
     inline static constexpr bool v3 = is_v3_v<version_t>;
     cdf_DR_header<version_t, cdf_record_type::CPR> header;
     cdf_compression_type cType;
-    uint32_t rfuA;
+    unused_field<uint32_t> rfuA;
     uint32_t pCount;
     table_field<uint32_t> cParms;
     std::size_t size(const table_field<uint32_t, 0>&) const

include/cdfpp/cdf-io/loading/loading.hpp

@@ -20,16 +20,16 @@
 /*-- Author : Alexis Jeandet
 -- Mail : alexis.jeandet@member.fsf.org
 ----------------------------------------------------------------------------*/
+#include "../common.hpp"
+#include "../decompression.hpp"
+#include "../desc-records.hpp"
 #include "../endianness.hpp"
-#include "cdfpp/cdf-enums.hpp"
-#include "cdfpp/cdf-file.hpp"
 #include "./attribute.hpp"
 #include "./buffers.hpp"
 #include "./records-loading.hpp"
 #include "./variable.hpp"
-#include "../common.hpp"
-#include "../decompression.hpp"
-#include "../desc-records.hpp"
+#include "cdfpp/cdf-enums.hpp"
+#include "cdfpp/cdf-file.hpp"
 #include <algorithm>
 #include <fstream>
 #include <iostream>
@@ -71,7 +71,7 @@ namespace
         cdf.variables = std::move(repr.variables);
         cdf.lazy_loaded = repr.lazy;
         cdf.compression = repr.compression_type;
-        //cdf.leap_second_last_updated = repr.leap_second_last_updated;
+        // cdf.leap_second_last_updated = repr.leap_second_last_updated;
         return cdf;
     }
 
@@ -123,9 +123,9 @@ namespace
     }
 
     template <typename buffer_t, typename iso_8859_1_to_utf8>
-    [[nodiscard]] auto _impl_load(
-        buffer_t&& buffer, iso_8859_1_to_utf8 iso_8859_1_to_utf8_tag, bool lazy_load = false)
-        -> decltype(buffer.read(std::declval<char*>(), 0UL, 0UL), std::optional<CDF> {})
+    [[nodiscard]] auto _impl_load(buffer_t&& buffer, iso_8859_1_to_utf8 iso_8859_1_to_utf8_tag,
+        bool lazy_load = false) -> decltype(buffer.read(std::declval<char*>(), 0UL, 0UL),
+                                    std::optional<CDF> {})
     {
         auto magic = get_magic(buffer);
         if (common::is_cdf(magic))
@@ -135,9 +135,14 @@ namespace
                 return parse_cdf<v3x_tag>(std::move(buffer), iso_8859_1_to_utf8_tag,
                     common::is_compressed(magic), lazy_load);
             }
+            else if (common::is_v2_5_or_more(magic))
+            {
+                return parse_cdf<v2_5_or_more_tag>(std::move(buffer), iso_8859_1_to_utf8_tag,
+                    common::is_compressed(magic), lazy_load);
+            }
             else
             {
-                return parse_cdf<v2x_tag>(std::move(buffer), iso_8859_1_to_utf8_tag,
+                return parse_cdf<v2_4_or_less_tag>(std::move(buffer), iso_8859_1_to_utf8_tag,
                     common::is_compressed(magic), lazy_load);
             }
         }

include/cdfpp/cdf-io/loading/records-loading.hpp

@@ -96,6 +96,29 @@ inline std::size_t load_field(
         }
     }
 }
+template <typename record_t, typename parsing_context_t, typename T>
+inline std::size_t load_field(const record_t& r, parsing_context_t& parsing_context,
+    std::size_t offset, unused_field<T>& field)
+{
+    (void)r;
+    (void)parsing_context;
+    if constexpr (is_string_field_v<T>)
+    {
+        return offset + T::max_len;
+    }
+    else
+    {
+        if constexpr (is_table_field_v<T>)
+        {
+            return offset + r.size(field.value);
+        }
+        else
+        {
+            return offset + sizeof(T);
+        }
+    }
+}
+
 
 template <typename parsing_context_t, typename version_t>
 inline std::size_t load_field(const cdf_rVDR_t<version_t>& r, parsing_context_t& parsing_context,
@@ -120,8 +143,8 @@ inline std::size_t load_fields(const record_t& r, parsing_context_t& parsing_con
 {
     using Field_t = std::remove_cv_t<std::remove_reference_t<T>>;
     static constexpr std::size_t count = count_members<Field_t>;
-    if constexpr (std::is_compound_v<Field_t> && (count > 1)
-        && (not is_string_field_v<Field_t>)&&(not is_table_field_v<Field_t>))
+    if constexpr (std::is_compound_v<Field_t> && (count > 1) && (not is_string_field_v<Field_t>)
+        && (not is_table_field_v<Field_t>))
         return load_record(field, parsing_context, offset);
     else
         return load_field(r, parsing_context, offset, std::forward<T>(field));

include/cdfpp/cdf-io/special-fields.hpp

@@ -20,10 +20,10 @@
 /*-- Author : Alexis Jeandet
 -- Mail : alexis.jeandet@member.fsf.org
 ----------------------------------------------------------------------------*/
+#include "../no_init_vector.hpp"
 #include <cstddef>
 #include <string>
 #include <vector>
-#include "../no_init_vector.hpp"
 
 template <std::size_t _max_len>
 struct string_field
@@ -56,6 +56,13 @@ struct table_field
     no_init_vector<T> values;
 };
 
+template <typename T>
+struct unused_field
+{
+    T value;
+};
+
+
 template <typename T, typename = void>
 struct is_table_field : std::false_type
 {

tests/records_loading/main.cpp

@@ -36,6 +36,22 @@ SCENARIO("record loading", "[CDF]")
             REQUIRE(s.b == 'd');
         }
     }
+    GIVEN("a simple two char fields record with an unused field")
+    {
+        struct two_chars
+        {
+            char a;
+            unused_field<char> b;
+        };
+        two_chars s { 'a', 'b' };
+        THEN("we can load it from a buffer")
+        {
+            std::string buffer { "cd" };
+            cdf::io::load_record(s, buffer.c_str(), 0);
+            REQUIRE(s.a == 'c');
+            REQUIRE(s.b.value == 'b');
+        }
+    }
     GIVEN("a more complex record")
     {
         struct complex_record

tests/simple_open/main.cpp

@@ -78,25 +78,25 @@ impl_compare_attribute_value(const cdf::VariableAttribute& attribute, const valu
     return attribute.get<value_type>() == no_init_vector<value_type> { value };
 }
 
-template <int index,typename attr_t, typename value_type>
-auto impl_compare_attribute_value(const attr_t& attribute, const value_type& value)
-    -> decltype(std::cbegin(value), value.at(0),attribute.value(), true)
+template <int index, typename attr_t, typename value_type>
+auto impl_compare_attribute_value(const attr_t& attribute,
+    const value_type& value) -> decltype(std::cbegin(value), value.at(0), attribute.value(), true)
 {
-    return attribute.template get<typename std::remove_const_t<std::remove_reference_t<value_type>>::value_type>()
+    return attribute.template get<
+               typename std::remove_const_t<std::remove_reference_t<value_type>>::value_type>()
         == value;
 }
 
-template <int index,typename attr_t, typename value_type>
-auto impl_compare_attribute_value(const attr_t& attribute, const value_type& value)
-    -> decltype(std::cbegin(value), value.at(0), attribute[0], true)
+template <int index, typename attr_t, typename value_type>
+auto impl_compare_attribute_value(const attr_t& attribute,
+    const value_type& value) -> decltype(std::cbegin(value), value.at(0), attribute[0], true)
 {
-    return attribute
-               .template get<typename std::remove_const_t<std::remove_reference_t<value_type>>::value_type>(
-                   index)
+    return attribute.template get<
+               typename std::remove_const_t<std::remove_reference_t<value_type>>::value_type>(index)
         == value;
 }
 
-template <int index,typename attr_t,  typename T>
+template <int index, typename attr_t, typename T>
 bool compare_attribute_value(const attr_t& attribute, const T& values)
 {
     auto value = std::get<index>(values);
@@ -532,5 +532,17 @@ SCENARIO("Loading cdf files", "[CDF]")
                 CHECK_VARIABLES(cd);
             }
         }
+        WHEN("file is a 2.4.x cdf")
+        {
+            auto path = std::string(DATA_PATH) + "/ia_k0_epi_19970102_v01.cdf";
+            REQUIRE(file_exists(path));
+            auto cd_opt = cdf::io::load(path);
+            REQUIRE(cd_opt != std::nullopt);
+            auto cd = *cd_opt;
+            THEN("All expected variables are found")
+            {
+                REQUIRE(std::size(cd.variables) == 10);
+            }
+        }
     }
 }