From 3a3a7cde99f245ad73d926b426f943c222f0ac0e Mon Sep 17 00:00:00 2001 From: Aastha Bagree Date: Tue, 18 Jul 2023 11:55:20 -0700 Subject: [PATCH] Added CLI for CombineH5 along with a test Made required modifications, added check_src flag Made newer changes Made newer changes Removed unnecessary includes Changes --- src/IO/H5/CombineH5.cpp | 10 +- src/IO/H5/CombineH5.hpp | 19 +--- src/IO/H5/Python/CombineH5.cpp | 2 +- src/IO/H5/Python/CombineH5.py | 71 ++++++++++++++ support/Python/__main__.py | 5 + tests/Unit/IO/H5/CMakeLists.txt | 4 +- tests/Unit/IO/H5/Test_CombineH5.py | 147 +++++++++++++++++++++++++++-- 7 files changed, 227 insertions(+), 31 deletions(-) create mode 100644 src/IO/H5/Python/CombineH5.py diff --git a/src/IO/H5/CombineH5.cpp b/src/IO/H5/CombineH5.cpp index 995d51cc6d67..a356abf6a418 100644 --- a/src/IO/H5/CombineH5.cpp +++ b/src/IO/H5/CombineH5.cpp @@ -1,6 +1,8 @@ // Distributed under the MIT License. // See LICENSE.txt for details. +#include "IO/H5/CombineH5.hpp" + #include #include #include @@ -46,11 +48,11 @@ size_t get_number_of_elements(const std::vector& input_filenames, } return total_elements; } -} +} //namespace namespace h5 { void combine_h5(const std::string& file_prefix, const std::string& subfile_name, - const std::string& output) { + const std::string& output, const bool check_src) { // Parses for and stores all input files to be looped over const std::vector& file_names = file_system::glob(file_prefix + "[0-9]*.h5"); @@ -58,11 +60,13 @@ void combine_h5(const std::string& file_prefix, const std::string& subfile_name, std::string{MakeString{} << file_names}.c_str()); // Checks that volume data was generated with identical versions of SpECTRE - if (!h5::check_src_files_match(file_names)) { + if (check_src){ + if (!h5::check_src_files_match(file_names)) { ERROR( "One or more of your files were found to have differing src.tar.gz " "files, meaning that they may be from differing versions of SpECTRE."); } + } // Checks that volume data files contain the same observation ids if (!h5::check_observation_ids_match(file_names, subfile_name)) { diff --git a/src/IO/H5/CombineH5.hpp b/src/IO/H5/CombineH5.hpp index ff71bddce464..30a7948fcf3b 100644 --- a/src/IO/H5/CombineH5.hpp +++ b/src/IO/H5/CombineH5.hpp @@ -1,28 +1,13 @@ // Distributed under the MIT License. // See LICENSE.txt for details. + #pragma once -#include -#include -#include -#include #include -#include - -#include "DataStructures/DataVector.hpp" -#include "IO/H5/AccessType.hpp" -#include "IO/H5/CheckH5PropertiesMatch.hpp" -#include "IO/H5/File.hpp" -#include "IO/H5/SourceArchive.hpp" -#include "IO/H5/VolumeData.hpp" -#include "Parallel/Printf.hpp" -#include "Utilities/FileSystem.hpp" -#include "Utilities/MakeString.hpp" -#include "Utilities/StdHelpers.hpp" namespace h5 { void combine_h5(const std::string& file_prefix, const std::string& subfile_name, - const std::string& output); + const std::string& output, const bool check_src = true); } // namespace h5 diff --git a/src/IO/H5/Python/CombineH5.cpp b/src/IO/H5/Python/CombineH5.cpp index 24ab02007c78..c59c2683866d 100644 --- a/src/IO/H5/Python/CombineH5.cpp +++ b/src/IO/H5/Python/CombineH5.cpp @@ -15,6 +15,6 @@ namespace py_bindings { void bind_h5combine(py::module& m) { // Wrapper for combining h5 files m.def("combine_h5", &h5::combine_h5, py::arg("file_prefix"), - py::arg("subfile_name"), py::arg("output")); + py::arg("subfile_name"), py::arg("output"), py::arg("check_src")); } } // namespace py_bindings diff --git a/src/IO/H5/Python/CombineH5.py b/src/IO/H5/Python/CombineH5.py new file mode 100644 index 000000000000..cfc6073f7680 --- /dev/null +++ b/src/IO/H5/Python/CombineH5.py @@ -0,0 +1,71 @@ +# Distributed under the MIT License. +# See LICENSE.txt for details. + +import logging +import os + +import click +import rich + +import spectre.IO.H5 as spectre_h5 + + +@click.command(name="combine-h5") +@click.option( + "--file-prefix", + required=True, + type=str, + help="prefix of the files to be combined (omit number and file extension)", +) +@click.option( + "--subfile-name", + "-d", + type=str, + help="subfile name of the volume file in the H5 file (omit file extension)", +) +@click.option( + "--output", + "-o", + required=True, + type=click.Path( + exists=False, + file_okay=True, + dir_okay=False, + writable=True, + readable=True, + ), + help="combined output filename (omit file extension)", +) +@click.option( + "--check-src/--no-check-src", + default=True, + show_default=True, + help=( + "flag to check src files, True implies src files exist and can be" + " checked, False implies no src files to check." + ), +) +def combine_h5_command(file_prefix, subfile_name, output, check_src): + """Combines multiple HDF5 volume files + + This executable is used for combining a series of HDF5 volume files into one + continuous dataset to be stored in a single HDF5 volume file.""" + + # Print available subfile names and exit + filename = file_prefix + "0.h5" + if not subfile_name: + if os.path.exists(filename): + spectre_file = spectre_h5.H5File(filename, "r") + import rich.columns + + rich.print(rich.columns.Columns(spectre_file.all_vol_files())) + return + + if subfile_name[0] == "/": + subfile_name = subfile_name[1:] + + spectre_h5.combine_h5(file_prefix, subfile_name, output, check_src) + + +if __name__ == "__main__": + combine_h5_command(help_option_names=["-h", "--help"]) diff --git a/support/Python/__main__.py b/support/Python/__main__.py index 3dbf7a7a9ed3..84d3f87245be 100644 --- a/support/Python/__main__.py +++ b/support/Python/__main__.py @@ -21,6 +21,7 @@ class Cli(click.MultiCommand): def list_commands(self, ctx): return [ "clean-output", + "combine-h5", "delete-subfiles", "extend-connectivity", "extract-dat", @@ -44,6 +45,10 @@ def get_command(self, ctx, name): from spectre.tools.CleanOutput import clean_output_command return clean_output_command + elif name == "combine-h5": + from spectre.IO.H5.CombineH5 import combine_h5_command + + return combine_h5_command elif name == "delete-subfiles": from spectre.IO.H5.DeleteSubfiles import delete_subfiles_command diff --git a/tests/Unit/IO/H5/CMakeLists.txt b/tests/Unit/IO/H5/CMakeLists.txt index c3526c11a672..5122ccc30c1d 100644 --- a/tests/Unit/IO/H5/CMakeLists.txt +++ b/tests/Unit/IO/H5/CMakeLists.txt @@ -42,12 +42,10 @@ spectre_add_python_bindings_test( ) if(${BUILD_PYTHON_BINDINGS}) - # Test is a bit slow because it writes a bunch of plot files to verify - # the argument handling works. set_tests_properties( "Unit.IO.H5.CombineH5.Python" PROPERTIES - TIMEOUT 80 + TIMEOUT 10 ) endif() diff --git a/tests/Unit/IO/H5/Test_CombineH5.py b/tests/Unit/IO/H5/Test_CombineH5.py index 83d150ae533b..9e00ca31bb07 100644 --- a/tests/Unit/IO/H5/Test_CombineH5.py +++ b/tests/Unit/IO/H5/Test_CombineH5.py @@ -5,11 +5,13 @@ import unittest import numpy as np +from click.testing import CliRunner import spectre.IO.H5 as spectre_h5 from spectre import Informer from spectre.DataStructures import DataVector from spectre.IO.H5 import ElementVolumeData, TensorComponent, combine_h5 +from spectre.IO.H5.CombineH5 import combine_h5_command from spectre.Spectral import Basis, Quadrature @@ -38,7 +40,7 @@ def setUp(self): # Initializing attributes grid_names1 = ["[B0(L0I0,L0I0,L1I0)]"] - grid_names2 = ["[B0(L1I0,L0I0,L0I0)]"] + grid_names2 = ["[B1(L1I0,L0I0,L0I0)]"] observation_values = {0: 7.0, 1: 1.3} basis = Basis.Legendre quad = Quadrature.Gauss @@ -46,7 +48,12 @@ def setUp(self): # Writing ElementVolume data and TensorComponent Data to first file self.h5_file1 = spectre_h5.H5File(file_name=self.file_name1, mode="a") - self.tensor_component_data1 = np.array([[0.0], [0.3], [0.6]]) + self.tensor_component_data1 = np.array( + [ + [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7], + [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08], + ] + ) self.h5_file1.insert_vol("/element_data", version=0) self.h5_file1.close_current_object() self.vol_file1 = self.h5_file1.get_vol(path="/element_data") @@ -60,7 +67,7 @@ def setUp(self): ), TensorComponent( "field_2", - DataVector(self.tensor_component_data1[i + 1]), + DataVector(self.tensor_component_data1[i]), ), ], extents=3 * [2], @@ -89,7 +96,12 @@ def setUp(self): # Writing ElementVolume data and TensorComponent Data to second file self.h5_file2 = spectre_h5.H5File(file_name=self.file_name2, mode="a") - self.tensor_component_data2 = np.array([[1.0], [0.13], [0.16]]) + self.tensor_component_data2 = np.array( + [ + [0.1, 0.12, 0.22, 0.32, 0.42, 0.52, 0.62, 0.72], + [0.011, 0.021, 0.031, 0.041, 0.051, 0.061, 0.079, 0.089], + ] + ) self.h5_file2.insert_vol("/element_data", version=0) self.h5_file2.close_current_object() self.vol_file2 = self.h5_file2.get_vol(path="/element_data") @@ -103,7 +115,7 @@ def setUp(self): ), TensorComponent( "field_2", - DataVector(self.tensor_component_data2[i + 1]), + DataVector(self.tensor_component_data2[i]), ), ], extents=3 * [2], @@ -138,9 +150,112 @@ def test_combine_h5(self): # Run the combine_h5 command and check if any feature (for eg. # connectivity length has increased due to combining two files) - combine_h5(self.file_name1[:-4], "element_data", self.output_file) + combine_h5( + self.file_name1[:-4], "element_data", self.output_file, False + ) + h5_output = spectre_h5.H5File( + file_name=self.output_file + "0.h5", mode="r" + ) + output_vol = h5_output.get_vol(path="/element_data") + + # Test observation ids + + actual_obs_ids = output_vol.list_observation_ids() + actual_obs_ids.sort() + + expected_obs_ids = [0, 1] + + self.assertEqual(actual_obs_ids, expected_obs_ids) + + # Test observation values + + obs_id = actual_obs_ids[0] + + actual_obs_value = output_vol.get_observation_value(obs_id) + expected_obs_value = 7.0 + + self.assertEqual(actual_obs_value, expected_obs_value) + + # Test tensor components + + actual_tensor_component_names = output_vol.list_tensor_components( + obs_id + ) + + expected_tensor_component_names = ["field_1", "field_2"] + + self.assertEqual( + actual_tensor_component_names, expected_tensor_component_names + ) + + expected_tensor_components = np.array( + [ + ( + 0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.1, + 0.12, + 0.22, + 0.32, + 0.42, + 0.52, + 0.62, + 0.72, + ), + ( + 0.01, + 0.02, + 0.03, + 0.04, + 0.05, + 0.06, + 0.07, + 0.08, + 0.011, + 0.021, + 0.031, + 0.041, + 0.051, + 0.061, + 0.079, + 0.089, + ), + ] + ) + for i in range(len(expected_tensor_components)): + value = ( + output_vol.get_tensor_component( + i, actual_tensor_component_names[i] + ).data + == expected_tensor_components[i] + ) + self.assertEqual(value, True) + + def test_cli(self): + # Checks if the CLI for CombineH5 runs properly + runner = CliRunner() + result = runner.invoke( + combine_h5_command, + [ + "--file-prefix", + self.file_name1[:-4], + "-d", + "element_data", + "-o", + self.output_file, + "--check-src", + ], + catch_exceptions=False, + ) + h5_output = spectre_h5.H5File( - file_name=self.output_file + "0.h5", mode="a" + file_name=self.output_file + "0.h5", mode="r" ) output_vol = h5_output.get_vol(path="/element_data") @@ -153,6 +268,24 @@ def test_combine_h5(self): ).data assert len(self.initial_h5_connectivity) < len(final_h5_connectivity) + self.assertEqual(result.exit_code, 0, result.output) + + def test_cli2(self): + # Checks that if no subfile name is given, the CLI prints + # available subfiles correctly + runner = CliRunner() + result = runner.invoke( + combine_h5_command, + [ + "--file-prefix", + self.file_name1[:-4], + "-o", + self.output_file, + "--check-src", + ], + catch_exceptions=False, + ) + assert result.output is not None if __name__ == "__main__":