Skip to content

Commit

Permalink
Soft disable deflate filter and add profiling regions to hdf5 output (#…
Browse files Browse the repository at this point in the history 
…899)

* Deflate compression filter is not called any more if compression is soft disabled

* Sprinkle profiling regions over hdf5 outputs
  • Loading branch information
@pgrete
pgrete authored Jun 27, 2023
1 parent d9d2212 commit 105c7d0
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 9 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
- [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing

### Changed (changing behavior/API/variables/...)
- [[PR 897]](https://github.com/parthenon-hpc-lab/parthenon/pull/897) Deflate compression filter is not called any more if compression is soft disabled
- [[PR 896]](https://github.com/parthenon-hpc-lab/parthenon/pull/896) Update Kokkos integration to support installed version. Use `serial` (flat MPI) host parallelization by default (instead of OpenMP)
- [[PR 894]](https://github.com/parthenon-hpc-lab/parthenon/pull/894) Demand that sparse pool order sparse ids
- [[PR 888]](https://github.com/parthenon-hpc-lab/parthenon/pull/888) Bump Kokkos submodule to 4.0.1
Expand Down
6 changes: 4 additions & 2 deletions doc/sphinx/src/outputs.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,10 @@ allocated only in a few blocks, because all other blocks would write
zeros of these variables, which can drastically increase output file
size (and decrease I/O performance) without compression. The optional
parameter ``hdf5_compression_level`` can be used to set the compression
level (between 1 and 9, default is 5). Compression can be disabled
altogether with the CMake build option
level (between 1 and 9, default is 5). If ``parthenon`` is compiled with
support for compression, this also enables (logical) chunking of the
data in blocks of ``nx1*nx2*nx3``. Compression (and thus chunking) can
be disabled altogether with the CMake build option
``PARTHENON_DISABLE_HDF5_COMPRESSION``.
See the :ref:`building` for more details.

Expand Down
54 changes: 47 additions & 7 deletions src/outputs/parthenon_hdf5.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
using namespace HDF5;
using namespace OutputUtils;

if constexpr (WRITE_SINGLE_PRECISION) {
Kokkos::Profiling::pushRegion("PHDF5::WriteOutputFileSinglePrec");
} else {
Kokkos::Profiling::pushRegion("PHDF5::WriteOutputFileRealPrec");
}

// writes all graphics variables to hdf file
// HDF5 structures
// Also writes companion xdmf file
Expand Down Expand Up @@ -111,7 +117,9 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
// -------------------------------------------------------------------------------- //
// WRITING ATTRIBUTES //
// -------------------------------------------------------------------------------- //
Kokkos::Profiling::pushRegion("write Attributes");
{
Kokkos::Profiling::pushRegion("write input");
// write input key-value pairs
std::ostringstream oss;
pin->ParameterDump(oss);
Expand All @@ -120,11 +128,13 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
const H5G input_group = MakeGroup(file, "/Input");

HDF5WriteAttribute("File", oss.str().c_str(), input_group);
} // Input section
Kokkos::Profiling::popRegion(); // write input
} // Input section

// we'll need this again at the end
const H5G info_group = MakeGroup(file, "/Info");
{
Kokkos::Profiling::pushRegion("write Info");
HDF5WriteAttribute("OutputFormatVersion", OUTPUT_VERSION_FORMAT, info_group);

if (tm != nullptr) {
Expand Down Expand Up @@ -178,18 +188,22 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
}

HDF5WriteAttribute("BoundaryConditions", boundary_condition_str, info_group);
} // Info section
Kokkos::Profiling::popRegion(); // write Info
} // Info section

// write Params
{
Kokkos::Profiling::pushRegion("behold: write Params");
const H5G params_group = MakeGroup(file, "/Params");

for (const auto &package : pm->packages.AllPackages()) {
const auto state = package.second;
// Write all params that can be written as HDF5 attributes
state->AllParams().WriteAllToHDF5(state->label(), params_group);
}
} // Params section
Kokkos::Profiling::popRegion(); // behold: write Params
} // Params section
Kokkos::Profiling::popRegion(); // write Attributes

// -------------------------------------------------------------------------------- //
// WRITING MESHBLOCK METADATA //
Expand Down Expand Up @@ -226,6 +240,7 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm

// write Blocks metadata
{
Kokkos::Profiling::pushRegion("write block metadata");
const H5G gBlocks = MakeGroup(file, "/Blocks");

// write Xmin[ndim] for blocks
Expand Down Expand Up @@ -255,9 +270,11 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
HDF5Write2D(gBlocks, "loc.level-gid-lid-cnghost-gflag", tmpID.data(), p_loc_offset,
p_loc_cnt, p_glob_cnt, pl_xfer);
}
} // Block section
Kokkos::Profiling::popRegion(); // write block metadata
} // Block section

// Write mesh coordinates to file
Kokkos::Profiling::pushRegion("write mesh coords");
for (const bool face : {true, false}) {
const H5G gLocations = MakeGroup(file, face ? "/Locations" : "/VolumeLocations");

Expand All @@ -280,10 +297,12 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
HDF5Write2D(gLocations, "z", loc_z.data(), p_loc_offset, p_loc_cnt, p_glob_cnt,
pl_xfer);
}
Kokkos::Profiling::popRegion(); // write mesh coords

// Write Levels and Logical Locations with the level for each Meshblock loclist contains
// levels and logical locations for all meshblocks on all ranks
{
Kokkos::Profiling::pushRegion("write levels and locations");
const auto &loclist = pm->GetLocList();

std::vector<std::int64_t> levels;
Expand All @@ -310,11 +329,13 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm

// reset for collective output
local_count[0] = num_blocks_local;
Kokkos::Profiling::popRegion(); // write levels and locations
}

// -------------------------------------------------------------------------------- //
// WRITING VARIABLES DATA //
// -------------------------------------------------------------------------------- //
Kokkos::Profiling::pushRegion("write all variable data");

// All blocks have the same list of variable metadata that exist in the entire
// simulation, but not all variables may be allocated on all blocks
Expand Down Expand Up @@ -391,6 +412,7 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm

// for each variable we write
for (auto &vinfo : all_vars_info) {
Kokkos::Profiling::pushRegion("write variable loop");
// not really necessary, but doesn't hurt
memset(tmpData.data(), 0, tmpData.size() * sizeof(OutT));

Expand Down Expand Up @@ -448,13 +470,18 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm

#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION
PARTHENON_HDF5_CHECK(H5Pset_chunk(pl_dcreate, ndim, chunk_size.data()));
PARTHENON_HDF5_CHECK(
H5Pset_deflate(pl_dcreate, std::min(9, output_params.hdf5_compression_level)));
// Do not run the pipeline if compression is soft disabled.
// By default data would still be passed, which may result in slower output.
if (output_params.hdf5_compression_level > 0) {
PARTHENON_HDF5_CHECK(
H5Pset_deflate(pl_dcreate, std::min(9, output_params.hdf5_compression_level)));
}
#endif

// load up data
hsize_t index = 0;

Kokkos::Profiling::pushRegion("fill host output buffer");
// for each local mesh block
for (size_t b_idx = 0; b_idx < num_blocks_local; ++b_idx) {
const auto &pmb = pm->block_list[b_idx];
Expand Down Expand Up @@ -520,11 +547,16 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
}
}
}
Kokkos::Profiling::popRegion(); // fill host output buffer

Kokkos::Profiling::pushRegion("write variable data");
// write data to file
HDF5WriteND(file, var_name, tmpData.data(), ndim, p_loc_offset, p_loc_cnt, p_glob_cnt,
pl_xfer, pl_dcreate);
Kokkos::Profiling::popRegion(); // write variable data
Kokkos::Profiling::popRegion(); // write variable loop
}
Kokkos::Profiling::popRegion(); // write all variable data

// names of variables
std::vector<std::string> var_names;
Expand Down Expand Up @@ -557,6 +589,7 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
// write SparseInfo and SparseFields (we can't write a zero-size dataset, so only write
// this if we have sparse fields)
if (num_sparse > 0) {
Kokkos::Profiling::pushRegion("write sparse info");
local_count[1] = global_count[1] = num_sparse;

HDF5Write2D(file, "SparseInfo", sparse_allocated.get(), p_loc_offset, p_loc_cnt,
Expand All @@ -569,12 +602,14 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm

const H5D dset = H5D::FromHIDCheck(H5Dopen2(file, "SparseInfo", H5P_DEFAULT));
HDF5WriteAttribute("SparseFields", names, dset);
} // SparseInfo and SparseFields sections
Kokkos::Profiling::popRegion(); // write sparse info
} // SparseInfo and SparseFields sections

// -------------------------------------------------------------------------------- //
// WRITING PARTICLE DATA //
// -------------------------------------------------------------------------------- //

Kokkos::Profiling::pushRegion("write particle data");
AllSwarmInfo swarm_info(pm->block_list, output_params.swarms, restart_);
for (auto &[swname, swinfo] : swarm_info.all_info) {
const H5G g_swm = MakeGroup(file, swname);
Expand Down Expand Up @@ -635,9 +670,14 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
pl_xfer);
}
}
Kokkos::Profiling::popRegion(); // write particle data

Kokkos::Profiling::pushRegion("genXDMF");
// generate XDMF companion file
XDMF::genXDMF(filename, pm, tm, nx1, nx2, nx3, all_vars_info, swarm_info);
Kokkos::Profiling::popRegion(); // genXDMF

Kokkos::Profiling::popRegion(); // WriteOutputFile???Prec
}

std::string PHDF5Output::GenerateFilename_(ParameterInput *pin, SimTime *tm,
Expand Down

0 comments on commit 105c7d0

Please sign in to comment.