From b97e479979c5a78aec7093aa7fc568480019df12 Mon Sep 17 00:00:00 2001
From: Tobias Meisel <tobias.meisel@ufz.de>
Date: Thu, 22 Apr 2021 16:01:19 +0200
Subject: [PATCH] [MeL/IO] HDF5: Compression can be turned off
---
MeshLib/IO/XDMF/HdfWriter.cpp | 57 ++++++++++++++++---------------
MeshLib/IO/XDMF/HdfWriter.h | 8 +++--
MeshLib/IO/XDMF/XdmfHdfWriter.cpp | 7 ++--
MeshLib/IO/XDMF/XdmfHdfWriter.h | 2 +-
MeshLib/IO/writeMeshToFile.cpp | 5 +--
ProcessLib/Output/Output.cpp | 2 +-
6 files changed, 44 insertions(+), 37 deletions(-)
diff --git a/MeshLib/IO/XDMF/HdfWriter.cpp b/MeshLib/IO/XDMF/HdfWriter.cpp
index 4fabdb25528..bab994ff7f3 100644
--- a/MeshLib/IO/XDMF/HdfWriter.cpp
+++ b/MeshLib/IO/XDMF/HdfWriter.cpp
@@ -29,7 +29,7 @@ void checkHdfStatus(const hid_t status, std::string const& formatting,
}
}
-static unsigned short int const compression_factor = 1;
+static unsigned short int const default_compression_factor = 1;
using namespace MeshLib::IO;
@@ -73,15 +73,16 @@ static hid_t createStepGroup(hid_t const& file, int const step)
H5P_DEFAULT);
}
-static hid_t writeDataSet(void const* nodes_data, // what
- hid_t const data_type,
- std::vector<Hdf5DimType> const& data_dims, // how ...
- std::vector<Hdf5DimType> const& dim_offsets,
- std::vector<Hdf5DimType> const& dim_maxs,
- [[maybe_unused]] std::vector<Hdf5DimType> const& chunk_dim,
- bool has_compression_lib,
- hid_t const section,
- std::string const& dataset_name) // where
+static hid_t writeDataSet(
+ void const* nodes_data, // what
+ hid_t const data_type,
+ std::vector<Hdf5DimType> const& data_dims, // how ...
+ std::vector<Hdf5DimType> const& dim_offsets,
+ std::vector<Hdf5DimType> const& dim_maxs,
+ [[maybe_unused]] std::vector<Hdf5DimType> const& chunk_dim,
+ bool use_compression,
+ hid_t const section,
+ std::string const& dataset_name) // where
{
int const dim_size = data_dims.size();
hid_t const memspace =
@@ -91,15 +92,15 @@ static hid_t writeDataSet(void const* nodes_data, // what
hid_t dataset_property = H5Pcreate(H5P_DATASET_CREATE);
- hid_t status = H5Pset_chunk(dataset_property, dim_size, dim_maxs.data());
- if (status != 0)
- {
- ERR("H5Pset_layout failed for data set: {:s}.", dataset_name);
- }
-
- if (has_compression_lib)
+ if (use_compression )
{
- H5Pset_deflate(dataset_property, compression_factor);
+ hid_t status =
+ H5Pset_chunk(dataset_property, dim_size, dim_maxs.data());
+ if (status != 0)
+ {
+ ERR("H5Pset_layout failed for data set: {:s}.", dataset_name);
+ }
+ H5Pset_deflate(dataset_property, default_compression_factor);
}
hid_t const dataset =
@@ -115,7 +116,7 @@ static hid_t writeDataSet(void const* nodes_data, // what
std::vector<hsize_t> const count(dim_size, 1);
std::vector<hsize_t> const block = data_dims;
- status = H5Sselect_hyperslab(dataset_filespace, H5S_SELECT_SET,
+ hid_t status = H5Sselect_hyperslab(dataset_filespace, H5S_SELECT_SET,
dim_offsets.data(), stride.data(),
count.data(), block.data());
if (status != 0)
@@ -143,14 +144,16 @@ HdfWriter::HdfWriter(std::vector<HdfData> constant_attributes,
std::vector<HdfData>
variable_attributes,
int const step,
- std::filesystem::path const& filepath)
+ std::filesystem::path const& filepath,
+ bool const compression_factor)
: _variable_attributes(std::move(variable_attributes)),
_hdf5_filepath(filepath),
- _has_compression(checkCompression())
+// _has_compression(checkCompression()),
+ _use_compression(checkCompression() && compression_factor)
{
- file = createFile(filepath);
+ _file = createFile(filepath);
std::string const& time_section = getTimeSection(step);
- hid_t const group_id = H5Gcreate2(file, time_section.c_str(), H5P_DEFAULT,
+ hid_t const group_id = H5Gcreate2(_file, time_section.c_str(), H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
for (auto const& attribute : constant_attributes)
@@ -158,7 +161,7 @@ HdfWriter::HdfWriter(std::vector<HdfData> constant_attributes,
hid_t status = writeDataSet(attribute.data_start, attribute.data_type,
attribute.data_space, attribute.offsets,
attribute.file_space, attribute.chunk_space,
- _has_compression, group_id, attribute.name);
+ _use_compression, group_id, attribute.name);
checkHdfStatus(status, "Writing HDF5 Dataset: {:s} failed.",
attribute.name);
@@ -171,12 +174,12 @@ HdfWriter::HdfWriter(std::vector<HdfData> constant_attributes,
HdfWriter::~HdfWriter()
{
- H5Fclose(file);
+ H5Fclose(_file);
}
bool HdfWriter::writeStep(int const step) const
{
- hid_t const group = createStepGroup(file, step);
+ hid_t const group = createStepGroup(_file, step);
hid_t status = 0;
for (auto const& attribute : _variable_attributes)
@@ -184,7 +187,7 @@ bool HdfWriter::writeStep(int const step) const
status = writeDataSet(attribute.data_start, attribute.data_type,
attribute.data_space, attribute.offsets,
attribute.file_space, attribute.chunk_space,
- _has_compression, group, attribute.name);
+ _use_compression, group, attribute.name);
checkHdfStatus(status, "Writing HDF5 Dataset: {:s} failed.",
attribute.name);
diff --git a/MeshLib/IO/XDMF/HdfWriter.h b/MeshLib/IO/XDMF/HdfWriter.h
index a74d082c0d8..97c6fa656bd 100644
--- a/MeshLib/IO/XDMF/HdfWriter.h
+++ b/MeshLib/IO/XDMF/HdfWriter.h
@@ -31,11 +31,13 @@ public:
* property), contains meta data for geometry and topology
* @param step number of the step (temporal collection)
* @param filepath absolute or relative filepath to the hdf5 file
+ * @param use_compression if true gzip compression is enabled
*/
HdfWriter(std::vector<HdfData> constant_attributes,
std::vector<HdfData> variable_attributes,
int const step,
- std::filesystem::path const& filepath);
+ std::filesystem::path const& filepath,
+ bool const use_compression);
/**
* \brief Writes attributes. The data
@@ -51,7 +53,7 @@ public:
private:
std::vector<HdfData> const _variable_attributes;
std::filesystem::path const _hdf5_filepath;
- bool const _has_compression;
- hid_t file;
+ bool const _use_compression;
+ hid_t _file;
};
} // namespace MeshLib::IO
\ No newline at end of file
diff --git a/MeshLib/IO/XDMF/XdmfHdfWriter.cpp b/MeshLib/IO/XDMF/XdmfHdfWriter.cpp
index c5e244d8aaf..5c512312c67 100644
--- a/MeshLib/IO/XDMF/XdmfHdfWriter.cpp
+++ b/MeshLib/IO/XDMF/XdmfHdfWriter.cpp
@@ -21,7 +21,8 @@ XdmfHdfWriter::XdmfHdfWriter(MeshLib::Mesh const& mesh,
std::filesystem::path const& filepath,
int const time_step,
std::set<std::string>
- variable_output_names)
+ variable_output_names,
+ bool const use_compression)
{
// transform Data into contiguous data and collect meta data
Geometry const& geometry = transformGeometry(mesh);
@@ -57,7 +58,7 @@ XdmfHdfWriter::XdmfHdfWriter(MeshLib::Mesh const& mesh,
// The hdf writer can write when data is out of scope.
_hdf_writer = std::make_unique<HdfWriter>(
std::move(hdf_constant_attributes), std::move(hdf_variable_attributes),
- time_step, hdf_filepath);
+ time_step, hdf_filepath, use_compression);
// XDMF
// The light data is only written by just one process
if (!isFileManager())
@@ -108,4 +109,4 @@ void XdmfHdfWriter::writeStep(int const time_step, double const time) const
_xdmf_writer->writeStep(time_step, time);
}
-} // namespace MeshLib::IO
\ No newline at end of file
+} // namespace MeshLib::IO
diff --git a/MeshLib/IO/XDMF/XdmfHdfWriter.h b/MeshLib/IO/XDMF/XdmfHdfWriter.h
index 8c61646214e..eece0c5b722 100644
--- a/MeshLib/IO/XDMF/XdmfHdfWriter.h
+++ b/MeshLib/IO/XDMF/XdmfHdfWriter.h
@@ -34,7 +34,7 @@ public:
*/
XdmfHdfWriter(MeshLib::Mesh const& mesh,
std::filesystem::path const& filepath, int time_step,
- std::set<std::string> variable_output_names);
+ std::set<std::string> variable_output_names, bool use_compression);
/**
* \brief Write attribute data that has modified to previous time step or
* initial
diff --git a/MeshLib/IO/writeMeshToFile.cpp b/MeshLib/IO/writeMeshToFile.cpp
index 12f2705ab0e..c0ba41acdbf 100644
--- a/MeshLib/IO/writeMeshToFile.cpp
+++ b/MeshLib/IO/writeMeshToFile.cpp
@@ -21,7 +21,8 @@ namespace MeshLib::IO
{
int writeMeshToFile(const MeshLib::Mesh& mesh,
std::filesystem::path const& file_path,
- std::set<std::string> names)
+ std::set<std::string>
+ names)
{
if (file_path.extension().string() == ".msh")
{
@@ -46,7 +47,7 @@ int writeMeshToFile(const MeshLib::Mesh& mesh,
if (file_path.extension().string() == ".xdmf")
{
auto writer = std::make_unique<MeshLib::IO::XdmfHdfWriter>(
- MeshLib::IO::XdmfHdfWriter(mesh, file_path, 0, names));
+ MeshLib::IO::XdmfHdfWriter(mesh, file_path, 0, names, true));
return 0;
}
#endif
diff --git a/ProcessLib/Output/Output.cpp b/ProcessLib/Output/Output.cpp
index 10f6bb40541..b1a97ba4de3 100644
--- a/ProcessLib/Output/Output.cpp
+++ b/ProcessLib/Output/Output.cpp
@@ -246,7 +246,7 @@ void Output::outputMeshXdmf(OutputFile const& output_file,
_mesh_xdmf_hdf_writer = std::make_unique<MeshLib::IO::XdmfHdfWriter>(
MeshLib::IO::XdmfHdfWriter(
mesh, path, timestep,
- _output_data_specification.output_variables));
+ _output_data_specification.output_variables, output_file.compression));
}
_mesh_xdmf_hdf_writer->writeStep(timestep, t);
}
--
GitLab