Commit abd88a52 authored by Tobias Meisel's avatar Tobias Meisel
Browse files

[MeL/IO] Separate dedicated class for writing HDF5

parent c0ef1fa8
/**
* \file
* \copyright
* Copyright (c) 2012-2020, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
* See accompanying file LICENSE.txt or
* http://www.opengeosys.org/project/license
*/
#include "HdfWriter.h"
#include <hdf5.h>
#include <string>
#include <utility>
#include <vector>
#include "BaseLib/Error.h"
#include "BaseLib/Logging.h"
#include "fileIO.h"
template <typename... Args>
void checkHdfStatus(const hid_t status, const std::string formatting,
Args... args)
{
if (status < 0)
{
OGS_FATAL(formatting, std::forward<Args>(args)...);
}
}
static unsigned short int const compression_factor = 5;
using namespace MeshLib::IO;
using namespace std::string_literals;
static std::string getTimeSection(int const step)
{
return "t_"s + std::to_string(step);
}
static bool checkCompression()
{
// Check if gzip compression is available and can be used for both
// compression and decompression.
if (htri_t avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE); !avail)
{
WARN("gzip filter not available.\n");
return false;
}
unsigned int filter_info;
H5Zget_filter_info(H5Z_FILTER_DEFLATE, &filter_info);
if (!(filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) ||
!(filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED))
{
WARN("gzip filter not available for encoding and decoding.\n");
return false;
}
return true;
}
static hid_t createStepGroup(hid_t const& file, int const step)
{
std::string const& time_section = getTimeSection(step);
// Open or create Group
if (H5Lexists(file, time_section.c_str(), H5P_DEFAULT) > 0)
{
return H5Gopen2(file, time_section.c_str(), H5P_DEFAULT);
}
return H5Gcreate2(file, time_section.c_str(), H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT);
}
static hid_t writeDataSet(
void const* nodes_data, // what
hid_t const data_type,
std::vector<Hdf5DimType> const& chunked_dims, // how ...
std::vector<Hdf5DimType> const& dim_offsets,
std::vector<Hdf5DimType> const& dim_maxs, bool has_compression_lib,
hid_t const section,
std::string const& dataset_name) // where
{
DBUG("Num global nodes : {:d} ", dim_maxs[0]);
int const dim_size = chunked_dims.size();
hid_t const memspace =
H5Screate_simple(dim_size, chunked_dims.data(), nullptr);
hid_t const filespace =
H5Screate_simple(dim_size, dim_maxs.data(), nullptr);
hid_t dataset_property = H5Pcreate(H5P_DATASET_CREATE);
if (has_compression_lib)
{
H5Pset_deflate(dataset_property, compression_factor);
DBUG("Compression will be used for {:s} with factor {:d}.",
dataset_name, compression_factor);
}
hid_t status =
H5Pset_chunk(dataset_property, dim_size, chunked_dims.data());
if (status != 0)
{
ERR("H5Pset_layout failed for data set: {:s}.", dataset_name);
}
// TODO (tm) Compression is not enabled!!
dataset_property = H5P_DEFAULT;
hid_t const dataset =
H5Dcreate2(section, dataset_name.c_str(), H5T_IEEE_F64BE, filespace,
H5P_DEFAULT, dataset_property, H5P_DEFAULT);
H5Pclose(dataset_property);
H5Sclose(filespace);
hid_t const dataset_filespace = H5Dget_space(dataset);
std::vector<hsize_t> const stride(dim_size, 1);
std::vector<hsize_t> const count(dim_size, 1);
std::vector<hsize_t> const block = chunked_dims;
DBUG("Offset in dataset '{:s}' is: {:d}.", dataset_name, dim_offsets[0]);
status = H5Sselect_hyperslab(dataset_filespace, H5S_SELECT_SET,
dim_offsets.data(), stride.data(),
count.data(), block.data());
if (status != 0)
{
ERR("H5Sselect_hyperslab failed in dataset '{:s}'.", dataset_name);
}
hid_t const io_transfer_property = createHDF5TransferPolicy();
status = H5Dwrite(dataset, data_type, memspace, dataset_filespace,
io_transfer_property, nodes_data);
if (status != 0)
{
ERR("H5Dwrite failed in dataset '{:s}'.", dataset_name);
}
H5Dclose(dataset);
H5Pclose(io_transfer_property);
status = H5Sclose(memspace);
return (status >= 0 ? 1 : 0);
}
namespace MeshLib::IO
{
HdfWriter::HdfWriter(HdfData const& geometry,
HdfData const& topology,
std::vector<HdfData>
attributes,
int const step,
std::filesystem::path const& filepath)
: _attributes(std::move(attributes)),
_hdf5_filepath(filepath),
_has_compression(checkCompression())
{
hid_t const file = createFile(filepath);
std::string const& time_section = getTimeSection(step);
hid_t const group_id = H5Gcreate2(file, time_section.c_str(), H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
// geometry
hid_t status =
writeDataSet(geometry.data_start, geometry.data_type,
geometry.data_space, geometry.offsets, geometry.file_space,
_has_compression, group_id, geometry.name);
checkHdfStatus(status, "Writing HDF5 Dataset: {:s} failed.", geometry.name);
// topology
status =
writeDataSet(topology.data_start, topology.data_type,
topology.data_space, topology.offsets, topology.file_space,
_has_compression, group_id, topology.name);
checkHdfStatus(status, "Writing HDF5 Dataset: {:s} failed.", topology.name);
H5Gclose(group_id);
status = H5Fclose(file);
checkHdfStatus(status, "HDF Writer could not be created!", topology.name);
}
bool HdfWriter::writeStep(int const step) const
{
hid_t const file = openHDF5File(_hdf5_filepath);
hid_t const group = createStepGroup(file, step);
hid_t status = 0;
for (auto const& attribute : _attributes)
{
status = writeDataSet(attribute.data_start, attribute.data_type,
attribute.data_space, attribute.offsets,
attribute.file_space, _has_compression, group,
attribute.name);
checkHdfStatus(status, "Not all attributes written to HDF5 file.");
}
H5Gclose(group);
status = H5Fclose(file);
return (status >= 0);
}
} // namespace MeshLib::IO
\ No newline at end of file
/**
* \file
* \author Tobias Meisel
* \date 2020-12-15
* \brief Writes vectorized data to HDF File
* \copyright Copyright (c) 2012-2020, OpenGeoSys Community
* (http://www.opengeosys.org) Distributed under a Modified BSD License. See
* accompanying file LICENSE.txt or http://www.opengeosys.org/project/license
*/
#pragma once
#include <filesystem.h>
#include <vector>
#include "HdfData.h"
namespace MeshLib::IO
{
class HdfWriter final
{
public:
/**
* \brief Write file with geometry and topology data. The data
* itself is held by a structure outside of this class. The writer assumes
* the data holder to not change during writing
* @param geometry contains meta data to coordinates
* @param topology contains meta data cells
* @param attributes vector of attributes (each attribute is a OGS mesh
* property)
* @param step number of the step (temporal collection)
* @param filepath absolute or relative filepath to the hdf5 file
*/
HdfWriter(HdfData const& geometry,
HdfData const& topology,
std::vector<HdfData>
attributes,
int const step,
std::filesystem::path const& filepath);
/**
* \brief Writes attributes. The data
* itself is hold by a structure outside of this class. The writer assumes
* the data holder to not change during writing and HdfData given to
* constructor to be still valid
* @param step number of the step (temporal collection)
* @return true = success, false = error
*/
bool writeStep(int step) const;
private:
std::vector<HdfData> const _attributes;
std::filesystem::path const _hdf5_filepath;
bool const _has_compression;
};
} // namespace MeshLib::IO
\ No newline at end of file
/**
* \file
* \copyright
* Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
* See accompanying file LICENSE.txt or
* http://www.opengeosys.org/project/license
*/
#include "writeHDF5.h"
#include <boost/shared_ptr.hpp>
#include <string>
#include "BaseLib/Error.h"
#include "BaseLib/Logging.h"
#include "XdmfArrayType.hpp"
#include "hdf5.h"
static unsigned short int const compression_factor = 5;
hid_t XdmfType2Hdf5Type(boost::shared_ptr<XdmfArrayType const> xdmf)
{
std::map<boost::shared_ptr<XdmfArrayType const>, hid_t> xdmf2HdfType = {
{XdmfArrayType::Float64(), H5T_IEEE_F64LE},
{XdmfArrayType::Float32(), H5T_IEEE_F32LE},
{XdmfArrayType::Int32(), H5T_STD_I32LE},
//{XdmfArrayType::Int64(), H5T_STD_I64LE},
{XdmfArrayType::UInt32(), H5T_STD_U32LE},
//{XdmfArrayType::UInt64(), H5T_STD_U64LE},
{XdmfArrayType::Int8(), H5T_STD_I8LE},
{XdmfArrayType::UInt8(), H5T_STD_U8LE}};
try
{
return xdmf2HdfType.at(xdmf);
}
catch (const std::exception& e)
{
OGS_FATAL("No known HDF5 type for XDMF type : {:s} .", xdmf->getName());
}
}
using namespace std::string_literals;
static std::string getTimeSection(int const step)
{
return "t_"s + std::to_string(step);
}
namespace MeshLib::IO
{
int writeHDF5Step(std::filesystem::path const& filepath, int const step,
std::string const& attribute_name, void const* attribute_data,
std::vector<unsigned long long> const& attribute_dims,
boost::shared_ptr<const XdmfArrayType> data_type,
bool const has_compression_lib)
{
// \TODO (tm) Errhandling, not implemented as we will change from CAPI to
// C++API negative value is failure
hid_t file = H5Fopen(filepath.string().c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
std::string const time_section = getTimeSection(step);
// Open or create Group
hid_t group_id = [](auto file, auto time_section) {
if (H5Lexists(file, time_section.c_str(), H5P_DEFAULT) > 0)
{
// negative value is failure
return H5Gopen2(file, time_section.c_str(), H5P_DEFAULT);
}
// negative value is failure
return H5Gcreate2(file, time_section.c_str(), H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT);
}(file, time_section);
// negative value is failure
hid_t dataset_property = H5Pcreate(H5P_DATASET_CREATE);
// Alternativle set H5Pset_layout(dcpl, H5D_CONTIGUOUS);
herr_t status;
if (has_compression_lib)
{
status = H5Pset_deflate(dataset_property, compression_factor);
}
status = H5Pset_chunk(dataset_property, attribute_dims.size(),
attribute_dims.data());
hid_t space =
H5Screate_simple(attribute_dims.size(), attribute_dims.data(), nullptr);
hid_t dset = H5Dcreate(group_id, attribute_name.c_str(),
XdmfType2Hdf5Type(data_type), space, H5P_DEFAULT,
dataset_property, H5P_DEFAULT);
status = H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT,
attribute_data);
status = H5Pclose(dataset_property);
status = H5Dclose(dset);
status = H5Sclose(space);
status = H5Gclose(group_id);
status = H5Fclose(file);
return status;
}
std::pair<int, bool> writeHDF5Initial(
std::vector<double> const& nodes,
std::vector<unsigned long long> const& geometry_dims,
std::vector<int> const& topology,
int const step,
std::filesystem::path const& filepath)
{
// Check if gzip compression is available and can be used for both
// compression and decompression.
bool has_compression_lib = true;
if (htri_t avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE); !avail)
{
WARN("gzip filter not available.\n");
has_compression_lib = false;
}
unsigned int filter_info;
H5Zget_filter_info(H5Z_FILTER_DEFLATE, &filter_info);
if (!(filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) ||
!(filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED))
{
WARN("gzip filter not available for encoding and decoding.\n");
has_compression_lib = false;
}
hid_t dataset_property_geometry = H5Pcreate(H5P_DATASET_CREATE);
herr_t status;
// status = H5Pset_layout(dcpl, H5D_CONTIGUOUS);
if (has_compression_lib)
{
status = H5Pset_deflate(dataset_property_geometry, compression_factor);
}
status = H5Pset_chunk(dataset_property_geometry, geometry_dims.size(),
geometry_dims.data());
if (status != 0)
{
ERR("H5Pset_layout failed for geometry");
}
hid_t geometry_space = H5Screate_simple(2, geometry_dims.data(), nullptr);
hsize_t topology_dims[1] = {topology.size()};
hid_t topology_space = H5Screate_simple(1, topology_dims, nullptr);
{
hid_t file = H5Fcreate(filepath.string().c_str(), H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT);
std::string const time_section = getTimeSection(step);
hid_t group_id = H5Gcreate2(file, time_section.c_str(), H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
hid_t dataset_geometry =
H5Dcreate(group_id, "geometry", H5T_IEEE_F64BE, geometry_space,
H5P_DEFAULT, dataset_property_geometry, H5P_DEFAULT);
status = H5Dwrite(dataset_geometry, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL,
H5P_DEFAULT, nodes.data());
hid_t dataset_property_topology = H5Pcreate(H5P_DATASET_CREATE);
// status = H5Pset_layout(dcpl, H5D_CONTIGUOUS);
status = H5Pset_deflate(dataset_property_topology, compression_factor);
status = H5Pset_chunk(dataset_property_topology, 1, topology_dims);
hid_t dataset_topology =
H5Dcreate(group_id, "topology", H5T_STD_I32LE, topology_space,
H5P_DEFAULT, dataset_property_topology, H5P_DEFAULT);
status = H5Dwrite(dataset_topology, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, topology.data());
status = H5Pclose(dataset_property_geometry);
status = H5Pclose(dataset_property_topology);
status = H5Dclose(dataset_geometry);
status = H5Dclose(dataset_topology);
status = H5Sclose(geometry_space);
status = H5Gclose(group_id);
status = H5Fclose(file);
}
return {0, has_compression_lib};
}
} // namespace MeshLib::IO
/**
* \file
* \author Tobias Meisel
* \date 2020-11-13
* \brief Write vectorized data into hdf5 file
* \copyright
* Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
* See accompanying file LICENSE.txt or
* http://www.opengeosys.org/project/license
*/
#pragma once
#include <boost/shared_ptr.hpp>
#include <filesystem>
#include <vector>
class XdmfArrayType;
namespace MeshLib::IO
{
/**
* \brief Writes the initial data (constant data) to the h5 file.
* The file will be overwritten if it already exists
* @param nodes flattened vector of all nodes (points)
* (containing point coordinates in form px0 py0 pz0 px1 py1 pz1)
* @param geometry_dims vector of values at the supporting points
* @param topology flattened vector of all cells
* (containg [cell1, cell2] with flattened values for each cell
* [celltype cellpoint_1..cellpoint_n]
* @param step number of the timestep
* @param filepath absolute or relativ filepath to the hdf5 file
* @return pair with pair.first: returns negative value on error
* pair.second return true if compression is found on system otherwise
* false
*/
std::pair<int, bool> writeHDF5Initial(
std::vector<double> const& nodes,
std::vector<unsigned long long> const& geometry_dims,
std::vector<int> const& topology,
int step,
std::filesystem::path const& filepath);
/**
* \brief Appends the data for the time step (constant data) to the h5 file.
* @param filepath absolute or relativ filepath to the hdf5 file
* @param step number of the timestep
* @param attribute_name name of the attribute (property)
* @param attribute_data pointer to first element of the actual attribute data
* @param attribute_dims vector of dimensions of the attribute data
* @param attribue_data_type XdmfArrayType of underlying (e.g. float32)
* @return returns negative value on error
*/
int writeHDF5Step(std::filesystem::path const& filepath, int step,
std::string const& attribute_name, void const* attribute_data,
std::vector<unsigned long long> const& attribute_dims,
boost::shared_ptr<const XdmfArrayType> attribue_data_type,
bool has_compression_lib);
} // namespace MeshLib::IO
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment