Commit de485c62 authored by Dmitry Yu. Naumov's avatar Dmitry Yu. Naumov
Browse files

Merge branch 'parallelHDF5Writer' into 'master'

[MeL/IO] Parallel writer based on HDF5 and MPI IO

See merge request ogs/ogs!3307
parents 88cd2898 62aedb85
......@@ -14,6 +14,11 @@ append_source_files(SOURCES IO/Legacy)
append_source_files(SOURCES IO/VtkIO)
if(OGS_USE_XDMF)
append_source_files(SOURCES IO/XDMF)
if(OGS_USE_PETSC)
append_source_files(SOURCES IO/XDMF/mpi)
else()
append_source_files(SOURCES IO/XDMF/posix)
endif()
endif()
append_source_files(SOURCES MeshQuality)
append_source_files(SOURCES Vtk)
......
/**
* \file
* \copyright
* Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
* See accompanying file LICENSE.txt or
* http://www.opengeosys.org/project/license
*/
#include "HdfData.h"
#include <hdf5.h>
#include <map>
#include "BaseLib/Error.h"
#include "BaseLib/Logging.h"
#include "partition.h"
namespace MeshLib::IO
{
static hid_t meshPropertyType2HdfType(MeshPropertyDataType const ogs_data_type)
{
std::map<MeshPropertyDataType const, hid_t> ogs_to_hdf_type = {
{MeshPropertyDataType::float64, H5T_IEEE_F64LE},
{MeshPropertyDataType::float32, H5T_IEEE_F32LE},
{MeshPropertyDataType::int32, H5T_STD_I32LE},
{MeshPropertyDataType::int64, H5T_STD_I64LE},
{MeshPropertyDataType::uint32, H5T_STD_U32LE},
{MeshPropertyDataType::uint64, H5T_STD_U64LE},
{MeshPropertyDataType::int8, H5T_STD_I8LE},
{MeshPropertyDataType::uint8, H5T_STD_U8LE}};
try
{
return ogs_to_hdf_type.at(ogs_data_type);
}
catch (std::exception const& e)
{
OGS_FATAL("No known HDF5 type for OGS type. {:s}", e.what());
}
}
HdfData::HdfData(void const* data_start, std::size_t const size_partitioned_dim,
std::size_t const size_tuple, std::string const& name,
MeshPropertyDataType const mesh_property_data_type)
: data_start(data_start),
data_space{size_partitioned_dim, size_tuple},
name(name)
{
auto const& partition_info = getPartitionInfo(size_partitioned_dim);
DBUG("HdfData: The partition of dataset {:s} has dimension {:d} and offset {:d}.",
name, size_partitioned_dim, partition_info.first);
auto const& offset_partitioned_dim = partition_info.first;
offsets = {offset_partitioned_dim, 0};
file_space = {partition_info.second, size_tuple};
data_type = meshPropertyType2HdfType(mesh_property_data_type);
}
} // namespace MeshLib::IO
\ No newline at end of file
/**
* \file
* \author Tobias Meisel
* \date 2020-12-08
* \brief Collects and holds all metadata for writing HDF5 file
* \copyright
* Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
* See accompanying file LICENSE.txt or
* http://www.opengeosys.org/project/license
*/
#pragma once
#include <string>
#include <vector>
#include "MeshPropertyDataType.h"
namespace MeshLib::IO
{
using Hdf5DimType = unsigned long long;
struct HdfData
{
HdfData(void const* data_start, std::size_t size_partitioned_dim,
std::size_t size_tuple, std::string const& name,
MeshPropertyDataType mesh_property_data_type);
void const* data_start;
std::vector<Hdf5DimType> const data_space;
std::vector<Hdf5DimType> offsets;
std::vector<Hdf5DimType> file_space;
std::string const name;
int64_t data_type;
};
} // namespace MeshLib::IO
\ No newline at end of file
/**
* \file
* \copyright
* Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
* See accompanying file LICENSE.txt or
* http://www.opengeosys.org/project/license
*/
#include "HdfWriter.h"
#include <hdf5.h>
#include <string>
#include <utility>
#include <vector>
#include "BaseLib/Error.h"
#include "BaseLib/Logging.h"
#include "fileIO.h"
template <typename... Args>
void checkHdfStatus(const hid_t status, const std::string formatting,
Args... args)
{
if (status < 0)
{
OGS_FATAL(formatting, std::forward<Args>(args)...);
}
}
static unsigned short int const compression_factor = 5;
using namespace MeshLib::IO;
using namespace std::string_literals;
static std::string getTimeSection(int const step)
{
return "t_"s + std::to_string(step);
}
static bool checkCompression()
{
// Check if gzip compression is available and can be used for both
// compression and decompression.
if (htri_t avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE); !avail)
{
WARN("gzip filter not available.\n");
return false;
}
unsigned int filter_info;
H5Zget_filter_info(H5Z_FILTER_DEFLATE, &filter_info);
if (!(filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) ||
!(filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED))
{
WARN("gzip filter not available for encoding and decoding.\n");
return false;
}
return true;
}
static hid_t createStepGroup(hid_t const& file, int const step)
{
std::string const& time_section = getTimeSection(step);
// Open or create Group
if (H5Lexists(file, time_section.c_str(), H5P_DEFAULT) > 0)
{
return H5Gopen2(file, time_section.c_str(), H5P_DEFAULT);
}
return H5Gcreate2(file, time_section.c_str(), H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT);
}
static hid_t writeDataSet(
void const* nodes_data, // what
hid_t const data_type,
std::vector<Hdf5DimType> const& chunked_dims, // how ...
std::vector<Hdf5DimType> const& dim_offsets,
std::vector<Hdf5DimType> const& dim_maxs, bool has_compression_lib,
hid_t const section,
std::string const& dataset_name) // where
{
DBUG("Num global nodes : {:d} ", dim_maxs[0]);
int const dim_size = chunked_dims.size();
hid_t const memspace =
H5Screate_simple(dim_size, chunked_dims.data(), nullptr);
hid_t const filespace =
H5Screate_simple(dim_size, dim_maxs.data(), nullptr);
hid_t dataset_property = H5Pcreate(H5P_DATASET_CREATE);
if (has_compression_lib)
{
H5Pset_deflate(dataset_property, compression_factor);
DBUG("Compression will be used for {:s} with factor {:d}.",
dataset_name, compression_factor);
}
hid_t status =
H5Pset_chunk(dataset_property, dim_size, chunked_dims.data());
if (status != 0)
{
ERR("H5Pset_layout failed for data set: {:s}.", dataset_name);
}
// TODO (tm) Compression is not enabled!!
dataset_property = H5P_DEFAULT;
hid_t const dataset =
H5Dcreate2(section, dataset_name.c_str(), H5T_IEEE_F64BE, filespace,
H5P_DEFAULT, dataset_property, H5P_DEFAULT);
H5Pclose(dataset_property);
H5Sclose(filespace);
hid_t const dataset_filespace = H5Dget_space(dataset);
std::vector<hsize_t> const stride(dim_size, 1);
std::vector<hsize_t> const count(dim_size, 1);
std::vector<hsize_t> const block = chunked_dims;
DBUG("Offset in dataset '{:s}' is: {:d}.", dataset_name, dim_offsets[0]);
status = H5Sselect_hyperslab(dataset_filespace, H5S_SELECT_SET,
dim_offsets.data(), stride.data(),
count.data(), block.data());
if (status != 0)
{
ERR("H5Sselect_hyperslab failed in dataset '{:s}'.", dataset_name);
}
hid_t const io_transfer_property = createHDF5TransferPolicy();
status = H5Dwrite(dataset, data_type, memspace, dataset_filespace,
io_transfer_property, nodes_data);
if (status != 0)
{
ERR("H5Dwrite failed in dataset '{:s}'.", dataset_name);
}
H5Dclose(dataset);
H5Pclose(io_transfer_property);
status = H5Sclose(memspace);
return (status >= 0 ? 1 : 0);
}
namespace MeshLib::IO
{
HdfWriter::HdfWriter(HdfData const& geometry,
HdfData const& topology,
std::vector<HdfData>
attributes,
int const step,
std::filesystem::path const& filepath)
: _attributes(std::move(attributes)),
_hdf5_filepath(filepath),
_has_compression(checkCompression())
{
hid_t const file = createFile(filepath);
std::string const& time_section = getTimeSection(step);
hid_t const group_id = H5Gcreate2(file, time_section.c_str(), H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
// geometry
hid_t status =
writeDataSet(geometry.data_start, geometry.data_type,
geometry.data_space, geometry.offsets, geometry.file_space,
_has_compression, group_id, geometry.name);
checkHdfStatus(status, "Writing HDF5 Dataset: {:s} failed.", geometry.name);
// topology
status =
writeDataSet(topology.data_start, topology.data_type,
topology.data_space, topology.offsets, topology.file_space,
_has_compression, group_id, topology.name);
checkHdfStatus(status, "Writing HDF5 Dataset: {:s} failed.", topology.name);
H5Gclose(group_id);
status = H5Fclose(file);
checkHdfStatus(status, "HDF Writer could not be created!", topology.name);
}
bool HdfWriter::writeStep(int const step) const
{
hid_t const file = openHDF5File(_hdf5_filepath);
hid_t const group = createStepGroup(file, step);
hid_t status = 0;
for (auto const& attribute : _attributes)
{
status = writeDataSet(attribute.data_start, attribute.data_type,
attribute.data_space, attribute.offsets,
attribute.file_space, _has_compression, group,
attribute.name);
checkHdfStatus(status, "Not all attributes written to HDF5 file.");
}
H5Gclose(group);
status = H5Fclose(file);
return (status >= 0);
}
} // namespace MeshLib::IO
\ No newline at end of file
/**
* \file
* \author Tobias Meisel
* \date 2020-12-15
* \brief Writes vectorized data to HDF File
* \copyright
* Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
* See accompanying file LICENSE.txt or
* http://www.opengeosys.org/project/license
*/
#pragma once
#include <filesystem.h>
#include <vector>
#include "HdfData.h"
namespace MeshLib::IO
{
class HdfWriter final
{
public:
/**
* \brief Write file with geometry and topology data. The data
* itself is held by a structure outside of this class. The writer assumes
* the data holder to not change during writing
* @param geometry contains meta data to coordinates
* @param topology contains meta data cells
* @param attributes vector of attributes (each attribute is a OGS mesh
* property)
* @param step number of the step (temporal collection)
* @param filepath absolute or relative filepath to the hdf5 file
*/
HdfWriter(HdfData const& geometry,
HdfData const& topology,
std::vector<HdfData>
attributes,
int const step,
std::filesystem::path const& filepath);
/**
* \brief Writes attributes. The data
* itself is hold by a structure outside of this class. The writer assumes
* the data holder to not change during writing and HdfData given to
* constructor to be still valid
* @param step number of the step (temporal collection)
* @return true = success, false = error
*/
bool writeStep(int step) const;
private:
std::vector<HdfData> const _attributes;
std::filesystem::path const _hdf5_filepath;
bool const _has_compression;
};
} // namespace MeshLib::IO
\ No newline at end of file
/**
* \file
* \author Tobias Meisel
* \date 2020-12-15
* \brief Enum for all propertyVector data types
* \copyright
* Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
* See accompanying file LICENSE.txt or
* http://www.opengeosys.org/project/license
*/
#pragma once
// TODO (tm) If used on several other places move definition of propertyVector
enum class MeshPropertyDataType
{
unknown,
float64,
float32,
int32,
int64,
uint32,
uint64,
int8,
uint8
};
\ No newline at end of file
......@@ -24,19 +24,33 @@
#include <string>
#include "InfoLib/GitInfo.h"
#include "XdmfData.h"
#include "writeHDF5.h"
using namespace MeshLib::IO;
using namespace std::string_literals;
boost::shared_ptr<const XdmfAttributeCenter> elemTypeOGS2XDMF(
MeshLib::MeshItemType const elem_type)
{
std::map<MeshLib::MeshItemType,
boost::shared_ptr<const XdmfAttributeCenter>>
mesh_item_type_ogs2xdmf = {
{MeshLib::MeshItemType::Cell, XdmfAttributeCenter::Cell()},
{MeshLib::MeshItemType::Edge, XdmfAttributeCenter::Edge()},
{MeshLib::MeshItemType::Face, XdmfAttributeCenter::Face()},
{MeshLib::MeshItemType::Node, XdmfAttributeCenter::Node()},
{MeshLib::MeshItemType::IntegrationPoint,
XdmfAttributeCenter::Other()}};
return mesh_item_type_ogs2xdmf.at(elem_type);
}
static std::string getTimeSection(int const step, std::string const& name)
{
return "t_"s + std::to_string(step) + "/"s + name;
}
static boost::shared_ptr<XdmfGeometry> getLightGeometry(
std::string const& hdf5filename, int const step, Geometry const& geometry)
std::string const& hdf5filename, int const step, XdmfData const& geometry)
{
auto xdmf_geometry = XdmfGeometry::New();
xdmf_geometry->setType(XdmfGeometryType::XYZ());
......@@ -46,14 +60,14 @@ static boost::shared_ptr<XdmfGeometry> getLightGeometry(
XdmfArrayType::Float64(),
geometry.starts,
geometry.strides,
geometry.vdims,
geometry.vdims);
geometry.block_dims,
geometry.block_dims);
xdmf_geometry->setHeavyDataController(geometry_controller);
return xdmf_geometry;
}
static boost::shared_ptr<XdmfTopology> getLightTopology(
std::string const& hdf5filename, int const step, Topology const& topology)
std::string const& hdf5filename, int const step, XdmfData const& topology)
{
auto xdmf_topology = XdmfTopology::New();
xdmf_topology->setType(XdmfTopologyType::Mixed());
......@@ -63,28 +77,27 @@ static boost::shared_ptr<XdmfTopology> getLightTopology(
XdmfArrayType::Int32(),
topology.starts,
topology.strides,
topology.vdims,
topology.vdims);
topology.block_dims,
topology.block_dims);
xdmf_topology->setHeavyDataController(topology_controller);
return xdmf_topology;
}
static boost::shared_ptr<XdmfAttribute> getLightAttribute(
std::string const& hdf5filename,
int const step,
AttributeMeta const& attribute)
std::string const& hdf5filename, int const step, XdmfData const& attribute)
{
auto attribute_controller =
auto const attribute_controller =
XdmfHDF5Controller::New(hdf5filename,
getTimeSection(step, attribute.name),
attribute.data_type,
attribute.starts,
attribute.strides,
attribute.vdims,
attribute.vdims);
attribute.block_dims,
attribute.block_dims);
auto xdmf_attribute = XdmfAttribute::New();
xdmf_attribute->setCenter(attribute.attribute_center);
auto const xdmf_attribute = XdmfAttribute::New();
auto const center = elemTypeOGS2XDMF(*(attribute.attribute_center));
xdmf_attribute->setCenter(center);
xdmf_attribute->setName(attribute.name);
xdmf_attribute->setHeavyDataController(attribute_controller);
return xdmf_attribute;
......@@ -92,32 +105,17 @@ static boost::shared_ptr<XdmfAttribute> getLightAttribute(
namespace MeshLib::IO
{
Xdmf3Writer::Xdmf3Writer(std::filesystem::path const& filepath,
Geometry const& geometry,
Topology const& topology,
std::vector<AttributeMeta>
attributes,
int const timestep)
Xdmf3Writer::Xdmf3Writer(XdmfData const& geometry, XdmfData const& topology,
std::vector<XdmfData> attributes,
std::filesystem::path const& filepath,
int const time_step)
: _attributes(std::move(attributes)),
_hdf5filepath(filepath.parent_path() /
(filepath.stem().string() + ".h5"))
_hdf5filename(filepath.stem().string() + ".h5")
{
std::filesystem::path const xdmf_filepath =
filepath.parent_path() / (filepath.stem().string() + ".xdmf");
auto const ret_hdf5 = writeHDF5Initial(geometry.flattend_values,
geometry.vldims,
topology.flattend_values,
timestep,
_hdf5filepath);
// If we find a library for compression we use it
_use_compression = ret_hdf5.second;
_initial_geometry =
getLightGeometry(_hdf5filepath.filename().string(), timestep, geometry);
_initial_topology =
getLightTopology(_hdf5filepath.filename().string(), timestep, topology);
_writer = XdmfWriter::New(xdmf_filepath.string());
_initial_geometry = getLightGeometry(_hdf5filename, time_step, geometry);
_initial_topology = getLightTopology(_hdf5filename, time_step, topology);
_writer = XdmfWriter::New(filepath.string());
_writer->setMode(XdmfWriter::DistributedHeavyData);
auto version = XdmfInformation::New();
......@@ -141,19 +139,11 @@ void Xdmf3Writer::writeStep(int const time_step, double const time)
for (auto const& attribute : _attributes)
{
writeHDF5Step(_hdf5filepath,
time_step,
attribute.name,
attribute.data_start,
attribute.vldims,
attribute.data_type,
_use_compression);
grid->insert(
getLightAttribute(_hdf5filepath.filename().string(), time_step, attribute));
grid->insert(getLightAttribute(_hdf5filename, time_step, attribute));
}
auto gridcollection = _root->getGridCollection(0);
gridcollection->insert(grid);
auto grid_collection = _root->getGridCollection(0);
grid_collection->insert(grid);
_root->accept(_writer);
}
} // namespace MeshLib::IO
......@@ -3,7 +3,6 @@
* \author Tobias Meisel
* \date 2020-11-13
* \brief XdmfWriter which takes contiguous data and writes 1 xdmf + 1 hdf file
*
* \copyright
* Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
* Distributed under a Modified BSD License.
......@@ -17,6 +16,7 @@
#include <boost/shared_ptr.hpp>
#include "MeshLib/Mesh.h"
#include "XdmfData.h"
class XdmfGridCollection;
......@@ -33,26 +33,26 @@ struct Topology;
namespace MeshLib::IO
{
class Xdmf3Writer
class Xdmf3Writer final
{
public:
/**
* \brief Write xdmf and h5 file with geometry and topology data.
* @param filepath absolute or relativ filepath to the hdf5 file
* @param time_step number of the step (temporal collection)
* @param geometry contains point coordinates
* @param topology contains cells
* The data itself is held by a structure outside of this class.
* The writer assumes a constant data holder (data itself can change, memory
* adress is the same)
* @param geometry contains meta data to coordinates
* @param topology contains meta data cells
* @param attributes vector of attributes (each attribute is a OGS property)
* @param filepath absolute or relative filepath to the hdf5 file
* @param time_step number of the step (temporal collection)
*/
Xdmf3Writer(std::filesystem::path const& filepath,
Geometry const& geometry,
Topology const& topology,
std::vector<AttributeMeta> attributes,
int time_step);
Xdmf3Writer(XdmfData const& geometry, XdmfData const& topology,
std::vector<XdmfData> attributes,
std::filesystem::path const& filepath, int time_step);
/**
* \brief Write attribute data that has modified to previous time step or
* initial
* @param filepath absolute or relativ filepath to the hdf5 file
* @param time_step number of the step (temporal collection)
* @param time time value of the current time_step