[MeL/IO] HDF: Add option to support writing into mulitple HDF5 files

Documentation/ProjectFile/prj/time_loop/output/hdf/i_hdf.md

+Group of parameters when XDMF/HDF writer is used.
+Type of output in time_loop must be XDMF

Documentation/ProjectFile/prj/time_loop/output/hdf/t_number_of_files.md

+\copydoc ProcessLib::Output::_n_files

MeshLib/IO/XDMF/HdfData.cpp

@@ -42,10 +42,12 @@ static hid_t meshPropertyType2HdfType(MeshPropertyDataType const ogs_data_type)
 
 HdfData::HdfData(void const* data_start, std::size_t const size_partitioned_dim,
                  std::size_t const size_tuple, std::string const& name,
-                 MeshPropertyDataType const mesh_property_data_type)
+                 MeshPropertyDataType const mesh_property_data_type,
+                 unsigned int const num_of_files)
     : data_start(data_start), name(name)
 {
-    auto const& partition_info = getPartitionInfo(size_partitioned_dim);
+    auto const& partition_info =
+        getPartitionInfo(size_partitioned_dim, num_of_files);
     auto const& offset_partitioned_dim = partition_info.local_offset;
     offsets = {offset_partitioned_dim, 0};
 

MeshLib/IO/XDMF/HdfData.h

@@ -25,7 +25,8 @@ struct HdfData final
 {
     HdfData(void const* data_start, std::size_t size_partitioned_dim,
             std::size_t size_tuple, std::string const& name,
-            MeshPropertyDataType mesh_property_data_type);
+            MeshPropertyDataType mesh_property_data_type,
+            unsigned int num_of_files);
     void const* data_start;
     std::vector<Hdf5DimType> data_space;
     std::vector<Hdf5DimType> offsets;

MeshLib/IO/XDMF/HdfWriter.cpp

@@ -213,9 +213,10 @@ HdfWriter::HdfWriter(std::vector<MeshHdfData> meshes,
                      unsigned long long const initial_step,
                      std::filesystem::path const& filepath,
                      bool const use_compression,
-                     bool const is_file_manager)
+                     bool const is_file_manager,
+                     unsigned int const num_of_files)
     : _hdf5_filepath(filepath),
-      _file(createFile(filepath)),
+      _file(createFile(filepath, num_of_files)),
       _meshes_group(
           H5Gcreate2(_file, "/meshes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)),
       _step_times{0},  // ToDo need to be initial time

MeshLib/IO/XDMF/HdfWriter.h

@@ -43,13 +43,15 @@ public:
      * @param filepath absolute or relative filepath to the hdf5 file
      * @param use_compression if true gzip compression is enabled
      * @param is_file_manager True if process (in parallel execution) is
+     * @param num_of_files Number of outputfiles
      * File_Manager
      */
     HdfWriter(std::vector<MeshHdfData> meshes,
               unsigned long long initial_step,
               std::filesystem::path const& filepath,
               bool use_compression,
-              bool is_file_manager);
+              bool is_file_manager,
+              unsigned int num_of_files);
     /**
      * \brief Writes attributes. The data
      * itself is hold by a structure outside of this class. The writer assumes

MeshLib/IO/XDMF/XdmfData.cpp

@@ -23,7 +23,8 @@ XdmfData::XdmfData(std::size_t const size_partitioned_dim,
                    MeshPropertyDataType const mesh_property_data_type,
                    std::string const& name,
                    std::optional<MeshLib::MeshItemType> const attribute_center,
-                   unsigned int const index)
+                   unsigned int const index,
+                   unsigned int const num_of_files)
     : starts(
           [&size_tuple]()
           {
@@ -53,7 +54,7 @@ XdmfData::XdmfData(std::size_t const size_partitioned_dim,
       attribute_center(attribute_center),
       index(index)
 {
-    auto partition_info = getPartitionInfo(size_partitioned_dim);
+    auto partition_info = getPartitionInfo(size_partitioned_dim, num_of_files);
     // TODO (tm) XdmfLib does not support 64 bit data types so far
     assert(partition_info.local_length <
            std::numeric_limits<unsigned int>::max());

MeshLib/IO/XDMF/XdmfData.h

@@ -46,13 +46,15 @@ struct XdmfData final
      * @param index The position of the DataItem parents in a grid
      * (representing a single step). Convention is: 1=Time, 2=
      * Geometry, 3=Topology, 4>=Attribute
+     * @param num_of_files If greater than 1 it groups the data of each process.
+     * The num_of_files specifies the number of groups
      *
      */
     XdmfData(std::size_t size_partitioned_dim, std::size_t size_tuple,
              MeshPropertyDataType mesh_property_data_type,
              std::string const& name,
              std::optional<MeshLib::MeshItemType> attribute_center,
-             unsigned int const index);
+             unsigned int const index, unsigned int num_of_files);
     // a hyperslab is defined by starts and strides see
     // https://www.xdmf.org/index.php/XDMF_Model_and_Format#HyperSlab
     std::vector<XdmfDimType> starts;

MeshLib/IO/XDMF/XdmfHdfWriter.cpp

@@ -39,7 +39,7 @@ XdmfHdfWriter::XdmfHdfWriter(
     std::filesystem::path const& filepath, unsigned long long const time_step,
     double const initial_time,
     std::set<std::string> const& variable_output_names,
-    bool const use_compression)
+    bool const use_compression, unsigned int const num_of_files)
 {
     // ogs meshes to vector of Xdmf/HDF meshes (we keep Xdmf and HDF together
     // because XDMF depends on HDF) to meta
@@ -79,12 +79,12 @@ XdmfHdfWriter::XdmfHdfWriter(
     // Transform the data to be written into a format conforming with the rules
     // of xdmf topology and geometry
     auto const transform_ogs_mesh_data_to_xdmf_conforming_data =
-        [](auto const& mesh)
+        [&num_of_files](auto const& mesh)
     {
         auto flattened_geometry_values = transformToXDMFGeometry(mesh);
         // actually this line is only needed to calculate the offset
-        XdmfHdfData const& geometry =
-            transformGeometry(mesh, flattened_geometry_values.data());
+        XdmfHdfData const& geometry = transformGeometry(
+            mesh, flattened_geometry_values.data(), num_of_files);
         auto const flattened_topology_values =
             transformToXDMFTopology(mesh, geometry.hdf.offsets[0]);
         return std::make_unique<TransformedMeshData>(
@@ -94,17 +94,19 @@ XdmfHdfWriter::XdmfHdfWriter(
 
     // create metadata for transformed data and original ogs mesh data
     auto const transform_to_meta_data =
-        [&transform_ogs_mesh_data_to_xdmf_conforming_data](auto const& mesh)
+        [&transform_ogs_mesh_data_to_xdmf_conforming_data,
+         &num_of_files](auto const& mesh)
     {
         // important: transformed data must survive and be unique, raw pointer
         // to its memory!
         std::unique_ptr<TransformedMeshData> xdmf_conforming_data =
             transform_ogs_mesh_data_to_xdmf_conforming_data(mesh);
         auto const geometry = transformGeometry(
-            mesh, xdmf_conforming_data->flattened_geometry_values.data());
-        auto const topology =
-            transformTopology(xdmf_conforming_data->flattened_topology_values);
-        auto const attributes = transformAttributes(mesh);
+            mesh, xdmf_conforming_data->flattened_geometry_values.data(),
+            num_of_files);
+        auto const topology = transformTopology(
+            xdmf_conforming_data->flattened_topology_values, num_of_files);
+        auto const attributes = transformAttributes(mesh, num_of_files);
         return XdmfHdfMesh{std::move(geometry), std::move(topology),
                            std::move(attributes), mesh.get().getName(),
                            std::move(xdmf_conforming_data)};
@@ -157,7 +159,7 @@ XdmfHdfWriter::XdmfHdfWriter(
     auto const is_file_manager = isFileManager();
     _hdf_writer = std::make_unique<HdfWriter>(std::move(hdf_meshes), time_step,
                                               hdf_filepath, use_compression,
-                                              is_file_manager);
+                                              is_file_manager, num_of_files);
 
     // --------------- XDMF ---------------------
     // The light data is only written by just one process
@@ -174,7 +176,8 @@ XdmfHdfWriter::XdmfHdfWriter(
     {
         std::string const xdmf_name = metamesh.name;
         std::filesystem::path const xdmf_filepath =
-            filepath.parent_path() / (xdmf_name + ".xdmf");
+            filepath.parent_path() /
+            (filepath.stem().string() + "_" + xdmf_name + ".xdmf");
 
         std::vector<XdmfData> xdmf_attributes;
         std::transform(metamesh.attributes.begin(), metamesh.attributes.end(),

MeshLib/IO/XDMF/XdmfHdfWriter.h

@@ -36,12 +36,13 @@ public:
      * that change over time
      * @param use_compression if true, zlib compression in HDFWriter component
      * is used
+     * @param num_of_files number of hdf5 output files
      */
     XdmfHdfWriter(
         std::vector<std::reference_wrapper<const MeshLib::Mesh>> meshes,
         std::filesystem::path const& filepath, unsigned long long time_step,
         double initial_time, std::set<std::string> const& variable_output_names,
-        bool use_compression);
+        bool use_compression, unsigned int num_of_files);
 
     /**
      * \brief Adds data for either lazy (xdmf) or eager (hdf) writing algorithm

MeshLib/IO/XDMF/fileIO.h

@@ -18,7 +18,9 @@
 
 namespace MeshLib::IO
 {
-int64_t createFile(std::filesystem::path const& filepath);
-int64_t openHDF5File(std::filesystem::path const& filepath);
+int64_t createFile(std::filesystem::path const& filepath,
+                   unsigned int num_of_files);
+int64_t openHDF5File(std::filesystem::path const& filepath,
+                     unsigned int num_of_files);
 int64_t createHDF5TransferPolicy();
 }  // namespace MeshLib::IO

MeshLib/IO/XDMF/mpi/fileIO.cpp

@@ -17,24 +17,47 @@
 #include <mpi.h>
 
 #include "BaseLib/Logging.h"
+#include "getCommunicator.h"
 
+using namespace std::string_literals;
 namespace MeshLib::IO
 {
-hid_t createFile(std::filesystem::path const& filepath)
+std::filesystem::path partitionFilename(
+    std::filesystem::path const& basic_filepath, int file_group)
 {
-    MPI_Comm comm = MPI_COMM_WORLD;
-    MPI_Info info = MPI_INFO_NULL;
+    std::string const filename = (file_group > 0)
+                                     ? basic_filepath.stem().string() + "_"s +
+                                           std::to_string(file_group) +
+                                           basic_filepath.extension().string()
+                                     : basic_filepath.filename().string();
+    std::filesystem::path const filepathwithextension =
+        basic_filepath.parent_path() / filename;
+    DBUG("HDF Filepath: {:s}.", filepathwithextension.string());
+    return filepathwithextension;
+};
+
+hid_t createFile(std::filesystem::path const& filepath,
+                 unsigned int const num_of_files)
+{
+    auto const communicator = getCommunicator(num_of_files);
+    MPI_Comm const comm = communicator.mpi_communicator;
+    MPI_Info const info = MPI_INFO_NULL;
     hid_t const plist_id = H5Pcreate(H5P_FILE_ACCESS);
+
     H5Pset_fapl_mpio(plist_id, comm, info);
-    hid_t file = H5Fcreate(filepath.string().c_str(), H5F_ACC_TRUNC,
+    std::filesystem::path const partition_filename =
+        partitionFilename(filepath, communicator.color);
+    hid_t file = H5Fcreate(partition_filename.string().c_str(), H5F_ACC_TRUNC,
                            H5P_DEFAULT, plist_id);
     H5Pclose(plist_id);
+
     return file;
 }
 
-hid_t openHDF5File(std::filesystem::path const& filepath)
+hid_t openHDF5File(std::filesystem::path const& filepath,
+                   unsigned int const num_of_files)
 {
-    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Comm const comm = getCommunicator(num_of_files).mpi_communicator;
     MPI_Info info = MPI_INFO_NULL;
     hid_t const plist_id = H5Pcreate(H5P_FILE_ACCESS);
     H5Pset_fapl_mpio(plist_id, comm, info);

MeshLib/IO/XDMF/mpi/getCommunicator.cpp

+/**
+ * \file
+ * \author Tobias Meisel
+ * \date   2020-12-08
+ * \brief  Function specific to execution with MPI, never include directly!!
+ * \copyright
+ * Copyright (c) 2012-2021, OpenGeoSys Community (http://www.opengeosys.org)
+ *            Distributed under a Modified BSD License.
+ *              See accompanying file LICENSE.txt or
+ *              http://www.opengeosys.org/project/license
+ *
+ */
+
+#include "getCommunicator.h"
+
+#include <hdf5.h>
+#include <math.h>
+#include <mpi.h>
+
+#include <cassert>
+
+#include "BaseLib/Logging.h"
+#include "MeshLib/IO/XDMF/fileIO.h"
+
+using namespace std::string_literals;
+
+namespace MeshLib::IO
+{
+int getGroupIndex(int const input_index, int const input_size,
+                  int const new_group_size)
+{
+    // A grouping algorithm that determines the number of groups and return the
+    // group idx of the specified input_index
+    assert(input_size >= new_group_size);
+    int const minimal_output_group_size =
+        std::lround(input_size / new_group_size);
+    int const maximal_output_group_size = (input_size % new_group_size)
+                                              ? minimal_output_group_size + 1
+                                              : minimal_output_group_size;
+    return std::lround(input_index / maximal_output_group_size);
+};
+
+FileCommunicator getCommunicator(unsigned int const num_of_files)
+{
+    int num_procs;
+    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+    int rank_id;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank_id);
+    int file_group_id = getGroupIndex(rank_id, num_procs, num_of_files);
+    MPI_Comm new_communicator;
+    MPI_Comm_split(MPI_COMM_WORLD, file_group_id, rank_id, &new_communicator);
+    return FileCommunicator{std::move(new_communicator),
+                            std::move(file_group_id), ""};
+}
+}  // namespace MeshLib::IO
\ No newline at end of file

MeshLib/IO/XDMF/mpi/getCommunicator.h

+#pragma once
+
+#include <mpi.h>
+
+#include <filesystem>
+
+namespace MeshLib::IO
+{
+struct FileCommunicator final
+{
+    MPI_Comm mpi_communicator;
+    int color;
+    std::filesystem::path output_filename;
+};
+FileCommunicator getCommunicator(unsigned int num_of_files);
+}  // namespace MeshLib::IO
\ No newline at end of file

MeshLib/IO/XDMF/mpi/partition.cpp

@@ -15,11 +15,11 @@
 
 #include <mpi.h>
 
-#include <deque>
 #include <numeric>
 
 #include "BaseLib/Logging.h"
 #include "MeshLib/IO/XDMF/fileIO.h"
+#include "getCommunicator.h"
 
 namespace MeshLib::IO
 {
@@ -30,9 +30,10 @@ bool isFileManager()
     return mpi_rank == 0;
 }
 
-PartitionInfo getPartitionInfo(std::size_t const size)
+PartitionInfo getPartitionInfo(std::size_t const size,
+                               unsigned int const num_of_files)
 {
-    MPI_Comm const mpi_comm = MPI_COMM_WORLD;
+    MPI_Comm const mpi_comm = getCommunicator(num_of_files).mpi_communicator;
     int mpi_size;
     int mpi_rank;
     MPI_Comm_size(mpi_comm, &mpi_size);

MeshLib/IO/XDMF/partition.h

@@ -24,6 +24,7 @@ struct PartitionInfo
     std::size_t global_length;
 };
 
-PartitionInfo getPartitionInfo(std::size_t const size);
+PartitionInfo getPartitionInfo(std::size_t const size,
+                               unsigned int const num_of_files);
 bool isFileManager();
 }  // namespace MeshLib::IO

MeshLib/IO/XDMF/posix/fileIO.cpp

@@ -16,7 +16,7 @@
 #include <hdf5.h>
 namespace MeshLib::IO
 {
-int64_t createFile(std::filesystem::path const& filepath)
+int64_t createFile(std::filesystem::path const& filepath, unsigned int)
 {
     return H5Fcreate(filepath.string().c_str(), H5F_ACC_TRUNC, H5P_DEFAULT,
                      H5P_DEFAULT);

MeshLib/IO/XDMF/posix/partition.cpp

@@ -17,7 +17,7 @@ bool isFileManager()
     return true;
 }
 
-PartitionInfo getPartitionInfo(std::size_t const size)
+PartitionInfo getPartitionInfo(std::size_t const size, unsigned int)
 {
         // local_offset, local_length, longest_local_length, global_number_process
     return {0, size, size, size};

MeshLib/IO/XDMF/transformData.cpp

@@ -70,7 +70,8 @@ constexpr auto cellTypeOGS2XDMF(MeshLib::CellType const& cell_type)
 }
 
 std::optional<XdmfHdfData> transformAttribute(
-    std::pair<std::string, PropertyVectorBase*> const& property_pair)
+    std::pair<std::string, PropertyVectorBase*> const& property_pair,
+    unsigned int const num_of_files)
 {
     // 3 data that will be captured and written by lambda f below
     MeshPropertyDataType data_type = MeshPropertyDataType::unknown;
@@ -190,16 +191,18 @@ std::optional<XdmfHdfData> transformAttribute(
 
     std::string const& name = property_base->getPropertyName();
 
-    HdfData hdf = {data_ptr, num_of_tuples, ui_global_components, name,
-                   data_type};
+    HdfData hdf = {data_ptr, num_of_tuples, ui_global_components,
+                   name,     data_type,     num_of_files};
 
     XdmfData xdmf = {num_of_tuples, ui_global_components, data_type,
-                     name,          mesh_item_type,       0};
+                     name,          mesh_item_type,       0,
+                     num_of_files};
 
     return XdmfHdfData{std::move(hdf), std::move(xdmf)};
 }
 
-std::vector<XdmfHdfData> transformAttributes(MeshLib::Mesh const& mesh)
+std::vector<XdmfHdfData> transformAttributes(MeshLib::Mesh const& mesh,
+                                             unsigned int const num_of_files)
 {
     MeshLib::Properties const& properties = mesh.getProperties();
 
@@ -213,8 +216,8 @@ std::vector<XdmfHdfData> transformAttributes(MeshLib::Mesh const& mesh)
             continue;
         }
 
-        if (auto const attribute =
-                transformAttribute(std::pair(name, property_base)))
+        if (auto const attribute = transformAttribute(
+                std::pair(name, property_base), num_of_files))
         {
             attributes.push_back(attribute.value());
         }
@@ -242,7 +245,9 @@ std::vector<double> transformToXDMFGeometry(MeshLib::Mesh const& mesh)
     return values;
 }
 
-XdmfHdfData transformGeometry(MeshLib::Mesh const& mesh, double const* data_ptr)
+XdmfHdfData transformGeometry(MeshLib::Mesh const& mesh,
+                              double const* data_ptr,
+                              unsigned int const num_of_files)
 {
     std::string const name = "geometry";
     std::vector<MeshLib::Node*> const& nodes = mesh.getNodes();
@@ -250,11 +255,16 @@ XdmfHdfData transformGeometry(MeshLib::Mesh const& mesh, double const* data_ptr)
     int const point_size = 3;
     auto const& partition_dim = nodes.size();
 
-    HdfData const hdf = {data_ptr, partition_dim, point_size, name,
-                         MeshPropertyDataType::float64};
+    HdfData const hdf = {data_ptr,
+                         partition_dim,
+                         point_size,
+                         name,
+                         MeshPropertyDataType::float64,
+                         num_of_files};
     XdmfData const xdmf = {
         partition_dim, point_size,   MeshPropertyDataType::float64,
-        name,          std::nullopt, 2};
+        name,          std::nullopt, 2,
+        num_of_files};
 
     return XdmfHdfData{std::move(hdf), std::move(xdmf)};
 }
@@ -286,13 +296,16 @@ std::vector<int> transformToXDMFTopology(MeshLib::Mesh const& mesh,
     return values;
 }
 
-XdmfHdfData transformTopology(std::vector<int> const& values)
+XdmfHdfData transformTopology(std::vector<int> const& values,
+                              unsigned int const num_of_files)
 {
     std::string const name = "topology";
-    HdfData const hdf = {values.data(), values.size(), 1, name,
-                         MeshPropertyDataType::int32};
-    XdmfData const xdmf = {values.size(), 1, MeshPropertyDataType::int32, name,
-                           std::nullopt,  3};
+    HdfData const hdf = {
+        values.data(), values.size(), 1, name, MeshPropertyDataType::int32,
+        num_of_files};
+    XdmfData const xdmf = {
+        values.size(), 1, MeshPropertyDataType::int32, name, std::nullopt, 3,
+        num_of_files};
 
     return XdmfHdfData{std::move(hdf), std::move(xdmf)};
 }

MeshLib/IO/XDMF/transformData.h

@@ -26,23 +26,31 @@ namespace MeshLib::IO
 /**
  * \brief Create meta data for attributes used for hdf5 and xdmf
  * @param mesh OGS mesh can be mesh or partitionedMesh
+ * @param num_of_files If greater than 1 it groups the data of each process. The
+ * num_of_files specifies the number of groups
  * @return vector of meta data
  */
-std::vector<XdmfHdfData> transformAttributes(MeshLib::Mesh const& mesh);
+std::vector<XdmfHdfData> transformAttributes(MeshLib::Mesh const& mesh,
+                                             unsigned int num_of_files);
 /**
  * \brief Create meta data for geometry used for hdf5 and xdmf
  * @param mesh OGS mesh can be mesh or partitionedMesh
  * @param data_ptr Memory location of geometry values.
+ * @param num_of_files If greater than 1 it groups the data of each process. The
+ * num_of_files specifies the number of groups
  * @return Geometry meta data
  */
-XdmfHdfData transformGeometry(MeshLib::Mesh const& mesh,
-                              double const* data_ptr);
+XdmfHdfData transformGeometry(MeshLib::Mesh const& mesh, double const* data_ptr,
+                              unsigned int num_of_files);
 /**
  * \brief  Create meta data for topology used for HDF5 and XDMF
  * @param values actual topology values to get size and memory location
+ * @param num_of_files If greater than 1 it groups the data of each process. The
+ * num_of_files specifies the number of groups
  * @return Topology meta data
  */
-XdmfHdfData transformTopology(std::vector<int> const& values);
+XdmfHdfData transformTopology(std::vector<int> const& values,
+                              unsigned int num_of_files);
 /**
  * \brief Copies all node points into a new vector. Contiguous data used for
  * writing. Conform with XDMF standard in