From f359e53e46e0f1d437632b98154ae3b46bc31fac Mon Sep 17 00:00:00 2001
From: Dmitri Naumov <dmitri.naumov@ufz.de>
Date: Fri, 9 Jan 2015 15:55:48 +0100
Subject: [PATCH] Squashed code review changes.
[IO] Replace pointer types with std::vector.
[T] Use Logog formatter pointers. Reduce indent.
[MeL] Remove default NodePartitionedMesh dtor definition.
[MeL] Const correctness of NodePartitionedMesh::isGhostNode().
[IO] Remove default NodePartitionedMeshRead definiton.
[IO] Split long lines.
[IO] Correct doxygen comments.
[IO] Correct private _mpi_comm variable name.
[IO] Correct typo in buildNodeStructTypeMPI() fct name.
[IO] Use NodeData reference instead of pointer.
[IO] Add std namespace identifier for std::getline.
[IO] Remove printMessage().
[IO] Remove unused MPI_status, make message_tag const.
[IO] Add private NodePartitionedMeshReader::readDataFromFile().
This encapsulates MPI IO code.
[MeL] NodePartMesh: Change global node id type to std::size_t.
[IO] Fix some of implicit int type conversions: sign/width.
[IO] Correct doxygen comments.
[IO] Move MPI communicator init to NodePartMeshReader ctor.
[MeL] Use size_t from std namespace.
[IO] Use postfix increment for counters.
[IO] Pass a vector to buildNodeStructType().
[IO] Use std::vector's data() instead of &v[0] in MPI calls.
[IO] Rewrite openASCIIFiles. Scopes and var names.
[IO] Improve const correctness.
[IO] Remove almost unused MeshLib namespace include.
[IO] Improve some comments.
[IO] Add missing includes and remove unused one.
[MeL] Remove unused forward declaration.
[IO] Replace _size_str with on-the-fly conversion.
[IO] Spelling
[IO] Remove duplicate information.
[IO] Improve const-correctness of member fcts.
[IO] Use std::vector in readElementASCII().
[IO] Move up MPI_node deallocation.
[IO] Rename variables.
anode -> node.
s_nodes -> nodes.
_size -> _mpi_comm_size
_rank -> _mpi_rank
[IO] Replace mesh_controls type with std::array.
In the readASCII().
[IO] Variable declaration at definition/use point.
[IO] Rewrite buildNodesStructTypeMPI().
Shorten the code, use MPI blocks for array (of three coordinates).
[IO] Make _mpi_node_type a class variable.
Move initialization to ctor and freeing to dtor.
Function renames.
The reader must exit scope before MPI_FINILIZE
is called.
[IO] Change variable's type long->int; MPI_Send().
[IO] Comment on element types.
[IO] Rename readBinaryDataFromFile(). Change container type.
---
FileIO/MPI_IO/NodePartitionedMeshReader.cpp | 460 ++++++++----------
FileIO/MPI_IO/NodePartitionedMeshReader.h | 121 +++--
MeshLib/NodePartitionedMesh.h | 45 +-
.../MPI/NodePartitionedMeshTester.cpp | 97 ++--
4 files changed, 347 insertions(+), 376 deletions(-)
diff --git a/FileIO/MPI_IO/NodePartitionedMeshReader.cpp b/FileIO/MPI_IO/NodePartitionedMeshReader.cpp
index 35b115bdc46..e79d29eafef 100644
--- a/FileIO/MPI_IO/NodePartitionedMeshReader.cpp
+++ b/FileIO/MPI_IO/NodePartitionedMeshReader.cpp
@@ -14,6 +14,8 @@
#include "NodePartitionedMeshReader.h"
+#include <array>
+
#include "logog/include/logog.hpp"
#include "BaseLib/RunTime.h"
@@ -28,24 +30,34 @@
#include "MeshLib/Elements/Tet.h"
#include "MeshLib/Elements/Tri.h"
-using namespace MeshLib;
-
namespace FileIO
{
-MeshLib::NodePartitionedMesh* NodePartitionedMeshReader::read(MPI_Comm comm, const std::string &file_name_base)
+
+NodePartitionedMeshReader::NodePartitionedMeshReader(MPI_Comm comm)
+ : _mpi_comm(comm)
+{
+ MPI_Comm_size(_mpi_comm, &_mpi_comm_size);
+ MPI_Comm_rank(_mpi_comm, &_mpi_rank);
+
+ registerNodeDataMpiType();
+}
+
+NodePartitionedMeshReader::~NodePartitionedMeshReader()
+{
+ MPI_Type_free(&_mpi_node_type);
+}
+
+MeshLib::NodePartitionedMesh* NodePartitionedMeshReader::read(
+ const std::string &file_name_base)
{
BaseLib::RunTime timer;
timer.start();
- mpi_comm_ = comm;
- MPI_Comm_size(mpi_comm_, &_size);
- _size_str = std::to_string(_size);
- MPI_Comm_rank(mpi_comm_, &_rank);
-
- NodePartitionedMesh *mesh = nullptr;
+ MeshLib::NodePartitionedMesh *mesh = nullptr;
// Always try binary file first
- std::string fname_new = file_name_base + "_partitioned_msh_cfg" + _size_str + ".bin";
+ std::string const fname_new = file_name_base + "_partitioned_msh_cfg" +
+ std::to_string(_mpi_comm_size) + ".bin";
if(!BaseLib::IsFileExisting(fname_new)) // doesn't exist binary file.
{
@@ -62,124 +74,96 @@ MeshLib::NodePartitionedMesh* NodePartitionedMeshReader::read(MPI_Comm comm, con
INFO("\t\n>>Total elapsed time in reading mesh:%f s\n", timer.elapsed());
- MPI_Barrier(comm);
+ MPI_Barrier(_mpi_comm);
return mesh;
}
-MeshLib::NodePartitionedMesh* NodePartitionedMeshReader
-::readBinary(const std::string &file_name_base)
+template <typename DATA>
+bool
+NodePartitionedMeshReader::readBinaryDataFromFile(std::string const& filename,
+ MPI_Offset offset, MPI_Datatype type, DATA& data) const
+{
+ // Open file
+ MPI_File file;
+
+ char* filename_char = const_cast<char*>(filename.data());
+ int const file_status = MPI_File_open(_mpi_comm, filename_char,
+ MPI_MODE_RDONLY, MPI_INFO_NULL, &file);
+
+ if(file_status != 0)
+ {
+ ERR("Error opening file %s. MPI error code %d", filename.c_str(), file_status);
+ return false;
+ }
+
+ // Read data
+ char file_mode[] = "native";
+ MPI_File_set_view(file, offset, type, type, file_mode, MPI_INFO_NULL);
+ MPI_File_read(file, data.data(), data.size(), type, MPI_STATUS_IGNORE);
+ MPI_File_close(&file);
+
+ return true;
+}
+
+MeshLib::NodePartitionedMesh* NodePartitionedMeshReader::readBinary(
+ const std::string &file_name_base)
{
//----------------------------------------------------------------------------------
// Read headers
- MPI_File fh;
- char ftype[] = "native";
- int file_status = 0;
+
const std::string fname_header = file_name_base + "_partitioned_msh_";
- const std::string fname_num_p_ext = _size_str + ".bin";
+ const std::string fname_num_p_ext = std::to_string(_mpi_comm_size) + ".bin";
- std::string fname_new_base = fname_header + "cfg" + fname_num_p_ext;
- file_status = MPI_File_open(mpi_comm_, &fname_new_base[0], MPI_MODE_RDONLY,
- MPI_INFO_NULL, &fh);
- if(file_status) // Failed to open the file
- {
- printMessage(fname_new_base);
- return nullptr;
- }
+ // Array that contains integer numbers of the partition data header.
+ // See description of readBinary() function.
+ std::array<long, 14> mesh_controls;
- const long num_controls = 14;
- /*
- Array that contains integer numbers of the partition data header
- and its size is 14.
- 0: Number of all nodes of a partition,
- 1: Number of nodes for linear element of a parition,
- 2: Number of non-ghost elements of a partition,
- 3: Number of ghost element of a partition,
- 4: Number of active nodes for linear element of a parition,
- 5: Number of all active nodes a parition,
- 6: Number of nodes for linear element of global mesh,
- 7: Number of all nodes of global mesh,
- 8~12: Offsets of positions of partitions in the data arrays,
- 13: Reserved for exra flag.
- */
- long mesh_controls[num_controls];
+ if (!readBinaryDataFromFile(fname_header + "cfg" + fname_num_p_ext,
+ static_cast<MPI_Offset>(
+ static_cast<unsigned>(_mpi_rank) * mesh_controls.size() * sizeof(long)),
+ MPI_LONG, mesh_controls))
+ return nullptr;
- MPI_Offset offset_new;
- offset_new = _rank * num_controls * sizeof(long);
- MPI_File_set_view(fh, offset_new, MPI_LONG, MPI_LONG, ftype, MPI_INFO_NULL);
- MPI_File_read(fh, mesh_controls, num_controls, MPI_LONG, MPI_STATUS_IGNORE); //_all
- MPI_File_close(&fh);
_num_nodes_part = mesh_controls[0];
_num_regular_elems_part = mesh_controls[2];
_num_ghost_elems_part = mesh_controls[3];
//----------------------------------------------------------------------------------
// Read Nodes
- fname_new_base = fname_header + "nod" + fname_num_p_ext;
- file_status = MPI_File_open(mpi_comm_, &fname_new_base[0], MPI_MODE_RDONLY,
- MPI_INFO_NULL, &fh);
- if(file_status) // Failed to open the file
- {
- printMessage(fname_new_base);
- return nullptr;
- }
-
- MPI_Datatype MPI_node;
- std::vector<NodeData> s_nodes(_num_nodes_part);
- buildNodeStrucTypeMPI(&s_nodes[0], &MPI_node);
+ std::vector<NodeData> nodes(static_cast<std::size_t>(_num_nodes_part));
- offset_new = mesh_controls[10];
- MPI_File_set_view(fh, offset_new, MPI_node, MPI_node, ftype, MPI_INFO_NULL);
- MPI_File_read(fh, &s_nodes[0], _num_nodes_part, MPI_node, MPI_STATUS_IGNORE);
- MPI_File_close(&fh);
+ if (!readBinaryDataFromFile(fname_header + "nod" + fname_num_p_ext,
+ static_cast<MPI_Offset>(mesh_controls[10]), _mpi_node_type, nodes))
+ return nullptr;
std::vector<MeshLib::Node*> mesh_nodes;
- std::vector<unsigned> glb_node_ids;
- setNodes(s_nodes, mesh_nodes, glb_node_ids);
-
- MPI_Type_free(&MPI_node);
+ std::vector<std::size_t> glb_node_ids;
+ setNodes(nodes, mesh_nodes, glb_node_ids);
//----------------------------------------------------------------------------------
// Read non-ghost elements
- fname_new_base = fname_header +"ele" + fname_num_p_ext;
- file_status = MPI_File_open(mpi_comm_, &fname_new_base[0], MPI_MODE_RDONLY,
- MPI_INFO_NULL, &fh);
- if(file_status) // Failed to open the file
- {
- printMessage(fname_new_base);
- return nullptr;
- }
- long size_elem_info = _num_regular_elems_part + mesh_controls[8];
- std::vector<long> elem_data(size_elem_info);
- offset_new = mesh_controls[11];
- MPI_File_set_view(fh, offset_new, MPI_LONG, MPI_LONG, ftype, MPI_INFO_NULL);
- MPI_File_read(fh, &elem_data[0], size_elem_info, MPI_LONG, MPI_STATUS_IGNORE);
- MPI_File_close(&fh);
+ std::vector<long> elem_data(static_cast<std::size_t>(
+ _num_regular_elems_part + mesh_controls[8]));
+ if (!readBinaryDataFromFile(fname_header +"ele" + fname_num_p_ext,
+ static_cast<MPI_Offset>(mesh_controls[11]), MPI_LONG, elem_data))
+ return nullptr;
std::vector<MeshLib::Element*> mesh_elems(_num_regular_elems_part + _num_ghost_elems_part);
- setElements(mesh_nodes, elem_data.data(), mesh_elems);
+ setElements(mesh_nodes, elem_data, mesh_elems);
//----------------------------------------------------------------------------------
//Read ghost element
- fname_new_base = fname_header + "ele_g" + fname_num_p_ext;
- file_status = MPI_File_open(mpi_comm_, &fname_new_base[0], MPI_MODE_RDONLY,
- MPI_INFO_NULL, &fh);
- if(file_status)
- {
- printMessage(fname_new_base);
- return nullptr;
- }
+ std::vector<long> ghost_elem_data(static_cast<std::size_t>(
+ _num_ghost_elems_part + mesh_controls[9]));
- size_elem_info = _num_ghost_elems_part + mesh_controls[9];
- elem_data.resize(size_elem_info);
- offset_new = mesh_controls[12];
- MPI_File_set_view(fh, offset_new, MPI_LONG, MPI_LONG, ftype, MPI_INFO_NULL);
- MPI_File_read(fh, &elem_data[0], size_elem_info, MPI_LONG, MPI_STATUS_IGNORE);
- MPI_File_close(&fh);
+ if (!readBinaryDataFromFile(fname_header + "ele_g" + fname_num_p_ext,
+ static_cast<MPI_Offset>(mesh_controls[12]), MPI_LONG, ghost_elem_data))
+ return nullptr;
const bool process_ghost = true;
- setElements(mesh_nodes, elem_data.data(), mesh_elems, process_ghost);
+ setElements(mesh_nodes, ghost_elem_data, mesh_elems, process_ghost);
//----------------------------------------------------------------------------------
//Create a mesh and return it
@@ -190,114 +174,110 @@ MeshLib::NodePartitionedMesh* NodePartitionedMeshReader
const unsigned n_active_base_nodes = static_cast<unsigned>(mesh_controls[4]);
const unsigned n_active_nodes = static_cast<unsigned>(mesh_controls[5]);
- return new NodePartitionedMesh(BaseLib::extractBaseName(file_name_base) + _size_str,
- mesh_nodes, glb_node_ids,
- mesh_elems, n_nghost_elem,
- n_global_base_nodes, n_global_nodes,
- n_base_nodes, n_active_base_nodes, n_active_nodes);
+ return new MeshLib::NodePartitionedMesh(
+ BaseLib::extractBaseName(file_name_base) + std::to_string(_mpi_comm_size),
+ mesh_nodes, glb_node_ids,
+ mesh_elems, n_nghost_elem,
+ n_global_base_nodes, n_global_nodes,
+ n_base_nodes, n_active_base_nodes, n_active_nodes);
}
-bool NodePartitionedMeshReader::
-openASCIIFiles(std::string const& file_name_base,
- std::ifstream& is_cfg, std::ifstream& is_node, std::ifstream& is_elem)
+bool NodePartitionedMeshReader::openASCIIFiles(std::string const& file_name_base,
+ std::ifstream& is_cfg, std::ifstream& is_node, std::ifstream& is_elem) const
{
const std::string fname_header = file_name_base + "_partitioned_";
- const std::string fname_num_p_ext = _size_str + ".msh";
+ const std::string fname_num_p_ext = std::to_string(_mpi_comm_size) + ".msh";
- std::string str_var = fname_header + "cfg"+ fname_num_p_ext;
- is_cfg.open(str_var.c_str());
+ { // Configuration.
+ std::string const filename = fname_header + "cfg"+ fname_num_p_ext;
+ is_cfg.open(filename);
- if( !is_cfg.good() )
- {
- printMessage(str_var);
- return false;
- }
+ if( !is_cfg.good() )
+ {
+ ERR("Error opening file %s for input.", filename.c_str());
+ return false;
+ }
- getline(is_cfg, str_var);
- int num_parts;
- is_cfg >> num_parts >> std::ws;
+ std::string tmp_line;
+ std::getline(is_cfg, tmp_line);
+ int num_parts;
+ is_cfg >> num_parts >> std::ws;
- if(num_parts != _size)
- {
- std::string str_var = "Aborting computation because of number of cores / subdomains mismatch.";
- printMessage(str_var, false);
- return false;
+ if(num_parts != _mpi_comm_size)
+ {
+ ERR("Aborting computation because of number of cores"
+ "/ subdomains mismatch.");
+ return false;
+ }
}
- str_var = fname_header + "nodes_" + fname_num_p_ext;
- is_node.open(str_var.c_str());
- if( !is_node.good() )
- {
- return false;
+ { // Nodes.
+ std::string const filename = fname_header + "nodes_" + fname_num_p_ext;
+ is_node.open(filename);
+ if( !is_node.good() )
+ {
+ ERR("Error opening file %s for input.", filename.c_str());
+ return false;
+ }
}
- str_var = fname_header + "elems_" + fname_num_p_ext;
- is_elem.open(str_var.c_str());
- if( !is_elem.good() )
- {
- printMessage(str_var);
- return false;
+ { // Elements.
+ std::string const filename = fname_header + "elems_" + fname_num_p_ext;
+ is_elem.open(filename);
+ if( !is_elem.good() )
+ {
+ ERR("Error opening file %s for input.", filename.c_str());
+ return false;
+ }
}
return true;
}
-void NodePartitionedMeshReader
-::readCastNodesASCII(std::ifstream& is_node, const int part_id,
- std::vector<MeshLib::Node*> &mesh_nodes,
- std::vector<unsigned> &glb_node_ids)
+void NodePartitionedMeshReader::readCastNodesASCII(std::ifstream& is_node,
+ const int part_id, std::vector<MeshLib::Node*> &mesh_nodes,
+ std::vector<std::size_t> &glb_node_ids) const
{
- int tag = 0;
- MPI_Status status;
+ int const message_tag = 0;
//----------------------------------------------------------------------------------
// Read Nodes
- std::vector<NodeData> s_nodes(_num_nodes_part);
-
- MPI_Datatype MPI_node;
- if(part_id > 0)
- buildNodeStrucTypeMPI(&s_nodes[0], &MPI_node);
+ std::vector<NodeData> nodes(static_cast<std::size_t>(_num_nodes_part));
- if(_rank == 0)
+ if(_mpi_rank == 0)
{
- for(long k=0; k<_num_nodes_part; k++)
+ for(std::size_t k=0; k<_num_nodes_part; k++)
{
- NodeData *anode = &s_nodes[k];
- is_node >> anode->index
- >> anode->x >> anode->y >> anode->z >> std::ws;
+ NodeData &node = nodes[k];
+ is_node >> node.index
+ >> node.x >> node.y >> node.z >> std::ws;
}
if(part_id == 0)
{
- setNodes(s_nodes, mesh_nodes, glb_node_ids);
+ setNodes(nodes, mesh_nodes, glb_node_ids);
}
else
{
- MPI_Send(&s_nodes[0], _num_nodes_part, MPI_node, part_id, tag, mpi_comm_);
+ MPI_Send(nodes.data(), _num_nodes_part, _mpi_node_type, part_id, message_tag, _mpi_comm);
}
}
- else if(part_id > 0 && _rank == part_id)
+ else if(part_id > 0 && _mpi_rank == part_id)
{
- MPI_Recv(&s_nodes[0], _num_nodes_part, MPI_node, 0, tag, mpi_comm_, &status);
- setNodes(s_nodes, mesh_nodes, glb_node_ids);
+ MPI_Recv(nodes.data(), _num_nodes_part, _mpi_node_type, 0, message_tag, _mpi_comm, MPI_STATUS_IGNORE);
+ setNodes(nodes, mesh_nodes, glb_node_ids);
}
-
- if(part_id > 0)
- MPI_Type_free(&MPI_node);
-
}
-void NodePartitionedMeshReader
-::readCastElemsASCII(std::ifstream& is_elem, const int part_id,
- const long data_size, const bool process_ghost,
- const std::vector<MeshLib::Node*> &mesh_nodes,
- std::vector<MeshLib::Element*> &mesh_elems)
+void NodePartitionedMeshReader::readCastElemsASCII(std::ifstream& is_elem,
+ const int part_id, const int data_size, const bool process_ghost,
+ const std::vector<MeshLib::Node*> &mesh_nodes,
+ std::vector<MeshLib::Element*> &mesh_elems) const
{
- int tag = 0;
- MPI_Status status;
+ int const message_tag = 0;
- long *elem_data = new long[data_size];
- if(_rank == 0)
+ std::vector<long> elem_data(static_cast<std::size_t>(data_size));
+ if(_mpi_rank == 0)
{
readElementASCII(is_elem, elem_data, process_ghost);
@@ -309,30 +289,28 @@ void NodePartitionedMeshReader
}
else
{
- MPI_Send(elem_data, data_size, MPI_LONG, part_id, tag, mpi_comm_);
+ MPI_Send(elem_data.data(), data_size, MPI_LONG, part_id, message_tag, _mpi_comm);
}
}
- else if(part_id > 0 && _rank == part_id)
+ else if(part_id > 0 && _mpi_rank == part_id)
{
- MPI_Recv(elem_data, data_size, MPI_LONG, 0, tag, mpi_comm_, &status);
+ MPI_Recv(elem_data.data(), data_size, MPI_LONG, 0, message_tag, _mpi_comm, MPI_STATUS_IGNORE);
if(!process_ghost)
mesh_elems.resize(_num_regular_elems_part + _num_ghost_elems_part);
setElements(mesh_nodes, elem_data, mesh_elems, process_ghost);
}
-
- delete [] elem_data;
}
-MeshLib::NodePartitionedMesh* NodePartitionedMeshReader
-::readASCII(const std::string &file_name_base)
+MeshLib::NodePartitionedMesh* NodePartitionedMeshReader::readASCII(
+ const std::string &file_name_base)
{
std::ifstream is_cfg;
std::ifstream is_node;
std::ifstream is_elem;
bool file_opened = false;
- if(_rank == 0)
+ if(_mpi_rank == 0)
{
file_opened = openASCIIFiles(file_name_base, is_cfg, is_node, is_elem);
@@ -344,12 +322,11 @@ MeshLib::NodePartitionedMesh* NodePartitionedMeshReader
}
}
- MPI_Bcast(&file_opened, 1, MPI_INT, 0, mpi_comm_);
+ MPI_Bcast(&file_opened, 1, MPI_INT, 0, _mpi_comm);
if(!file_opened)
return nullptr;
- const long num_controls = 11;
/*
Array that contains integer numbers of the partition data header
and its size is 11.
@@ -364,25 +341,25 @@ MeshLib::NodePartitionedMesh* NodePartitionedMeshReader
8~9: Offsets of positions of partitions in the data arrays,
11: Reserved for exra flag.
*/
- long mesh_controls[num_controls];
+ std::array<long, 11> mesh_controls;
- NodePartitionedMesh *np_mesh = nullptr;
+ MeshLib::NodePartitionedMesh *np_mesh = nullptr;
std::vector<MeshLib::Node*> mesh_nodes;
- std::vector<unsigned> glb_node_ids;
+ std::vector<std::size_t> glb_node_ids;
std::vector<MeshLib::Element*> mesh_elems;
- for(int i=0; i<_size; i++)
+ for(int i = 0; i < _mpi_comm_size; i++)
{
- if(_rank == 0)
+ if(_mpi_rank == 0)
{
INFO("-->Parallel reading the partitioned mesh: ");
- for(long j=0; j< num_controls; j++)
- is_cfg >> mesh_controls[j];
+ for(long& v : mesh_controls)
+ is_cfg >> v;
is_cfg >> std::ws;
}
- MPI_Bcast(mesh_controls, num_controls, MPI_LONG, 0, mpi_comm_);
+ MPI_Bcast(mesh_controls.data(), mesh_controls.size(), MPI_LONG, 0, _mpi_comm);
_num_nodes_part = mesh_controls[0];
_num_regular_elems_part = mesh_controls[2];
@@ -406,39 +383,40 @@ MeshLib::NodePartitionedMesh* NodePartitionedMeshReader
readCastElemsASCII(is_elem, i, size_elem_g_info, process_ghost,
mesh_nodes, mesh_elems);
- if(_rank == i)
+ if(_mpi_rank == i)
{
//----------------------------------------------------------------------------------
//Create a mesh
- const std::size_t n_nghost_elem = static_cast<size_t>(mesh_controls[2]);
+ const std::size_t n_nghost_elem = static_cast<std::size_t>(mesh_controls[2]);
const unsigned n_global_base_nodes = static_cast<unsigned>(mesh_controls[6]);
const unsigned n_global_nodes = static_cast<unsigned>(mesh_controls[7]);
const unsigned n_base_nodes = static_cast<unsigned>(mesh_controls[1]);
const unsigned n_active_base_nodes = static_cast<unsigned>(mesh_controls[4]);
const unsigned n_active_nodes = static_cast<unsigned>(mesh_controls[5]);
- np_mesh = new NodePartitionedMesh(BaseLib::extractBaseName(file_name_base) + _size_str,
- mesh_nodes, glb_node_ids,
- mesh_elems, n_nghost_elem,
- n_global_base_nodes, n_global_nodes,
- n_base_nodes, n_active_base_nodes, n_active_nodes);
+ np_mesh = new MeshLib::NodePartitionedMesh(
+ BaseLib::extractBaseName(file_name_base) + std::to_string(_mpi_comm_size),
+ mesh_nodes, glb_node_ids,
+ mesh_elems, n_nghost_elem,
+ n_global_base_nodes, n_global_nodes,
+ n_base_nodes, n_active_base_nodes, n_active_nodes);
}
}
- if(_rank == 0)
+ if(_mpi_rank == 0)
{
is_cfg.close();
is_node.close();
is_elem.close();
}
- MPI_Barrier(mpi_comm_);
+ MPI_Barrier(_mpi_comm);
return np_mesh;
}
void NodePartitionedMeshReader::readElementASCII(std::ifstream &ins,
- long *elem_data, const bool ghost)
+ std::vector<long>& elem_data, const bool ghost) const
{
// Set number of elements.
const long ne = ghost ? _num_ghost_elems_part : _num_regular_elems_part;
@@ -446,62 +424,55 @@ void NodePartitionedMeshReader::readElementASCII(std::ifstream &ins,
for(long j=0; j<ne; j++)
{
elem_data[j] = counter;
- ins >> elem_data[counter]; //mat. idx
- counter++;
- ins >> elem_data[counter]; //type
- counter++;
+ ins >> elem_data[counter++]; //mat. idx
+ ins >> elem_data[counter++]; //type
ins >> elem_data[counter]; //nnodes
- const long nn_e = elem_data[counter];
- counter++;
+ const long nn_e = elem_data[counter++];
for(long k=0; k<nn_e; k++)
- {
- ins >> elem_data[counter];
- counter++;
- }
+ ins >> elem_data[counter++];
}
}
void NodePartitionedMeshReader::setNodes(const std::vector<NodeData> &node_data,
- std::vector<MeshLib::Node*> &mesh_node, std::vector<unsigned> &glb_node_ids)
+ std::vector<MeshLib::Node*> &mesh_node,
+ std::vector<std::size_t> &glb_node_ids) const
{
mesh_node.resize( _num_nodes_part );
glb_node_ids.resize( _num_nodes_part );
- for(size_t i=0; i<mesh_node.size(); i++)
+ for(std::size_t i=0; i<mesh_node.size(); i++)
{
- const NodeData *nd = &node_data[i];
- glb_node_ids[i] = static_cast<unsigned>(nd->index);
- mesh_node[i] = new MeshLib::Node(nd->x, nd->y, nd->z, i);
+ NodeData const& nd = node_data[i];
+ glb_node_ids[i] = nd.index;
+ mesh_node[i] = new MeshLib::Node(nd.x, nd.y, nd.z, i);
}
}
-void NodePartitionedMeshReader::setElements(const std::vector<MeshLib::Node*> &mesh_nodes,
- const long *elem_data, std::vector<MeshLib::Element*> &mesh_elems, const bool ghost)
+void NodePartitionedMeshReader::setElements(
+ const std::vector<MeshLib::Node*> &mesh_nodes,
+ const std::vector<long> &elem_data,
+ std::vector<MeshLib::Element*> &mesh_elems, const bool ghost) const
{
// Number of elements, ether ghost or regular
const long ne = ghost ? _num_ghost_elems_part : _num_regular_elems_part;
const long id_offset = ghost ? _num_regular_elems_part : 0;
- long counter;
- for(long i=0; i<ne; i++)
+
+ for(std::size_t i=0; i < ne; i++)
{
- counter = elem_data[i];
+ std::size_t counter = elem_data[i];
- const long mat_idx = static_cast<size_t>( elem_data[counter] );
- counter++;
- const long e_type = elem_data[counter];
- counter++;
- const long nnodes = elem_data[counter];
- counter++;
+ const unsigned mat_idx = static_cast<unsigned>( elem_data[counter++] );
+ const long e_type = elem_data[counter++];
+ const long nnodes = elem_data[counter++];
MeshLib::Node **elem_nodes = new MeshLib::Node*[nnodes];
- for(long k=0; k<nnodes; k++)
- {
- elem_nodes[k] = mesh_nodes[ elem_data[counter] ];
- counter++;
- }
+ for(std::size_t k=0; k<nnodes; k++)
+ elem_nodes[k] = mesh_nodes[ elem_data[counter++] ];
MeshLib::Element *elem = nullptr;
+ // The element types below are defined by the mesh_partition tool
+ // available at https://github.com/ufz/mesh_partition .
switch(e_type)
{
case 1:
@@ -531,42 +502,15 @@ void NodePartitionedMeshReader::setElements(const std::vector<MeshLib::Node*> &m
}
}
-void NodePartitionedMeshReader::buildNodeStrucTypeMPI(NodeData *anode, MPI_Datatype *mpi_node_ptr)
+void NodePartitionedMeshReader::registerNodeDataMpiType()
{
- int nblocklen[4] = {1, 1, 1, 1};
-
- MPI_Aint disp[4], base;
- MPI_Get_address(anode, disp);
- MPI_Get_address(&(anode[0].x), disp+1);
- MPI_Get_address(&(anode[0].y), disp+2);
- MPI_Get_address(&(anode[0].z), disp+3);
- base = disp[0];
- for(int j=0; j <4; j++)
- {
- disp[j] -= base;
- }
-
- MPI_Datatype my_comp_type[4];
- my_comp_type[0] = MPI_LONG;
- my_comp_type[1] = MPI_DOUBLE;
- my_comp_type[2] = MPI_DOUBLE;
- my_comp_type[3] = MPI_DOUBLE;
+ int const count = 2;
+ int blocks[count] = {1, 3};
+ MPI_Datatype types[count] = {MPI_UNSIGNED_LONG, MPI_DOUBLE};
+ MPI_Aint displacements[count] = {0, sizeof(NodeData::index)};
- // build datatype describing structure
- MPI_Type_create_struct(4, nblocklen, disp, my_comp_type, mpi_node_ptr);
- MPI_Type_commit(mpi_node_ptr);
-}
-
-void NodePartitionedMeshReader::printMessage(const std::string & err_message, const bool for_fileopen)
-{
- if( for_fileopen )
- {
- ERR("! File %s does not exist.", err_message.c_str());
- }
- else
- {
- ERR( err_message.c_str() );
- }
+ MPI_Type_create_struct(count, blocks, displacements, types, &_mpi_node_type);
+ MPI_Type_commit(&_mpi_node_type);
}
} // namespace FileIO
diff --git a/FileIO/MPI_IO/NodePartitionedMeshReader.h b/FileIO/MPI_IO/NodePartitionedMeshReader.h
index 8756e70a68d..0fbfd221779 100644
--- a/FileIO/MPI_IO/NodePartitionedMeshReader.h
+++ b/FileIO/MPI_IO/NodePartitionedMeshReader.h
@@ -14,13 +14,13 @@
#ifndef NODE_PARTITIONED_MESH_READER_H
#define NODE_PARTITIONED_MESH_READER_H
-#include <string>
#include <fstream>
+#include <string>
+#include <vector>
#include <mpi.h>
#include "MeshLib/NodePartitionedMesh.h"
-#include "MeshLib/Node.h"
namespace MeshLib
{
@@ -30,24 +30,42 @@ class Element;
namespace FileIO
{
-/// Class to handle reading data of partitioned mesh.
+/// Class for reading of ascii or binary partitioned mesh files into a
+/// NodePartitionedMesh.
class NodePartitionedMeshReader
{
public:
- NodePartitionedMeshReader() = default;
+ /*!
+ \param comm MPI Communicator.
+ */
+ NodePartitionedMeshReader(MPI_Comm comm);
+
+ ~NodePartitionedMeshReader();
/*!
\brief Create a NodePartitionedMesh object, read data to it,
and return a pointer to it. Data files are either in
ASCII format or binary format.
- \param comm MPI Communicator.
\param file_name_base Name of file to be read, and it must be base name without name extension.
\return Pointer to Mesh object. If the creation of mesh object
fails, return a null pointer.
*/
- MeshLib::NodePartitionedMesh* read(MPI_Comm comm, const std::string &file_name_base);
+ MeshLib::NodePartitionedMesh* read(const std::string &file_name_base);
private:
+ /// Pointer to MPI commumicator;
+ MPI_Comm _mpi_comm = MPI_COMM_WORLD;
+
+ /// Number of MPI processes
+ int _mpi_comm_size;
+
+ /// Rank of compute core
+ int _mpi_rank;
+
+ /// MPI data type description for sending/receiving of node data.
+ MPI_Datatype _mpi_node_type;
+
+
/// Number of all nodes of a partition.
long _num_nodes_part;
@@ -57,18 +75,6 @@ class NodePartitionedMeshReader
/// Number of ghost elements of a partition.
long _num_ghost_elems_part;
- /// Number of MPI processes
- int _size;
-
- /// _size converted to string
- std::string _size_str;
-
- /// MPI commumicator;
- MPI_Comm mpi_comm_ = MPI_COMM_WORLD;
-
- /// Rank of compute core
- int _rank;
-
/*!
\brief Create a NodePartitionedMesh object, read binary mesh data
in the manner of parallel, and return a pointer to it.
@@ -88,7 +94,7 @@ class NodePartitionedMeshReader
6: Number of nodes for linear element of global mesh,
7: Number of all nodes of global mesh,
8~12: Offsets of positions of partitions in the data arrays,
- 13: Reserved for exra flag.
+ 13: Reserved for extra flag.
for all partitions
the second file contains a struct type (long, double double double) array of
@@ -105,7 +111,7 @@ class NodePartitionedMeshReader
*/
MeshLib::NodePartitionedMesh* readBinary(const std::string &file_name_base);
- /*
+ /*!
\brief Open ASCII files of node partitioned mesh data.
\param file_name_base Name of file to be read, which must be a name with the
@@ -116,34 +122,35 @@ class NodePartitionedMeshReader
\return Return true if all files are good.
*/
bool openASCIIFiles(std::string const& file_name_base,std::ifstream& is_cfg,
- std::ifstream& is_node, std::ifstream& is_elem);
+ std::ifstream& is_node, std::ifstream& is_elem) const;
- /*
- \brief Read mesh nodes from an ASCII file and cast to the correponding rank.
+ /*!
+ \brief Read mesh nodes from an ASCII file and cast to the corresponding rank.
\param is_node Input stream for the file contains node data.
\param part_id Partition ID.
\param mesh_nodes Node vector to be filled.
- \param part_id Global Node ID to be filled.
+ \param glb_node_ids Global Node IDs to be filled.
*/
void readCastNodesASCII(std::ifstream& is_node, const int part_id,
std::vector<MeshLib::Node*> &mesh_nodes,
- std::vector<unsigned> &glb_node_ids);
+ std::vector<std::size_t> &glb_node_ids) const;
- /*
- \brief Read mesh elements from an ASCII file and cast to the correponding rank.
+ /*!
+ \brief Read mesh elements from an ASCII file and cast to the corresponding rank.
\param is_elem Input stream for the file contains element data.
\param part_id Partition ID.
- \param data_size Total size of the data to be read.
- \param proc_ghost Flag to process ghost element.
+ \param data_size Total size of the data to be read. This type is an
+ int because of MPI_Send() implicit cast.
+ \param process_ghost Flag to process ghost element.
\param mesh_nodes Node vector to be filled.
\param mesh_elems Element vector to be filled.
*/
void readCastElemsASCII(std::ifstream& is_elem, const int part_id,
- const long data_size, const bool process_ghost,
+ const int data_size, const bool process_ghost,
const std::vector<MeshLib::Node*> &mesh_nodes,
- std::vector<MeshLib::Element*> &mesh_elems);
+ std::vector<MeshLib::Element*> &mesh_elems) const;
/*!
\brief Create a NodePartitionedMesh object, read ASCII mesh data,
@@ -163,7 +170,7 @@ class NodePartitionedMeshReader
6: Number of nodes for linear element of global mesh,
7: Number of all nodes of global mesh,
8~9: Offsets of positions of partitions in the data arrays,
- 11: Reserved for exra flag.
+ 11: Reserved for extra flag.
for all partitions
the second file contains nodes information of global IDs and coordinates
@@ -183,31 +190,31 @@ class NodePartitionedMeshReader
\param elem_data Pointer to array that contains element data, which to be filled.
\param ghost Flag to read ghost elements.
*/
- void readElementASCII(std::ifstream &ins, long *elem_data,
- const bool ghost = false);
+ void readElementASCII(std::ifstream &ins,
+ std::vector<long>& elem_data,
+ const bool ghost = false) const;
/// Node data only for parallel reading.
struct NodeData
{
- long index; ///< Global node index.
+ std::size_t index; ///< Global node index.
double x;
double y;
double z;
};
- /*! Define MPI data type, mpi_node_ptr, for struct MeshNode for palllel reading of nodes
- \param anode a NodeData variable.
- \param mpi_node_ptr Defined MPI data type of struct NodeData.
- */
- void buildNodeStrucTypeMPI(NodeData *anode, MPI_Datatype *mpi_node_ptr);
+ /// Define MPI data type for NodeData struct.
+ void registerNodeDataMpiType();
+
/*!
\brief Set mesh nodes from a tempory array containing node data read from file.
\param node_data Array containing node data read from file.
\param mesh_node Vector of mesh nodes to be set.
\param glb_node_ids Global IDs of nodes of a partition.
*/
- void setNodes(const std::vector<NodeData> &node_data, std::vector<MeshLib::Node*> &mesh_node,
- std::vector<unsigned> &glb_node_ids);
+ void setNodes(const std::vector<NodeData> &node_data,
+ std::vector<MeshLib::Node*> &mesh_node,
+ std::vector<std::size_t> &glb_node_ids) const;
/*!
\brief Set mesh elements from a tempory array containing node data read from file.
@@ -216,13 +223,29 @@ class NodePartitionedMeshReader
\param mesh_elems Vector of mesh elements to be set.
\param ghost Flag of processing ghost elements.
*/
- void setElements(const std::vector<MeshLib::Node*> &mesh_nodes, const long *elem_data,
- std::vector<MeshLib::Element*> &mesh_elems, const bool ghost = false);
-
- /// Print message when file opening fails or the requested numbers mismatch
- void printMessage(const std::string & err_message, const bool for_fileopen = true);
+ void setElements(const std::vector<MeshLib::Node*> &mesh_nodes,
+ const std::vector<long> &elem_data,
+ std::vector<MeshLib::Element*> &mesh_elems,
+ const bool ghost = false) const;
+
+ /*! Fills the given data vector with data read from file starting at an
+ offset.
+ \note In case of failure during opening of the file, an
+ error message is printed.
+ \param filename File name containing data.
+ \param offset Displacement of the data accessible from the view;
+ see MPI_File_set_view() documentation.
+ \param type Type of data.
+ \param data A container to be filled with data. Its size is used
+ to determine how many values should be read.
+ \tparam DATA A homogeneous contaner type supporting data() and size().
+ \return True on success and false otherwise.
+ */
+ template <typename DATA>
+ bool readBinaryDataFromFile(std::string const& filename, MPI_Offset offset,
+ MPI_Datatype type, DATA& data) const;
};
-} // End of namespace
+} // FileIO
-#endif // end of #ifndef READ_NODE_PARTITIONED_MESH_H
+#endif // READ_NODE_PARTITIONED_MESH_H
diff --git a/MeshLib/NodePartitionedMesh.h b/MeshLib/NodePartitionedMesh.h
index 505f8630275..d8a391ae360 100644
--- a/MeshLib/NodePartitionedMesh.h
+++ b/MeshLib/NodePartitionedMesh.h
@@ -21,11 +21,6 @@
#include "Mesh.h"
-namespace FileIO
-{
-class readNodePartitionedMesh;
-};
-
namespace MeshLib
{
class Node;
@@ -55,14 +50,14 @@ class NodePartitionedMesh : public Mesh
*/
NodePartitionedMesh(const std::string &name,
const std::vector<Node*> &nodes,
- const std::vector<unsigned> &glb_node_ids,
+ const std::vector<std::size_t> &glb_node_ids,
const std::vector<Element*> &elements,
const std::size_t n_nghost_elem,
- const unsigned n_global_base_nodes,
- const unsigned n_global_nodes,
- const unsigned n_base_nodes,
- const unsigned n_active_base_nodes,
- const unsigned n_active_nodes)
+ const std::size_t n_global_base_nodes,
+ const std::size_t n_global_nodes,
+ const std::size_t n_base_nodes,
+ const std::size_t n_active_base_nodes,
+ const std::size_t n_active_nodes)
: Mesh(name, nodes, elements, n_base_nodes),
_global_node_ids(glb_node_ids), _n_nghost_elem(n_nghost_elem),
_n_global_base_nodes(n_global_base_nodes),
@@ -72,36 +67,32 @@ class NodePartitionedMesh : public Mesh
{
}
- ~NodePartitionedMesh()
- {
- }
-
/// Get the number of nodes of the global mesh for linear elements.
- unsigned getNGlobalBaseNodes() const
+ std::size_t getNGlobalBaseNodes() const
{
return _n_global_base_nodes;
}
/// Get the number of all nodes of the global mesh.
- unsigned getNGlobalNodes() const
+ std::size_t getNGlobalNodes() const
{
return _n_global_nodes;
}
/// Get the number of the active nodes of the partition for linear elements.
- unsigned getNActiveBaseNodes() const
+ std::size_t getNActiveBaseNodes() const
{
return _n_active_base_nodes;
}
/// Get the number of all active nodes of the partition.
- unsigned getNActiveNodes() const
+ std::size_t getNActiveNodes() const
{
return _n_active_nodes;
}
/// Check whether a node with ID of node_id is a ghost node
- bool isGhostNode(const unsigned node_id)
+ bool isGhostNode(const std::size_t node_id) const
{
if(node_id < _n_active_base_nodes)
return true;
@@ -112,35 +103,35 @@ class NodePartitionedMesh : public Mesh
}
/// Get the largest ID of active nodes for higher order elements in a partition.
- unsigned getLargestActiveNodeID() const
+ std::size_t getLargestActiveNodeID() const
{
return _n_base_nodes + _n_active_nodes - _n_active_base_nodes;
}
/// Get the number of non-ghost elements, or the start entry ID of ghost elements in element vector.
- size_t getNNonGhostElements() const
+ std::size_t getNNonGhostElements() const
{
return _n_nghost_elem;
}
private:
/// Global IDs of nodes of a partition
- std::vector<unsigned> _global_node_ids;
+ std::vector<std::size_t> _global_node_ids;
/// Number of non-ghost elements, or the ID of the start entry of ghost elements in _elements vector.
std::size_t _n_nghost_elem;
/// Number of the nodes of the global mesh linear interpolations.
- unsigned _n_global_base_nodes;
+ std::size_t _n_global_base_nodes;
/// Number of all nodes of the global mesh.
- unsigned _n_global_nodes;
+ std::size_t _n_global_nodes;
/// Number of the active nodes for linear interpolations
- unsigned _n_active_base_nodes;
+ std::size_t _n_active_base_nodes;
/// Number of the all active nodes.
- unsigned _n_active_nodes;
+ std::size_t _n_active_nodes;
};
} // namespace MeshLib
diff --git a/SimpleTests/MeshTests/MPI/NodePartitionedMeshTester.cpp b/SimpleTests/MeshTests/MPI/NodePartitionedMeshTester.cpp
index 3495b70ff43..c7d9421ac57 100644
--- a/SimpleTests/MeshTests/MPI/NodePartitionedMeshTester.cpp
+++ b/SimpleTests/MeshTests/MPI/NodePartitionedMeshTester.cpp
@@ -37,65 +37,78 @@ using namespace FileIO;
int main(int argc, char *argv[])
{
LOGOG_INITIALIZE();
- {
- MPI_Init(&argc, &argv);
+
+ MPI_Init(&argc, &argv);
#ifdef USE_PETSC
- char help[] = "ogs6 with PETSc \n";
- PetscInitialize(&argc, &argv, nullptr, help);
+ char help[] = "ogs6 with PETSc \n";
+ PetscInitialize(&argc, &argv, nullptr, help);
#endif
- BaseLib::LogogCustomCout out(1);
- BaseLib::TemplateLogogFormatterSuppressedGCC<TOPIC_LEVEL_FLAG | TOPIC_FILE_NAME_FLAG | TOPIC_LINE_NUMBER_FLAG> custom_format;
- out.SetFormatter(custom_format);
+ BaseLib::LogogCustomCout* out = new BaseLib::LogogCustomCout(1);
+ using LogogFormatter = BaseLib::TemplateLogogFormatterSuppressedGCC
+ <TOPIC_LEVEL_FLAG | TOPIC_FILE_NAME_FLAG | TOPIC_LINE_NUMBER_FLAG>;
+ LogogFormatter* fmt = new LogogFormatter();
- const std::string file_name = argv[1];
+ out->SetFormatter(*fmt);
- NodePartitionedMeshReader read_pmesh;
- NodePartitionedMesh *mesh = read_pmesh.read(MPI_COMM_WORLD, file_name);
+ const std::string file_name = argv[1];
- int rank;
- MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- const std::string rank_str = std::to_string(rank);
- const std::string ofile_name = file_name + "_partition_" + rank_str + ".msh";
- std::ofstream os(ofile_name.data(), std::ios::trunc);
+ NodePartitionedMesh *mesh = nullptr;
+ {
+ NodePartitionedMeshReader read_pmesh(MPI_COMM_WORLD);
+ mesh = read_pmesh.read(file_name);
+ }
+ if (!mesh)
+ {
+ ERR("Could not read mesh from files with prefix %s.", file_name.c_str());
+ return EXIT_FAILURE;
+ }
+
+ int rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ const std::string rank_str = std::to_string(rank);
+ const std::string ofile_name = file_name + "_partition_" + rank_str + ".msh";
+ std::ofstream os(ofile_name.data(), std::ios::trunc);
+
+ // Output nodes
+ os.setf(std::ios::scientific, std::ios::floatfield);
+ std::setprecision(10);
+ const std::size_t nn = mesh->getNNodes();
+ for(std::size_t i=0; i<nn; i++)
+ {
+ const double *x = mesh->getNode(i)->getCoords();
+ os << mesh->getNode(i)->getID() << " "
+ << std::setw(14) << x[0] << " " << x[1] << " "<< x[2] << "\n";
+ }
+ os.flush();
+
+ // Output elements
+ const std::size_t ne = mesh->getNElements();
+ for(std::size_t i=0; i<ne; i++)
+ {
+ const Element *elem = mesh->getElement(i);
+ Node* const* ele_nodes = elem->getNodes();
- // Output nodes
- os.setf(std::ios::scientific, std::ios::floatfield);
- std::setprecision(10);
- const size_t nn = mesh->getNNodes();
- for(size_t i=0; i<nn; i++)
+ for(unsigned j=0; j<elem->getNNodes(); j++)
{
- const double *x = mesh->getNode(i)->getCoords();
- os << mesh->getNode(i)->getID() << " "
- << std::setw(14) << x[0] << " " << x[1] << " "<< x[2] << "\n";
+ os << ele_nodes[j]->getID() << " ";
}
- os.flush();
+ os << "\n";
+ }
+ os.flush();
- // Output elements
- const size_t ne = mesh->getNElements();
- for(size_t i=0; i<ne; i++)
- {
- const Element *elem = mesh->getElement(i);
- Node* const* ele_nodes = elem->getNodes();
-
- for(unsigned j=0; j<elem->getNNodes(); j++)
- {
- os << ele_nodes[j]->getID() << " ";
- }
- os << "\n";
- }
- os.flush();
+ delete mesh;
- delete mesh;
+ delete out;
+ delete fmt;
#ifdef USE_PETSC
- PetscFinalize();
+ PetscFinalize();
#endif
- MPI_Finalize();
+ MPI_Finalize();
- } // make sure no logog objects exist when LOGOG_SHUTDOWN() is called.
LOGOG_SHUTDOWN();
}
--
GitLab