diff --git a/Applications/Utils/ModelPreparation/CMakeLists.txt b/Applications/Utils/ModelPreparation/CMakeLists.txt index c90970e913a401aab725cdf346dcb5d441fcbfa7..1d8e282ea56814621d4c70c2b6a3161d5c894755 100644 --- a/Applications/Utils/ModelPreparation/CMakeLists.txt +++ b/Applications/Utils/ModelPreparation/CMakeLists.txt @@ -19,6 +19,4 @@ target_link_libraries(createNeumannBc ${OGS_VTK_REQUIRED_LIBS} ) -if(OGS_BUILD_METIS) - add_subdirectory(PartitionMesh) -endif() +add_subdirectory(PartitionMesh) diff --git a/Applications/Utils/ModelPreparation/PartitionMesh/CMakeLists.txt b/Applications/Utils/ModelPreparation/PartitionMesh/CMakeLists.txt index b331c0db9310b76826aa4d367fc56874c190a6c5..e0888ee1d606c7fb4ccc5c8597336cadb01cd997 100644 --- a/Applications/Utils/ModelPreparation/PartitionMesh/CMakeLists.txt +++ b/Applications/Utils/ModelPreparation/PartitionMesh/CMakeLists.txt @@ -1,6 +1,8 @@ -add_executable(partmesh PartitionMesh.cpp NodeWiseMeshPartitioner.h NodeWiseMeshPartitioner.cpp) +add_executable(partmesh PartitionMesh.cpp Metis.cpp NodeWiseMeshPartitioner.cpp) + set_target_properties(partmesh PROPERTIES FOLDER Utilities) target_link_libraries(partmesh MeshLib) +add_dependencies(partmesh mpmetis) #################### ### Installation ### diff --git a/Applications/Utils/ModelPreparation/PartitionMesh/Metis.cpp b/Applications/Utils/ModelPreparation/PartitionMesh/Metis.cpp new file mode 100644 index 0000000000000000000000000000000000000000..51bc8aed815d58cf571ae796937c402fb8c00b3b --- /dev/null +++ b/Applications/Utils/ModelPreparation/PartitionMesh/Metis.cpp @@ -0,0 +1,99 @@ +/** + * \file + * + * \copyright + * Copyright (c) 2012-2018, OpenGeoSys Community (http://www.opengeosys.org) + * Distributed under a Modified BSD License. + * See accompanying file LICENSE.txt or + * http://www.opengeosys.org/project/license + * + */ + +#include <iostream> + +#include "BaseLib/Error.h" +#include "MeshLib/Elements/Element.h" + +namespace ApplicationUtils +{ +void writeMETIS(std::vector<MeshLib::Element*> const& elements, + const std::string& file_name) +{ + std::ofstream os(file_name, std::ios::trunc); + if (!os.is_open()) + { + OGS_FATAL("Error: cannot open file %s.", file_name.data()); + } + + if (!os.good()) + { + OGS_FATAL("Error: Cannot write in file %s.", file_name.data()); + } + + os << elements.size() << " \n"; + for (const auto* elem : elements) + { + os << elem->getNodeIndex(0) + 1; + for (unsigned j = 1; j < elem->getNumberOfNodes(); j++) + { + os << " " << elem->getNodeIndex(j) + 1; + } + os << "\n"; + } +} + +std::vector<std::size_t> readMetisData(const std::string& file_name_base, + long const number_of_partitions, + std::size_t const number_of_nodes) +{ + const std::string npartitions_str = std::to_string(number_of_partitions); + + // Read partitioned mesh data from METIS + const std::string fname_parts = + file_name_base + ".mesh.npart." + npartitions_str; + + std::ifstream npart_in(fname_parts); + if (!npart_in.is_open()) + { + OGS_FATAL( + "Error: cannot open file %s. It may not exist!\n" + "Run mpmetis beforehand or use option -m", + fname_parts.data()); + } + + std::vector<std::size_t> partition_ids(number_of_nodes); + + + std::size_t counter = 0; + while (!npart_in.eof()) + { + npart_in >> partition_ids[counter++] >> std::ws; + if (counter == number_of_nodes) + { + break; + } + } + + if (npart_in.bad()) + { + OGS_FATAL("Error while reading file %s.", fname_parts.data()); + } + + if (counter != number_of_nodes) + { + OGS_FATAL("Error: data in %s are less than expected.", + fname_parts.data()); + } + + return partition_ids; +} + +void removeMetisPartitioningFiles(std::string const& file_name_base, + long const number_of_partitions) +{ + const std::string npartitions_str = std::to_string(number_of_partitions); + + std::remove((file_name_base + ".mesh.npart." + npartitions_str).c_str()); + std::remove((file_name_base + ".mesh.epart." + npartitions_str).c_str()); +} +} // namespace ApplicationUtils diff --git a/Applications/Utils/ModelPreparation/PartitionMesh/Metis.h b/Applications/Utils/ModelPreparation/PartitionMesh/Metis.h new file mode 100644 index 0000000000000000000000000000000000000000..85eb164e68269b2dd7fe1cb01084ed252fd72bf2 --- /dev/null +++ b/Applications/Utils/ModelPreparation/PartitionMesh/Metis.h @@ -0,0 +1,44 @@ +/** + * \file + * + * \copyright + * Copyright (c) 2012-2018, OpenGeoSys Community (http://www.opengeosys.org) + * Distributed under a Modified BSD License. + * See accompanying file LICENSE.txt or + * http://www.opengeosys.org/project/license + * + */ + +#pragma once + +#include <string> +#include <vector> + +namespace MeshLib +{ +class Element; +} + +namespace ApplicationUtils +{ +/// Write elements as METIS graph file +/// \param elements The mesh elements. +/// \param file_name File name with an extension of mesh. +void writeMETIS(std::vector<MeshLib::Element*> const& elements, + const std::string& file_name); + +/// Read metis data +/// \param file_name_base The prefix of the filename. +/// \param number_of_partitions The number is used to compose the full filename +/// and forms the postfix. +/// \param number_of_nodes Expected/required number of nodes to be read. +std::vector<std::size_t> readMetisData(const std::string& file_name_base, + long number_of_partitions, + std::size_t number_of_nodes); + +/// Removes the F.mesh.npart.P and F.mesh.epart.P files, where F is file name +/// base and P is the number of partitions. +void removeMetisPartitioningFiles(std::string const& file_name_base, + long number_of_partitions); + +} // namespace ApplicationUtils diff --git a/Applications/Utils/ModelPreparation/PartitionMesh/NodeWiseMeshPartitioner.cpp b/Applications/Utils/ModelPreparation/PartitionMesh/NodeWiseMeshPartitioner.cpp index 26f249a7a8df5c50ccb6584a5d2be063e6ed5ade..de71ebd4842f9b27203a50a0e432a60834d23fc2 100644 --- a/Applications/Utils/ModelPreparation/PartitionMesh/NodeWiseMeshPartitioner.cpp +++ b/Applications/Utils/ModelPreparation/PartitionMesh/NodeWiseMeshPartitioner.cpp @@ -15,9 +15,8 @@ #include "NodeWiseMeshPartitioner.h" #include <limits> -#include <iomanip> -#include <cstdio> // for binary output #include <numeric> +#include <unordered_map> #include <logog/include/logog.hpp> @@ -29,57 +28,84 @@ namespace ApplicationUtils { struct NodeStruct { + NodeStruct(NodeWiseMeshPartitioner::IntegerType const id_, + double const x_, + double const y_, + double const z_) + : id(id_), x(x_), y(y_), z(z_) + { + } + NodeWiseMeshPartitioner::IntegerType id; double x; double y; double z; }; -void NodeWiseMeshPartitioner::readMetisData(const std::string& file_name_base) +std::size_t Partition::numberOfMeshItems( + MeshLib::MeshItemType const item_type) const { - const std::string npartitions_str = std::to_string(_npartitions); - - // Read partitioned mesh data from METIS - const std::string fname_parts = file_name_base + ".mesh.npart." + npartitions_str; - - std::ifstream npart_in(fname_parts); - if (!npart_in.is_open()) + if (item_type == MeshLib::MeshItemType::Node) { - OGS_FATAL( - "Error: cannot open file %s. It may not exist!\n" - "Run mpmetis beforehand or use option -m", - fname_parts.data()); + return nodes.size(); } - const std::size_t nnodes = _mesh->getNumberOfNodes(); - - std::size_t counter = 0; - while (!npart_in.eof()) + if (item_type == MeshLib::MeshItemType::Cell) { - npart_in >> _nodes_partition_ids[counter++] >> std::ws; - if (counter == nnodes) - break; - } - - if (npart_in.bad()) - { - OGS_FATAL( - "Error while reading file %s.", fname_parts.data()); + return regular_elements.size() + ghost_elements.size(); } + OGS_FATAL("Mesh items other than nodes and cells are not supported."); +} - npart_in.close(); +std::ostream& Partition::writeNodesBinary( + std::ostream& os, std::vector<std::size_t> const& global_node_ids) const +{ + std::vector<NodeStruct> nodes_buffer; + nodes_buffer.reserve(nodes.size()); - if( counter != nnodes) + for (const auto* node : nodes) { - OGS_FATAL( - "Error: data in %s are less than expected.", fname_parts.data()); + double const* coords = node->getCoords(); + nodes_buffer.emplace_back(global_node_ids[node->getID()], coords[0], + coords[1], coords[2]); } + return os.write(reinterpret_cast<const char*>(nodes_buffer.data()), + sizeof(NodeStruct) * nodes_buffer.size()); +} + +/// Calculate the total number of integer variables of an element vector. Each +/// element has three integer variables for element ID, element type, number of +/// nodes of the element. Therefore the total number of the integers in +/// \c elements is 3 * elements.size() + sum (number of nodes of each element). +NodeWiseMeshPartitioner::IntegerType getNumberOfIntegerVariablesOfElements( + std::vector<const MeshLib::Element*> const& elements) +{ + return 3 * elements.size() + + std::accumulate(begin(elements), end(elements), 0, + [](auto const nnodes, auto const* e) { + return nnodes + e->getNumberOfNodes(); + }); +} - // remove metis files. - std::remove(fname_parts.c_str()); - const std::string fname_eparts = file_name_base + ".mesh.epart." - + npartitions_str; - std::remove(fname_eparts.c_str()); +std::ostream& Partition::writeConfigBinary(std::ostream& os) const +{ + long const data[] = { + static_cast<long>(nodes.size()), + static_cast<long>(number_of_base_nodes), + static_cast<long>(regular_elements.size()), + static_cast<long>(ghost_elements.size()), + static_cast<long>(number_of_non_ghost_base_nodes), + static_cast<long>(number_of_non_ghost_nodes), + static_cast<long>(number_of_mesh_base_nodes), + static_cast<long>(number_of_mesh_all_nodes), + static_cast<long>( + getNumberOfIntegerVariablesOfElements(regular_elements)), + static_cast<long>( + getNumberOfIntegerVariablesOfElements(ghost_elements)), + }; + + return os.write(reinterpret_cast<const char*>(data), + sizeof(data)); } void NodeWiseMeshPartitioner::findNonGhostNodesInPartition( @@ -94,8 +120,8 @@ void NodeWiseMeshPartitioner::findNonGhostNodesInPartition( { if (_nodes_partition_ids[i] == part_id) { - splitOfHigherOrderNode(nodes, is_mixed_high_order_linear_elems, - i, partition.nodes, extra_nodes); + splitOffHigherOrderNode(nodes, is_mixed_high_order_linear_elems, i, + partition.nodes, extra_nodes); } } partition.number_of_non_ghost_base_nodes = partition.nodes.size(); @@ -113,7 +139,9 @@ void NodeWiseMeshPartitioner::findElementsInPartition(std::size_t const part_id) { const auto* elem = elements[elem_id]; if (_is_regular_element[elem_id]) + { continue; + } std::size_t non_ghost_node_number = 0; for (unsigned i = 0; i < elem->getNumberOfNodes(); i++) @@ -125,7 +153,9 @@ void NodeWiseMeshPartitioner::findElementsInPartition(std::size_t const part_id) } if (non_ghost_node_number == 0) + { continue; + } if (non_ghost_node_number == elem->getNumberOfNodes()) { @@ -153,40 +183,40 @@ void NodeWiseMeshPartitioner::findGhostNodesInPartition( { const unsigned node_id = ghost_elem->getNodeIndex(i); if (nodes_reserved[node_id]) + { continue; + } if (_nodes_partition_ids[node_id] != part_id) { - splitOfHigherOrderNode(nodes, is_mixed_high_order_linear_elems, - node_id, partition.nodes, extra_nodes); + splitOffHigherOrderNode(nodes, is_mixed_high_order_linear_elems, + node_id, partition.nodes, extra_nodes); nodes_reserved[node_id] = true; } } } } -void NodeWiseMeshPartitioner::splitOfHigherOrderNode( +void NodeWiseMeshPartitioner::splitOffHigherOrderNode( std::vector<MeshLib::Node*> const& nodes, bool const is_mixed_high_order_linear_elems, unsigned const node_id, std::vector<MeshLib::Node*>& base_nodes, std::vector<MeshLib::Node*>& extra_nodes) { - if (is_mixed_high_order_linear_elems) + auto const n_base_nodes = _mesh->getNumberOfBaseNodes(); + if (!is_mixed_high_order_linear_elems || node_id > n_base_nodes) { - if (node_id < _mesh->getNumberOfBaseNodes()) - base_nodes.push_back(nodes[node_id]); - else - extra_nodes.push_back(nodes[node_id]); + base_nodes.push_back(nodes[node_id]); } else { - base_nodes.push_back(nodes[node_id]); + extra_nodes.push_back(nodes[node_id]); } } -void NodeWiseMeshPartitioner::processPartition(std::size_t const part_id, - const bool is_mixed_high_order_linear_elems) +void NodeWiseMeshPartitioner::processPartition( + std::size_t const part_id, const bool is_mixed_high_order_linear_elems) { std::vector<MeshLib::Node*> extra_nodes; findNonGhostNodesInPartition(part_id, is_mixed_high_order_linear_elems, @@ -199,78 +229,157 @@ void NodeWiseMeshPartitioner::processPartition(std::size_t const part_id, partition.number_of_base_nodes = partition.nodes.size(); if (is_mixed_high_order_linear_elems) + { partition.nodes.insert(partition.nodes.end(), extra_nodes.begin(), extra_nodes.end()); + } + + // Set the node numbers of base and all mesh nodes. + partition.number_of_mesh_base_nodes = _mesh->getNumberOfBaseNodes(); + partition.number_of_mesh_all_nodes = _mesh->getNumberOfNodes(); } -void NodeWiseMeshPartitioner::processNodeProperties() +/// Copies the properties from global property vector \c pv to the +/// partition-local one \c partitioned_pv. +template <typename T> +std::size_t copyNodePropertyVectorValues( + Partition const& p, + std::size_t const offset, + MeshLib::PropertyVector<T> const& pv, + MeshLib::PropertyVector<T>& partitioned_pv) { - std::size_t const total_number_of_tuples = - std::accumulate(std::begin(_partitions), std::end(_partitions), 0, - [](std::size_t const sum, Partition const& p) { - return sum + p.nodes.size(); - }); + auto const& nodes = p.nodes; + auto const nnodes = nodes.size(); + for (std::size_t i = 0; i < nnodes; ++i) + { + const auto global_id = nodes[i]->getID(); + partitioned_pv[offset + i] = pv[global_id]; + } + return nnodes; +} - DBUG("total number of node-based tuples after partitioning: %d ", - total_number_of_tuples); - // 1 create new PV - // 2 resize the PV with total_number_of_tuples - // 3 copy the values according to the partition info - auto const& original_properties(_mesh->getProperties()); - auto const property_names = - original_properties.getPropertyVectorNames(MeshLib::MeshItemType::Node); +/// Copies the properties from global property vector \c pv to the +/// partition-local one \c partitioned_pv. Regular elements' and ghost elements' +/// values are copied. +template <typename T> +std::size_t copyCellPropertyVectorValues( + Partition const& p, + std::size_t const offset, + MeshLib::PropertyVector<T> const& pv, + MeshLib::PropertyVector<T>& partitioned_pv) +{ + std::size_t const n_regular(p.regular_elements.size()); + for (std::size_t i = 0; i < n_regular; ++i) + { + const auto id = p.regular_elements[i]->getID(); + partitioned_pv[offset + i] = pv[id]; + } + + std::size_t const n_ghost(p.ghost_elements.size()); + for (std::size_t i = 0; i < n_ghost; ++i) + { + const auto id = p.ghost_elements[i]->getID(); + partitioned_pv[offset + n_regular + i] = pv[id]; + } + return n_regular + n_ghost; +} + +template <typename T> +bool copyPropertyVector(MeshLib::Properties const& original_properties, + MeshLib::Properties& partitioned_properties, + std::vector<Partition> const& partitions, + std::string const& name, + std::size_t const total_number_of_tuples) +{ + if (!original_properties.existsPropertyVector<T>(name)) + return false; + + auto const& pv = original_properties.getPropertyVector<T>(name); + auto partitioned_pv = partitioned_properties.createNewPropertyVector<T>( + name, pv->getMeshItemType(), pv->getNumberOfComponents()); + partitioned_pv->resize(total_number_of_tuples * + pv->getNumberOfComponents()); + + auto copy_property_vector_values = [&](Partition const& p, + std::size_t offset) { + if (pv->getMeshItemType() == MeshLib::MeshItemType::Node) + { + return copyNodePropertyVectorValues(p, offset, *pv, + *partitioned_pv); + } + if (pv->getMeshItemType() == MeshLib::MeshItemType::Cell) + { + return copyCellPropertyVectorValues(p, offset, *pv, + *partitioned_pv); + } + OGS_FATAL( + "Copying of property vector values for mesh item type %s is " + "not implemented.", + pv->getMeshItemType()); + }; + + std::size_t position_offset(0); + for (auto p : partitions) + { + position_offset += copy_property_vector_values(p, position_offset); + } + return true; +} + +/// Applies a function of the form f(type, name) -> bool for each of the +/// properties names. +/// The type argument is used to call f<decltype(type)>(name). +/// At least one of the functions must return the 'true' value, but at most one +/// is executed. +template <typename Function> +void applyToPropertyVectors(std::vector<std::string> const& property_names, + Function f) +{ for (auto const& name : property_names) { + // Open question, why is the 'unsigned long' case not compiling giving + // an error "expected '(' for function-style cast or type construction" + // with clang-7, and "error C4576: a parenthesized type followed by an + // initializer list is a non-standard explicit type conversion syntax" + // with MSVC-15. bool success = - copyNodePropertyVector<double>(name, total_number_of_tuples) || - copyNodePropertyVector<float>(name, total_number_of_tuples) || - copyNodePropertyVector<int>(name, total_number_of_tuples) || - copyNodePropertyVector<long>(name, total_number_of_tuples) || - copyNodePropertyVector<unsigned>(name, total_number_of_tuples) || - copyNodePropertyVector<unsigned long>(name, - total_number_of_tuples) || - copyNodePropertyVector<std::size_t>(name, total_number_of_tuples); + f(double{}, name) || f(float{}, name) || f(int{}, name) || + f(long{}, name) || f(unsigned{}, name) || + f(static_cast<unsigned long>(0), name) || f(std::size_t{}, name); if (!success) - WARN( - "processNodeProperties: Could not create partitioned " - "PropertyVector '%s'.", - name.c_str()); + { + OGS_FATAL("Could not apply function to PropertyVector '%s'.", + name.c_str()); + } } } -void NodeWiseMeshPartitioner::processCellProperties() +void NodeWiseMeshPartitioner::processProperties( + MeshLib::MeshItemType const mesh_item_type) { - std::size_t const total_number_of_tuples = std::accumulate( - std::begin(_partitions), std::end(_partitions), 0, - [](std::size_t const sum, Partition const& p) { - return sum + p.regular_elements.size() + p.ghost_elements.size(); - }); + std::size_t const total_number_of_tuples = + std::accumulate(std::begin(_partitions), std::end(_partitions), 0, + [&](std::size_t const sum, Partition const& p) { + return sum + p.numberOfMeshItems(mesh_item_type); + }); + + DBUG( + "total number of tuples define on mesh item type '%d' after " + "partitioning: %d ", + mesh_item_type, total_number_of_tuples); - DBUG("total number of cell-based tuples after partitioning: %d ", - total_number_of_tuples); // 1 create new PV // 2 resize the PV with total_number_of_tuples // 3 copy the values according to the partition info auto const& original_properties(_mesh->getProperties()); - auto const property_names = - original_properties.getPropertyVectorNames(MeshLib::MeshItemType::Cell); - for (auto const& name : property_names) - { - bool success = - copyCellPropertyVector<double>(name, total_number_of_tuples) || - copyCellPropertyVector<float>(name, total_number_of_tuples) || - copyCellPropertyVector<int>(name, total_number_of_tuples) || - copyCellPropertyVector<long>(name, total_number_of_tuples) || - copyCellPropertyVector<unsigned>(name, total_number_of_tuples) || - copyCellPropertyVector<unsigned long>(name, - total_number_of_tuples) || - copyCellPropertyVector<std::size_t>(name, total_number_of_tuples); - if (!success) - WARN( - "processCellProperties: Could not create partitioned " - "PropertyVector '%s'.", - name.c_str()); - } + + applyToPropertyVectors( + original_properties.getPropertyVectorNames(mesh_item_type), + [&](auto type, std::string const& name) { + return copyPropertyVector<decltype(type)>( + original_properties, _partitioned_properties, _partitions, name, + total_number_of_tuples); + }); } void NodeWiseMeshPartitioner::partitionByMETIS( @@ -284,8 +393,8 @@ void NodeWiseMeshPartitioner::partitionByMETIS( renumberNodeIndices(is_mixed_high_order_linear_elems); - processNodeProperties(); - processCellProperties(); + processProperties(MeshLib::MeshItemType::Node); + processProperties(MeshLib::MeshItemType::Cell); } void NodeWiseMeshPartitioner::renumberNodeIndices( @@ -306,7 +415,9 @@ void NodeWiseMeshPartitioner::renumberNodeIndices( } if (!is_mixed_high_order_linear_elems) + { return; + } // -- Nodes for high order elements. for (auto& partition : _partitions) @@ -323,92 +434,82 @@ void NodeWiseMeshPartitioner::renumberNodeIndices( } } -void NodeWiseMeshPartitioner::writeMETIS(const std::string& file_name) +template <typename T> +void writePropertyVectorValuesBinary(std::ostream& os, + MeshLib::PropertyVector<T> const& pv) { - std::ofstream os(file_name, std::ios::trunc); - if (!os.is_open()) - { - OGS_FATAL("Error: cannot open file %s.", - file_name.data()); - } - - if (!os.good()) - { - OGS_FATAL("Error: Cannot write in file %s.", file_name.data()); - } - - std::vector<MeshLib::Element*> const& elements = _mesh->getElements(); - os << elements.size() << " \n"; - for (const auto* elem : elements) - { - os << elem->getNodeIndex(0) + 1; - for (unsigned j = 1; j < elem->getNumberOfNodes(); j++) - { - os << " " << elem->getNodeIndex(j) + 1; - } - os << "\n"; - } + os.write(reinterpret_cast<const char*>(pv.data()), + pv.size() * sizeof(T)); } -NodeWiseMeshPartitioner::IntegerType -NodeWiseMeshPartitioner::getNumberOfIntegerVariablesOfElements( - const std::vector<const MeshLib::Element*>& elements) const +template <typename T> +bool writePropertyVectorBinary( + MeshLib::Properties const& partitioned_properties, std::string const& name, + std::ostream& out_val, std::ostream& out_meta) { - // Element ID, element type, and number of the nodes of - // an element of all elements in the current partition. - IntegerType nmb_element_idxs = 3 * elements.size(); - for (const auto* elem : elements) - { - nmb_element_idxs += elem->getNumberOfNodes(); - } - return nmb_element_idxs; + if (!partitioned_properties.existsPropertyVector<T>(name)) + return false; + + MeshLib::IO::PropertyVectorMetaData pvmd; + pvmd.property_name = name; + auto* pv = partitioned_properties.getPropertyVector<T>(name); + pvmd.fillPropertyVectorMetaDataTypeInfo<T>(); + pvmd.number_of_components = pv->getNumberOfComponents(); + pvmd.number_of_tuples = pv->getNumberOfTuples(); + writePropertyVectorValuesBinary(out_val, *pv); + MeshLib::IO::writePropertyVectorMetaDataBinary(out_meta, pvmd); + return true; } -void NodeWiseMeshPartitioner::writeNodePropertiesBinary( - const std::string& file_name_base) const +void writePropertiesBinary(const std::string& file_name_base, + MeshLib::Properties const& partitioned_properties, + std::vector<Partition> const& partitions, + MeshLib::MeshItemType const mesh_item_type) { - auto const& property_names(_partitioned_properties.getPropertyVectorNames( - MeshLib::MeshItemType::Node)); + auto const& property_names = + partitioned_properties.getPropertyVectorNames(mesh_item_type); if (property_names.empty()) + { return; + } - std::size_t const number_of_properties(property_names.size()); - - const std::string fname_cfg = file_name_base + - "_partitioned_node_properties_cfg" + - std::to_string(_npartitions) + ".bin"; - std::ofstream out(fname_cfg.c_str(), std::ios::binary | std::ios::out); + auto const file_name_infix = toString(mesh_item_type); - const std::string fname_val = file_name_base + - "_partitioned_node_properties_val" + - std::to_string(_npartitions) + ".bin"; - std::ofstream out_val(fname_val.c_str(), std::ios::binary | std::ios::out); + auto const file_name_cfg = file_name_base + "_partitioned_" + + file_name_infix + "_properties_cfg" + + std::to_string(partitions.size()) + ".bin"; + std::ofstream out(file_name_cfg, std::ios::binary); + if (!out) + { + OGS_FATAL("Could not open file '%s' for output.", + file_name_cfg.c_str()); + } - out.write(reinterpret_cast<const char*>(&number_of_properties), - sizeof(number_of_properties)); - for (auto const& name : property_names) + auto const file_name_val = file_name_base + "_partitioned_" + + file_name_infix + "_properties_val" + + std::to_string(partitions.size()) + ".bin"; + std::ofstream out_val(file_name_val, std::ios::binary); + if (!out_val) { - bool success = - writePropertyVectorBinary<double>(name, out_val, out) || - writePropertyVectorBinary<float>(name, out_val, out) || - writePropertyVectorBinary<int>(name, out_val, out) || - writePropertyVectorBinary<long>(name, out_val, out) || - writePropertyVectorBinary<unsigned>(name, out_val, out) || - writePropertyVectorBinary<unsigned long>(name, out_val, out) || - writePropertyVectorBinary<std::size_t>(name, out_val, out); - if (!success) - OGS_FATAL( - "writeNodePropertiesBinary: Could not write PropertyVector " - "'%s'.", - name.c_str()); + OGS_FATAL("Could not open file '%s' for output.", + file_name_val.c_str()); } - out_val.close(); + + std::size_t const number_of_properties(property_names.size()); + BaseLib::writeValueBinary(out, number_of_properties); + + applyToPropertyVectors(property_names, + [&](auto type, std::string const& name) { + return writePropertyVectorBinary<decltype(type)>( + partitioned_properties, name, out_val, out); + }); + unsigned long offset = 0; - for (const auto& partition : _partitions) + for (const auto& partition : partitions) { - MeshLib::IO::PropertyVectorPartitionMetaData pvpmd; - pvpmd.offset = offset; - pvpmd.number_of_tuples = partition.nodes.size(); + MeshLib::IO::PropertyVectorPartitionMetaData pvpmd{ + offset, static_cast<unsigned long>( + partition.numberOfMeshItems(mesh_item_type))}; DBUG( "Write meta data for node-based PropertyVector: global offset %d, " "number of tuples %d", @@ -416,168 +517,194 @@ void NodeWiseMeshPartitioner::writeNodePropertiesBinary( MeshLib::IO::writePropertyVectorPartitionMetaData(out, pvpmd); offset += pvpmd.number_of_tuples; } - out.close(); } -void NodeWiseMeshPartitioner::writeCellPropertiesBinary( - const std::string& file_name_base) const +struct ConfigOffsets { - auto const& property_names(_partitioned_properties.getPropertyVectorNames( - MeshLib::MeshItemType::Cell)); - if (property_names.empty()) - return; + long node_rank_offset; + long element_rank_offset; + long ghost_element_rank_offset; - std::size_t const number_of_properties(property_names.size()); + std::ostream& writeConfigBinary(std::ostream& os) const; +}; - const std::string fname_cfg = file_name_base + - "_partitioned_cell_properties_cfg" + - std::to_string(_npartitions) + ".bin"; - std::ofstream out(fname_cfg.c_str(), std::ios::binary | std::ios::out); +std::ostream& ConfigOffsets::writeConfigBinary(std::ostream& os) const +{ + os.write(reinterpret_cast<const char*>(this), sizeof(ConfigOffsets)); - const std::string fname_val = file_name_base + - "_partitioned_cell_properties_val" + - std::to_string(_npartitions) + ".bin"; - std::ofstream out_val(fname_val.c_str(), std::ios::binary | std::ios::out); + static long reserved = 0; // Value reserved in the binary format, not used + // in the partitioning process. + return os.write(reinterpret_cast<const char*>(&reserved), sizeof(long)); +} - out.write(reinterpret_cast<const char*>(&number_of_properties), - sizeof(number_of_properties)); - for (auto const& name : property_names) - { - bool success = - writePropertyVectorBinary<double>(name, out_val, out) || - writePropertyVectorBinary<float>(name, out_val, out) || - writePropertyVectorBinary<int>(name, out_val, out) || - writePropertyVectorBinary<long>(name, out_val, out) || - writePropertyVectorBinary<unsigned>(name, out_val, out) || - writePropertyVectorBinary<unsigned long>(name, out_val, out) || - writePropertyVectorBinary<std::size_t>(name, out_val, out); - if (!success) - OGS_FATAL( - "writeCellPropertiesBinary: Could not write PropertyVector " - "'%s'.", - name.c_str()); - } - out_val.close(); - unsigned long offset = 0; - for (const auto& partition : _partitions) - { - MeshLib::IO::PropertyVectorPartitionMetaData pvpmd; - pvpmd.offset = offset; - pvpmd.number_of_tuples = - partition.regular_elements.size() + partition.ghost_elements.size(); - DBUG( - "Write meta data for cell-based PropertyVector: global offset %d, " - "number of tuples %d", - pvpmd.offset, pvpmd.number_of_tuples); - MeshLib::IO::writePropertyVectorPartitionMetaData(out, pvpmd); - offset += pvpmd.number_of_tuples; - } - out.close(); +struct PartitionOffsets +{ + long node; + long regular_elements; + long ghost_elements; +}; + +PartitionOffsets +computePartitionElementOffsets(Partition const& partition) +{ + return {static_cast<long>(partition.nodes.size()), + static_cast<long>(partition.regular_elements.size() + + getNumberOfIntegerVariablesOfElements( + partition.regular_elements)), + static_cast<long>(partition.ghost_elements.size() + + getNumberOfIntegerVariablesOfElements( + partition.ghost_elements))}; } -std::tuple<std::vector<NodeWiseMeshPartitioner::IntegerType>, - std::vector<NodeWiseMeshPartitioner::IntegerType>> -NodeWiseMeshPartitioner::writeConfigDataBinary( - const std::string& file_name_base) +ConfigOffsets incrementConfigOffsets(ConfigOffsets const& oldConfig, + PartitionOffsets const& offsets) { - const std::string fname = file_name_base + "_partitioned_msh_cfg" - + std::to_string(_npartitions) + ".bin"; - FILE* of_bin_cfg = fopen(fname.c_str(), "wb"); - - const IntegerType num_config_data = 14; - IntegerType config_data[num_config_data]; - // node rank offset - config_data[10] = 0; - // element rank offset - config_data[11] = 0; - // ghost element rank offset - config_data[12] = 0; - // Reserved - config_data[13] = 0; - std::vector<IntegerType> num_elem_integers(_partitions.size()); - std::vector<IntegerType> num_g_elem_integers(_partitions.size()); - std::size_t loop_id = 0; - for (const auto& partition : _partitions) - { - config_data[0] = partition.nodes.size(); - config_data[1] = partition.number_of_base_nodes; - config_data[2] = partition.regular_elements.size(); - config_data[3] = partition.ghost_elements.size(); - config_data[4] = partition.number_of_non_ghost_base_nodes; - config_data[5] = partition.number_of_non_ghost_nodes; - config_data[6] = _mesh->getNumberOfBaseNodes(); - config_data[7] = _mesh->getNumberOfNodes(); - config_data[8] = - getNumberOfIntegerVariablesOfElements(partition.regular_elements); - config_data[9] = - getNumberOfIntegerVariablesOfElements(partition.ghost_elements); - - fwrite(config_data, 1, num_config_data * sizeof(IntegerType), of_bin_cfg); - - config_data[10] += config_data[0] * sizeof(NodeStruct); - - // Update offsets - num_elem_integers[loop_id] = - partition.regular_elements.size() + config_data[8]; + return { + static_cast<long>(oldConfig.node_rank_offset + + offsets.node * sizeof(NodeStruct)), // Offset the ending entry of the element integer variales of // the non-ghost elements of this partition in the vector of elem_info. - config_data[11] += num_elem_integers[loop_id] * sizeof(IntegerType); + static_cast<long>(oldConfig.element_rank_offset + + offsets.regular_elements * sizeof(long)), + // Offset the ending entry of the element integer variales of // the ghost elements of this partition in the vector of elem_info. - num_g_elem_integers[loop_id] = - partition.ghost_elements.size() + config_data[9]; - config_data[12] += num_g_elem_integers[loop_id] * sizeof(IntegerType); + static_cast<long>(oldConfig.ghost_element_rank_offset + + offsets.ghost_elements * sizeof(long))}; +} - loop_id++; +/// Write the configuration data of the partition data in binary files. +/// \return a pair of vectors for: +/// 1. The number of all non-ghost element integer variables for each +/// partition. +/// 2. The number of all ghost element integer variables for each partition. +std::tuple<std::vector<long>, std::vector<long>> writeConfigDataBinary( + const std::string& file_name_base, + std::vector<Partition> const& partitions) +{ + auto const file_name_cfg = file_name_base + "_partitioned_msh_cfg" + + std::to_string(partitions.size()) + ".bin"; + std::ofstream of_bin_cfg(file_name_cfg, std::ios::binary); + if (!of_bin_cfg) + { + OGS_FATAL("Could not open file '%s' for output.", + file_name_cfg.c_str()); } - fclose(of_bin_cfg); + std::vector<long> num_elem_integers; + num_elem_integers.reserve(partitions.size()); + std::vector<long> num_g_elem_integers; + num_g_elem_integers.reserve(partitions.size()); + + ConfigOffsets config_offsets = {0, 0, 0}; // 0 for first partition. + for (const auto& partition : partitions) + { + partition.writeConfigBinary(of_bin_cfg); + + config_offsets.writeConfigBinary(of_bin_cfg); + auto const& new_offsets = computePartitionElementOffsets(partition); + config_offsets = incrementConfigOffsets(config_offsets, new_offsets); - return std::make_tuple(num_elem_integers, num_g_elem_integers); + num_elem_integers.push_back(new_offsets.regular_elements); + num_g_elem_integers.push_back(new_offsets.ghost_elements); + } + + return std::make_tuple(num_elem_integers, num_g_elem_integers); } -void NodeWiseMeshPartitioner::writeElementsBinary - (const std::string& file_name_base, - const std::vector<IntegerType>& num_elem_integers, - const std::vector<IntegerType>& num_g_elem_integers) +/// Get integer variables, which are used to define an element +/// +/// \param elem Element +/// \param local_node_ids Local node indices of a partition +/// \param elem_info A vector holds all integer variables of +/// element definitions +/// \param counter Recorder of the number of integer variables. +void getElementIntegerVariables( + const MeshLib::Element& elem, + const std::unordered_map<std::size_t, long>& local_node_ids, + std::vector<long>& elem_info, + long& counter) { - const std::string npartitions_str = std::to_string(_npartitions); - std::string fname = file_name_base + "_partitioned_msh_ele" - + npartitions_str + ".bin"; - FILE* of_bin_ele = fopen(fname.c_str(), "wb"); - fname = + unsigned mat_id = 0; // TODO: Material ID to be set from the mesh data + const long nn = elem.getNumberOfNodes(); + elem_info[counter++] = mat_id; + elem_info[counter++] = static_cast<long>(elem.getCellType()); + elem_info[counter++] = nn; + + for (long i = 0; i < nn; i++) + { + elem_info[counter++] = local_node_ids.at(elem.getNodeIndex(i)); + } +} + +/// Generates a mapping of given node ids to a new local (renumbered) node ids. +std::unordered_map<std::size_t, long> enumerateLocalNodeIds( + std::vector<MeshLib::Node*> const& nodes) +{ + std::unordered_map<std::size_t, long> local_ids; + local_ids.reserve(nodes.size()); + + long local_node_id = 0; + for (const auto* node : nodes) + { + local_ids[node->getID()] = local_node_id++; + } + return local_ids; +} + +/// Write the element integer variables of all partitions into binary files. +/// \param file_name_base The prefix of the file name. +/// \param partitions Partitions vector. +/// \param num_elem_integers The numbers of all non-ghost element +/// integer variables of each partitions. +/// \param num_g_elem_integers The numbers of all ghost element +void writeElementsBinary(std::string const& file_name_base, + std::vector<Partition> const& partitions, + std::vector<long> const& num_elem_integers, + std::vector<long> const& num_g_elem_integers) +{ + const std::string npartitions_str = std::to_string(partitions.size()); + + auto const file_name_ele = + file_name_base + "_partitioned_msh_ele" + npartitions_str + ".bin"; + std::ofstream element_info_os(file_name_ele, std::ios::binary); + if (!element_info_os) + { + OGS_FATAL("Could not open file '%s' for output.", + file_name_ele.c_str()); + } + + auto const file_name_ele_g = file_name_base + "_partitioned_msh_ele_g" + npartitions_str + ".bin"; - FILE* of_bin_ele_g = fopen(fname.c_str(), "wb"); - for (std::size_t i = 0; i < _partitions.size(); i++) + std::ofstream ghost_element_info_os(file_name_ele_g, std::ios::binary); + if (!ghost_element_info_os) { - const auto& partition = _partitions[i]; + OGS_FATAL("Could not open file '%s' for output.", + file_name_ele_g.c_str()); + } - // Set the local node indices of the current partition. - IntegerType node_local_id_offset = 0; - std::vector<IntegerType> nodes_local_ids(_mesh->getNumberOfNodes(), -1); - for (const auto* node : partition.nodes) - { - nodes_local_ids[node->getID()] = node_local_id_offset; - node_local_id_offset++; - } + for (std::size_t i = 0; i < partitions.size(); i++) + { + const auto& partition = partitions[i]; + auto const local_node_ids = enumerateLocalNodeIds(partition.nodes); - // A vector contians all element integer variales of + // A vector contians all element integer variables of // the non-ghost elements of this partition - std::vector<IntegerType> ele_info(num_elem_integers[i]); + std::vector<long> ele_info(num_elem_integers[i]); // Non-ghost elements. - IntegerType counter = partition.regular_elements.size(); + long counter = partition.regular_elements.size(); for (std::size_t j = 0; j < partition.regular_elements.size(); j++) { const auto* elem = partition.regular_elements[j]; ele_info[j] = counter; - getElementIntegerVariables(*elem, nodes_local_ids, ele_info, + getElementIntegerVariables(*elem, local_node_ids, ele_info, counter); } // Write vector data of non-ghost elements - fwrite(ele_info.data(), 1, (num_elem_integers[i]) * sizeof(IntegerType), - of_bin_ele); + element_info_os.write(reinterpret_cast<const char*>(ele_info.data()), + ele_info.size() * sizeof(long)); // Ghost elements ele_info.resize(num_g_elem_integers[i]); @@ -588,67 +715,63 @@ void NodeWiseMeshPartitioner::writeElementsBinary { const auto* elem = partition.ghost_elements[j]; ele_info[j] = counter; - getElementIntegerVariables(*elem, nodes_local_ids, ele_info, + getElementIntegerVariables(*elem, local_node_ids, ele_info, counter); } // Write vector data of ghost elements - fwrite(ele_info.data(), 1, (num_g_elem_integers[i]) * sizeof(IntegerType), - of_bin_ele_g); + ghost_element_info_os.write( + reinterpret_cast<const char*>(ele_info.data()), + ele_info.size() * sizeof(long)); } - - fclose(of_bin_ele); - fclose(of_bin_ele_g); } -void NodeWiseMeshPartitioner::writeNodesBinary(const std::string& file_name_base) +/// Write the nodes of all partitions into a binary file. +/// \param file_name_base The prefix of the file name. +/// \param partitions the list of partitions +/// \param global_node_ids global numbering of nodes +void writeNodesBinary(const std::string& file_name_base, + std::vector<Partition> const& partitions, + std::vector<std::size_t> const& global_node_ids) { - const std::string fname = file_name_base + "_partitioned_msh_nod" - + std::to_string(_npartitions) + ".bin"; - FILE* of_bin_nod = fopen(fname.c_str(), "wb"); - - for (const auto& partition : _partitions) + auto const file_name = file_name_base + "_partitioned_msh_nod" + + std::to_string(partitions.size()) + ".bin"; + std::ofstream os(file_name, std::ios::binary); + if (!os) { - std::vector<NodeStruct> nodes_buffer; - nodes_buffer.reserve(partition.nodes.size()); + OGS_FATAL("Could not open file '%s' for output.", file_name.c_str()); + } - for (const auto* node : partition.nodes) - { - double const* coords = node->getCoords(); - NodeStruct node_struct; - node_struct.id = _nodes_global_ids[node->getID()]; - node_struct.x = coords[0]; - node_struct.y = coords[1]; - node_struct.z = coords[2]; - nodes_buffer.emplace_back(node_struct); - } - fwrite(nodes_buffer.data(), sizeof(NodeStruct), partition.nodes.size(), - of_bin_nod); + for (const auto& partition : partitions) + { + partition.writeNodesBinary(os, global_node_ids); } - fclose(of_bin_nod); } void NodeWiseMeshPartitioner::writeBinary(const std::string& file_name_base) { - writeNodePropertiesBinary(file_name_base); - writeCellPropertiesBinary(file_name_base); - const auto elem_integers = writeConfigDataBinary(file_name_base); - - const std::vector<IntegerType>& num_elem_integers - = std::get<0>(elem_integers); - const std::vector<IntegerType>& num_g_elem_integers - = std::get<1>(elem_integers); - writeElementsBinary(file_name_base, num_elem_integers, + writePropertiesBinary(file_name_base, _partitioned_properties, _partitions, + MeshLib::MeshItemType::Node); + writePropertiesBinary(file_name_base, _partitioned_properties, _partitions, + MeshLib::MeshItemType::Cell); + + const auto elem_integers = + writeConfigDataBinary(file_name_base, _partitions); + + const std::vector<IntegerType>& num_elem_integers = + std::get<0>(elem_integers); + const std::vector<IntegerType>& num_g_elem_integers = + std::get<1>(elem_integers); + writeElementsBinary(file_name_base, _partitions, num_elem_integers, num_g_elem_integers); - writeNodesBinary(file_name_base); + writeNodesBinary(file_name_base, _partitions, _nodes_global_ids); } -void NodeWiseMeshPartitioner::writeConfigDataASCII - (const std::string& file_name_base) +void NodeWiseMeshPartitioner::writeConfigDataASCII( + const std::string& file_name_base) { - const std::string fname = - file_name_base + "_partitioned_cfg" - + std::to_string(_npartitions) + ".msh"; + const std::string fname = file_name_base + "_partitioned_cfg" + + std::to_string(_npartitions) + ".msh"; std::fstream os_subd_head(fname, std::ios::out | std::ios::trunc); const std::string mesh_info = "Subdomain mesh (" @@ -672,18 +795,21 @@ void NodeWiseMeshPartitioner::writeConfigDataASCII os_subd_head << " " << partition.number_of_non_ghost_nodes; os_subd_head << " " << _mesh->getNumberOfBaseNodes(); os_subd_head << " " << _mesh->getNumberOfNodes(); - os_subd_head << " " << getNumberOfIntegerVariablesOfElements( - partition.regular_elements); - os_subd_head << " " << getNumberOfIntegerVariablesOfElements( - partition.ghost_elements) + os_subd_head << " " + << getNumberOfIntegerVariablesOfElements( + partition.regular_elements); + os_subd_head << " " + << getNumberOfIntegerVariablesOfElements( + partition.ghost_elements) << " 0\n"; } } -void NodeWiseMeshPartitioner::writeElementsASCII(const std::string& file_name_base) +void NodeWiseMeshPartitioner::writeElementsASCII( + const std::string& file_name_base) { - const std::string fname = file_name_base + "_partitioned_elems_" - + std::to_string(_npartitions) + ".msh"; + const std::string fname = file_name_base + "_partitioned_elems_" + + std::to_string(_npartitions) + ".msh"; std::fstream os_subd(fname, std::ios::out | std::ios::trunc); for (const auto& partition : _partitions) { @@ -710,8 +836,8 @@ void NodeWiseMeshPartitioner::writeElementsASCII(const std::string& file_name_ba void NodeWiseMeshPartitioner::writeNodesASCII(const std::string& file_name_base) { - const std::string fname = file_name_base + "_partitioned_nodes_" - + std::to_string(_npartitions) + ".msh"; + const std::string fname = file_name_base + "_partitioned_nodes_" + + std::to_string(_npartitions) + ".msh"; std::fstream os_subd_node(fname, std::ios::out | std::ios::trunc); os_subd_node.precision(std::numeric_limits<double>::digits10); os_subd_node.setf(std::ios::scientific); @@ -735,32 +861,14 @@ void NodeWiseMeshPartitioner::writeASCII(const std::string& file_name_base) writeNodesASCII(file_name_base); } -void NodeWiseMeshPartitioner::getElementIntegerVariables( - const MeshLib::Element& elem, - const std::vector<IntegerType>& local_node_ids, - std::vector<IntegerType>& elem_info, - IntegerType& counter) -{ - unsigned mat_id = 0; // TODO: Material ID to be set from the mesh data - const IntegerType nn = elem.getNumberOfNodes(); - elem_info[counter++] = mat_id; - elem_info[counter++] = static_cast<unsigned>(elem.getCellType()); - elem_info[counter++] = nn; - - for (IntegerType i = 0; i < nn; i++) - { - elem_info[counter++] = local_node_ids[elem.getNodeIndex(i)]; - } -} - void NodeWiseMeshPartitioner::writeLocalElementNodeIndices( std::ostream& os, const MeshLib::Element& elem, const std::vector<IntegerType>& local_node_ids) { unsigned mat_id = 0; // TODO: Material ID to be set from the mesh data - os << mat_id << " " << static_cast<unsigned>(elem.getCellType()) - << " " << elem.getNumberOfNodes() << " "; + os << mat_id << " " << static_cast<unsigned>(elem.getCellType()) << " " + << elem.getNumberOfNodes() << " "; for (unsigned i = 0; i < elem.getNumberOfNodes(); i++) { os << " " << local_node_ids[elem.getNodeIndex(i)]; @@ -768,4 +876,4 @@ void NodeWiseMeshPartitioner::writeLocalElementNodeIndices( os << "\n"; } -} // namespace MeshLib +} // namespace ApplicationUtils diff --git a/Applications/Utils/ModelPreparation/PartitionMesh/NodeWiseMeshPartitioner.h b/Applications/Utils/ModelPreparation/PartitionMesh/NodeWiseMeshPartitioner.h index 2ec13d5fbef335f3491eb302dbc1d9151845e621..c7439c4e93086c3268dfdb20946ffc7fe76e3b1f 100644 --- a/Applications/Utils/ModelPreparation/PartitionMesh/NodeWiseMeshPartitioner.h +++ b/Applications/Utils/ModelPreparation/PartitionMesh/NodeWiseMeshPartitioner.h @@ -15,15 +15,14 @@ #pragma once #include <memory> +#include <string> #include <tuple> #include <vector> -#include <string> -#include <fstream> -#include "MeshLib/Mesh.h" -#include "MeshLib/Node.h" #include "MeshLib/Elements/Element.h" #include "MeshLib/IO/MPI_IO/PropertyVectorMetaData.h" +#include "MeshLib/Mesh.h" +#include "MeshLib/Node.h" namespace ApplicationUtils { @@ -34,9 +33,19 @@ struct Partition std::size_t number_of_non_ghost_base_nodes; std::size_t number_of_non_ghost_nodes; std::size_t number_of_base_nodes; + std::size_t number_of_mesh_base_nodes; + std::size_t number_of_mesh_all_nodes; /// Non ghost elements std::vector<const MeshLib::Element*> regular_elements; std::vector<const MeshLib::Element*> ghost_elements; + + std::size_t numberOfMeshItems(MeshLib::MeshItemType const item_type) const; + + std::ostream& writeNodesBinary( + std::ostream& os, + std::vector<std::size_t> const& global_node_ids) const; + + std::ostream& writeConfigBinary(std::ostream& os) const; }; /// Mesh partitioner. @@ -67,14 +76,6 @@ public: /// interpolation void partitionByMETIS(const bool is_mixed_high_order_linear_elems); - /// Read metis data - /// \param file_name_base The prefix of the file name. - void readMetisData(const std::string& file_name_base); - - /// Write mesh to METIS input file - /// \param file_name File name with an extension of mesh. - void writeMETIS(const std::string& file_name); - /// Write the partitions into ASCII files /// \param file_name_base The prefix of the file name. void writeASCII(const std::string& file_name_base); @@ -83,6 +84,14 @@ public: /// \param file_name_base The prefix of the file name. void writeBinary(const std::string& file_name_base); + void resetPartitionIdsForNodes( + std::vector<std::size_t>&& node_partition_ids) + { + _nodes_partition_ids = std::move(node_partition_ids); + } + + MeshLib::Mesh const& mesh() const { return *_mesh; } + private: /// Number of partitions. IntegerType _npartitions; @@ -108,32 +117,6 @@ private: /// interpolation void renumberNodeIndices(const bool is_mixed_high_order_linear_elems); - /*! - Calculate the total number of integer variables of an element - vector. Each element has three integer variables for element ID, - element type, number of nodes of the element. Therefore - the total number of the integers in an element vector is - 3 * vector size + sum (number of nodes of each element) - */ - IntegerType getNumberOfIntegerVariablesOfElements( - const std::vector<const MeshLib::Element*>& elements) const; - - /*! - \brief Get integer variables, which are used to define an element - \param elem Element - \param local_node_ids Local node indices of a partition - \param elem_info A vector holds all integer variables of - element definitions - \param counter Recorder of the number of integer variables. - */ - void getElementIntegerVariables(const MeshLib::Element& elem, - const std::vector<IntegerType>& local_node_ids, - std::vector<IntegerType>& elem_info, - IntegerType& counter); - - void writeNodePropertiesBinary(std::string const& file_name_base) const; - void writeCellPropertiesBinary(std::string const& file_name_base) const; - /// 1 copy pointers to nodes belonging to the partition part_id /// 2 collect non-linear element nodes belonging to the partition part_id in /// extra_nodes @@ -156,146 +139,16 @@ private: const bool is_mixed_high_order_linear_elems, std::vector<MeshLib::Node*>& extra_nodes); - void splitOfHigherOrderNode(std::vector<MeshLib::Node*> const& nodes, - bool const is_mixed_high_order_linear_elems, - unsigned const node_id, - std::vector<MeshLib::Node*>& base_nodes, - std::vector<MeshLib::Node*>& extra_nodes); + void splitOffHigherOrderNode(std::vector<MeshLib::Node*> const& nodes, + bool const is_mixed_high_order_linear_elems, + unsigned const node_id, + std::vector<MeshLib::Node*>& base_nodes, + std::vector<MeshLib::Node*>& extra_nodes); void processPartition(std::size_t const part_id, const bool is_mixed_high_order_linear_elems); - void processNodeProperties(); - void processCellProperties(); - - template <typename T> - bool copyNodePropertyVector(std::string const& name, - std::size_t const total_number_of_tuples) - { - auto const& original_properties(_mesh->getProperties()); - if (!original_properties.existsPropertyVector<T>(name)) - return false; - - auto const& pv(original_properties.getPropertyVector<T>(name)); - auto partitioned_pv = - _partitioned_properties.createNewPropertyVector<T>( - name, pv->getMeshItemType(), pv->getNumberOfComponents()); - partitioned_pv->resize(total_number_of_tuples * - pv->getNumberOfComponents()); - std::size_t position_offset(0); - for (auto p : _partitions) - { - for (std::size_t i = 0; i < p.nodes.size(); ++i) - { - const auto global_id = p.nodes[i]->getID(); - (*partitioned_pv)[position_offset + i] = (*pv)[global_id]; - } - position_offset += p.nodes.size(); - } - return true; - } - - template <typename T> - bool copyCellPropertyVector(std::string const& name, - std::size_t const total_number_of_tuples) - { - auto const& original_properties(_mesh->getProperties()); - if (!original_properties.existsPropertyVector<T>(name)) - return false; - - auto const& pv(original_properties.getPropertyVector<T>(name)); - auto partitioned_pv = - _partitioned_properties.createNewPropertyVector<T>( - name, pv->getMeshItemType(), pv->getNumberOfComponents()); - partitioned_pv->resize(total_number_of_tuples * - pv->getNumberOfComponents()); - std::size_t position_offset(0); - for (auto const& p : _partitions) - { - std::size_t const n_regular(p.regular_elements.size()); - for (std::size_t i = 0; i < n_regular; ++i) - { - const auto id = p.regular_elements[i]->getID(); - (*partitioned_pv)[position_offset + i] = (*pv)[id]; - } - position_offset += n_regular; - std::size_t const n_ghost(p.ghost_elements.size()); - for (std::size_t i = 0; i < n_ghost; ++i) - { - const auto id = p.ghost_elements[i]->getID(); - (*partitioned_pv)[position_offset + i] = (*pv)[id]; - } - position_offset += n_ghost; - } - return true; - } - - template <typename T> - void writePropertyVectorValuesBinary( - std::ostream& os, MeshLib::PropertyVector<T> const& pv) const - { - std::size_t number_of_components(pv.getNumberOfComponents()); - std::size_t number_of_tuples(pv.getNumberOfTuples()); - std::vector<T> property_vector_buffer; - property_vector_buffer.resize(number_of_tuples * number_of_components); - for (std::size_t i = 0; i < pv.getNumberOfTuples(); ++i) - { - for (std::size_t c(0); c < number_of_components; ++c) - property_vector_buffer[i * number_of_components + c] = - pv.getComponent(i, c); - } - os.write(reinterpret_cast<char*>(property_vector_buffer.data()), - number_of_components * number_of_tuples * sizeof(T)); - } - - template <typename T> - bool writePropertyVectorBinary(std::string const& name, - std::ostream& out_val, - std::ostream& out_meta) const - { - if (!_partitioned_properties.existsPropertyVector<T>(name)) - return false; - - MeshLib::IO::PropertyVectorMetaData pvmd; - pvmd.property_name = name; - auto* pv = _partitioned_properties.getPropertyVector<T>(name); - pvmd.fillPropertyVectorMetaDataTypeInfo<T>(); - pvmd.number_of_components = pv->getNumberOfComponents(); - pvmd.number_of_tuples = pv->getNumberOfTuples(); - writePropertyVectorValuesBinary(out_val, *pv); - MeshLib::IO::writePropertyVectorMetaDataBinary(out_meta, pvmd); - return true; - } - - /*! - \brief Write the configuration data of the partition data in - binary files. - \param file_name_base The prefix of the file name. - \return element 1: The numbers of all non-ghost element integer - variables of each partitions. - element 2: The numbers of all ghost element integer - variables of each partitions. - */ - std::tuple<std::vector<IntegerType>, std::vector<IntegerType>> - writeConfigDataBinary(const std::string& file_name_base); - - /*! - \brief Write the element integer variables of all partitions - into binary files. - \param file_name_base The prefix of the file name. - \param num_elem_integers The numbers of all non-ghost element - integer variables of each partitions. - \param num_g_elem_integers The numbers of all ghost element - integer variables of each partitions. - */ - void writeElementsBinary(const std::string& file_name_base, - const std::vector<IntegerType>& num_elem_integers, - const std::vector<IntegerType>& num_g_elem_integers); - - /// Write the nodes of all partitions into a binary file. - /// \param file_name_base The prefix of the file name. - void writeNodesBinary(const std::string& file_name_base); - + void processProperties(MeshLib::MeshItemType const mesh_item_type); /// Write the configuration data of the partition data in ASCII files. /// \param file_name_base The prefix of the file name. @@ -322,4 +175,4 @@ private: const std::vector<IntegerType>& local_node_ids); }; -} // namespace MeshLib +} // namespace ApplicationUtils diff --git a/Applications/Utils/ModelPreparation/PartitionMesh/PartitionMesh.cpp b/Applications/Utils/ModelPreparation/PartitionMesh/PartitionMesh.cpp index 4bf3d86e181da8fae86e3496d802f8637541c50b..f6224fccfa04225322b6adf2dc43a684114385b6 100644 --- a/Applications/Utils/ModelPreparation/PartitionMesh/PartitionMesh.cpp +++ b/Applications/Utils/ModelPreparation/PartitionMesh/PartitionMesh.cpp @@ -21,13 +21,16 @@ #endif #include "Applications/ApplicationsLib/LogogSetup.h" -#include "BaseLib/FileTools.h" #include "BaseLib/CPUTime.h" +#include "BaseLib/FileTools.h" #include "BaseLib/RunTime.h" #include "MeshLib/IO/readMeshFromFile.h" #include "NodeWiseMeshPartitioner.h" +#include "Metis.h" + +using namespace ApplicationUtils; int main(int argc, char* argv[]) { @@ -50,6 +53,12 @@ int main(int argc, char* argv[]) "the name of the file containing the input mesh", true, "", "file name of input mesh"); cmd.add(mesh_input); + + TCLAP::ValueArg<std::string> output_directory_arg( + "o", "output", "directory name for the output files", false, "", + "directory"); + cmd.add(output_directory_arg); + TCLAP::ValueArg<int> nparts("n", "np", "the number of partitions", false, 2, "integer"); cmd.add(nparts); @@ -78,84 +87,84 @@ int main(int argc, char* argv[]) BaseLib::CPUTime CPU_timer; CPU_timer.start(); - const std::string ifile_name = mesh_input.getValue(); - const std::string file_name_base = BaseLib::dropFileExtension(ifile_name); + const std::string input_file_name_wo_extension = + BaseLib::dropFileExtension(mesh_input.getValue()); std::unique_ptr<MeshLib::Mesh> mesh_ptr( - MeshLib::IO::readMeshFromFile(file_name_base + ".vtu")); + MeshLib::IO::readMeshFromFile(input_file_name_wo_extension + ".vtu")); INFO("Mesh read: %d nodes, %d elements.", mesh_ptr->getNumberOfNodes(), mesh_ptr->getNumberOfElements()); - std::size_t const number_of_nodes(mesh_ptr->getNumberOfNodes()); - std::size_t const number_of_elements(mesh_ptr->getNumberOfElements()); + if (ogs2metis_flag.getValue()) + { + INFO("Write the mesh into METIS input file."); + ApplicationUtils::writeMETIS(mesh_ptr->getElements(), + input_file_name_wo_extension + ".mesh"); + INFO("Total runtime: %g s.", run_timer.elapsed()); + INFO("Total CPU time: %g s.", CPU_timer.elapsed()); + + return EXIT_SUCCESS; + } ApplicationUtils::NodeWiseMeshPartitioner mesh_partitioner( nparts.getValue(), std::move(mesh_ptr)); - if (ogs2metis_flag.getValue()) + std::string const output_file_name_wo_extension = BaseLib::joinPaths( + output_directory_arg.getValue(), + BaseLib::extractBaseNameWithoutExtension(mesh_input.getValue())); + const int num_partitions = nparts.getValue(); + + if (num_partitions < 1) { - INFO("Write the mesh into METIS input file."); - mesh_partitioner.writeMETIS(BaseLib::dropFileExtension(ifile_name) + - ".mesh"); + OGS_FATAL("Number of partitions must be positive."); } - else + + if (num_partitions == 1) { - const int num_partitions = nparts.getValue(); + OGS_FATAL( + "Partitioning the mesh into one domain is unnecessary because OGS " + "reads vtu mesh data directly when called with 'mpirun bin/ogs " + "-np=1'."); + } - // Execute mpmetis via system(...) - if (num_partitions > 1 && exe_metis_flag.getValue()) - { - INFO("METIS is running ..."); - const std::string exe_name = argv[0]; - const std::string exe_path = BaseLib::extractPath(exe_name); - INFO("Path to mpmetis is: \n\t%s", exe_path.c_str()); - - const std::string mpmetis_com = - exe_path + "/mpmetis " + " -gtype=nodal " + file_name_base + - ".mesh " + std::to_string(nparts.getValue()); - - const int status = system(mpmetis_com.c_str()); - if (status != 0) - { - INFO("Failed in system calling."); - INFO("Return value of system call %d ", status); - return EXIT_FAILURE; - } - } - else if (num_partitions == 1 && exe_metis_flag.getValue()) + // Execute mpmetis via system(...) + if (exe_metis_flag.getValue()) + { + INFO("METIS is running ..."); + const std::string exe_name = argv[0]; + const std::string exe_path = BaseLib::extractPath(exe_name); + INFO("Path to mpmetis is: \n\t%s", exe_path.c_str()); + + const std::string mpmetis_com = + BaseLib::joinPaths(exe_path, "mpmetis") + " -gtype=nodal " + "'" + + input_file_name_wo_extension + ".mesh" + "' " + + std::to_string(nparts.getValue()); + + const int status = system(mpmetis_com.c_str()); + if (status != 0) { - // The mpmetis tool can not be used for 'partitioning' in only one - // domain. For this reason the according files are written for just - // one domain in the metis output format in the following. - auto writePartitionFile = [&file_name_base]( - std::string const& file_name_extension, std::size_t number) { - std::string const name(file_name_base + file_name_extension); - std::ofstream os(name); - if (!os) - OGS_FATAL("Couldn't open file '%s' for writing.", - name.c_str()); - for (std::size_t n(0); n < number; ++n) - os << "0\n"; - }; - - writePartitionFile(".mesh.npart.1", number_of_nodes); - writePartitionFile(".mesh.epart.1", number_of_elements); + INFO("Failed in system calling."); + INFO("Return value of system call %d ", status); + return EXIT_FAILURE; } + } + mesh_partitioner.resetPartitionIdsForNodes( + readMetisData(input_file_name_wo_extension, num_partitions, + mesh_partitioner.mesh().getNumberOfNodes())); - mesh_partitioner.readMetisData(file_name_base); + removeMetisPartitioningFiles(input_file_name_wo_extension, num_partitions); - INFO("Partitioning the mesh in the node wise way ..."); - mesh_partitioner.partitionByMETIS(lh_elems_flag.getValue()); - if (ascii_flag.getValue()) - { - INFO("Write the data of partitions into ASCII files ..."); - mesh_partitioner.writeASCII(file_name_base); - } - else - { - INFO("Write the data of partitions into binary files ..."); - mesh_partitioner.writeBinary(file_name_base); - } + INFO("Partitioning the mesh in the node wise way ..."); + mesh_partitioner.partitionByMETIS(lh_elems_flag.getValue()); + if (ascii_flag.getValue()) + { + INFO("Write the data of partitions into ASCII files ..."); + mesh_partitioner.writeASCII(output_file_name_wo_extension); + } + else + { + INFO("Write the data of partitions into binary files ..."); + mesh_partitioner.writeBinary(output_file_name_wo_extension); } INFO("Total runtime: %g s.", run_timer.elapsed()); diff --git a/Applications/Utils/Tests.cmake b/Applications/Utils/Tests.cmake index ea7fb182a8eaefacc6d8a20d47bc80bc17904a20..dfaf1bcaa9120adf7f478424917f971c16f4b1e8 100644 --- a/Applications/Utils/Tests.cmake +++ b/Applications/Utils/Tests.cmake @@ -39,3 +39,36 @@ AddTest( DIFF_DATA expected_post_single_joint_pcs_0_ts_1_t_1.000000.vtu post_single_joint_pcs_0_ts_1_t_1.000000.vtu u u 1e-14 1e-14 ) + +# Mac is producing slightly different partitioning, so the results are not +# comparable. +AddTest( + NAME partmesh_2Dmesh_3partitions_ascii + PATH NodePartitionedMesh/partmesh_2Dmesh_3partitions + EXECUTABLE partmesh + EXECUTABLE_ARGS -a -m -n 3 -i 2Dmesh.vtu -o ${Data_BINARY_DIR}/NodePartitionedMesh/partmesh_2Dmesh_3partitions + REQUIREMENTS NOT (OGS_USE_MPI OR APPLE) + TESTER diff + DIFF_DATA 2Dmesh_partitioned_elems_3.msh + 2Dmesh_partitioned_cfg3.msh + 2Dmesh_partitioned_nodes_3.msh +) + +# Mac is producing slightly different partitioning, so the results are not +# comparable. +AddTest( + NAME partmesh_2Dmesh_3partitions_binary + PATH NodePartitionedMesh/partmesh_2Dmesh_3partitions + EXECUTABLE partmesh + EXECUTABLE_ARGS -m -n 3 -i 2Dmesh.vtu -o ${Data_BINARY_DIR}/NodePartitionedMesh/partmesh_2Dmesh_3partitions + REQUIREMENTS NOT (OGS_USE_MPI OR APPLE) + TESTER diff + DIFF_DATA 2Dmesh_partitioned_node_properties_val3.bin + 2Dmesh_partitioned_node_properties_cfg3.bin + 2Dmesh_partitioned_msh_cfg3.bin + 2Dmesh_partitioned_cell_properties_val3.bin + 2Dmesh_partitioned_cell_properties_cfg3.bin + 2Dmesh_partitioned_msh_ele_g3.bin + 2Dmesh_partitioned_msh_ele3.bin + 2Dmesh_partitioned_msh_nod3.bin +) diff --git a/BaseLib/FileTools.cpp b/BaseLib/FileTools.cpp index 7ed7c38676ab0a6532b99ad8aab73964a091ff17..7a50c5e6cc0b999f6d54e20097009ff1dd1497e2 100644 --- a/BaseLib/FileTools.cpp +++ b/BaseLib/FileTools.cpp @@ -13,6 +13,7 @@ */ #include "FileTools.h" +#include "Error.h" #include "StringTools.h" #include <sys/stat.h> diff --git a/BaseLib/FileTools.h b/BaseLib/FileTools.h index 791736beea60f52c193459104873fb47939aeb69..b3a2c9ec0d0e83bcc79a623e0c7bb078b11a476c 100644 --- a/BaseLib/FileTools.h +++ b/BaseLib/FileTools.h @@ -38,7 +38,7 @@ bool IsFileExisting(const std::string &strFilename); */ template <typename T> void writeValueBinary(std::ostream &out, T const& val) { - out.write(static_cast<const char*>(&val), sizeof(T)); + out.write(reinterpret_cast<const char*>(&val), sizeof(T)); } template <typename T> diff --git a/CMakeLists.txt b/CMakeLists.txt index 2566fdea8ad273f902f120e983b205e22c73c0ab..3d87e8ba12738e631d4dfbb34c53744ff2ab4dc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,8 +119,6 @@ if(NOT WIN32 AND OGS_BUILD_SWMM) message(FATAL_ERROR "OGS_BUILD_SWMM requires Windows!") endif() -option(OGS_BUILD_METIS "Should metis and the partmesh util be built?" OFF) - option(OGS_NO_EXTERNAL_LIBS "Builds OGS without any external dependencies." OFF) option(OGS_INSITU "Builds OGS with insitu visualization capabilities." OFF) diff --git a/Jenkinsfile b/Jenkinsfile index 5224df282d7c010ae2c6ab471bbc8fc1271438cb..dcbbc0b0be74e0743adde3f0cf9f9f68604ec01d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -97,8 +97,7 @@ pipeline { '-DOGS_USE_PCH=OFF ' + // see #1992 '-DOGS_BUILD_GUI=ON ' + '-DOGS_BUILD_UTILS=ON ' + - '-DOGS_BUILD_TESTS=OFF ' + - '-DOGS_BUILD_METIS=ON ' + '-DOGS_BUILD_TESTS=OFF ' keepDir = true } build { } @@ -169,7 +168,6 @@ pipeline { configure { cmakeOptions = '-DOGS_BUILD_UTILS=ON ' + - '-DOGS_BUILD_METIS=ON ' + '-DBUILD_SHARED_LIBS=ON ' env = 'envinf1/cli.sh' } @@ -202,7 +200,6 @@ pipeline { configure { cmakeOptions = '-DOGS_BUILD_UTILS=ON ' + - '-DOGS_BUILD_METIS=ON ' + '-DBUILD_SHARED_LIBS=ON ' + '-DOGS_USE_PETSC=ON ' env = 'envinf1/petsc.sh' @@ -254,8 +251,7 @@ pipeline { '-DOGS_BUILD_GUI=ON ' + '-DOGS_BUILD_UTILS=ON ' + '-DOGS_BUILD_TESTS=OFF ' + - '-DOGS_BUILD_SWMM=ON ' + - '-DOGS_BUILD_METIS=ON ' + '-DOGS_BUILD_SWMM=ON ' keepDir = true } build { } @@ -290,7 +286,6 @@ pipeline { '-DOGS_DOWNLOAD_ADDITIONAL_CONTENT=ON ' + '-DOGS_BUILD_GUI=ON ' + '-DOGS_BUILD_UTILS=ON ' + - '-DOGS_BUILD_METIS=ON ' + '-DCMAKE_OSX_DEPLOYMENT_TARGET="10.13" ' } build { @@ -400,7 +395,6 @@ pipeline { configure { cmakeOptions = '-DOGS_BUILD_UTILS=ON ' + - '-DOGS_BUILD_METIS=ON ' + '-DBUILD_SHARED_LIBS=ON ' + '-DCMAKE_INSTALL_PREFIX=/global/apps/ogs/head/standard ' + '-DOGS_MODULEFILE=/global/apps/modulefiles/ogs/head/standard ' + @@ -428,7 +422,6 @@ pipeline { cmakeOptions = '-DOGS_USE_PETSC=ON ' + '-DOGS_BUILD_UTILS=ON ' + - '-DOGS_BUILD_METIS=ON ' + '-DBUILD_SHARED_LIBS=ON ' + '-DCMAKE_INSTALL_PREFIX=/global/apps/ogs/head/petsc ' + '-DOGS_MODULEFILE=/global/apps/modulefiles/ogs/head/petsc ' + diff --git a/MeshLib/Location.h b/MeshLib/Location.h index 7a1bb5dff65b70b1675df96e387f845f5c7fc737..33ec8e5bd513d38ed23f2a3798fbf50b99fdb7d4 100644 --- a/MeshLib/Location.h +++ b/MeshLib/Location.h @@ -20,6 +20,16 @@ namespace MeshLib enum class MeshItemType { Node, Edge, Face, Cell, IntegrationPoint }; +/// Char array names for all of MeshItemType values. +static constexpr char const* mesh_item_type_strings[] = { + "node", "edge", "face", "cell", "integration_point"}; + +/// Returns a char array for a specific MeshItemType. +static constexpr char const* toString(const MeshItemType t) +{ + return mesh_item_type_strings[static_cast<int>(t)]; +} + std::ostream& operator<<(std::ostream& os, MeshItemType const& t); /// Spatial location description. diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh.mesh b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh.mesh new file mode 100644 index 0000000000000000000000000000000000000000..729ff5e143a9ce27b9f2b0339f2b221542497cd7 --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh.mesh @@ -0,0 +1,1176 @@ +1175 +1 2 1223 +2 3 1226 +3 4 1229 +4 5 1232 +5 6 1235 +6 7 1238 +7 8 1241 +8 9 1244 +9 10 1247 +10 11 1250 +11 12 1253 +12 13 1256 +13 14 1259 +14 15 1262 +15 16 1265 +16 17 1268 +17 18 1271 +18 19 1274 +19 20 1277 +20 21 1280 +21 22 1283 +22 23 1286 +23 24 1289 +24 25 1292 +25 26 1295 +30 29 28 27 1304 1302 1300 1301 +32 31 29 30 1308 1305 1304 1307 +34 33 31 32 1312 1309 1308 1311 +36 35 33 34 1316 1313 1312 1315 +38 37 35 36 1320 1317 1316 1319 +40 39 37 38 1324 1321 1320 1323 +42 41 39 40 1328 1325 1324 1327 +44 43 41 42 1332 1329 1328 1331 +46 45 43 44 1336 1333 1332 1335 +48 47 45 46 1340 1337 1336 1339 +50 49 47 48 1344 1341 1340 1343 +52 51 49 50 1348 1345 1344 1347 +54 53 51 52 1352 1349 1348 1351 +56 55 53 54 1356 1353 1352 1355 +58 57 55 56 1360 1357 1356 1359 +60 59 57 58 1364 1361 1360 1363 +62 61 59 60 1368 1365 1364 1367 +64 63 61 62 1372 1369 1368 1371 +66 65 63 64 1376 1373 1372 1375 +68 67 65 66 1380 1377 1376 1379 +70 69 67 68 1384 1381 1380 1383 +72 71 69 70 1388 1385 1384 1387 +74 73 71 72 1392 1389 1388 1391 +76 75 73 74 1396 1393 1392 1395 +78 77 75 76 1400 1397 1396 1399 +29 80 79 28 1306 1402 1303 1302 +31 81 80 29 1310 1404 1306 1305 +33 82 81 31 1314 1406 1310 1309 +35 83 82 33 1318 1408 1314 1313 +37 84 83 35 1322 1410 1318 1317 +39 85 84 37 1326 1412 1322 1321 +41 86 85 39 1330 1414 1326 1325 +43 87 86 41 1334 1416 1330 1329 +45 88 87 43 1338 1418 1334 1333 +47 89 88 45 1342 1420 1338 1337 +49 90 89 47 1346 1422 1342 1341 +51 91 90 49 1350 1424 1346 1345 +53 92 91 51 1354 1426 1350 1349 +55 93 92 53 1358 1428 1354 1353 +57 94 93 55 1362 1430 1358 1357 +59 95 94 57 1366 1432 1362 1361 +61 96 95 59 1370 1434 1366 1365 +63 97 96 61 1374 1436 1370 1369 +65 98 97 63 1378 1438 1374 1373 +67 99 98 65 1382 1440 1378 1377 +69 100 99 67 1386 1442 1382 1381 +71 101 100 69 1390 1444 1386 1385 +73 102 101 71 1394 1446 1390 1389 +75 103 102 73 1398 1448 1394 1393 +77 104 103 75 1401 1450 1398 1397 +80 106 105 79 1405 1453 1403 1402 +81 107 106 80 1407 1455 1405 1404 +82 108 107 81 1409 1457 1407 1406 +83 109 108 82 1411 1459 1409 1408 +84 110 109 83 1413 1461 1411 1410 +85 111 110 84 1415 1463 1413 1412 +86 112 111 85 1417 1465 1415 1414 +87 113 112 86 1419 1467 1417 1416 +88 114 113 87 1421 1469 1419 1418 +89 115 114 88 1423 1471 1421 1420 +90 116 115 89 1425 1473 1423 1422 +91 117 116 90 1427 1475 1425 1424 +92 118 117 91 1429 1477 1427 1426 +93 119 118 92 1431 1479 1429 1428 +94 120 119 93 1433 1481 1431 1430 +95 121 120 94 1435 1483 1433 1432 +96 122 121 95 1437 1485 1435 1434 +97 123 122 96 1439 1487 1437 1436 +98 124 123 97 1441 1489 1439 1438 +99 125 124 98 1443 1491 1441 1440 +100 126 125 99 1445 1493 1443 1442 +101 127 126 100 1447 1495 1445 1444 +102 128 127 101 1449 1497 1447 1446 +103 129 128 102 1451 1499 1449 1448 +104 130 129 103 1452 1501 1451 1450 +106 132 131 105 1456 1504 1454 1453 +107 133 132 106 1458 1506 1456 1455 +108 134 133 107 1460 1508 1458 1457 +109 135 134 108 1462 1510 1460 1459 +110 136 135 109 1464 1512 1462 1461 +111 137 136 110 1466 1514 1464 1463 +112 138 137 111 1468 1516 1466 1465 +113 139 138 112 1470 1518 1468 1467 +114 140 139 113 1472 1520 1470 1469 +115 141 140 114 1474 1522 1472 1471 +116 142 141 115 1476 1524 1474 1473 +117 143 142 116 1478 1526 1476 1475 +118 144 143 117 1480 1528 1478 1477 +119 145 144 118 1482 1530 1480 1479 +120 146 145 119 1484 1532 1482 1481 +121 147 146 120 1486 1534 1484 1483 +122 148 147 121 1488 1536 1486 1485 +123 149 148 122 1490 1538 1488 1487 +124 150 149 123 1492 1540 1490 1489 +125 151 150 124 1494 1542 1492 1491 +126 152 151 125 1496 1544 1494 1493 +127 153 152 126 1498 1546 1496 1495 +128 154 153 127 1500 1548 1498 1497 +129 155 154 128 1502 1550 1500 1499 +130 156 155 129 1503 1552 1502 1501 +132 158 157 131 1507 1555 1505 1504 +133 159 158 132 1509 1557 1507 1506 +134 160 159 133 1511 1559 1509 1508 +135 161 160 134 1513 1561 1511 1510 +136 162 161 135 1515 1563 1513 1512 +137 163 162 136 1517 1565 1515 1514 +138 164 163 137 1519 1567 1517 1516 +139 165 164 138 1521 1569 1519 1518 +140 166 165 139 1523 1571 1521 1520 +141 167 166 140 1525 1573 1523 1522 +142 168 167 141 1527 1575 1525 1524 +143 169 168 142 1529 1577 1527 1526 +144 170 169 143 1531 1579 1529 1528 +145 171 170 144 1533 1581 1531 1530 +146 172 171 145 1535 1583 1533 1532 +147 173 172 146 1537 1585 1535 1534 +148 174 173 147 1539 1587 1537 1536 +149 175 174 148 1541 1589 1539 1538 +150 176 175 149 1543 1591 1541 1540 +151 177 176 150 1545 1593 1543 1542 +152 178 177 151 1547 1595 1545 1544 +153 179 178 152 1549 1597 1547 1546 +154 180 179 153 1551 1599 1549 1548 +155 181 180 154 1553 1601 1551 1550 +156 182 181 155 1554 1603 1553 1552 +158 184 183 157 1558 1606 1556 1555 +159 185 184 158 1560 1608 1558 1557 +160 186 185 159 1562 1610 1560 1559 +161 187 186 160 1564 1612 1562 1561 +162 188 187 161 1566 1614 1564 1563 +163 189 188 162 1568 1616 1566 1565 +164 190 189 163 1570 1618 1568 1567 +165 191 190 164 1572 1620 1570 1569 +166 192 191 165 1574 1622 1572 1571 +167 193 192 166 1576 1624 1574 1573 +168 194 193 167 1578 1626 1576 1575 +169 195 194 168 1580 1628 1578 1577 +170 196 195 169 1582 1630 1580 1579 +171 197 196 170 1584 1632 1582 1581 +172 198 197 171 1586 1634 1584 1583 +173 199 198 172 1588 1636 1586 1585 +174 200 199 173 1590 1638 1588 1587 +175 201 200 174 1592 1640 1590 1589 +176 202 201 175 1594 1642 1592 1591 +177 203 202 176 1596 1644 1594 1593 +178 204 203 177 1598 1646 1596 1595 +179 205 204 178 1600 1648 1598 1597 +180 206 205 179 1602 1650 1600 1599 +181 207 206 180 1604 1652 1602 1601 +182 208 207 181 1605 1654 1604 1603 +184 210 209 183 1609 1657 1607 1606 +185 211 210 184 1611 1659 1609 1608 +186 212 211 185 1613 1661 1611 1610 +187 213 212 186 1615 1663 1613 1612 +188 214 213 187 1617 1665 1615 1614 +189 215 214 188 1619 1667 1617 1616 +190 216 215 189 1621 1669 1619 1618 +191 217 216 190 1623 1671 1621 1620 +192 218 217 191 1625 1673 1623 1622 +193 219 218 192 1627 1675 1625 1624 +194 220 219 193 1629 1677 1627 1626 +195 221 220 194 1631 1679 1629 1628 +196 222 221 195 1633 1681 1631 1630 +197 223 222 196 1635 1683 1633 1632 +198 224 223 197 1637 1685 1635 1634 +199 225 224 198 1639 1687 1637 1636 +200 226 225 199 1641 1689 1639 1638 +201 227 226 200 1643 1691 1641 1640 +202 228 227 201 1645 1693 1643 1642 +203 229 228 202 1647 1695 1645 1644 +204 230 229 203 1649 1697 1647 1646 +205 231 230 204 1651 1699 1649 1648 +206 232 231 205 1653 1701 1651 1650 +207 233 232 206 1655 1703 1653 1652 +208 234 233 207 1656 1705 1655 1654 +210 236 235 209 1660 1708 1658 1657 +211 237 236 210 1662 1710 1660 1659 +212 238 237 211 1664 1712 1662 1661 +213 239 238 212 1666 1714 1664 1663 +214 240 239 213 1668 1716 1666 1665 +215 241 240 214 1670 1718 1668 1667 +216 242 241 215 1672 1720 1670 1669 +217 243 242 216 1674 1722 1672 1671 +218 244 243 217 1676 1724 1674 1673 +219 245 244 218 1678 1726 1676 1675 +220 246 245 219 1680 1728 1678 1677 +221 247 246 220 1682 1730 1680 1679 +222 248 247 221 1684 1732 1682 1681 +223 249 248 222 1686 1734 1684 1683 +224 250 249 223 1688 1736 1686 1685 +225 251 250 224 1690 1738 1688 1687 +226 252 251 225 1692 1740 1690 1689 +227 253 252 226 1694 1742 1692 1691 +228 254 253 227 1696 1744 1694 1693 +229 255 254 228 1698 1746 1696 1695 +230 256 255 229 1700 1748 1698 1697 +231 257 256 230 1702 1750 1700 1699 +232 258 257 231 1704 1752 1702 1701 +233 259 258 232 1706 1754 1704 1703 +234 260 259 233 1707 1756 1706 1705 +236 262 261 235 1711 1759 1709 1708 +237 263 262 236 1713 1761 1711 1710 +238 264 263 237 1715 1763 1713 1712 +239 265 264 238 1717 1765 1715 1714 +240 266 265 239 1719 1767 1717 1716 +241 267 266 240 1721 1769 1719 1718 +242 268 267 241 1723 1771 1721 1720 +243 269 268 242 1725 1773 1723 1722 +244 270 269 243 1727 1775 1725 1724 +245 271 270 244 1729 1777 1727 1726 +246 272 271 245 1731 1779 1729 1728 +247 273 272 246 1733 1781 1731 1730 +248 274 273 247 1735 1783 1733 1732 +249 275 274 248 1737 1785 1735 1734 +250 276 275 249 1739 1787 1737 1736 +251 277 276 250 1741 1789 1739 1738 +252 278 277 251 1743 1791 1741 1740 +253 279 278 252 1745 1793 1743 1742 +254 280 279 253 1747 1795 1745 1744 +255 281 280 254 1749 1797 1747 1746 +256 282 281 255 1751 1799 1749 1748 +257 283 282 256 1753 1801 1751 1750 +258 284 283 257 1755 1803 1753 1752 +259 285 284 258 1757 1805 1755 1754 +260 286 285 259 1758 1807 1757 1756 +262 288 287 261 1762 1810 1760 1759 +288 290 289 287 1812 1814 1811 1810 +290 292 291 289 1816 1818 1815 1814 +292 294 293 291 1820 1822 1819 1818 +294 296 295 293 1824 1826 1823 1822 +296 298 297 295 1828 1830 1827 1826 +298 300 299 297 1832 1834 1831 1830 +300 302 301 299 1836 1838 1835 1834 +302 304 303 301 1840 1842 1839 1838 +304 306 305 303 1844 1846 1843 1842 +306 308 307 305 1848 1850 1847 1846 +308 310 309 307 1852 1854 1851 1850 +310 312 311 309 1856 1858 1855 1854 +312 2 1 311 1227 1223 1224 1858 +263 313 288 262 1764 1813 1762 1761 +313 314 290 288 1860 1817 1812 1813 +314 315 292 290 1862 1821 1816 1817 +315 316 294 292 1864 1825 1820 1821 +316 317 296 294 1866 1829 1824 1825 +317 318 298 296 1868 1833 1828 1829 +318 319 300 298 1870 1837 1832 1833 +319 320 302 300 1872 1841 1836 1837 +320 321 304 302 1874 1845 1840 1841 +321 322 306 304 1876 1849 1844 1845 +322 323 308 306 1878 1853 1848 1849 +323 324 310 308 1880 1857 1852 1853 +324 325 312 310 1882 1859 1856 1857 +325 3 2 312 1230 1226 1227 1859 +264 326 313 263 1766 1861 1764 1763 +326 327 314 313 1885 1863 1860 1861 +327 328 315 314 1887 1865 1862 1863 +328 329 316 315 1889 1867 1864 1865 +329 330 317 316 1891 1869 1866 1867 +330 331 318 317 1893 1871 1868 1869 +331 332 319 318 1895 1873 1870 1871 +332 333 320 319 1897 1875 1872 1873 +333 334 321 320 1899 1877 1874 1875 +334 335 322 321 1901 1879 1876 1877 +335 336 323 322 1903 1881 1878 1879 +336 337 324 323 1905 1883 1880 1881 +337 338 325 324 1907 1884 1882 1883 +338 4 3 325 1233 1229 1230 1884 +265 339 326 264 1768 1886 1766 1765 +339 340 327 326 1910 1888 1885 1886 +340 341 328 327 1912 1890 1887 1888 +341 342 329 328 1914 1892 1889 1890 +342 343 330 329 1916 1894 1891 1892 +343 344 331 330 1918 1896 1893 1894 +344 345 332 331 1920 1898 1895 1896 +345 346 333 332 1922 1900 1897 1898 +346 347 334 333 1924 1902 1899 1900 +347 348 335 334 1926 1904 1901 1902 +348 349 336 335 1928 1906 1903 1904 +349 350 337 336 1930 1908 1905 1906 +350 351 338 337 1932 1909 1907 1908 +351 5 4 338 1236 1232 1233 1909 +266 352 339 265 1770 1911 1768 1767 +352 353 340 339 1935 1913 1910 1911 +353 354 341 340 1937 1915 1912 1913 +354 355 342 341 1939 1917 1914 1915 +355 356 343 342 1941 1919 1916 1917 +356 357 344 343 1943 1921 1918 1919 +357 358 345 344 1945 1923 1920 1921 +358 359 346 345 1947 1925 1922 1923 +359 360 347 346 1949 1927 1924 1925 +360 361 348 347 1951 1929 1926 1927 +361 362 349 348 1953 1931 1928 1929 +362 363 350 349 1955 1933 1930 1931 +363 364 351 350 1957 1934 1932 1933 +364 6 5 351 1239 1235 1236 1934 +267 365 352 266 1772 1936 1770 1769 +365 366 353 352 1960 1938 1935 1936 +366 367 354 353 1962 1940 1937 1938 +367 368 355 354 1964 1942 1939 1940 +368 369 356 355 1966 1944 1941 1942 +369 370 357 356 1968 1946 1943 1944 +370 371 358 357 1970 1948 1945 1946 +371 372 359 358 1972 1950 1947 1948 +372 373 360 359 1974 1952 1949 1950 +373 374 361 360 1976 1954 1951 1952 +374 375 362 361 1978 1956 1953 1954 +375 376 363 362 1980 1958 1955 1956 +376 377 364 363 1982 1959 1957 1958 +377 7 6 364 1242 1238 1239 1959 +268 378 365 267 1774 1961 1772 1771 +378 379 366 365 1985 1963 1960 1961 +379 380 367 366 1987 1965 1962 1963 +380 381 368 367 1989 1967 1964 1965 +381 382 369 368 1991 1969 1966 1967 +382 383 370 369 1993 1971 1968 1969 +383 384 371 370 1995 1973 1970 1971 +384 385 372 371 1997 1975 1972 1973 +385 386 373 372 1999 1977 1974 1975 +386 387 374 373 2001 1979 1976 1977 +387 388 375 374 2003 1981 1978 1979 +388 389 376 375 2005 1983 1980 1981 +389 390 377 376 2007 1984 1982 1983 +390 8 7 377 1245 1241 1242 1984 +269 391 378 268 1776 1986 1774 1773 +391 392 379 378 2010 1988 1985 1986 +392 393 380 379 2012 1990 1987 1988 +393 394 381 380 2014 1992 1989 1990 +394 395 382 381 2016 1994 1991 1992 +395 396 383 382 2018 1996 1993 1994 +396 397 384 383 2020 1998 1995 1996 +397 398 385 384 2022 2000 1997 1998 +398 399 386 385 2024 2002 1999 2000 +399 400 387 386 2026 2004 2001 2002 +400 401 388 387 2028 2006 2003 2004 +401 402 389 388 2030 2008 2005 2006 +402 403 390 389 2032 2009 2007 2008 +403 9 8 390 1248 1244 1245 2009 +270 404 391 269 1778 2011 1776 1775 +404 405 392 391 2035 2013 2010 2011 +405 406 393 392 2037 2015 2012 2013 +406 407 394 393 2039 2017 2014 2015 +407 408 395 394 2041 2019 2016 2017 +408 409 396 395 2043 2021 2018 2019 +409 410 397 396 2045 2023 2020 2021 +410 411 398 397 2047 2025 2022 2023 +411 412 399 398 2049 2027 2024 2025 +412 413 400 399 2051 2029 2026 2027 +413 414 401 400 2053 2031 2028 2029 +414 415 402 401 2055 2033 2030 2031 +415 416 403 402 2057 2034 2032 2033 +416 10 9 403 1251 1247 1248 2034 +271 417 404 270 1780 2036 1778 1777 +417 418 405 404 2060 2038 2035 2036 +418 419 406 405 2062 2040 2037 2038 +419 420 407 406 2064 2042 2039 2040 +420 421 408 407 2066 2044 2041 2042 +421 422 409 408 2068 2046 2043 2044 +422 423 410 409 2070 2048 2045 2046 +423 424 411 410 2072 2050 2047 2048 +424 425 412 411 2074 2052 2049 2050 +425 426 413 412 2076 2054 2051 2052 +426 427 414 413 2078 2056 2053 2054 +427 428 415 414 2080 2058 2055 2056 +428 429 416 415 2082 2059 2057 2058 +429 11 10 416 1254 1250 1251 2059 +272 430 417 271 1782 2061 1780 1779 +430 431 418 417 2085 2063 2060 2061 +431 432 419 418 2087 2065 2062 2063 +432 433 420 419 2089 2067 2064 2065 +433 434 421 420 2091 2069 2066 2067 +434 435 422 421 2093 2071 2068 2069 +435 436 423 422 2095 2073 2070 2071 +436 437 424 423 2097 2075 2072 2073 +437 438 425 424 2099 2077 2074 2075 +438 439 426 425 2101 2079 2076 2077 +439 440 427 426 2103 2081 2078 2079 +440 441 428 427 2105 2083 2080 2081 +441 442 429 428 2107 2084 2082 2083 +442 12 11 429 1257 1253 1254 2084 +273 443 430 272 1784 2086 1782 1781 +443 444 431 430 2110 2088 2085 2086 +444 445 432 431 2112 2090 2087 2088 +445 446 433 432 2114 2092 2089 2090 +446 447 434 433 2116 2094 2091 2092 +447 448 435 434 2118 2096 2093 2094 +448 449 436 435 2120 2098 2095 2096 +449 450 437 436 2122 2100 2097 2098 +450 451 438 437 2124 2102 2099 2100 +451 452 439 438 2126 2104 2101 2102 +452 453 440 439 2128 2106 2103 2104 +453 454 441 440 2130 2108 2105 2106 +454 455 442 441 2132 2109 2107 2108 +455 13 12 442 1260 1256 1257 2109 +274 456 443 273 1786 2111 1784 1783 +456 457 444 443 2135 2113 2110 2111 +457 458 445 444 2137 2115 2112 2113 +458 459 446 445 2139 2117 2114 2115 +459 460 447 446 2141 2119 2116 2117 +460 461 448 447 2143 2121 2118 2119 +461 462 449 448 2145 2123 2120 2121 +462 463 450 449 2147 2125 2122 2123 +463 464 451 450 2149 2127 2124 2125 +464 465 452 451 2151 2129 2126 2127 +465 466 453 452 2153 2131 2128 2129 +466 467 454 453 2155 2133 2130 2131 +467 468 455 454 2157 2134 2132 2133 +468 14 13 455 1263 1259 1260 2134 +275 469 456 274 1788 2136 1786 1785 +469 470 457 456 2160 2138 2135 2136 +470 471 458 457 2162 2140 2137 2138 +471 472 459 458 2164 2142 2139 2140 +472 473 460 459 2166 2144 2141 2142 +473 474 461 460 2168 2146 2143 2144 +474 475 462 461 2170 2148 2145 2146 +475 476 463 462 2172 2150 2147 2148 +476 477 464 463 2174 2152 2149 2150 +477 478 465 464 2176 2154 2151 2152 +478 479 466 465 2178 2156 2153 2154 +479 480 467 466 2180 2158 2155 2156 +480 481 468 467 2182 2159 2157 2158 +481 15 14 468 1266 1262 1263 2159 +276 482 469 275 1790 2161 1788 1787 +482 483 470 469 2185 2163 2160 2161 +483 484 471 470 2187 2165 2162 2163 +484 485 472 471 2189 2167 2164 2165 +485 486 473 472 2191 2169 2166 2167 +486 487 474 473 2193 2171 2168 2169 +487 488 475 474 2195 2173 2170 2171 +488 489 476 475 2197 2175 2172 2173 +489 490 477 476 2199 2177 2174 2175 +490 491 478 477 2201 2179 2176 2177 +491 492 479 478 2203 2181 2178 2179 +492 493 480 479 2205 2183 2180 2181 +493 494 481 480 2207 2184 2182 2183 +494 16 15 481 1269 1265 1266 2184 +277 495 482 276 1792 2186 1790 1789 +495 496 483 482 2210 2188 2185 2186 +496 497 484 483 2212 2190 2187 2188 +497 498 485 484 2214 2192 2189 2190 +498 499 486 485 2216 2194 2191 2192 +499 500 487 486 2218 2196 2193 2194 +500 501 488 487 2220 2198 2195 2196 +501 502 489 488 2222 2200 2197 2198 +502 503 490 489 2224 2202 2199 2200 +503 504 491 490 2226 2204 2201 2202 +504 505 492 491 2228 2206 2203 2204 +505 506 493 492 2230 2208 2205 2206 +506 507 494 493 2232 2209 2207 2208 +507 17 16 494 1272 1268 1269 2209 +278 508 495 277 1794 2211 1792 1791 +508 509 496 495 2235 2213 2210 2211 +509 510 497 496 2237 2215 2212 2213 +510 511 498 497 2239 2217 2214 2215 +511 512 499 498 2241 2219 2216 2217 +512 513 500 499 2243 2221 2218 2219 +513 514 501 500 2245 2223 2220 2221 +514 515 502 501 2247 2225 2222 2223 +515 516 503 502 2249 2227 2224 2225 +516 517 504 503 2251 2229 2226 2227 +517 518 505 504 2253 2231 2228 2229 +518 519 506 505 2255 2233 2230 2231 +519 520 507 506 2257 2234 2232 2233 +520 18 17 507 1275 1271 1272 2234 +279 521 508 278 1796 2236 1794 1793 +521 522 509 508 2260 2238 2235 2236 +522 523 510 509 2262 2240 2237 2238 +523 524 511 510 2264 2242 2239 2240 +524 525 512 511 2266 2244 2241 2242 +525 526 513 512 2268 2246 2243 2244 +526 527 514 513 2270 2248 2245 2246 +527 528 515 514 2272 2250 2247 2248 +528 529 516 515 2274 2252 2249 2250 +529 530 517 516 2276 2254 2251 2252 +530 531 518 517 2278 2256 2253 2254 +531 532 519 518 2280 2258 2255 2256 +532 533 520 519 2282 2259 2257 2258 +533 19 18 520 1278 1274 1275 2259 +280 534 521 279 1798 2261 1796 1795 +534 535 522 521 2285 2263 2260 2261 +535 536 523 522 2287 2265 2262 2263 +536 537 524 523 2289 2267 2264 2265 +537 538 525 524 2291 2269 2266 2267 +538 539 526 525 2293 2271 2268 2269 +539 540 527 526 2295 2273 2270 2271 +540 541 528 527 2297 2275 2272 2273 +541 542 529 528 2299 2277 2274 2275 +542 543 530 529 2301 2279 2276 2277 +543 544 531 530 2303 2281 2278 2279 +544 545 532 531 2305 2283 2280 2281 +545 546 533 532 2307 2284 2282 2283 +546 20 19 533 1281 1277 1278 2284 +281 547 534 280 1800 2286 1798 1797 +547 548 535 534 2310 2288 2285 2286 +548 549 536 535 2312 2290 2287 2288 +549 550 537 536 2314 2292 2289 2290 +550 551 538 537 2316 2294 2291 2292 +551 552 539 538 2318 2296 2293 2294 +552 553 540 539 2320 2298 2295 2296 +553 554 541 540 2322 2300 2297 2298 +554 555 542 541 2324 2302 2299 2300 +555 556 543 542 2326 2304 2301 2302 +556 557 544 543 2328 2306 2303 2304 +557 558 545 544 2330 2308 2305 2306 +558 559 546 545 2332 2309 2307 2308 +559 21 20 546 1284 1280 1281 2309 +282 560 547 281 1802 2311 1800 1799 +560 561 548 547 2335 2313 2310 2311 +561 562 549 548 2337 2315 2312 2313 +562 563 550 549 2339 2317 2314 2315 +563 564 551 550 2341 2319 2316 2317 +564 565 552 551 2343 2321 2318 2319 +565 566 553 552 2345 2323 2320 2321 +566 567 554 553 2347 2325 2322 2323 +567 568 555 554 2349 2327 2324 2325 +568 569 556 555 2351 2329 2326 2327 +569 570 557 556 2353 2331 2328 2329 +570 571 558 557 2355 2333 2330 2331 +571 572 559 558 2357 2334 2332 2333 +572 22 21 559 1287 1283 1284 2334 +283 573 560 282 1804 2336 1802 1801 +573 574 561 560 2360 2338 2335 2336 +574 575 562 561 2362 2340 2337 2338 +575 576 563 562 2364 2342 2339 2340 +576 577 564 563 2366 2344 2341 2342 +577 578 565 564 2368 2346 2343 2344 +578 579 566 565 2370 2348 2345 2346 +579 580 567 566 2372 2350 2347 2348 +580 581 568 567 2374 2352 2349 2350 +581 582 569 568 2376 2354 2351 2352 +582 583 570 569 2378 2356 2353 2354 +583 584 571 570 2380 2358 2355 2356 +584 585 572 571 2382 2359 2357 2358 +585 23 22 572 1290 1286 1287 2359 +284 586 573 283 1806 2361 1804 1803 +586 587 574 573 2385 2363 2360 2361 +587 588 575 574 2387 2365 2362 2363 +588 589 576 575 2389 2367 2364 2365 +589 590 577 576 2391 2369 2366 2367 +590 591 578 577 2393 2371 2368 2369 +591 592 579 578 2395 2373 2370 2371 +592 593 580 579 2397 2375 2372 2373 +593 594 581 580 2399 2377 2374 2375 +594 595 582 581 2401 2379 2376 2377 +595 596 583 582 2403 2381 2378 2379 +596 597 584 583 2405 2383 2380 2381 +597 598 585 584 2407 2384 2382 2383 +598 24 23 585 1293 1289 1290 2384 +285 599 586 284 1808 2386 1806 1805 +599 600 587 586 2410 2388 2385 2386 +600 601 588 587 2412 2390 2387 2388 +601 602 589 588 2414 2392 2389 2390 +602 603 590 589 2416 2394 2391 2392 +603 604 591 590 2418 2396 2393 2394 +604 605 592 591 2420 2398 2395 2396 +605 606 593 592 2422 2400 2397 2398 +606 607 594 593 2424 2402 2399 2400 +607 608 595 594 2426 2404 2401 2402 +608 609 596 595 2428 2406 2403 2404 +609 610 597 596 2430 2408 2405 2406 +610 611 598 597 2432 2409 2407 2408 +611 25 24 598 1296 1292 1293 2409 +286 612 599 285 1809 2411 1808 1807 +612 613 600 599 2435 2413 2410 2411 +613 614 601 600 2436 2415 2412 2413 +614 615 602 601 2437 2417 2414 2415 +615 616 603 602 2438 2419 2416 2417 +616 617 604 603 2439 2421 2418 2419 +617 618 605 604 2440 2423 2420 2421 +618 619 606 605 2441 2425 2422 2423 +619 620 607 606 2442 2427 2424 2425 +620 621 608 607 2443 2429 2426 2427 +621 622 609 608 2444 2431 2428 2429 +622 623 610 609 2445 2433 2430 2431 +623 624 611 610 2446 2434 2432 2433 +624 26 25 611 1298 1295 1296 2434 +628 627 626 625 2451 2449 2447 2448 +627 630 629 626 2452 2455 2450 2449 +630 632 631 629 2457 2459 2456 2455 +632 634 633 631 2461 2463 2460 2459 +634 636 635 633 2465 2467 2464 2463 +636 638 637 635 2469 2471 2468 2467 +638 640 639 637 2473 2475 2472 2471 +640 642 641 639 2477 2479 2476 2475 +642 644 643 641 2481 2483 2480 2479 +644 646 645 643 2485 2487 2484 2483 +646 648 647 645 2489 2491 2488 2487 +648 650 649 647 2493 2495 2492 2491 +650 652 651 649 2497 2499 2496 2495 +652 654 653 651 2501 2503 2500 2499 +654 656 655 653 2505 2507 2504 2503 +656 658 657 655 2509 2511 2508 2507 +658 660 659 657 2513 2515 2512 2511 +660 662 661 659 2517 2519 2516 2515 +662 664 663 661 2521 2523 2520 2519 +664 666 665 663 2525 2527 2524 2523 +666 668 667 665 2529 2531 2528 2527 +668 670 669 667 2533 2535 2532 2531 +670 672 671 669 2537 2539 2536 2535 +672 674 673 671 2541 2543 2540 2539 +674 676 675 673 2545 2547 2544 2543 +678 677 627 628 2549 2453 2451 2454 +677 679 630 627 2550 2458 2452 2453 +679 680 632 630 2553 2462 2457 2458 +680 681 634 632 2555 2466 2461 2462 +681 682 636 634 2557 2470 2465 2466 +682 683 638 636 2559 2474 2469 2470 +683 684 640 638 2561 2478 2473 2474 +684 685 642 640 2563 2482 2477 2478 +685 686 644 642 2565 2486 2481 2482 +686 687 646 644 2567 2490 2485 2486 +687 688 648 646 2569 2494 2489 2490 +688 689 650 648 2571 2498 2493 2494 +689 690 652 650 2573 2502 2497 2498 +690 691 654 652 2575 2506 2501 2502 +691 692 656 654 2577 2510 2505 2506 +692 693 658 656 2579 2514 2509 2510 +693 694 660 658 2581 2518 2513 2514 +694 695 662 660 2583 2522 2517 2518 +695 696 664 662 2585 2526 2521 2522 +696 697 666 664 2587 2530 2525 2526 +697 698 668 666 2589 2534 2529 2530 +698 699 670 668 2591 2538 2533 2534 +699 700 672 670 2593 2542 2537 2538 +700 701 674 672 2595 2546 2541 2542 +701 702 676 674 2597 2548 2545 2546 +704 703 677 678 2600 2551 2549 2552 +703 705 679 677 2601 2554 2550 2551 +705 706 680 679 2604 2556 2553 2554 +706 707 681 680 2606 2558 2555 2556 +707 708 682 681 2608 2560 2557 2558 +708 709 683 682 2610 2562 2559 2560 +709 710 684 683 2612 2564 2561 2562 +710 711 685 684 2614 2566 2563 2564 +711 712 686 685 2616 2568 2565 2566 +712 713 687 686 2618 2570 2567 2568 +713 714 688 687 2620 2572 2569 2570 +714 715 689 688 2622 2574 2571 2572 +715 716 690 689 2624 2576 2573 2574 +716 717 691 690 2626 2578 2575 2576 +717 718 692 691 2628 2580 2577 2578 +718 719 693 692 2630 2582 2579 2580 +719 720 694 693 2632 2584 2581 2582 +720 721 695 694 2634 2586 2583 2584 +721 722 696 695 2636 2588 2585 2586 +722 723 697 696 2638 2590 2587 2588 +723 724 698 697 2640 2592 2589 2590 +724 725 699 698 2642 2594 2591 2592 +725 726 700 699 2644 2596 2593 2594 +726 727 701 700 2646 2598 2595 2596 +727 728 702 701 2648 2599 2597 2598 +730 729 703 704 2651 2602 2600 2603 +729 731 705 703 2652 2605 2601 2602 +731 732 706 705 2655 2607 2604 2605 +732 733 707 706 2657 2609 2606 2607 +733 734 708 707 2659 2611 2608 2609 +734 735 709 708 2661 2613 2610 2611 +735 736 710 709 2663 2615 2612 2613 +736 737 711 710 2665 2617 2614 2615 +737 738 712 711 2667 2619 2616 2617 +738 739 713 712 2669 2621 2618 2619 +739 740 714 713 2671 2623 2620 2621 +740 741 715 714 2673 2625 2622 2623 +741 742 716 715 2675 2627 2624 2625 +742 743 717 716 2677 2629 2626 2627 +743 744 718 717 2679 2631 2628 2629 +744 745 719 718 2681 2633 2630 2631 +745 746 720 719 2683 2635 2632 2633 +746 747 721 720 2685 2637 2634 2635 +747 748 722 721 2687 2639 2636 2637 +748 749 723 722 2689 2641 2638 2639 +749 750 724 723 2691 2643 2640 2641 +750 751 725 724 2693 2645 2642 2643 +751 752 726 725 2695 2647 2644 2645 +752 753 727 726 2697 2649 2646 2647 +753 754 728 727 2699 2650 2648 2649 +756 755 729 730 2702 2653 2651 2654 +755 757 731 729 2703 2656 2652 2653 +757 758 732 731 2706 2658 2655 2656 +758 759 733 732 2708 2660 2657 2658 +759 760 734 733 2710 2662 2659 2660 +760 761 735 734 2712 2664 2661 2662 +761 762 736 735 2714 2666 2663 2664 +762 763 737 736 2716 2668 2665 2666 +763 764 738 737 2718 2670 2667 2668 +764 765 739 738 2720 2672 2669 2670 +765 766 740 739 2722 2674 2671 2672 +766 767 741 740 2724 2676 2673 2674 +767 768 742 741 2726 2678 2675 2676 +768 769 743 742 2728 2680 2677 2678 +769 770 744 743 2730 2682 2679 2680 +770 771 745 744 2732 2684 2681 2682 +771 772 746 745 2734 2686 2683 2684 +772 773 747 746 2736 2688 2685 2686 +773 774 748 747 2738 2690 2687 2688 +774 775 749 748 2740 2692 2689 2690 +775 776 750 749 2742 2694 2691 2692 +776 777 751 750 2744 2696 2693 2694 +777 778 752 751 2746 2698 2695 2696 +778 779 753 752 2748 2700 2697 2698 +779 780 754 753 2750 2701 2699 2700 +782 781 755 756 2753 2704 2702 2705 +781 783 757 755 2754 2707 2703 2704 +783 784 758 757 2757 2709 2706 2707 +784 785 759 758 2759 2711 2708 2709 +785 786 760 759 2761 2713 2710 2711 +786 787 761 760 2763 2715 2712 2713 +787 788 762 761 2765 2717 2714 2715 +788 789 763 762 2767 2719 2716 2717 +789 790 764 763 2769 2721 2718 2719 +790 791 765 764 2771 2723 2720 2721 +791 792 766 765 2773 2725 2722 2723 +792 793 767 766 2775 2727 2724 2725 +793 794 768 767 2777 2729 2726 2727 +794 795 769 768 2779 2731 2728 2729 +795 796 770 769 2781 2733 2730 2731 +796 797 771 770 2783 2735 2732 2733 +797 798 772 771 2785 2737 2734 2735 +798 799 773 772 2787 2739 2736 2737 +799 800 774 773 2789 2741 2738 2739 +800 801 775 774 2791 2743 2740 2741 +801 802 776 775 2793 2745 2742 2743 +802 803 777 776 2795 2747 2744 2745 +803 804 778 777 2797 2749 2746 2747 +804 805 779 778 2799 2751 2748 2749 +805 806 780 779 2801 2752 2750 2751 +808 807 781 782 2804 2755 2753 2756 +807 809 783 781 2805 2758 2754 2755 +809 810 784 783 2808 2760 2757 2758 +810 811 785 784 2810 2762 2759 2760 +811 812 786 785 2812 2764 2761 2762 +812 813 787 786 2814 2766 2763 2764 +813 814 788 787 2816 2768 2765 2766 +814 815 789 788 2818 2770 2767 2768 +815 816 790 789 2820 2772 2769 2770 +816 817 791 790 2822 2774 2771 2772 +817 818 792 791 2824 2776 2773 2774 +818 819 793 792 2826 2778 2775 2776 +819 820 794 793 2828 2780 2777 2778 +820 821 795 794 2830 2782 2779 2780 +821 822 796 795 2832 2784 2781 2782 +822 823 797 796 2834 2786 2783 2784 +823 824 798 797 2836 2788 2785 2786 +824 825 799 798 2838 2790 2787 2788 +825 826 800 799 2840 2792 2789 2790 +826 827 801 800 2842 2794 2791 2792 +827 828 802 801 2844 2796 2793 2794 +828 829 803 802 2846 2798 2795 2796 +829 830 804 803 2848 2800 2797 2798 +830 831 805 804 2850 2802 2799 2800 +831 832 806 805 2852 2803 2801 2802 +834 833 807 808 2855 2806 2804 2807 +833 835 809 807 2856 2809 2805 2806 +835 836 810 809 2859 2811 2808 2809 +836 837 811 810 2861 2813 2810 2811 +837 838 812 811 2863 2815 2812 2813 +838 839 813 812 2865 2817 2814 2815 +839 840 814 813 2867 2819 2816 2817 +840 841 815 814 2869 2821 2818 2819 +841 842 816 815 2871 2823 2820 2821 +842 843 817 816 2873 2825 2822 2823 +843 844 818 817 2875 2827 2824 2825 +844 845 819 818 2877 2829 2826 2827 +845 846 820 819 2879 2831 2828 2829 +846 847 821 820 2881 2833 2830 2831 +847 848 822 821 2883 2835 2832 2833 +848 849 823 822 2885 2837 2834 2835 +849 850 824 823 2887 2839 2836 2837 +850 851 825 824 2889 2841 2838 2839 +851 852 826 825 2891 2843 2840 2841 +852 853 827 826 2893 2845 2842 2843 +853 854 828 827 2895 2847 2844 2845 +854 855 829 828 2897 2849 2846 2847 +855 856 830 829 2899 2851 2848 2849 +856 857 831 830 2901 2853 2850 2851 +857 858 832 831 2903 2854 2852 2853 +860 859 833 834 2906 2857 2855 2858 +859 861 835 833 2907 2860 2856 2857 +861 862 836 835 2910 2862 2859 2860 +862 863 837 836 2912 2864 2861 2862 +863 864 838 837 2914 2866 2863 2864 +864 865 839 838 2916 2868 2865 2866 +865 866 840 839 2918 2870 2867 2868 +866 867 841 840 2920 2872 2869 2870 +867 868 842 841 2922 2874 2871 2872 +868 869 843 842 2924 2876 2873 2874 +869 870 844 843 2926 2878 2875 2876 +870 871 845 844 2928 2880 2877 2878 +871 872 846 845 2930 2882 2879 2880 +872 873 847 846 2932 2884 2881 2882 +873 874 848 847 2934 2886 2883 2884 +874 875 849 848 2936 2888 2885 2886 +875 876 850 849 2938 2890 2887 2888 +876 877 851 850 2940 2892 2889 2890 +877 878 852 851 2942 2894 2891 2892 +878 879 853 852 2944 2896 2893 2894 +879 880 854 853 2946 2898 2895 2896 +880 881 855 854 2948 2900 2897 2898 +881 882 856 855 2950 2902 2899 2900 +882 883 857 856 2952 2904 2901 2902 +883 884 858 857 2954 2905 2903 2904 +886 885 859 860 2957 2908 2906 2909 +888 887 885 886 2961 2958 2957 2960 +890 889 887 888 2965 2962 2961 2964 +892 891 889 890 2969 2966 2965 2968 +894 893 891 892 2973 2970 2969 2972 +896 895 893 894 2977 2974 2973 2976 +898 897 895 896 2981 2978 2977 2980 +900 899 897 898 2985 2982 2981 2984 +902 901 899 900 2989 2986 2985 2988 +904 903 901 902 2993 2990 2989 2992 +906 905 903 904 2997 2994 2993 2996 +908 907 905 906 3001 2998 2997 3000 +910 909 907 908 3005 3002 3001 3004 +1 2 909 910 1223 1228 3005 1225 +885 911 861 859 2959 2911 2907 2908 +887 912 911 885 2963 3007 2959 2958 +889 913 912 887 2967 3009 2963 2962 +891 914 913 889 2971 3011 2967 2966 +893 915 914 891 2975 3013 2971 2970 +895 916 915 893 2979 3015 2975 2974 +897 917 916 895 2983 3017 2979 2978 +899 918 917 897 2987 3019 2983 2982 +901 919 918 899 2991 3021 2987 2986 +903 920 919 901 2995 3023 2991 2990 +905 921 920 903 2999 3025 2995 2994 +907 922 921 905 3003 3027 2999 2998 +909 923 922 907 3006 3029 3003 3002 +2 3 923 909 1226 1231 3006 1228 +911 924 862 861 3008 2913 2910 2911 +912 925 924 911 3010 3032 3008 3007 +913 926 925 912 3012 3034 3010 3009 +914 927 926 913 3014 3036 3012 3011 +915 928 927 914 3016 3038 3014 3013 +916 929 928 915 3018 3040 3016 3015 +917 930 929 916 3020 3042 3018 3017 +918 931 930 917 3022 3044 3020 3019 +919 932 931 918 3024 3046 3022 3021 +920 933 932 919 3026 3048 3024 3023 +921 934 933 920 3028 3050 3026 3025 +922 935 934 921 3030 3052 3028 3027 +923 936 935 922 3031 3054 3030 3029 +3 4 936 923 1229 1234 3031 1231 +924 937 863 862 3033 2915 2912 2913 +925 938 937 924 3035 3057 3033 3032 +926 939 938 925 3037 3059 3035 3034 +927 940 939 926 3039 3061 3037 3036 +928 941 940 927 3041 3063 3039 3038 +929 942 941 928 3043 3065 3041 3040 +930 943 942 929 3045 3067 3043 3042 +931 944 943 930 3047 3069 3045 3044 +932 945 944 931 3049 3071 3047 3046 +933 946 945 932 3051 3073 3049 3048 +934 947 946 933 3053 3075 3051 3050 +935 948 947 934 3055 3077 3053 3052 +936 949 948 935 3056 3079 3055 3054 +4 5 949 936 1232 1237 3056 1234 +937 950 864 863 3058 2917 2914 2915 +938 951 950 937 3060 3082 3058 3057 +939 952 951 938 3062 3084 3060 3059 +940 953 952 939 3064 3086 3062 3061 +941 954 953 940 3066 3088 3064 3063 +942 955 954 941 3068 3090 3066 3065 +943 956 955 942 3070 3092 3068 3067 +944 957 956 943 3072 3094 3070 3069 +945 958 957 944 3074 3096 3072 3071 +946 959 958 945 3076 3098 3074 3073 +947 960 959 946 3078 3100 3076 3075 +948 961 960 947 3080 3102 3078 3077 +949 962 961 948 3081 3104 3080 3079 +5 6 962 949 1235 1240 3081 1237 +950 963 865 864 3083 2919 2916 2917 +951 964 963 950 3085 3107 3083 3082 +952 965 964 951 3087 3109 3085 3084 +953 966 965 952 3089 3111 3087 3086 +954 967 966 953 3091 3113 3089 3088 +955 968 967 954 3093 3115 3091 3090 +956 969 968 955 3095 3117 3093 3092 +957 970 969 956 3097 3119 3095 3094 +958 971 970 957 3099 3121 3097 3096 +959 972 971 958 3101 3123 3099 3098 +960 973 972 959 3103 3125 3101 3100 +961 974 973 960 3105 3127 3103 3102 +962 975 974 961 3106 3129 3105 3104 +6 7 975 962 1238 1243 3106 1240 +963 976 866 865 3108 2921 2918 2919 +964 977 976 963 3110 3132 3108 3107 +965 978 977 964 3112 3134 3110 3109 +966 979 978 965 3114 3136 3112 3111 +967 980 979 966 3116 3138 3114 3113 +968 981 980 967 3118 3140 3116 3115 +969 982 981 968 3120 3142 3118 3117 +970 983 982 969 3122 3144 3120 3119 +971 984 983 970 3124 3146 3122 3121 +972 985 984 971 3126 3148 3124 3123 +973 986 985 972 3128 3150 3126 3125 +974 987 986 973 3130 3152 3128 3127 +975 988 987 974 3131 3154 3130 3129 +7 8 988 975 1241 1246 3131 1243 +976 989 867 866 3133 2923 2920 2921 +977 990 989 976 3135 3157 3133 3132 +978 991 990 977 3137 3159 3135 3134 +979 992 991 978 3139 3161 3137 3136 +980 993 992 979 3141 3163 3139 3138 +981 994 993 980 3143 3165 3141 3140 +982 995 994 981 3145 3167 3143 3142 +983 996 995 982 3147 3169 3145 3144 +984 997 996 983 3149 3171 3147 3146 +985 998 997 984 3151 3173 3149 3148 +986 999 998 985 3153 3175 3151 3150 +987 1000 999 986 3155 3177 3153 3152 +988 1001 1000 987 3156 3179 3155 3154 +8 9 1001 988 1244 1249 3156 1246 +989 1002 868 867 3158 2925 2922 2923 +990 1003 1002 989 3160 3182 3158 3157 +991 1004 1003 990 3162 3184 3160 3159 +992 1005 1004 991 3164 3186 3162 3161 +993 1006 1005 992 3166 3188 3164 3163 +994 1007 1006 993 3168 3190 3166 3165 +995 1008 1007 994 3170 3192 3168 3167 +996 1009 1008 995 3172 3194 3170 3169 +997 1010 1009 996 3174 3196 3172 3171 +998 1011 1010 997 3176 3198 3174 3173 +999 1012 1011 998 3178 3200 3176 3175 +1000 1013 1012 999 3180 3202 3178 3177 +1001 1014 1013 1000 3181 3204 3180 3179 +9 10 1014 1001 1247 1252 3181 1249 +1002 1015 869 868 3183 2927 2924 2925 +1003 1016 1015 1002 3185 3207 3183 3182 +1004 1017 1016 1003 3187 3209 3185 3184 +1005 1018 1017 1004 3189 3211 3187 3186 +1006 1019 1018 1005 3191 3213 3189 3188 +1007 1020 1019 1006 3193 3215 3191 3190 +1008 1021 1020 1007 3195 3217 3193 3192 +1009 1022 1021 1008 3197 3219 3195 3194 +1010 1023 1022 1009 3199 3221 3197 3196 +1011 1024 1023 1010 3201 3223 3199 3198 +1012 1025 1024 1011 3203 3225 3201 3200 +1013 1026 1025 1012 3205 3227 3203 3202 +1014 1027 1026 1013 3206 3229 3205 3204 +10 11 1027 1014 1250 1255 3206 1252 +1015 1028 870 869 3208 2929 2926 2927 +1016 1029 1028 1015 3210 3232 3208 3207 +1017 1030 1029 1016 3212 3234 3210 3209 +1018 1031 1030 1017 3214 3236 3212 3211 +1019 1032 1031 1018 3216 3238 3214 3213 +1020 1033 1032 1019 3218 3240 3216 3215 +1021 1034 1033 1020 3220 3242 3218 3217 +1022 1035 1034 1021 3222 3244 3220 3219 +1023 1036 1035 1022 3224 3246 3222 3221 +1024 1037 1036 1023 3226 3248 3224 3223 +1025 1038 1037 1024 3228 3250 3226 3225 +1026 1039 1038 1025 3230 3252 3228 3227 +1027 1040 1039 1026 3231 3254 3230 3229 +11 12 1040 1027 1253 1258 3231 1255 +1028 1041 871 870 3233 2931 2928 2929 +1029 1042 1041 1028 3235 3257 3233 3232 +1030 1043 1042 1029 3237 3259 3235 3234 +1031 1044 1043 1030 3239 3261 3237 3236 +1032 1045 1044 1031 3241 3263 3239 3238 +1033 1046 1045 1032 3243 3265 3241 3240 +1034 1047 1046 1033 3245 3267 3243 3242 +1035 1048 1047 1034 3247 3269 3245 3244 +1036 1049 1048 1035 3249 3271 3247 3246 +1037 1050 1049 1036 3251 3273 3249 3248 +1038 1051 1050 1037 3253 3275 3251 3250 +1039 1052 1051 1038 3255 3277 3253 3252 +1040 1053 1052 1039 3256 3279 3255 3254 +12 13 1053 1040 1256 1261 3256 1258 +1041 1054 872 871 3258 2933 2930 2931 +1042 1055 1054 1041 3260 3282 3258 3257 +1043 1056 1055 1042 3262 3284 3260 3259 +1044 1057 1056 1043 3264 3286 3262 3261 +1045 1058 1057 1044 3266 3288 3264 3263 +1046 1059 1058 1045 3268 3290 3266 3265 +1047 1060 1059 1046 3270 3292 3268 3267 +1048 1061 1060 1047 3272 3294 3270 3269 +1049 1062 1061 1048 3274 3296 3272 3271 +1050 1063 1062 1049 3276 3298 3274 3273 +1051 1064 1063 1050 3278 3300 3276 3275 +1052 1065 1064 1051 3280 3302 3278 3277 +1053 1066 1065 1052 3281 3304 3280 3279 +13 14 1066 1053 1259 1264 3281 1261 +1054 1067 873 872 3283 2935 2932 2933 +1055 1068 1067 1054 3285 3307 3283 3282 +1056 1069 1068 1055 3287 3309 3285 3284 +1057 1070 1069 1056 3289 3311 3287 3286 +1058 1071 1070 1057 3291 3313 3289 3288 +1059 1072 1071 1058 3293 3315 3291 3290 +1060 1073 1072 1059 3295 3317 3293 3292 +1061 1074 1073 1060 3297 3319 3295 3294 +1062 1075 1074 1061 3299 3321 3297 3296 +1063 1076 1075 1062 3301 3323 3299 3298 +1064 1077 1076 1063 3303 3325 3301 3300 +1065 1078 1077 1064 3305 3327 3303 3302 +1066 1079 1078 1065 3306 3329 3305 3304 +14 15 1079 1066 1262 1267 3306 1264 +1067 1080 874 873 3308 2937 2934 2935 +1068 1081 1080 1067 3310 3332 3308 3307 +1069 1082 1081 1068 3312 3334 3310 3309 +1070 1083 1082 1069 3314 3336 3312 3311 +1071 1084 1083 1070 3316 3338 3314 3313 +1072 1085 1084 1071 3318 3340 3316 3315 +1073 1086 1085 1072 3320 3342 3318 3317 +1074 1087 1086 1073 3322 3344 3320 3319 +1075 1088 1087 1074 3324 3346 3322 3321 +1076 1089 1088 1075 3326 3348 3324 3323 +1077 1090 1089 1076 3328 3350 3326 3325 +1078 1091 1090 1077 3330 3352 3328 3327 +1079 1092 1091 1078 3331 3354 3330 3329 +15 16 1092 1079 1265 1270 3331 1267 +1080 1093 875 874 3333 2939 2936 2937 +1081 1094 1093 1080 3335 3357 3333 3332 +1082 1095 1094 1081 3337 3359 3335 3334 +1083 1096 1095 1082 3339 3361 3337 3336 +1084 1097 1096 1083 3341 3363 3339 3338 +1085 1098 1097 1084 3343 3365 3341 3340 +1086 1099 1098 1085 3345 3367 3343 3342 +1087 1100 1099 1086 3347 3369 3345 3344 +1088 1101 1100 1087 3349 3371 3347 3346 +1089 1102 1101 1088 3351 3373 3349 3348 +1090 1103 1102 1089 3353 3375 3351 3350 +1091 1104 1103 1090 3355 3377 3353 3352 +1092 1105 1104 1091 3356 3379 3355 3354 +16 17 1105 1092 1268 1273 3356 1270 +1093 1106 876 875 3358 2941 2938 2939 +1094 1107 1106 1093 3360 3382 3358 3357 +1095 1108 1107 1094 3362 3384 3360 3359 +1096 1109 1108 1095 3364 3386 3362 3361 +1097 1110 1109 1096 3366 3388 3364 3363 +1098 1111 1110 1097 3368 3390 3366 3365 +1099 1112 1111 1098 3370 3392 3368 3367 +1100 1113 1112 1099 3372 3394 3370 3369 +1101 1114 1113 1100 3374 3396 3372 3371 +1102 1115 1114 1101 3376 3398 3374 3373 +1103 1116 1115 1102 3378 3400 3376 3375 +1104 1117 1116 1103 3380 3402 3378 3377 +1105 1118 1117 1104 3381 3404 3380 3379 +17 18 1118 1105 1271 1276 3381 1273 +1106 1119 877 876 3383 2943 2940 2941 +1107 1120 1119 1106 3385 3407 3383 3382 +1108 1121 1120 1107 3387 3409 3385 3384 +1109 1122 1121 1108 3389 3411 3387 3386 +1110 1123 1122 1109 3391 3413 3389 3388 +1111 1124 1123 1110 3393 3415 3391 3390 +1112 1125 1124 1111 3395 3417 3393 3392 +1113 1126 1125 1112 3397 3419 3395 3394 +1114 1127 1126 1113 3399 3421 3397 3396 +1115 1128 1127 1114 3401 3423 3399 3398 +1116 1129 1128 1115 3403 3425 3401 3400 +1117 1130 1129 1116 3405 3427 3403 3402 +1118 1131 1130 1117 3406 3429 3405 3404 +18 19 1131 1118 1274 1279 3406 1276 +1119 1132 878 877 3408 2945 2942 2943 +1120 1133 1132 1119 3410 3432 3408 3407 +1121 1134 1133 1120 3412 3434 3410 3409 +1122 1135 1134 1121 3414 3436 3412 3411 +1123 1136 1135 1122 3416 3438 3414 3413 +1124 1137 1136 1123 3418 3440 3416 3415 +1125 1138 1137 1124 3420 3442 3418 3417 +1126 1139 1138 1125 3422 3444 3420 3419 +1127 1140 1139 1126 3424 3446 3422 3421 +1128 1141 1140 1127 3426 3448 3424 3423 +1129 1142 1141 1128 3428 3450 3426 3425 +1130 1143 1142 1129 3430 3452 3428 3427 +1131 1144 1143 1130 3431 3454 3430 3429 +19 20 1144 1131 1277 1282 3431 1279 +1132 1145 879 878 3433 2947 2944 2945 +1133 1146 1145 1132 3435 3457 3433 3432 +1134 1147 1146 1133 3437 3459 3435 3434 +1135 1148 1147 1134 3439 3461 3437 3436 +1136 1149 1148 1135 3441 3463 3439 3438 +1137 1150 1149 1136 3443 3465 3441 3440 +1138 1151 1150 1137 3445 3467 3443 3442 +1139 1152 1151 1138 3447 3469 3445 3444 +1140 1153 1152 1139 3449 3471 3447 3446 +1141 1154 1153 1140 3451 3473 3449 3448 +1142 1155 1154 1141 3453 3475 3451 3450 +1143 1156 1155 1142 3455 3477 3453 3452 +1144 1157 1156 1143 3456 3479 3455 3454 +20 21 1157 1144 1280 1285 3456 1282 +1145 1158 880 879 3458 2949 2946 2947 +1146 1159 1158 1145 3460 3482 3458 3457 +1147 1160 1159 1146 3462 3484 3460 3459 +1148 1161 1160 1147 3464 3486 3462 3461 +1149 1162 1161 1148 3466 3488 3464 3463 +1150 1163 1162 1149 3468 3490 3466 3465 +1151 1164 1163 1150 3470 3492 3468 3467 +1152 1165 1164 1151 3472 3494 3470 3469 +1153 1166 1165 1152 3474 3496 3472 3471 +1154 1167 1166 1153 3476 3498 3474 3473 +1155 1168 1167 1154 3478 3500 3476 3475 +1156 1169 1168 1155 3480 3502 3478 3477 +1157 1170 1169 1156 3481 3504 3480 3479 +21 22 1170 1157 1283 1288 3481 1285 +1158 1171 881 880 3483 2951 2948 2949 +1159 1172 1171 1158 3485 3507 3483 3482 +1160 1173 1172 1159 3487 3509 3485 3484 +1161 1174 1173 1160 3489 3511 3487 3486 +1162 1175 1174 1161 3491 3513 3489 3488 +1163 1176 1175 1162 3493 3515 3491 3490 +1164 1177 1176 1163 3495 3517 3493 3492 +1165 1178 1177 1164 3497 3519 3495 3494 +1166 1179 1178 1165 3499 3521 3497 3496 +1167 1180 1179 1166 3501 3523 3499 3498 +1168 1181 1180 1167 3503 3525 3501 3500 +1169 1182 1181 1168 3505 3527 3503 3502 +1170 1183 1182 1169 3506 3529 3505 3504 +22 23 1183 1170 1286 1291 3506 1288 +1171 1184 882 881 3508 2953 2950 2951 +1172 1185 1184 1171 3510 3532 3508 3507 +1173 1186 1185 1172 3512 3534 3510 3509 +1174 1187 1186 1173 3514 3536 3512 3511 +1175 1188 1187 1174 3516 3538 3514 3513 +1176 1189 1188 1175 3518 3540 3516 3515 +1177 1190 1189 1176 3520 3542 3518 3517 +1178 1191 1190 1177 3522 3544 3520 3519 +1179 1192 1191 1178 3524 3546 3522 3521 +1180 1193 1192 1179 3526 3548 3524 3523 +1181 1194 1193 1180 3528 3550 3526 3525 +1182 1195 1194 1181 3530 3552 3528 3527 +1183 1196 1195 1182 3531 3554 3530 3529 +23 24 1196 1183 1289 1294 3531 1291 +1184 1197 883 882 3533 2955 2952 2953 +1185 1198 1197 1184 3535 3557 3533 3532 +1186 1199 1198 1185 3537 3559 3535 3534 +1187 1200 1199 1186 3539 3561 3537 3536 +1188 1201 1200 1187 3541 3563 3539 3538 +1189 1202 1201 1188 3543 3565 3541 3540 +1190 1203 1202 1189 3545 3567 3543 3542 +1191 1204 1203 1190 3547 3569 3545 3544 +1192 1205 1204 1191 3549 3571 3547 3546 +1193 1206 1205 1192 3551 3573 3549 3548 +1194 1207 1206 1193 3553 3575 3551 3550 +1195 1208 1207 1194 3555 3577 3553 3552 +1196 1209 1208 1195 3556 3579 3555 3554 +24 25 1209 1196 1292 1297 3556 1294 +1197 1210 884 883 3558 2956 2954 2955 +1198 1211 1210 1197 3560 3582 3558 3557 +1199 1212 1211 1198 3562 3583 3560 3559 +1200 1213 1212 1199 3564 3584 3562 3561 +1201 1214 1213 1200 3566 3585 3564 3563 +1202 1215 1214 1201 3568 3586 3566 3565 +1203 1216 1215 1202 3570 3587 3568 3567 +1204 1217 1216 1203 3572 3588 3570 3569 +1205 1218 1217 1204 3574 3589 3572 3571 +1206 1219 1218 1205 3576 3590 3574 3573 +1207 1220 1219 1206 3578 3591 3576 3575 +1208 1221 1220 1207 3580 3592 3578 3577 +1209 1222 1221 1208 3581 3593 3580 3579 +25 26 1222 1209 1295 1299 3581 1297 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh.vtu b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh.vtu new file mode 100644 index 0000000000000000000000000000000000000000..7b972e776937449e6a4bd8b446e45acaf115b473 --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh.vtu @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08780419dcfbb694657fd9809a81b7fc1d79de453c06896dbbf2c47c3a37305c +size 192799 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cell_properties_cfg3.bin b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cell_properties_cfg3.bin new file mode 100644 index 0000000000000000000000000000000000000000..fa6b9230e4e642555a2399dd1f482fcec1de802a --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cell_properties_cfg3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b67da24d2c90f3c54b467f274707f56b19aeb46637730f82b6f8de4cf37e8fe7 +size 228 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cell_properties_val3.bin b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cell_properties_val3.bin new file mode 100644 index 0000000000000000000000000000000000000000..7c2a7404a4a4d5bafa263b0745dc41aaeae5247c --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cell_properties_val3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:220c30f2ec39422ac36797b4f5e41f179c8214932e19cdde4ee572f82e8828d8 +size 54032 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cfg3.msh b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cfg3.msh new file mode 100644 index 0000000000000000000000000000000000000000..1f974e5370a5d3329f292b6bacab73e001f44f76 --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_cfg3.msh @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8de456010fdf4e4025b4f03ee4a8478c9f4e87a90d878c5725564b171f7a8e30 +size 503 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_elems_3.msh b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_elems_3.msh new file mode 100644 index 0000000000000000000000000000000000000000..78215cb8ffe85e80a45fd0f59e6e8bdfdcd4d4ab --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_elems_3.msh @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:139a4f33f6037430911e858c921d2b88a0fb130cd5a342a78ec51302cb19524e +size 47773 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_cfg3.bin b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_cfg3.bin new file mode 100644 index 0000000000000000000000000000000000000000..f0bab41be88a09f5a11b763f823201d719a9269e --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_cfg3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:800935fadd27ff85acdf6b6cf5ea5fabeddfa2ed3baee1f0ae5154e77ae59122 +size 336 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_ele3.bin b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_ele3.bin new file mode 100644 index 0000000000000000000000000000000000000000..020a1d78b94cb71344a9a5bedf1dcbdafc7042ea --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_ele3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29197c0b1baae92114a588ccc4cf54608750f9d6fa047b1c47648cab1a996d2f +size 106712 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_ele_g3.bin b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_ele_g3.bin new file mode 100644 index 0000000000000000000000000000000000000000..323502d0dd78d13397503d07fd51725b9e6d8ac1 --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_ele_g3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f1c0cb22f614126be323256cc178d86553a250713bf2f5524a660d6946c19c6 +size 10176 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_nod3.bin b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_nod3.bin new file mode 100644 index 0000000000000000000000000000000000000000..e14b73aac7d7164127b6b4c345d8a29f4f89f95d --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_msh_nod3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8470cc34280dceb20d270055b5f9c877f578205a052a95e5b2ad6b79a12a2bd0 +size 123648 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_node_properties_cfg3.bin b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_node_properties_cfg3.bin new file mode 100644 index 0000000000000000000000000000000000000000..dc691d8a50825d66dc1d1efa9a27dbd1dfd27974 --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_node_properties_cfg3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f2180cf64c8feec456019ba05792066b9d8af42b37e2cb4db630465f9392103 +size 291 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_node_properties_val3.bin b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_node_properties_val3.bin new file mode 100644 index 0000000000000000000000000000000000000000..b3ef390ac39b5e81508b9a1c0b1ae2488a3c2185 --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_node_properties_val3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d2d56e4eebf083f24c5376b537b468941f927cc8d2325210852aeed9578efe6 +size 216384 diff --git a/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_nodes_3.msh b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_nodes_3.msh new file mode 100644 index 0000000000000000000000000000000000000000..62095a3012387a5821d99bc4be3b59e8216b0ab7 --- /dev/null +++ b/Tests/Data/NodePartitionedMesh/partmesh_2Dmesh_3partitions/2Dmesh_partitioned_nodes_3.msh @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b155e753587d6a997f32270a246e5abb0d2a91f088daa81d09dccf0f52ae7f5 +size 273187 diff --git a/ThirdParty/CMakeLists.txt b/ThirdParty/CMakeLists.txt index a78c2eb6eb641320d9463edc291ddf3c352c4d19..ea686f5ee2fa134513f1f41aa989f790cf94806b 100644 --- a/ThirdParty/CMakeLists.txt +++ b/ThirdParty/CMakeLists.txt @@ -45,6 +45,6 @@ if(OGS_BUILD_SWMM) set_target_properties(SWMM SwmmInterface PROPERTIES COMPILE_FLAGS /W0) endif() -if(OGS_BUILD_METIS) +if(OGS_BUILD_UTILS) include(${PROJECT_SOURCE_DIR}/scripts/cmake/MetisSetup.cmake) endif() diff --git a/scripts/cmake/SubmoduleSetup.cmake b/scripts/cmake/SubmoduleSetup.cmake index 6871e34ee6da0faf4501dd551dc5225e2ba92218..09d8d084c1f9ae92197c939cbc2f69bab8629af3 100644 --- a/scripts/cmake/SubmoduleSetup.cmake +++ b/scripts/cmake/SubmoduleSetup.cmake @@ -12,7 +12,8 @@ set(REQUIRED_SUBMODULES ThirdParty/tetgen ${OGS_ADDITIONAL_SUBMODULES_TO_CHECKOUT} ) -if(OGS_BUILD_METIS) +if(OGS_BUILD_UTILS) + # Required by the partmesh tool, which is build with utils only. list(APPEND REQUIRED_SUBMODULES ThirdParty/metis) endif() if(OGS_BUILD_SWMM) diff --git a/scripts/cmake/test/Test.cmake b/scripts/cmake/test/Test.cmake index 1c1f108fb741905fded290147f6588a1cbadc612..81185a61d2ed9333ce2f1193ab189974c4db0f2f 100644 --- a/scripts/cmake/test/Test.cmake +++ b/scripts/cmake/test/Test.cmake @@ -53,6 +53,10 @@ add_custom_target( DEPENDS ogs vtkdiff ctest-cleanup USES_TERMINAL ) +if(OGS_BUILD_UTILS) + add_dependencies(ctest partmesh MapGeometryToMeshSurface) +endif() + add_custom_target( ctest-serial COMMAND ${CMAKE_CTEST_COMMAND} -T Test @@ -64,6 +68,10 @@ add_custom_target( DEPENDS ogs vtkdiff ctest-cleanup USES_TERMINAL ) +if(OGS_BUILD_UTILS) + add_dependencies(ctest-serial partmesh MapGeometryToMeshSurface) +endif() + add_custom_target(ctest-large-cleanup ${CMAKE_COMMAND} -E remove -f Tests/ctest-large.log) add_custom_target( ctest-large @@ -74,6 +82,10 @@ add_custom_target( DEPENDS ogs vtkdiff ctest-large-cleanup USES_TERMINAL ) +if(OGS_BUILD_UTILS) + add_dependencies(ctest-large partmesh MapGeometryToMeshSurface) +endif() + add_custom_target( ctest-large-serial COMMAND ${CMAKE_CTEST_COMMAND} -T Test @@ -83,6 +95,9 @@ add_custom_target( DEPENDS ogs vtkdiff ctest-large-cleanup USES_TERMINAL ) +if(OGS_BUILD_UTILS) + add_dependencies(ctest-large-serial partmesh MapGeometryToMeshSurface) +endif() set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${PROJECT_BINARY_DIR}/Tests/Data ) diff --git a/scripts/env/envinf1/mpi.sh b/scripts/env/envinf1/mpi.sh index 5c99bf4abe3de4d2430c85b4b57c37a5322a8c3c..f03066b37a8a8bd9093350706a9f48ea74715c2b 100644 --- a/scripts/env/envinf1/mpi.sh +++ b/scripts/env/envinf1/mpi.sh @@ -1,4 +1,2 @@ DIR=$( cd $(dirname "${BASH_SOURCE[0]}") ; pwd -P ) source $DIR/cli.sh -export CC=`which mpicc` -export CXX=`which mpicxx` diff --git a/web/content/docs/tools/workflows/create-a-simple-parallel-model/index.pandoc b/web/content/docs/tools/workflows/create-a-simple-parallel-model/index.pandoc index 28b0b458936054821a4a033d826ea655f0f32917..dbc666c530132c554f15be7f2f9d18c8b3d3f095 100644 --- a/web/content/docs/tools/workflows/create-a-simple-parallel-model/index.pandoc +++ b/web/content/docs/tools/workflows/create-a-simple-parallel-model/index.pandoc @@ -10,18 +10,16 @@ author = "Thomas Fischer" ## Software Modules and Compilation on EVE -- Modules to load: - - `module load gcc/4.8.1-3` - - `module load boost` - - `module load vtk/7.1.0_openmpi-1.8.4-1` - Checkout the OGS-6 source code -- Build utils with metis enable to + - Source the environment scripts `scripts/env/envinf1/cli.sh` +- Configure with utils enabled to: - Create the binary for generation of the structured mesh - - Cretae the binary for the partioner - - `cmake <path_to_source> -DOGS_BUILD_METIS=ON -DOGS_BUILD_UTILS=ON && make` -- Build OGS for parallel usage: - - `module load petsc/3.7.2_maint_petsc_maint_3.7.2_gcc_openmpi_1.8.4-2` - - `cmake <path_to_source> -DOGS_USE_PETSC=ON && make -j` + - Create the binary for the mesh partioner tool `partmesh` + - `cmake <path_to_source> -DOGS_BUILD_UTILS=ON` +- Configure OGS for parallel usage with PETSc: + - Source the environment scripts `scripts/env/envinf1/petsc.sh` + - `cmake <path_to_source> -DOGS_USE_PETSC=ON` +- Build the source code with `make` ## Create a structured mesh