Commit 9094e58e authored by Dmitry Yu. Naumov's avatar Dmitry Yu. Naumov
Browse files

Merge branch 'FixParallelOutputBug' into 'master'

[Output] Fix bug in parallel output of non-bulk meshes distributed to multiple domains.

Closes #3126

See merge request ogs/ogs!3576
parents 3034c39f 6d28eb89
......@@ -447,6 +447,26 @@ AddTest(
LF_square_1x1_tri_1.8e1_surfaceflux_ts_2_t_0.864000_expected.vtu LF_square_1x1_tri_1.8e1_surfaceflux_square_1x1_tri_1.8e1_ts_2_t_0.864000.vtu pressure pressure 1e-7 1e-13
)
AddTest(
NAME LiquidFlow_Flux_3D_HEX_Parallel_2
PATH Parabolic/LiquidFlow/Flux/3D/Hex/Parallel
EXECUTABLE ogs
EXECUTABLE_ARGS cuboid_1x1x1_hex_27_Dirichlet_Dirichlet.prj
WRAPPER mpirun
WRAPPER_ARGS -np 2
TESTER vtkdiff
REQUIREMENTS OGS_USE_MPI
DIFF_DATA
top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_ts_2_t_86400_000000_0.vtu top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_ts_2_t_86400_000000_0.vtu pressure pressure 1e-10 1e-15
top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_ts_2_t_86400_000000_1.vtu top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_ts_2_t_86400_000000_1.vtu pressure pressure 1e-10 1e-15
top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_bottom_boundary_ts_2_t_86400_000000_0.vtu top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_bottom_boundary_ts_2_t_86400_000000_0.vtu pressure pressure 1e-7 1e-13
top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_bottom_boundary_ts_2_t_86400_000000_1.vtu top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_bottom_boundary_ts_2_t_86400_000000_1.vtu pressure pressure 1e-7 1e-13
top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_top_boundary_ts_2_t_86400_000000_0.vtu top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_top_boundary_ts_2_t_86400_000000_0.vtu pressure pressure 1e-7 1e-13
top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_top_boundary_ts_2_t_86400_000000_1.vtu top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_top_boundary_ts_2_t_86400_000000_1.vtu pressure pressure 1e-7 1e-13
top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_entire_boundary_ts_2_t_86400_000000_0.vtu top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_entire_boundary_ts_2_t_86400_000000_0.vtu specific_flux specific_flux 1e-7 1e-13
top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_entire_boundary_ts_2_t_86400_000000_1.vtu top_boundary_to_bottom_boundary_cuboid_1x1x1_hex_27_entire_boundary_ts_2_t_86400_000000_1.vtu specific_flux specific_flux 1e-7 1e-13
)
AddTest(
NAME SimpleSynthetics_XDMF
PATH Parabolic/LiquidFlow/SimpleSynthetics/XDMF
......
......@@ -284,7 +284,7 @@ void Output::doOutputAlways(Process const& process,
bool output_secondary_variable = true;
// Need to add variables of process to vtu even no output takes place.
addProcessDataToMesh(t, x, process_id, process.getMesh(), dof_tables,
process.getProcessVariables(process_id),
dof_tables, process.getProcessVariables(process_id),
process.getSecondaryVariables(),
output_secondary_variable,
process.getIntegrationPointWriter(process.getMesh()),
......@@ -339,16 +339,16 @@ void Output::doOutputAlways(Process const& process,
continue;
}
// mesh related output
auto& mesh = *BaseLib::findElementOrError(
auto& non_bulk_mesh = *BaseLib::findElementOrError(
begin(_meshes), end(_meshes),
[&mesh_output_name](auto const& m) {
return m->getName() == mesh_output_name;
},
"Need mesh '" + mesh_output_name + "' for the output.");
std::vector<MeshLib::Node*> const& nodes = mesh.getNodes();
std::vector<MeshLib::Node*> const& nodes = non_bulk_mesh.getNodes();
DBUG("Found {:d} nodes for output at mesh '{:s}'.", nodes.size(),
mesh.getName());
non_bulk_mesh.getName());
std::vector<std::unique_ptr<NumLib::LocalToGlobalIndexMap>>
mesh_dof_tables;
......@@ -357,7 +357,7 @@ void Output::doOutputAlways(Process const& process,
{
mesh_dof_tables.push_back(
process.getDOFTable(i).deriveBoundaryConstrainedMap(
MeshLib::MeshSubset{mesh, nodes}));
MeshLib::MeshSubset{non_bulk_mesh, nodes}));
}
std::vector<NumLib::LocalToGlobalIndexMap const*>
mesh_dof_table_pointers;
......@@ -369,18 +369,14 @@ void Output::doOutputAlways(Process const& process,
});
output_secondary_variable = false;
addProcessDataToMesh(t, x, process_id, mesh, mesh_dof_table_pointers,
process.getProcessVariables(process_id),
process.getSecondaryVariables(),
output_secondary_variable,
process.getIntegrationPointWriter(mesh),
_output_data_specification);
// TODO (TomFischer): add pvd support here. This can be done if the
// output is mesh related instead of process related. This would also
// allow for merging bulk mesh output and arbitrary mesh output.
output_bulk_mesh(mesh);
addProcessDataToMesh(
t, x, process_id, non_bulk_mesh, dof_tables,
mesh_dof_table_pointers, process.getProcessVariables(process_id),
process.getSecondaryVariables(), output_secondary_variable,
process.getIntegrationPointWriter(non_bulk_mesh),
_output_data_specification);
output_bulk_mesh(non_bulk_mesh);
}
INFO("[time] Output of timestep {:d} took {:g} s.", timestep,
time_output.elapsed());
......@@ -444,7 +440,7 @@ void Output::doOutputNonlinearIteration(Process const& process,
bool const output_secondary_variable = true;
addProcessDataToMesh(t, x, process_id, process.getMesh(), dof_tables,
process.getProcessVariables(process_id),
dof_tables, process.getProcessVariables(process_id),
process.getSecondaryVariables(),
output_secondary_variable,
process.getIntegrationPointWriter(process.getMesh()),
......
......@@ -19,6 +19,9 @@
#include "InfoLib/GitInfo.h"
#include "IntegrationPointWriter.h"
#include "MathLib/LinAlg/LinAlg.h"
#ifdef USE_PETSC
#include "MeshLib/NodePartitionedMesh.h"
#endif
#include "MeshLib/IO/VtkIO/VtuInterface.h"
#include "MeshLib/IO/XDMF/XdmfHdfWriter.h"
#include "NumLib/DOF/LocalToGlobalIndexMap.h"
......@@ -136,6 +139,8 @@ namespace ProcessLib
void addProcessDataToMesh(
const double t, std::vector<GlobalVector*> const& x, int const process_id,
MeshLib::Mesh& mesh,
[[maybe_unused]] std::vector<NumLib::LocalToGlobalIndexMap const*> const&
bulk_dof_table,
std::vector<NumLib::LocalToGlobalIndexMap const*> const& dof_table,
std::vector<std::reference_wrapper<ProcessVariable>> const&
process_variables,
......@@ -207,6 +212,48 @@ void addProcessDataToMesh(
auto const mesh_id = mesh_subset.getMeshID();
for (auto const* node : mesh_subset.getNodes())
{
#ifdef USE_PETSC
if (bulk_dof_table[process_id] != dof_table[process_id])
{
if (!mesh.getProperties().existsPropertyVector<std::size_t>(
"bulk_node_ids"))
{
OGS_FATAL(
"The required bulk node ids map does not exist in "
"the boundary mesh '{:s}' or has the wrong data "
"type (should be equivalent to C++ data type "
"std::size_t which is an unsigned integer of size "
"{:d} or UInt64 in vtk terminology).",
mesh.getName(), sizeof(std::size_t));
}
auto const bulk_node_id_map =
*mesh.getProperties().getPropertyVector<std::size_t>(
"bulk_node_ids");
if (static_cast<MeshLib::NodePartitionedMesh const&>(mesh)
.isGhostNode(node->getID()))
{
auto const bulk_node_id =
bulk_node_id_map[node->getID()];
// use bulk_dof_table to find information
std::size_t const bulk_mesh_id = 0;
MeshLib::Location const l(bulk_mesh_id,
MeshLib::MeshItemType::Node,
bulk_node_id);
auto const global_component_id =
global_component_offset + component_id;
auto const index =
bulk_dof_table[process_id]->getLocalIndex(
l, global_component_id,
x[process_id]->getRangeBegin(),
x[process_id]->getRangeEnd());
output_data[node->getID() * n_components +
component_id] = x_copy[index];
continue;
}
}
#endif
MeshLib::Location const l(mesh_id, MeshLib::MeshItemType::Node,
node->getID());
......
......@@ -33,7 +33,8 @@ struct OutputDataSpecification final
void addProcessDataToMesh(
const double t, std::vector<GlobalVector*> const& x, int const process_id,
MeshLib::Mesh& mesh,
std::vector<NumLib::LocalToGlobalIndexMap const*> const& dof_table,
std::vector<NumLib::LocalToGlobalIndexMap const*> const& bulk_dof_tables,
std::vector<NumLib::LocalToGlobalIndexMap const*> const& dof_tables,
std::vector<std::reference_wrapper<ProcessVariable>> const&
process_variables,
SecondaryVariableCollection const& secondary_variables,
......
<?xml version="1.0" encoding="ISO-8859-1"?>
<OpenGeoSysProject>
<meshes>
<mesh>cuboid_1x1x1_hex_27.vtu</mesh>
<mesh>cuboid_1x1x1_hex_27_entire_boundary.vtu</mesh>
<mesh>cuboid_1x1x1_hex_27_top_boundary.vtu</mesh>
<mesh>cuboid_1x1x1_hex_27_bottom_boundary.vtu</mesh>
</meshes>
<processes>
<process>
<name>LiquidFlow</name>
<type>LIQUID_FLOW</type>
<integration_order>2</integration_order>
<darcy_gravity>
<!-- axis_id: 0, 1, or the dimension of space minus one -->
<axis_id>0</axis_id>
<!-- g>=0. g=0: non gravity term -->
<g>0.</g>
</darcy_gravity>
<process_variables>
<process_variable>pressure</process_variable>
</process_variables>
<secondary_variables>
<secondary_variable internal_name="darcy_velocity" output_name="v"/>
</secondary_variables>
<calculatesurfaceflux>
<mesh>cuboid_1x1x1_hex_27_entire_boundary</mesh>
<property_name>specific_flux</property_name>
</calculatesurfaceflux>
</process>
</processes>
<time_loop>
<processes>
<process ref="LiquidFlow">
<nonlinear_solver>basic_picard</nonlinear_solver>
<convergence_criterion>
<type>DeltaX</type>
<norm_type>NORM2</norm_type>
<abstol>1.e-10</abstol>
</convergence_criterion>
<time_discretization>
<type>BackwardEuler</type>
</time_discretization>
<time_stepping>
<type>FixedTimeStepping</type>
<t_initial> 0.0 </t_initial>
<t_end> 86400 </t_end>
<timesteps>
<pair>
<repeat>2</repeat>
<delta_t>43200</delta_t>
</pair>
</timesteps>
</time_stepping>
</process>
</processes>
<output>
<type>VTK</type>
<prefix>top_boundary_to_bottom_boundary_{:meshname}</prefix>
<suffix>_ts_{:timestep}_t_{:time}</suffix>
<timesteps>
<pair>
<repeat> 1 </repeat>
<each_steps> 1 </each_steps>
</pair>
</timesteps>
<variables>
<variable> pressure </variable>
<variable> v </variable>
</variables>
<meshes>
<mesh>cuboid_1x1x1_hex_27</mesh>
<mesh>cuboid_1x1x1_hex_27_top_boundary</mesh>
<mesh>cuboid_1x1x1_hex_27_bottom_boundary</mesh>
<mesh>cuboid_1x1x1_hex_27_entire_boundary</mesh>
</meshes>
</output>
</time_loop>
<media>
<medium id="0">
<phases>
<phase>
<type>AqueousLiquid</type>
<properties>
<property>
<name>viscosity</name>
<type>Constant</type>
<value> 1.295e-4 </value>
</property>
<property>
<name>density</name>
<type>Constant</type>
<value> 78.68 </value>
</property>
</properties>
</phase>
</phases>
<properties>
<property>
<name>permeability</name>
<type>Constant</type>
<value>9.2e-12 0 0 0 9.2e-12 0 0 0 9.2e-12</value>
</property>
<property>
<name>reference_temperature</name>
<type>Constant</type>
<value>293.15</value>
</property>
<property>
<name>porosity</name>
<type>Constant</type>
<value>1</value>
</property>
<property>
<name>storage</name>
<type>Constant</type>
<value> 0.00e-10 </value>
</property>
</properties>
</medium>
</media>
<parameters>
<parameter>
<name>p0</name>
<type>Constant</type>
<value>5e6</value>
</parameter>
<parameter>
<name>top_boundary_Dirichlet</name>
<type>Constant</type>
<value>1e7</value>
</parameter>
<parameter>
<name>bottom_boundary_Dirichlet</name>
<type>Constant</type>
<value>1e6</value>
</parameter>
<parameter>
<name>constant_porosity_parameter</name>
<type>Constant</type>
<value>1</value>
</parameter>
<parameter>
<name>kappa1</name>
<type>Constant</type>
<values>9.2e-12 0 0 0 9.2e-12 0 0 0 9.2e-12</values>
</parameter>
<parameter>
<name>p_spatial</name>
<type>Constant</type>
<value>1</value>
</parameter>
</parameters>
<process_variables>
<process_variable>
<name>pressure</name>
<components>1</components>
<order>1</order>
<initial_condition>p0</initial_condition>
<boundary_conditions>
<boundary_condition>
<mesh>cuboid_1x1x1_hex_27_top_boundary</mesh>
<type>Dirichlet</type>
<parameter>top_boundary_Dirichlet</parameter>
</boundary_condition>
<boundary_condition>
<mesh>cuboid_1x1x1_hex_27_bottom_boundary</mesh>
<type>Dirichlet</type>
<parameter>bottom_boundary_Dirichlet</parameter>
</boundary_condition>
</boundary_conditions>
</process_variable>
</process_variables>
<nonlinear_solvers>
<nonlinear_solver>
<name>basic_picard</name>
<type>Picard</type>
<max_iter>10</max_iter>
<linear_solver>general_linear_solver</linear_solver>
</nonlinear_solver>
</nonlinear_solvers>
<linear_solvers>
<linear_solver>
<name>general_linear_solver</name>
<lis>-i cg -p jacobi -tol 1e-16 -maxiter 10000</lis>
<eigen>
<solver_type>CG</solver_type>
<precon_type>DIAGONAL</precon_type>
<max_iteration_step>10000</max_iteration_step>
<error_tolerance>1e-20</error_tolerance>
</eigen>
<petsc>
<prefix>lf</prefix>
<parameters>-lf_ksp_type cg -lf_pc_type bjacobi -lf_ksp_rtol 1e-16 -lf_ksp_max_it 10000</parameters>
</petsc>
</linear_solver>
</linear_solvers>
</OpenGeoSysProject>
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment