Eigen iterative solvers use only single thread
Description
Usually, Eigen iterative solvers use shared-memory parallelization. However, since some time they use only one thread only. For Pardiso the parallelization works fine. It seems to be broken already before !4132 (6dc09812) A version that is about some weeks old works. Therefore, I think it is unrelated to #3296 (closed). This behaviour was also reported by @skai95. Multiple threads are also not working with clang, but this issue might even be older.
Steps to Reproduce the Problem
- OGS is compiled with
cmake -DOGS_USE_MKL="ON" -DCMAKE_BUILD_TYPE="Release" -G Ninja ..
- MKL BLAS variables were emptied such that openblas can be used
- I used the following benchmark for my tests: https://gitlab.opengeosys.org/ogs/ogs/-/blob/master/Tests/Data/ThermoRichardsMechanics/TaskCDECOVALEX2023/Decovalex-0.prj with BiCGSTAB and DIAGONAL preconditioning.
- Only one thread is used in the newer version, and it is causing a significant runtime increase:
- the older version running with 4 threads and takes 46.231 s
- the newer version runs with 1 thread and takes 73.2987 s.
ldd of working OGS version:
linux-vdso.so.1 (0x00007ffc8a3f7000)
libmpi_cxx.so.40 => /usr/lib/openmpi/libmpi_cxx.so.40 (0x00007f014957c000)
libmpi.so.40 => /usr/lib/openmpi/libmpi.so.40 (0x00007f0149453000)
libhdf5.so.200 => /usr/lib/libhdf5.so.200 (0x00007f0149000000)
libsz.so.2 => /usr/lib/libsz.so.2 (0x00007f0148ff4000)
libz.so.1 => /usr/lib/libz.so.1 (0x00007f0148fd8000)
libvtkIOXML.so.1 => /usr/lib/libvtkIOXML.so.1 (0x00007f0148ed9000)
libvtkIOXMLParser.so.1 => /usr/lib/libvtkIOXMLParser.so.1 (0x00007f0148ec1000)
libvtkIOLegacy.so.1 => /usr/lib/libvtkIOLegacy.so.1 (0x00007f0148e1d000)
libvtkIOCore.so.1 => /usr/lib/libvtkIOCore.so.1 (0x00007f0148dac000)
libvtkCommonExecutionModel.so.1 => /usr/lib/libvtkCommonExecutionModel.so.1 (0x00007f0148c59000)
libvtkCommonDataModel.so.1 => /usr/lib/libvtkCommonDataModel.so.1 (0x00007f0148730000)
libvtkCommonTransforms.so.1 => /usr/lib/libvtkCommonTransforms.so.1 (0x00007f01486df000)
libvtkCommonMath.so.1 => /usr/lib/libvtkCommonMath.so.1 (0x00007f01486ba000)
libvtkCommonCore.so.1 => /usr/lib/libvtkCommonCore.so.1 (0x00007f0147bce000)
libvtksys.so.1 => /usr/lib/libvtksys.so.1 (0x00007f0147b83000)
libtbb.so.12 => /opt/intel/oneapi/tbb/2021.5.1/env/../lib/intel64/gcc4.8/libtbb.so.12 (0x00007f0147800000)
libvtkkissfft.so.1 => /usr/lib/libvtkkissfft.so.1 (0x00007f0147b79000)
libxml2.so.2 => /usr/lib/libxml2.so.2 (0x00007f0147676000)
libopenblas.so.3 => /usr/lib/libopenblas.so.3 (0x00007f01462c0000)
libmkl_intel_lp64.so.2 => /opt/intel/oneapi/mkl/2022.0.2/lib/intel64/libmkl_intel_lp64.so.2 (0x00007f0145420000)
libmkl_gnu_thread.so.2 => /opt/intel/oneapi/mkl/2022.0.2/lib/intel64/libmkl_gnu_thread.so.2 (0x00007f014388d000)
libmkl_core.so.2 => /opt/intel/oneapi/mkl/2022.0.2/lib/intel64/libmkl_core.so.2 (0x00007f013f4d8000)
libm.so.6 => /usr/lib/libm.so.6 (0x00007f0147a8f000)
libgomp.so.1 => /usr/lib/libgomp.so.1 (0x00007f013f493000)
libpython3.10.so.1.0 => /usr/lib/libpython3.10.so.1.0 (0x00007f013f0d8000)
libstdc++.so.6 => /usr/lib/libstdc++.so.6 (0x00007f013eeb2000)
libgcc_s.so.1 => /usr/lib/libgcc_s.so.1 (0x00007f013ee97000)
libc.so.6 => /usr/lib/libc.so.6 (0x00007f013ec8d000)
libopen-rte.so.40 => /usr/lib/openmpi/libopen-rte.so.40 (0x00007f013ebd3000)
libopen-pal.so.40 => /usr/lib/openmpi/libopen-pal.so.40 (0x00007f013eb28000)
libdl.so.2 => /usr/lib/libdl.so.2 (0x00007f0147a88000)
librt.so.1 => /usr/lib/librt.so.1 (0x00007f0147a83000)
libutil.so.1 => /usr/lib/libutil.so.1 (0x00007f0147a7e000)
libhwloc.so.15 => /home/buchwalj/github/petsc/arch-linux-c-opt/lib/libhwloc.so.15 (0x00007f013eac5000)
libevent_core-2.1.so.7 => /usr/lib/libevent_core-2.1.so.7 (0x00007f013ea8c000)
libevent_pthreads-2.1.so.7 => /usr/lib/libevent_pthreads-2.1.so.7 (0x00007f0147671000)
libpthread.so.0 => /usr/lib/libpthread.so.0 (0x00007f013ea87000)
libvtkCommonSystem.so.1 => /usr/lib/libvtkCommonSystem.so.1 (0x00007f013ea4a000)
libvtkCommonMisc.so.1 => /usr/lib/libvtkCommonMisc.so.1 (0x00007f013e638000)
libexpat.so.1 => /usr/lib/libexpat.so.1 (0x00007f013e607000)
libdouble-conversion.so.3 => /usr/lib/libdouble-conversion.so.3 (0x00007f013e5f1000)
liblz4.so.1 => /usr/lib/liblz4.so.1 (0x00007f013e5ce000)
liblzma.so.5 => /usr/lib/liblzma.so.5 (0x00007f013e5a4000)
libpugixml.so.1 => /usr/lib/libpugixml.so.1 (0x00007f013e567000)
libvtkloguru.so.1 => /usr/lib/libvtkloguru.so.1 (0x00007f013e531000)
/lib64/ld-linux-x86-64.so.2 => /usr/lib64/ld-linux-x86-64.so.2 (0x00007f014a2b5000)
libicuuc.so.71 => /usr/lib/libicuuc.so.71 (0x00007f013e332000)
libudev.so.1 => /usr/lib/libudev.so.1 (0x00007f013e306000)
libpciaccess.so.0 => /usr/lib/libpciaccess.so.0 (0x00007f013e2fa000)
libX11.so.6 => /usr/lib/libX11.so.6 (0x00007f013e1b6000)
libicudata.so.71 => /usr/lib/libicudata.so.71 (0x00007f013c4b1000)
libxcb.so.1 => /usr/lib/libxcb.so.1 (0x00007f013c486000)
libXau.so.6 => /usr/lib/libXau.so.6 (0x00007f013c481000)
libXdmcp.so.6 => /usr/lib/libXdmcp.so.6 (0x00007f013c479000)
ldd of not working version
linux-vdso.so.1 (0x00007ffe74bf5000)
libvtkParallelMPI.so.1 => /usr/lib/libvtkParallelMPI.so.1 (0x00007f73ad7ac000)
libmpi_cxx.so.40 => /usr/lib/openmpi/libmpi_cxx.so.40 (0x00007f73ad78f000)
libmpi.so.40 => /usr/lib/openmpi/libmpi.so.40 (0x00007f73ad666000)
libhdf5.so.200 => /usr/lib/libhdf5.so.200 (0x00007f73ad24d000)
libsz.so.2 => /usr/lib/libsz.so.2 (0x00007f73ad23f000)
libz.so.1 => /usr/lib/libz.so.1 (0x00007f73ad225000)
libvtkIOParallelXML.so.1 => /usr/lib/libvtkIOParallelXML.so.1 (0x00007f73ad1e1000)
libvtkIOXML.so.1 => /usr/lib/libvtkIOXML.so.1 (0x00007f73ad0e2000)
libvtkIOXMLParser.so.1 => /usr/lib/libvtkIOXMLParser.so.1 (0x00007f73ad0ca000)
libvtkParallelCore.so.1 => /usr/lib/libvtkParallelCore.so.1 (0x00007f73ad077000)
libvtkIOLegacy.so.1 => /usr/lib/libvtkIOLegacy.so.1 (0x00007f73acfd1000)
libvtkIOCore.so.1 => /usr/lib/libvtkIOCore.so.1 (0x00007f73acf60000)
libvtkCommonExecutionModel.so.1 => /usr/lib/libvtkCommonExecutionModel.so.1 (0x00007f73ace0d000)
libvtkCommonDataModel.so.1 => /usr/lib/libvtkCommonDataModel.so.1 (0x00007f73ac8e6000)
libvtkCommonTransforms.so.1 => /usr/lib/libvtkCommonTransforms.so.1 (0x00007f73ac895000)
libvtkCommonMath.so.1 => /usr/lib/libvtkCommonMath.so.1 (0x00007f73ac870000)
libvtkCommonCore.so.1 => /usr/lib/libvtkCommonCore.so.1 (0x00007f73abd82000)
libvtksys.so.1 => /usr/lib/libvtksys.so.1 (0x00007f73abd37000)
libtbb.so.12 => /opt/intel/oneapi/tbb/2021.5.1/env/../lib/intel64/gcc4.8/libtbb.so.12 (0x00007f73aba00000)
libvtkkissfft.so.1 => /usr/lib/libvtkkissfft.so.1 (0x00007f73abd2f000)
libxml2.so.2 => /usr/lib/libxml2.so.2 (0x00007f73ab876000)
libopenblas.so.3 => /usr/lib/libopenblas.so.3 (0x00007f73aa4c0000)
libmkl_intel_lp64.so.2 => /opt/intel/oneapi/mkl/2022.0.2/lib/intel64/libmkl_intel_lp64.so.2 (0x00007f73a9620000)
libmkl_gnu_thread.so.2 => /opt/intel/oneapi/mkl/2022.0.2/lib/intel64/libmkl_gnu_thread.so.2 (0x00007f73a7a8d000)
libmkl_core.so.2 => /opt/intel/oneapi/mkl/2022.0.2/lib/intel64/libmkl_core.so.2 (0x00007f73a36d8000)
libm.so.6 => /usr/lib/libm.so.6 (0x00007f73a35f0000)
libgomp.so.1 => /usr/lib/libgomp.so.1 (0x00007f73abce8000)
libpython3.10.so.1.0 => /usr/lib/libpython3.10.so.1.0 (0x00007f73a3235000)
libstdc++.so.6 => /usr/lib/libstdc++.so.6 (0x00007f73a300f000)
libgcc_s.so.1 => /usr/lib/libgcc_s.so.1 (0x00007f73abccb000)
libc.so.6 => /usr/lib/libc.so.6 (0x00007f73a2e05000)
libopen-rte.so.40 => /usr/lib/openmpi/libopen-rte.so.40 (0x00007f73a2d4b000)
libopen-pal.so.40 => /usr/lib/openmpi/libopen-pal.so.40 (0x00007f73a2ca0000)
libdl.so.2 => /usr/lib/libdl.so.2 (0x00007f73abcc6000)
librt.so.1 => /usr/lib/librt.so.1 (0x00007f73abcbf000)
libutil.so.1 => /usr/lib/libutil.so.1 (0x00007f73abcba000)
libhwloc.so.15 => /home/buchwalj/github/petsc/arch-linux-c-opt/lib/libhwloc.so.15 (0x00007f73a2c3d000)
libevent_core-2.1.so.7 => /usr/lib/libevent_core-2.1.so.7 (0x00007f73abc81000)
libevent_pthreads-2.1.so.7 => /usr/lib/libevent_pthreads-2.1.so.7 (0x00007f73abc7c000)
libpthread.so.0 => /usr/lib/libpthread.so.0 (0x00007f73ab871000)
libvtkCommonMisc.so.1 => /usr/lib/libvtkCommonMisc.so.1 (0x00007f73a282b000)
libvtkCommonSystem.so.1 => /usr/lib/libvtkCommonSystem.so.1 (0x00007f73a27ee000)
libexpat.so.1 => /usr/lib/libexpat.so.1 (0x00007f73a27bd000)
libdouble-conversion.so.3 => /usr/lib/libdouble-conversion.so.3 (0x00007f73a27a7000)
liblz4.so.1 => /usr/lib/liblz4.so.1 (0x00007f73a2784000)
liblzma.so.5 => /usr/lib/liblzma.so.5 (0x00007f73a275a000)
libpugixml.so.1 => /usr/lib/libpugixml.so.1 (0x00007f73a271d000)
libvtkloguru.so.1 => /usr/lib/libvtkloguru.so.1 (0x00007f73a26e7000)
/lib64/ld-linux-x86-64.so.2 => /usr/lib64/ld-linux-x86-64.so.2 (0x00007f73ae763000)
libicuuc.so.71 => /usr/lib/libicuuc.so.71 (0x00007f73a24e8000)
libudev.so.1 => /usr/lib/libudev.so.1 (0x00007f73a24bc000)
libpciaccess.so.0 => /usr/lib/libpciaccess.so.0 (0x00007f73a24b0000)
libX11.so.6 => /usr/lib/libX11.so.6 (0x00007f73a236c000)
libicudata.so.71 => /usr/lib/libicudata.so.71 (0x00007f73a0667000)
libxcb.so.1 => /usr/lib/libxcb.so.1 (0x00007f73a063c000)
libXau.so.6 => /usr/lib/libXau.so.6 (0x00007f73a0635000)
libXdmcp.so.6 => /usr/lib/libXdmcp.so.6 (0x00007f73a062d000)
Although there are differences, I don't see anything suspicious missing.