Merge branch 'fix-iomp' into 'master'

[ci] Fix builds (MKL, Python mac / win) See merge request ogs/ogs!4269

Merge branch 'fix-iomp' into 'master'
9474baf9 · Lars Bilke · 78080728 · 9c7b4ba3 · 9474baf9 · 9474baf9
Commit 9474baf9 authored 2 years ago by Lars Bilke
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -252,8 +252,18 @@ if(OGS_USE_PIP)
        file(WRITE ${PROJECT_BINARY_DIR}/requirements.txt
             ${REQUIREMENTS_CONTENT}
        )
+        message(
+            STATUS
+                "Installing Python packages into local virtual environment..."
+        )
+        if(APPLE)
+            # CC=/Library/Developer/CommandLineTools/usr/bin/cc and this somehow
+            # breaks wheel builds ...
+            set(_apple_env ${CMAKE_COMMAND} -E env CC=clang CXX=clang)
+        endif()
        execute_process(
-            COMMAND ${LOCAL_VIRTUALENV_BIN_DIR}/pip install -r requirements.txt
+            COMMAND ${_apple_env} ${LOCAL_VIRTUALENV_BIN_DIR}/pip install -r
+                    requirements.txt
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
            RESULT_VARIABLE _return_code
            OUTPUT_VARIABLE _out
@@ -263,6 +273,7 @@ if(OGS_USE_PIP)
            set(_OGS_PYTHON_PACKAGES_SHA1 "${_ogs_python_packages_sha1}"
                CACHE INTERNAL ""
            )
+            message(STATUS "${_out}")
        else()
            message(
                FATAL_ERROR

--- a/scripts/ci/extends/template-build-linux.yml
+++ b/scripts/ci/extends/template-build-linux.yml
@@ -4,7 +4,7 @@
    max: 2
    when: runner_system_failure

-  before_script:
+  script:
    - 'echo "For debugging run: docker run --rm -it -v $CI_BUILDS_DIR:/builds -w=$CI_PROJECT_DIR $CI_JOB_IMAGE"'
    - build_dir_full=../build/${CMAKE_PRESET}
    - log_file=$build_dir_full/make.txt
@@ -14,7 +14,6 @@
    - ln -s `$readlink_cmd -f $build_dir_full` build/${CMAKE_PRESET}
    - ([ "${CONAN_USER_HOME}" == "${CI_PROJECT_DIR}/.conan" ]) && conan remove --system-reqs '*'
    - ([[ $BUILD_CTEST_LARGE_ON_MASTER && "${CI_COMMIT_BRANCH}" == "master" && "${CMAKE_ARGS}" == *"USE_PYTHON=ON"* ]]) && export BUILD_CTEST_LARGE=true
-  script:
    - cmake --preset=$CMAKE_PRESET -Wno-dev $CMAKE_ARGS
    - |
      if [[ -z "$TARGETS" ]]; then

--- a/scripts/ci/jobs/build-linux-frontend.yml
+++ b/scripts/ci/jobs/build-linux-frontend.yml
@@ -14,7 +14,6 @@ build linux frontend:
    CTEST_ARGS: "--label-exclude python_modules" # pip installed modules segfault
    CMAKE_ARGS: "-DOGS_CPU_ARCHITECTURE=generic"
  before_script:
-    - !reference [.template-build-linux, before_script]
    - source scripts/env/eve/cli.sh

 build linux frontend petsc:
@@ -32,5 +31,4 @@ build linux frontend petsc:
    CMAKE_PRESET: release-petsc
    CMAKE_ARGS: "-DOGS_CPU_ARCHITECTURE=generic"
  before_script:
-    - !reference [.template-build-linux, before_script]
    - source scripts/env/eve/petsc.sh
--- a/scripts/ci/jobs/code-coverage.yml
+++ b/scripts/ci/jobs/code-coverage.yml
@@ -12,7 +12,6 @@ code coverage:
      -DOGS_USE_PIP=ON
      -DBUILD_SHARED_LIBS=ON
  before_script:
-    - !reference [.template-build-linux, before_script]
    # Shortcut to the generated pages:
    - echo '<meta http-equiv="REFRESH" content="0;URL=build/coverage/coverage_report/index.html">' >> Coverage.html
  after_script:

--- a/scripts/ci/jobs/jupyter.yml
+++ b/scripts/ci/jobs/jupyter.yml
 # Built for Sandy Bridge (envinf1) and newer
 build jupyter:
  stage: build
-  tags: [envinf3-shell]
+  tags: [envinf, shell]
  needs: [meta]
  extends:
    - .container-maker-setup

--- a/scripts/cmake/CMakeSetup.cmake
+++ b/scripts/cmake/CMakeSetup.cmake
@@ -21,7 +21,7 @@ CPMAddPackage(
 CPMAddPackage(
    NAME findmkl_cmake
    GITHUB_REPOSITORY bilke/findmkl_cmake
-    GIT_TAG 7702795e84190a2a63adaa07520292dbfefbc6f6
+    GIT_TAG ee49c4f973f66bb7bfd644658d14e43459f557fa
    DOWNLOAD_ONLY YES
 )
 set(CMAKE_MODULE_PATH

--- a/scripts/cmake/PythonCreateVirtualEnv.cmake
+++ b/scripts/cmake/PythonCreateVirtualEnv.cmake
@@ -10,8 +10,12 @@ set(Python_FIND_STRATEGY VERSION)

 find_package(Python ${python_version} COMPONENTS Interpreter REQUIRED)

+if(${Python_VERSION} VERSION_GREATER_EQUAL 3.9)
+    set(_upgrade_deps --upgrade-deps)
+endif()
+
 execute_process(
-    COMMAND ${Python_EXECUTABLE} -m venv .venv
+    COMMAND ${Python_EXECUTABLE} -m venv ${_upgrade_deps} .venv
    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
 )


--- a/scripts/cmake/PythonSetup.cmake
+++ b/scripts/cmake/PythonSetup.cmake
 # cmake-lint: disable=C0103

+set(_python_version_max "...<3.11")
 if(OGS_USE_PIP)
    set(Python_ROOT_DIR ${PROJECT_BINARY_DIR}/.venv)
    set(CMAKE_REQUIRE_FIND_PACKAGE_Python TRUE)
@@ -7,13 +8,26 @@ if(OGS_USE_PIP)
        execute_process(
            COMMAND
                ${CMAKE_COMMAND} -DPROJECT_BINARY_DIR=${PROJECT_BINARY_DIR}
-                -Dpython_version=${ogs.minimum_version.python} -P
+                -Dpython_version=${ogs.minimum_version.python}${_python_version_max}
+                -P
                ${PROJECT_SOURCE_DIR}/scripts/cmake/PythonCreateVirtualEnv.cmake
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR} COMMAND_ECHO STDOUT
                              ECHO_OUTPUT_VARIABLE ECHO_ERROR_VARIABLE
        )
        unset(_OGS_PYTHON_PACKAGES_SHA1 CACHE)
    endif()
+    set(_venv_bin_dir "bin")
+    if(MSVC)
+        set(_venv_bin_dir "Scripts")
+    endif()
+    set(LOCAL_VIRTUALENV_BIN_DIR ${PROJECT_BINARY_DIR}/.venv/${_venv_bin_dir}
+        CACHE INTERNAL ""
+    )
+    # Fixes macOS install issues
+    execute_process(
+        COMMAND ${LOCAL_VIRTUALENV_BIN_DIR}/pip install wheel
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+    )
 else()
    # Prefer unix location over frameworks (Apple-only)
    set(Python_FIND_FRAMEWORK LAST)
@@ -28,13 +42,16 @@ if(OGS_USE_PYTHON)
    list(APPEND _python_componets Development)
 endif()
 find_package(
-    Python ${ogs.minimum_version.python} COMPONENTS ${_python_componets}
+    Python ${ogs.minimum_version.python}${_python_version_max}
+    COMPONENTS ${_python_componets}
 )

 if(OGS_USE_PIP)
    set(Python_SITEARCH_NATIVE ${Python_SITEARCH})
    if(WIN32)
-        string(REPLACE "\\" "\\\\" Python_SITEARCH_NATIVE ${Python_SITEARCH_NATIVE})
+        string(REPLACE "\\" "\\\\" Python_SITEARCH_NATIVE
+                       ${Python_SITEARCH_NATIVE}
+        )
    endif()
    set(OGS_PYTHON_PACKAGES ""
        CACHE INTERNAL "List of Python packages to be installed via pip."
@@ -42,14 +59,8 @@ if(OGS_USE_PIP)
    set(Python_ROOT_DIR ${PROJECT_BINARY_DIR}/.venv)
    if(MSVC)
        set(Python_EXECUTABLE ${Python_ROOT_DIR}/Scripts/python.exe)
-        set(LOCAL_VIRTUALENV_BIN_DIR ${PROJECT_BINARY_DIR}/.venv/Scripts
-            CACHE INTERNAL ""
-        )
    else()
        set(Python_EXECUTABLE ${Python_ROOT_DIR}/bin/python)
-        set(LOCAL_VIRTUALENV_BIN_DIR ${PROJECT_BINARY_DIR}/.venv/bin
-            CACHE INTERNAL ""
-        )
    endif()
    if(OGS_BUILD_TESTING)
        # Notebook requirements from versions.json

--- a/scripts/cmake/packaging/Pack.cmake
+++ b/scripts/cmake/packaging/Pack.cmake
@@ -72,8 +72,9 @@ endif()
 if(OGS_USE_PYTHON)
    if(WIN32)
        file(GLOB PYTHON_RUNTIME_LIBS "${Python_RUNTIME_LIBRARY_DIRS}/*.dll")
-        message(STATUS "Install Python: ${PYTHON_RUNTIME_LIBS}")
+        message(STATUS "Install Python into bin-dir: ${PYTHON_RUNTIME_LIBS}")
        install(FILES ${PYTHON_RUNTIME_LIBS} DESTINATION bin)
+        file(COPY ${PYTHON_RUNTIME_LIBS} DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
    else()
        install(FILES ${Python_LIBRARIES} DESTINATION bin)
    endif()

--- a/scripts/cmake/packaging/PackagingDependencies.cmake
+++ b/scripts/cmake/packaging/PackagingDependencies.cmake
@@ -25,7 +25,7 @@ install(
    RESOLVED_DEPENDENCIES_VAR _r_deps
    UNRESOLVED_DEPENDENCIES_VAR _u_deps
    PRE_EXCLUDE_REGEXES "api-ms-" "ext-ms-"
-    POST_EXCLUDE_REGEXES ".*system32/.*\\.dll"
+    POST_EXCLUDE_REGEXES ".*system32/.*\\.dll" "/usr/lib/libtbb.so.12"
  )
  find_program(PATCHELF_TOOL patchelf)
  foreach(_lib ${_r_deps})

--- a/web/data/versions.json
+++ b/web/data/versions.json
@@ -71,8 +71,8 @@
    }
  },
  "cpm": {
-    "package_file_id": 81,
-    "package_file_sha256": "c9ce6ab6f229ab34045d3cf86889dbddb2f400ebe39c0796b18a2dbebbae8193"
+    "package_file_id": 119,
+    "package_file_sha256": "7d98b148e6d24acd72d17d2503f3ce3bab74029da5b3328788b2d4c379e9dcac"
  },
  "ext": {
    "cache_hash": "e6f3f1f4c29c6c5f096f89785e6e245bdf39ac1a"