.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *$py.class
+*.egg-info
 
 .idea
 _out/

.gitlab-ci.yml

@@ -7,7 +7,7 @@ variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   BUILDKIT_PROGRESS: "plain"
   OGSCM_ARGUMENTS:
-    value: "compiler.py ogs.py --pm system --ogs ogs/ogs@master --cmake_args ' -DOGS_CPU_ARCHITECTURE=sandybridge -DBUILD_TESTING=OFF'"
+    value: "compiler.py ogs.py --ogs ogs/ogs@master --cmake_args ' -DOGS_CPU_ARCHITECTURE=sandybridge -DBUILD_TESTING=OFF'"
     description: "Arguments to ogscm. -B -C -R --ccache are always passed automatically. For more args see README.md!"
 
 dev images:
@@ -22,8 +22,8 @@ dev images:
   script:
     - black --check ogscm
     - poetry install
-    - poetry run ogscm compiler.py ogs.py -B -C --sif_file ubuntu-dev.sif --pm system --ogs off
-    - poetry run ogscm compiler.py ogs.py -B -C --sif_file centos-dev.sif --base_image centos:8 --pm system --ogs off
+    - poetry run ogscm compiler.py ogs.py -B -C --sif_file ubuntu-dev.sif --ogs off
+    - poetry run ogscm compiler.py ogs.py -B -C --sif_file centos-dev.sif --base_image centos:8 --ogs off
   artifacts:
     # example URL:
     # https://gitlab.opengeosys.org/ogs/container-maker/-/jobs/artifacts/master/raw/_out/images/ubuntu-dev.sif?job=dev+images

README.md

@@ -103,13 +103,10 @@ $ ogscm compiler.py mpi.py ogs.py --base_image 'centos:8' --help
 Evaluating compiler.py
 Evaluating mpi.py
 Evaluating ogs.py
-usage: ogscm [-h] [--version] [--out OUT] [--file FILE] [--sif_file SIF_FILE] [--print] [--format {docker,singularity}]
-             [--base_image BASE_IMAGE] [--build] [--build_args BUILD_ARGS] [--upload] [--registry REGISTRY] [--tag TAG]
-             [--convert] [--runtime-only] [--clean] [--deploy [DEPLOY]] [--pip [package [package ...]]]
-             [--packages [packages [packages ...]]] [--compiler COMPILER] [--compiler_version COMPILER_VERSION]
-             [--iwyy] [--ompi OMPI] [--mpi_benchmarks] [--pm {system,conan,off}] [--ogs OGS] [--cmake_args CMAKE_ARGS]
-             [--ccache] [--parallel PARALLEL] [--gui] [--docs] [--cvode] [--cppcheck] [--gcovr] [--tfel] [--insitu]
-             [--dev]
+usage: ogscm [-h] [--version] [--out OUT] [--file FILE] [--sif_file SIF_FILE] [--print] [--format {docker,singularity}] [--base_image BASE_IMAGE] [--build] [--build_args BUILD_ARGS] [--upload]
+             [--registry REGISTRY] [--tag TAG] [--convert] [--runtime-only] [--clean] [--deploy [DEPLOY]] [--pip [package ...]] [--packages [packages ...]] [--compiler COMPILER]
+             [--compiler_version COMPILER_VERSION] [--iwyy] [--ompi OMPI] [--mpi_benchmarks] [--pm {system,conan,off}] [--ogs OGS] [--cmake_args CMAKE_ARGS] [--ccache] [--parallel PARALLEL] [--gui] [--docs]
+             [--cvode] [--cppcheck] [--gcovr] [--tfel] [--insitu] [--dev] [--mkl] [--version_file VERSION_FILE]
              recipe [recipe ...]
 
 positional arguments:
@@ -131,11 +128,9 @@ General image config:
 Image build options:
   --build, -B           Build the images from the definition files (default: False)
   --build_args BUILD_ARGS
-                        Arguments to the build command. Have to be quoted and **must** start with a space. E.g.
-                        --build_args ' --no-cache' (default: )
+                        Arguments to the build command. Have to be quoted and **must** start with a space. E.g. --build_args ' --no-cache' (default: )
   --upload, -U          Upload Docker image to registry (default: False)
-  --registry REGISTRY   The docker registry the image is tagged and uploaded to. (default:
-                        registry.opengeosys.org/ogs/ogs)
+  --registry REGISTRY   The docker registry the image is tagged and uploaded to. (default: registry.opengeosys.org/ogs/ogs)
   --tag TAG             The full docker image tag. Overwrites --registry. (default: )
   --convert, -C         Convert Docker image to Singularity image (default: False)
   --runtime-only, -R    Generate multi-stage Dockerfiles for small runtime images (default: False)
@@ -145,13 +140,11 @@ Maintenance:
 
 Image deployment:
   --deploy [DEPLOY], -D [DEPLOY]
-                        Deploys to all configured hosts (in config/deploy_hosts.yml) with no additional arguments or to
-                        the specified host. Implies --build and --convert arguments. (default: )
+                        Deploys to all configured hosts (in config/deploy_hosts.yml) with no additional arguments or to the specified host. Implies --build and --convert arguments. (default: )
 
 Packages to install:
-  --pip [package [package ...]]
-                        Install additional Python packages (default: [])
-  --packages [packages [packages ...]]
+  --pip [package ...]   Install additional Python packages (default: [])
+  --packages [packages ...]
                         Install additional OS packages (default: [])
 
 compiler.py:
@@ -166,17 +159,14 @@ mpi.py:
 
 ogs.py:
   --pm {system,conan,off}
-                        Package manager to install third-party dependencies (default: conan)
-  --ogs OGS             OGS repo on gitlab.opengeosys.org in the form 'user/repo@branch' OR 'user/repo@@commit' to
-                        checkout a specific commit OR a path to a local subdirectory to the git cloned OGS sources OR
-                        'off' to disable OGS building OR 'clean' to disable OGS and all its dev dependencies (default:
-                        ogs/ogs@master)
+                        Package manager to install third-party dependencies (default: system)
+  --ogs OGS             OGS repo on gitlab.opengeosys.org in the form 'user/repo@branch' OR 'user/repo@@commit' to checkout a specific commit OR a path to a local subdirectory to the git cloned OGS sources OR
+                        'off' to disable OGS building OR 'clean' to disable OGS and all its dev dependencies (default: ogs/ogs@master)
   --cmake_args CMAKE_ARGS
-                        CMake argument set has to be quoted and **must** start with a space. e.g. --cmake_args '
-                        -DFIRST=TRUE -DFOO=BAR' (default: )
+                        CMake argument set has to be quoted and **must** start with a space. e.g. --cmake_args ' -DFIRST=TRUE -DFOO=BAR' (default: )
   --ccache              Enables ccache build caching. (default: False)
   --parallel PARALLEL, -j PARALLEL
-                        The number of cores to use for compilation. (default: 4)
+                        The number of cores to use for compilation. (default: 32)
   --gui                 Builds the GUI (Data Explorer) (default: False)
   --docs                Setup documentation requirements (Doxygen) (default: False)
   --cvode               Install and configure with cvode (default: False)
@@ -185,6 +175,9 @@ ogs.py:
   --tfel                Install tfel (default: False)
   --insitu              Builds with insitu capabilities (default: False)
   --dev                 Installs development tools (vim, gdb) (default: False)
+  --mkl                 Use MKL. By setting this option, you agree to the [Intel End User License Agreement](https://software.intel.com/en-us/articles/end-user-license-agreement). (default: False)
+  --version_file VERSION_FILE
+                        OGS versions.json file (default: None)
 ```
 
 ## Advanced usage
@@ -197,7 +190,7 @@ You can use the ogs-container-maker to build multiple container images from your
 virtualenv .venv
 source .venv/bin/activate
 pip install ogscm
-ogscm compiler.py ogs.py -B -C -R --ogs [path to ogs sources] --pm system
+ogscm compiler.py ogs.py -B -C -R --ogs [path to ogs sources]
 ```
 
 ### Deploy image files

codemeta.json

@@ -10,7 +10,7 @@
   "downloadUrl": "https://pypi.org/project/ogscm",
   "issueTracker": "https://gitlab.opengeosys.org/ogs/container-maker/-/issues",
   "name": "ogs-container-maker",
-  "version": "2.0.2",
+  "version": "2.1.1",
   "description": "An extension of hpc-container-maker to generate container definition files for the OpenGeoSys software.",
   "developmentStatus": "active",
   "isPartOf": "https://www.opengeosys.org",

ogscm/building_blocks/pm_conan.py

@@ -25,6 +25,7 @@ class pm_conan(bb_base):
         super(pm_conan, self).__init__()
 
         self.__user_home = kwargs.get("user_home", "")
+        self.__version = kwargs.get("version", "")
 
         self.__commands = []
 
@@ -38,16 +39,15 @@ class pm_conan(bb_base):
         self += packages(ospackages=["pkg-config"])
         # For building curl:
         self += packages(ospackages=["autoconf-archive", "libtool"])
-        conan_version = "1.33.0"
 
-        self += pip(pip="pip3", packages=["conan=={}".format(conan_version)])
+        self += pip(pip="pip3", packages=[f"conan=={self.__version}"])
         self += shell(commands=self.__commands)
         if self.__user_home != "":
             self += environment(variables={"CONAN_USER_HOME": self.__user_home})
         self += label(
             metadata={
                 "org.opengeosys.pm": "conan",
-                "org.opengeosys.pm.conan.version": conan_version,
+                "org.opengeosys.pm.conan.version": self.__version,
             }
         )
         if self.__user_home != "":

ogscm/cli.py

 #!/usr/bin/env python3
 import argparse
 import os
+import traceback
+import sys
 
 import hpccm
 from hpccm.building_blocks import (
@@ -180,7 +182,12 @@ def main():  # pragma: no cover
         try:
             recipe_builtin = pkg_resources.read_text(recipes, recipe)
             exec(compile(recipe_builtin, recipe, "exec"), locals(), ldict)
-        except:
+        except Exception as err:
+            error_class = err.__class__.__name__
+            detail = err.args[0]
+            cl, exc, tb = sys.exc_info()
+            line_number = traceback.extract_tb(tb)[-1][1]
+            print(f"{error_class} at line {line_number}: {detail}")
             if not os.path.exists(recipe):
                 print(f"{recipe} does not exist!")
                 exit(1)
@@ -235,6 +242,7 @@ def main():  # pragma: no cover
 
     if args.packages:
         Stage0 += packages(ospackages=args.packages)
+        Stage1 += packages(ospackages=args.packages)
 
     # Create definition
     hpccm.config.set_container_format(args.format)

ogscm/recipes/ogs.py

@@ -13,6 +13,7 @@ from hpccm.building_blocks import (
     generic_autotools,
     generic_cmake,
     hdf5,
+    mkl,
     packages,
     pip,
     scif,
@@ -37,7 +38,7 @@ parse_g.add_argument(
     "--pm",
     type=str,
     choices=["system", "conan", "off"],
-    default="conan",
+    default="system",
     help="Package manager to install third-party " "dependencies",
 )
 parse_g.add_argument(
@@ -93,7 +94,12 @@ parse_g.add_argument(
     "--cppcheck", dest="cppcheck", action="store_true", help="Install cppcheck"
 )
 parse_g.add_argument("--gcovr", dest="gcovr", action="store_true", help="Install gcovr")
-parse_g.add_argument("--tfel", dest="tfel", action="store_true", help="Install tfel")
+parse_g.add_argument(
+    "--mfront",
+    dest="mfront",
+    action="store_true",
+    help="Install tfel and build OGS with -DOGS_USE_MFRONT=ON",
+)
 parse_g.add_argument(
     "--insitu",
     dest="insitu",
@@ -106,6 +112,13 @@ parse_g.add_argument(
     action="store_true",
     help="Installs development tools (vim, gdb)",
 )
+parse_g.add_argument(
+    "--mkl",
+    dest="mkl",
+    action="store_true",
+    help="Use MKL. By setting this option, you agree to the [Intel End User License Agreement](https://software.intel.com/en-us/articles/end-user-license-agreement).",
+)
+parse_g.add_argument("--version_file", type=str, help="OGS versions.json file")
 
 # Parse local args
 local_args = parser.parse_known_args()[0]
@@ -114,6 +127,7 @@ branch_is_release = False
 git_version = ""
 name_start = ""
 repo = None
+versions = None
 
 if local_args.ogs not in ["off", "clean"]:  # != "off" and local_args.ogs != "clean":
     if os.path.isdir(local_args.ogs):
@@ -124,6 +138,8 @@ if local_args.ogs not in ["off", "clean"]:  # != "off" and local_args.ogs != "cl
             text=True,
             shell=True,
         ).stdout.rstrip()
+        with open(f"{local_args.ogs}/web/data/versions.json") as fp:
+            versions = json.load(fp)
         if "GITLAB_CI" in os.environ:
             if "CI_COMMIT_BRANCH" in os.environ:
                 branch = os.environ["CI_COMMIT_BRANCH"]
@@ -155,6 +171,11 @@ if local_args.ogs not in ["off", "clean"]:  # != "off" and local_args.ogs != "cl
             commit_hash = commit[0]
             if branch == "":
                 branch = "master"
+            versions = json.loads(
+                requests.get(
+                    f"https://gitlab.opengeosys.org/{repo}/-/raw/{commit_hash}/web/data/versions.json"
+                ).text
+            )
         else:
             if re.search(r"[\d.]+", branch):
                 branch_is_release = True
@@ -165,12 +186,27 @@ if local_args.ogs not in ["off", "clean"]:  # != "off" and local_args.ogs != "cl
             response_data = json.loads(response.text)
             commit_hash = response_data[0]["id"]
             # ogs_tag = args.ogs.replace('/', '.').replace('@', '.')
+            versions = json.loads(
+                requests.get(
+                    f"https://gitlab.opengeosys.org/{repo}/-/raw/{branch}/web/data/versions.json"
+                ).text
+            )
 
         if branch_is_release:
             name_start = f"ogs-{branch}"
         else:
             name_start = f"ogs-{commit_hash[:8]}"
 
+if local_args.version_file:
+    with open(local_args.version_file) as fp:
+        versions = json.load(fp)
+if versions == None:
+    versions = json.loads(
+        requests.get(
+            f"https://gitlab.opengeosys.org/ogs/ogs/-/raw/master/web/data/versions.json"
+        ).text
+    )
+
 folder = f"/{name_start}/{local_args.pm}".replace("//", "/")
 
 if len(local_args.cmake_args) > 0:
@@ -233,11 +269,15 @@ if local_args.ogs != "clean":
         conan_user_home = "/opt/conan"
         if local_args.dev:
             conan_user_home = ""
-        Stage0 += pm_conan(user_home=conan_user_home)
+        Stage0 += pm_conan(
+            user_home=conan_user_home, version=versions["minimum_version"]["conan"]
+        )
         Stage0 += environment(variables={"CONAN_SYSREQUIRES_SUDO": 0})
     elif local_args.pm == "system":
         Stage0 += cmake(eula=True, version="3.19.4")
-        Stage0 += boost(version="1.67.0", bootstrap_opts=["headers"])
+        Stage0 += boost(
+            version=versions["minimum_version"]["boost"], bootstrap_opts=["headers"]
+        )
         Stage0 += environment(variables={"BOOST_ROOT": "/usr/local/boost"})
         Stage0 += packages(apt=["libxml2-dev"], yum=["libxml2-devel"])
         vtk_cmake_args = [
@@ -284,7 +324,7 @@ if local_args.ogs != "clean":
             )
             # TODO: will not work with clang
             qt_install_dir = "/opt/qt"
-            qt_version = "5.14.2"
+            qt_version = versions["minimum_version"]["qt"]
             qt_dir = f"{qt_install_dir}/{qt_version}/gcc_64"
             Stage0 += pip(pip="pip3", packages=["aqtinstall"])
             Stage0 += shell(
@@ -340,17 +380,19 @@ if local_args.ogs != "clean":
                         "-DModule_vtkParallelMPI=ON",
                     ]
                 )
+            vtk_version = versions["tested_version"]["vtk"]
             Stage0 += generic_cmake(
                 cmake_opts=vtk_cmake_args,
                 devel_environment={"VTK_ROOT": "/usr/local/vtk"},
-                directory="VTK-8.2.0",
+                directory=f"VTK-{vtk_version}",
                 ldconfig=True,
                 prefix="/usr/local/vtk",
                 toolchain=toolchain,
-                url="https://www.vtk.org/files/release/8.2/VTK-8.2.0.tar.gz",
+                url=f"https://www.vtk.org/files/release/{vtk_version[:-2]}/VTK-{vtk_version}.tar.gz",
             )
         if toolchain.CC == "mpicc":
             Stage0 += packages(yum=["diffutils"])
+            petsc_version = versions["minimum_version"]["petsc"]
             Stage0 += generic_autotools(
                 configure_opts=[
                     f"CC={toolchain.CC}",
@@ -363,26 +405,31 @@ if local_args.ogs != "clean":
                     "--download-f2cblaslapack=1",
                 ],
                 devel_environment={"PETSC_DIR": "/usr/local/petsc"},
-                directory="petsc-3.11.3",
+                directory=f"petsc-{petsc_version}",
                 ldconfig=True,
                 preconfigure=["sed -i -- 's/python/python3/g' configure"],
                 prefix="/usr/local/petsc",
                 toolchain=toolchain,
-                url="http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/"
-                "petsc-lite-3.11.3.tar.gz",
+                url=f"http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-lite-{petsc_version}.tar.gz",
             )
 
+        eigen_version = versions["minimum_version"]["eigen"]
         Stage0 += generic_cmake(
             devel_environment={
                 "Eigen3_ROOT": "/usr/local/eigen",
                 "Eigen3_DIR": "/usr/local/eigen",
             },
-            directory="eigen-3.3.9",
+            directory=f"eigen-{eigen_version}",
             prefix="/usr/local/eigen",
-            url="https://gitlab.com/libeigen/eigen/-/archive/3.3.9/eigen-3.3.9.tar.gz",
+            url=f"https://gitlab.com/libeigen/eigen/-/archive/{eigen_version}/eigen-{eigen_version}.tar.gz",
+        )
+        Stage0 += hdf5(
+            configure_opts=["--enable-cxx"],
+            toolchain=toolchain,
+            version=versions["minimum_version"]["hdf5"],
         )
-        Stage0 += hdf5(configure_opts=["--enable-cxx"], toolchain=toolchain)
 if local_args.cvode:
+    # TODO version
     Stage0 += generic_cmake(
         cmake_opts=[
             "-D EXAMPLES_INSTALL=OFF",
@@ -414,7 +461,7 @@ if local_args.dev:
         ospackages=["neovim", "gdb", "silversearcher-ag", "ssh-client", "less"]
     )
 
-if local_args.tfel:
+if local_args.mfront:
     Stage0 += generic_cmake(
         directory="tfel-TFEL-3.3.0",
         ldconfig=True,
@@ -422,6 +469,11 @@ if local_args.tfel:
         prefix="/usr/local/tfel",
     )
     Stage0 += environment(variables={"TFELHOME": "/usr/local/tfel"})
+    cmake_args.append("-DOGS_USE_MFRONT=ON")
+
+if local_args.mkl:
+    Stage0 += mkl(eula=True)
+    cmake_args.append("-DOGS_USE_MKL=ON")
 
 if local_args.ccache:
     Stage0 += ccache(cache_size="15G")

poetry.lock

@@ -14,10 +14,6 @@ category = "dev"
 optional = false
 python-versions = ">=3.6"
 
-[package.extras]
-colorama = ["colorama (>=0.4.3)"]
-d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
-
 [package.dependencies]
 appdirs = "*"
 click = ">=7.1.2"
@@ -28,6 +24,10 @@ toml = ">=0.10.1"
 typed-ast = ">=1.4.0"
 typing-extensions = ">=3.7.4"
 
+[package.extras]
+colorama = ["colorama (>=0.4.3)"]
+d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
+
 [[package]]
 name = "certifi"
 version = "2020.12.5"
@@ -54,7 +54,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
 [[package]]
 name = "hpccm"
-version = "20.11.0"
+version = "21.2.0"
 description = "HPC Container Maker"
 category = "main"
 optional = false
@@ -130,16 +130,16 @@ category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
-[package.extras]
-security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
-socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"]
-
 [package.dependencies]
 certifi = ">=2017.4.17"
 chardet = ">=3.0.2,<4"
 idna = ">=2.5,<3"
 urllib3 = ">=1.21.1,<1.27"
 
+[package.extras]
+security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
+socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
+
 [[package]]
 name = "six"
 version = "1.15.0"
@@ -183,12 +183,12 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
 [package.extras]
 brotli = ["brotlipy (>=0.6.0)"]
 secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
-socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 
 [metadata]
-lock-version = "1.0"
+lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "8b299ba22cf9511cf3f8ab9382e5b8f459a71d23c5885c9faed3ddee4a8f7dcc"
+content-hash = "ea3f165d18a099487234f970cf33354f47b3c55c0b24a5a32cccaf18456e58c2"
 
 [metadata.files]
 appdirs = [
@@ -211,7 +211,7 @@ click = [
     {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
 ]
 hpccm = [
-    {file = "hpccm-20.11.0.tar.gz", hash = "sha256:33667f924a9c70dea66a9bf928e683e5476eedca1b48b7f4cf0b9301d9af3c6b"},
+    {file = "hpccm-21.2.0.tar.gz", hash = "sha256:37b1ab1c4f680d67e9916350842f3a1de7e6e8c75754a21cd05bab4e0e0b0f12"},
 ]
 idna = [
     {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},

pyproject.toml

 [tool.poetry]
 name = "ogscm"
-version = "2.0.2"
+version = "2.1.1"
 description = "OGS Container Maker"
 authors = ["Lars Bilke <lars.bilke@ufz.de>"]
 license = "BSD 3-clause"
@@ -9,13 +9,13 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.7"
-hpccm = "^20.10.0"
+hpccm = "^21.2.0"
 requests = "^2.24.0"
 PyYAML = "^5.3.1"
 packaging = "^20.4"
 
 [tool.poetry.dev-dependencies]
-black = {version = "^20.8b1", allow-prereleases = true}
+black = { version = "^20.8b1", allow-prereleases = true }
 
 [tool.poetry.scripts]
 ogscm = "ogscm.cli:main"