.gitlab-ci.yml

@@ -22,13 +22,14 @@ dev images:
   script:
     - black --check ogscm
     - poetry install
-    - poetry run ogscm compiler.py ogs.py -B -C --sif_file ubuntu-dev.sif --ogs off
-    - poetry run ogscm compiler.py ogs.py -B -C --sif_file centos-dev.sif --base_image centos:8 --ogs off
+    - poetry run ogscm compiler.py ogs.py -B -C --sif_file ubuntu-dev.sif -E --enroot_file ubuntu-dev.sqsh --ogs off
+    - poetry run ogscm compiler.py ogs.py -B -C --sif_file centos-dev.sif -E --enroot_file centos-dev.sqsh --base_image centos:8 --ogs off
   artifacts:
     # example URL:
     # https://gitlab.opengeosys.org/ogs/container-maker/-/jobs/artifacts/master/raw/_out/images/ubuntu-dev.sif?job=dev+images
     paths:
       - _out/images/*.sif
+      - _out/images/*.sqsh
 
 release:
   stage: release

README.md

@@ -103,10 +103,10 @@ $ ogscm compiler.py mpi.py ogs.py --base_image 'centos:8' --help
 Evaluating compiler.py
 Evaluating mpi.py
 Evaluating ogs.py
-usage: ogscm [-h] [--version] [--out OUT] [--file FILE] [--sif_file SIF_FILE] [--print] [--format {docker,singularity}] [--base_image BASE_IMAGE] [--build] [--build_args BUILD_ARGS] [--upload]
-             [--registry REGISTRY] [--tag TAG] [--convert] [--runtime-only] [--clean] [--deploy [DEPLOY]] [--pip [package ...]] [--packages [packages ...]] [--compiler COMPILER]
+usage: ogscm [-h] [--version] [--out OUT] [--file FILE] [--print] [--format {docker,singularity}] [--base_image BASE_IMAGE] [--build] [--build_args BUILD_ARGS] [--upload] [--registry REGISTRY] [--tag TAG]
+             [--convert] [--sif_file SIF_FILE] [--convert-enroot] [--enroot_file ENROOT_FILE] [--runtime-only] [--clean] [--deploy [DEPLOY]] [--pip [package ...]] [--packages [packages ...]] [--compiler COMPILER]
              [--compiler_version COMPILER_VERSION] [--iwyy] [--ompi OMPI] [--mpi_benchmarks] [--pm {system,conan,off}] [--ogs OGS] [--cmake_args CMAKE_ARGS] [--ccache] [--parallel PARALLEL] [--gui] [--docs]
-             [--cvode] [--cppcheck] [--gcovr] [--tfel] [--insitu] [--dev] [--mkl] [--version_file VERSION_FILE]
+             [--cvode] [--cppcheck] [--gcovr] [--mfront] [--insitu] [--dev] [--mkl] [--version_file VERSION_FILE]
              recipe [recipe ...]
 
 positional arguments:
@@ -117,7 +117,6 @@ optional arguments:
   --version             show program's version number and exit
   --out OUT             Output directory (default: _out)
   --file FILE           Overwrite output recipe file name (default: )
-  --sif_file SIF_FILE   Overwrite output singularity image file name (default: )
   --print, -P           Print the definition to stdout (default: False)
 
 General image config:
@@ -133,6 +132,10 @@ Image build options:
   --registry REGISTRY   The docker registry the image is tagged and uploaded to. (default: registry.opengeosys.org/ogs/ogs)
   --tag TAG             The full docker image tag. Overwrites --registry. (default: )
   --convert, -C         Convert Docker image to Singularity image (default: False)
+  --sif_file SIF_FILE   Overwrite output singularity image file name (default: )
+  --convert-enroot, -E  Convert Docker image to enroot image (default: False)
+  --enroot_file ENROOT_FILE
+                        Overwrite output enroot image file name (default: )
   --runtime-only, -R    Generate multi-stage Dockerfiles for small runtime images (default: False)
 
 Maintenance:
@@ -155,7 +158,7 @@ compiler.py:
 
 mpi.py:
   --ompi OMPI           OpenMPI version, e.g. 2.1.1, 2.1.5, 3.0.1, 3.1.2 (default: 4.0.5)
-  --mpi_benchmarks      Installs OSU MPI benchmarks as scif app and mpi_bw, mpi_ring,mpi_hello (default: False)
+  --mpi_benchmarks      Installs OSU MPI benchmarks and mpi_bw, mpi_ring, mpi_hello (default: False)
 
 ogs.py:
   --pm {system,conan,off}
@@ -172,7 +175,7 @@ ogs.py:
   --cvode               Install and configure with cvode (default: False)
   --cppcheck            Install cppcheck (default: False)
   --gcovr               Install gcovr (default: False)
-  --tfel                Install tfel (default: False)
+  --mfront              Install tfel and build OGS with -DOGS_USE_MFRONT=ON (default: False)
   --insitu              Builds with insitu capabilities (default: False)
   --dev                 Installs development tools (vim, gdb) (default: False)
   --mkl                 Use MKL. By setting this option, you agree to the [Intel End User License Agreement](https://software.intel.com/en-us/articles/end-user-license-agreement). (default: False)

codemeta.json

@@ -10,7 +10,7 @@
   "downloadUrl": "https://pypi.org/project/ogscm",
   "issueTracker": "https://gitlab.opengeosys.org/ogs/container-maker/-/issues",
   "name": "ogs-container-maker",
-  "version": "2.1.4",
+  "version": "2.2.0",
   "description": "An extension of hpc-container-maker to generate container definition files for the OpenGeoSys software.",
   "developmentStatus": "active",
   "isPartOf": "https://www.opengeosys.org",

ogscm/app/builder.py

@@ -28,11 +28,9 @@ class builder(object):
             f"sudo `which singularity` build --force {sif_file}"
             f"{self.__definition_file_path}",
             shell=True,
+            check=True,
         )
-        subprocess.run(
-            f"sudo chown $USER:$USER {sif_file}",
-            shell=True,
-        )
+        subprocess.run(f"sudo chown $USER:$USER {sif_file}", shell=True, check=True)
         # TODO: adapt this
         exit(0)
 
@@ -42,7 +40,7 @@ class builder(object):
             f"-t {self.__tag} -f {self.__definition_file_path} ."
         )
         print(f"Running: {build_cmd}")
-        subprocess.run(build_cmd, shell=True)
+        subprocess.run(build_cmd, shell=True, check=True)
         inspect_out = subprocess.check_output(
             f"docker inspect {self.__tag} | grep Id", shell=True
         ).decode(sys.stdout.encoding)
@@ -50,15 +48,43 @@ class builder(object):
         image_id_short = image_id[0:12]
 
         if self.__args.upload:
-            subprocess.run(f"docker push {self.__tag}", shell=True)
+            subprocess.run(f"docker push {self.__tag}", shell=True, check=True)
+        image_base_name = f"{self.__images_out_dir}/{self.__img_file}-{image_id_short}"
         if self.__args.sif_file:
             self.image_file = f"{self.__images_out_dir}/{self.__args.sif_file}"
         else:
-            self.image_file = (
-                f"{self.__images_out_dir}/{self.__img_file}-{image_id_short}.sif"
-            )
-        if self.__args.convert and not os.path.exists(self.image_file):
+            self.image_file = f"{image_base_name}.sif"
+        if self.__args.convert and (
+            not os.path.exists(self.image_file) or self.__args.force
+        ):
             subprocess.run(
                 f"cd {self.__cwd} && singularity build --force {self.image_file} docker-daemon:{self.__tag}",
                 shell=True,
+                check=True,
+            )
+
+        if self.__args.enroot_file:
+            self.image_file = f"{self.__images_out_dir}/{self.__args.enroot_file}"
+        else:
+            self.image_file = f"{image_base_name}.sqsh"
+        if self.__args.convert_enroot and (
+            not os.path.exists(self.image_file) or self.__args.force
+        ):
+            subprocess.run(
+                # See https://www.mankier.com/1/mksquashfs for options.
+                f"cd {self.__cwd} && rm -f {self.image_file} && ENROOT_SQUASH_OPTIONS='-comp xz -b 512K' enroot import -o {self.image_file} dockerd://{self.__tag}",
+                shell=True,
+                check=True,
+            )
+            print(f"Wrote image file {self.image_file}")
+
+        bundle_file = f"{self.image_file[:-5]}.run"
+        if self.__args.enroot_bundle and (
+            not os.path.exists(bundle_file) or self.__args.force
+        ):
+            subprocess.run(
+                f"cd {self.__cwd} && rm -f {bundle_file} && enroot bundle -o {bundle_file} {self.image_file}",
+                shell=True,
+                check=True,
             )
+            print(f"Wrote bundle file {bundle_file}")

ogscm/building_blocks/ogs.py

@@ -48,10 +48,14 @@ class ogs(bb_base, hpccm.templates.CMakeBuild, hpccm.templates.rm):
         self.__commit = kwargs.get("commit")
         self.__git_version = kwargs.get("git_version")
         self.__conan = kwargs.get("conan", False)
+        self.__shell_args = ""
 
         if self.__repo == "local":
             self.__skip_clone = True
             self.__remove_source = False
+            self.__shell_args = "--mount=type=bind,target={}/src,rw".format(
+                self.__prefix
+            )
 
         # Filled in by __setup():
         self.__commands = []
@@ -72,9 +76,8 @@ class ogs(bb_base, hpccm.templates.CMakeBuild, hpccm.templates.rm):
         self += packages(ospackages=self.__ospackages)
         self += shell(
             commands=self.__commands,
-            _arguments="--mount=type=bind,target=/scif/apps/ogs/src,rw",
+            _arguments=self.__shell_args,
         )
-        self += runscript(commands=["ogs"])
 
         if self.__environment_variables:
             self += environment(variables=self.__environment_variables)
@@ -154,7 +157,7 @@ class ogs(bb_base, hpccm.templates.CMakeBuild, hpccm.templates.rm):
         if self.__repo == "local":
             self.__labels["version"] = self.__git_version
         else:
-            self.__labels["version"] = "{0}@{1} ({2})".format(
+            self.__labels["version"] = "'{0}@{1} ({2})'".format(
                 self.__repo, self.__branch, self.__commit
             )
         self.__labels["cmake_args"] = "'" + " ".join(self.__cmake_args) + "'"

ogscm/cli.py

@@ -38,12 +38,6 @@ def main():  # pragma: no cover
     parser.add_argument(
         "--file", type=str, default="", help="Overwrite output recipe file name"
     )
-    parser.add_argument(
-        "--sif_file",
-        type=str,
-        default="",
-        help="Overwrite output singularity image file name",
-    )
     parser.add_argument(
         "--print",
         "-P",
@@ -103,6 +97,37 @@ def main():  # pragma: no cover
         action="store_true",
         help="Convert Docker image to Singularity image",
     )
+    build_g.add_argument(
+        "--sif_file",
+        type=str,
+        default="",
+        help="Overwrite output singularity image file name",
+    )
+    build_g.add_argument(
+        "--convert-enroot",
+        "-E",
+        dest="convert_enroot",
+        action="store_true",
+        help="Convert Docker image to enroot image",
+    )
+    build_g.add_argument(
+        "--enroot-bundle",
+        dest="enroot_bundle",
+        action="store_true",
+        help="Convert enroot image to enroot bundle",
+    )
+    build_g.add_argument(
+        "--enroot_file",
+        type=str,
+        default="",
+        help="Overwrite output enroot image file name",
+    )
+    build_g.add_argument(
+        "--force",
+        dest="force",
+        action="store_true",
+        help="Forces overwriting of image files!",
+    )
     build_g.add_argument(
         "--runtime-only",
         "-R",

ogscm/recipes/mpi.py

@@ -35,7 +35,7 @@ parse_g.add_argument(
     "--mpi_benchmarks",
     dest="mpi_benchmarks",
     action="store_true",
-    help="Installs OSU MPI " "benchmarks as scif app and mpi_bw, mpi_ring," "mpi_hello",
+    help="Installs OSU MPI benchmarks and mpi_bw, mpi_ring, mpi_hello",
 )
 # parse_g.add_argument(
 #     "--ucx",
@@ -207,19 +207,5 @@ if local_args.mpi_benchmarks:
             )
         }
     )
-    # Stage0 += osu_benchmarks(toolchain=toolchain)
-    # Stage0 += shell(
-    #     commands=[
-    #         "mkdir -p /usr/local/mpi-examples",
-    #         "cd /usr/local/mpi-examples",
-    #         "curl -O https://raw.githubusercontent.com/hpc/charliecloud/674b3b4e4ad243be5565f200d8f5fb92b7544480/examples/mpihello/hello.c",
-    #         "curl -O https://computing.llnl.gov/tutorials/mpi/samples/C/mpi_bandwidth.c",
-    #         "curl -O https://raw.githubusercontent.com/mpitutorial/mpitutorial/gh-pages/tutorials/mpi-send-and-receive/code/ring.c",
-    #         "mpicc -o /usr/local/bin/mpi-hello /usr/local/mpi-examples/hello.c",
-    #         "mpicc -o /usr/local/bin/mpi-ring /usr/local/mpi-examples/ring.c",
-    #         "mpicc -o /usr/local/bin/mpi-bandwidth /usr/local/mpi-examples/mpi_bandwidth.c",
-    #     ]
-    # )
-    # Stage1 += copy(_from="build", src="/usr/local/bin/mpi-*", dest="/usr/local/bin/")
 
 Stage0 += comment(f"--- End {filename} ---")

ogscm/recipes/ogs.py

@@ -16,7 +16,6 @@ from hpccm.building_blocks import (
     mkl,
     packages,
     pip,
-    scif,
 )
 from ogscm.building_blocks.pm_conan import pm_conan
 
@@ -229,11 +228,6 @@ if local_args.ogs != "off" and not args.runtime_only:
 # Optionally set out_dir
 out_dir += folder
 
-if repo == "local":
-    scif_file = f"{out_dir}/ogs.scif"  # TODO
-else:
-    scif_file = f"{out_dir}/ogs.scif"
-
 # Implement recipe
 Stage0 += comment(f"--- Begin {filename} ---")
 
@@ -276,7 +270,7 @@ if local_args.ogs != "clean":
         )
         Stage0 += environment(variables={"CONAN_SYSREQUIRES_SUDO": 0})
     elif local_args.pm == "system":
-        Stage0 += cmake(eula=True, version="3.19.4")
+        Stage0 += cmake(eula=True, version="3.20.1")
         Stage0 += boost(
             version=versions["minimum_version"]["boost"],
             bootstrap_opts=["--with-toolset=clang"] if toolchain.CC == "clang" else [],
@@ -497,23 +491,16 @@ if local_args.ogs != "off" and local_args.ogs != "clean":
     if repo == "local":
         print(f"chdir to {local_args.ogs}")
         os.chdir(local_args.ogs)
-        mount_args += f" --mount=type=bind,target=/scif/apps/ogs/src,rw"
 
-    ogs_app = scif(_arguments=mount_args, name="ogs", file=scif_file)
-    ogs_app += ogs(
+    Stage0 += ogs(
         repo=repo,
         branch=branch,
         commit=commit_hash,
         conan=(True if local_args.pm == "conan" else False),
         git_version=git_version,
         toolchain=toolchain,
-        prefix="/scif/apps/ogs",
         cmake_args=cmake_args,
         parallel=local_args.parallel,
         remove_build=True,
         remove_source=True,
     )
-    # Install scif in all stages
-    Stage0 += pip(packages=["scif"], pip="pip3")
-    Stage1 += pip(packages=["scif"], pip="pip3")
-    Stage0 += ogs_app

pyproject.toml

 [tool.poetry]
 name = "ogscm"
-version = "2.1.4"
+version = "2.2.0"
 description = "OGS Container Maker"
 authors = ["Lars Bilke <lars.bilke@ufz.de>"]
 license = "BSD 3-clause"
@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.7"
-hpccm = "^21.2.0"
+hpccm = "^21.3.0"
 requests = "^2.24.0"
 PyYAML = "^5.3.1"
 packaging = "^20.4"