Merge branch '53-manual-ci-job-to-run-all-benchmarks' into '51-update-to-ogsv6-4-1'

Add CI/CD Pipeline to check if benchmarks work with OGS version 6.4.1

See merge request !37

.CI-setup.sh

+#!/bin/bash
+
+### Setup singularity ###
+# adapted from: https://sylabs.io/guides/3.0/user-guide/installation.html
+# install dependencies
+apt-get update
+apt-get install -y \
+    build-essential \
+    libssl-dev \
+    uuid-dev \
+    libgpgme11-dev \
+    squashfs-tools \
+    libseccomp-dev \
+    pkg-config \
+    wget
+# yum update -y && \
+#     yum groupinstall -y 'Development Tools' && \
+#     yum install -y \
+#     openssl-devel \
+#     libuuid-devel \
+#     libseccomp-devel \
+#     wget \
+#     squashfs-tools
+
+
+
+# install go
+export GOVERSION=1.15.6 OS=linux ARCH=amd64
+wget https://dl.google.com/go/go$GOVERSION.$OS-$ARCH.tar.gz
+tar -C /usr/local -xzvf go$GOVERSION.$OS-$ARCH.tar.gz
+rm go$GOVERSION.$OS-$ARCH.tar.gz
+
+# install singularity
+export VERSION=3.9.1
+mkdir -p $GOPATH/src/github.com/sylabs
+cd $GOPATH/src/github.com/sylabs
+wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-ce-${VERSION}.tar.gz
+tar -xzf singularity-ce-${VERSION}.tar.gz
+cd ./singularity
+./mconfig
+make -C ./builddir
+make -C ./builddir install
\ No newline at end of file

.Rbuildignore

@@ -10,3 +10,4 @@
 ^doc$
 ^Meta$
 ^data-raw$
+^ci$
\ No newline at end of file

.gitlab-ci.yml

-image: rocker/tidyverse
+# helpful Links:
+# https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+# https://www.youtube.com/watch?v=7I6tHw68DMQ
+
+stages:
+  - test
+  - test-benchmarks
 
 test:
+  image: $CI_REGISTRY_IMAGE
   stage: test
   tags:
     - envinf2
@@ -8,16 +15,13 @@ test:
     R_LIBS_USER: "$CI_PROJECT_DIR/ci/lib/"
     RETICULATE_MINICONDA_PATH: "/root/.local/share/r-miniconda/" #default path
     RETICULATE_PYTHON_ENV: "/root/.local/share/r-miniconda/envs/r-reticulate/"
-    # RETICULATE_PYTHON: "$CI_PROJECT_DIR/ci/r-miniconda/envs/r-reticulate/bin/python3"
   script:
     - if [ ! -d "$R_LIBS_USER" ]; then mkdir -p "$R_LIBS_USER"; fi
     - R -e 'withr::with_libpaths(new = Sys.getenv("R_LIBS_USER"), devtools::install_deps(dependencies = T))'
     # checks if the packages from the Imports, Suggests, LinkinTo field
     # are already installed in the cache
     - R -e '.libPaths(new = Sys.getenv("R_LIBS_USER"))'
-    # python dependencies
-    - if [ ! -d "$RETICULATE_MINICONDA_PATH" ]; then R -e 'reticulate::install_miniconda()'; fi
-    - R -e 'if(tryCatch(reticulate::import("vtk"), error = function(e) return(TRUE))) reticulate::py_install("vtk")'
+    # Build and Check r2ogs6
     - R CMD build . --no-build-vignettes --no-manual
     - R CMD check $(ls -1t *.tar.gz | head -n 1) --no-build-vignettes --no-manual
   cache:
@@ -25,8 +29,79 @@ test:
     untracked: true
     paths:
       - "$R_LIBS_USER"
-      - "$RETICULATE_MINICONDA_PATH"
-      - "$RETICULATE_PYTHON_ENV"
-  # script will only run then attempting to merge/push to master
-  only:
-    - merge_requests
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "web"'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+benchmark:ref:
+  stage: test-benchmarks
\ No newline at end of file
+  tags:
+    - envinf1
+  image: "$CI_REGISTRY_IMAGE"
+  variables:
+    R_LIBS_USER: "$CI_PROJECT_DIR/ci/lib/"
+    # change this as needed
+    OGS_VERSION: "6.4.1"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+      when: on_success
+  script:
+    - if [ ! -d "$R_LIBS_USER" ]; then mkdir -p "$R_LIBS_USER"; fi
+    - R -e 'withr::with_libpaths(new = Sys.getenv("R_LIBS_USER"), devtools::install_deps(dependencies = T))'
+    - R -e '.libPaths(new = Sys.getenv("R_LIBS_USER"))'
+    - Rscript ./ci/test_bm_script.R "$OGS_VERSION" "ref"
+  cache:
+    key: "$CI_COMMIT_REF_SLUG"
+    untracked: true
+    paths:
+      - "$R_LIBS_USER"
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 4 hrs
+    paths:
+      - ref_exit.rda
+      - out_ref/logfiles/
+
+benchmark:r2ogs6:
+  stage: test-benchmarks
+  tags:
+    - envinf1
+  image: "$CI_REGISTRY_IMAGE"
+  variables:
+    R_LIBS_USER: "$CI_PROJECT_DIR/ci/lib/"
+    # change this as needed
+    OGS_VERSION: "6.4.1"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+      when: on_success
+  script:
+    - if [ ! -d "$R_LIBS_USER" ]; then mkdir -p "$R_LIBS_USER"; fi
+    - R -e 'withr::with_libpaths(new = Sys.getenv("R_LIBS_USER"), devtools::install_deps(dependencies = T))'
+    - R -e '.libPaths(new = Sys.getenv("R_LIBS_USER"))'
+    - Rscript ./ci/test_bm_script.R "$OGS_VERSION" "r2ogs6"
+  cache:
+    key: "$CI_COMMIT_REF_SLUG"
+    untracked: true
+    paths:
+      - "$R_LIBS_USER"
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 4 hrs
+    paths:
+      - test_exit.rda
+      - out_test/logfiles/
+
+compare-benchmarks:
+  needs: ["benchmark:ref", "benchmark:r2ogs6"]
+  stage: test-benchmarks
+  tags:
+    - envinf1
+  image: "$CI_REGISTRY_IMAGE"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+      when: on_success
+  dependencies:
+    - benchmark:ref
+    - benchmark:r2ogs6
+  script:
+    - Rscript ./ci/test_bm_script.R "$OGS_VERSION" "test"
+

R/analyse_xml.R

@@ -271,11 +271,11 @@ get_required <- function(names, occurence_probabilities){
 
     for(i in seq_len(length(names))) {
         if(occurence_probabilities[[i]] < 1) {
-            required[[names[[i]]]] <- FALSE
+            required[[paste0(names[[i]])]] <- FALSE
         }else{
-            required[[names[[i]]]] <- TRUE
+            required[[paste0(names[[i]])]] <- TRUE
         }
     }
 
-    return(invisible(required))
+    return(required)
 }

ci/test_bm_script.R

+
+# git clone --depth 1 --branch 6.4.1 https://gitlab.opengeosys.org/ogs/ogs
+# wget https://ogsstorage.blob.core.windows.net/binaries/ogs6/6.4.1/ogs-6.4.1-serial.sif
+
+# Setup -------------------------------------------------------------------
+if (commandArgs(trailingOnly=TRUE)[2] == "ref" |
+    commandArgs(trailingOnly=TRUE)[2 ]== "r2ogs6") {
+
+    ogs_version <- commandArgs(trailingOnly=TRUE)[1]
+
+    if (!dir.exists("/root/ogs")) {
+        system2(
+            command = c(
+                "git",
+                "clone",
+                "--depth",
+                "1",
+                "--branch",
+                ogs_version,
+                "https://gitlab.opengeosys.org/ogs/ogs.git/",
+                "/root/ogs"
+            )
+        )
+    }
+    if (!file.exists(paste0("/root/ogs-", ogs_version, "-serial.sif"))) {
+        system2(
+            command = c(
+                "wget",
+                "-nv", # no verbose
+                "-P",
+                "/root",
+                paste0("https://ogsstorage.blob.core.windows.net/binaries/ogs6/",
+                       ogs_version, "/ogs-", ogs_version, "-serial.sif")
+            )
+        )
+    }
+    devtools::load_all(".")
+    library(dplyr)
+
+    ogs_repo <- "/root/ogs/"
+    prjs <- r2ogs6:::get_benchmark_paths(paste0(ogs_repo, "ProcessLib"))
+    # prjs <- prjs[c(6, 45, 46, 47, 249, 250, 251, 252)] # for testing
+
+    #### Exceptions #####
+    # exclude some prj files
+    prjs <- prjs[-grep("\\.xml$", prjs)]
+    prjs <- prjs[-grep("TH2M/THM/slab/THM_1d_dirichlet.prj", prjs)]
+    prjs <- prjs[-grep("InvalidProjectFiles/", prjs)]
+    #####################
+
+    ogs6_container <- paste0("/root/ogs-", ogs_version, "-serial.sif")
+
+    # include call with --app ogs flag for ogs < 6.4.1
+    if (as.numeric(gsub("[.]", "", ogs_version)) <= 640) {
+        ogs6_command_str <- c("singularity", "exec", "--app", "ogs",
+                              ogs6_container, "ogs")
+    } else {
+        ogs6_command_str <- c("singularity", "exec", ogs6_container, "ogs")
+    }
+
+}
+
+# reference run -----------------------------------------------------------
+if (commandArgs(trailingOnly=TRUE)[2] == "ref") {
+
+    out_ref <- "-o /root/out_ref"
+    dir.create("/root/out_ref")
+    dir.create("/root/out_ref/logfiles/")
+
+    ref_exit <- tibble(benchmark = character(),
+                       ref = numeric())
+
+    for (prj in prjs) {
+
+        print(paste0("Running benchmark ", prj))
+        prj_path <- paste0(ogs_repo, "Tests/Data/", prj)
+
+        log_ref <- paste0("> /root/out_ref/logfiles/",
+                          sub(".prj",".txt", basename(prj)))
+
+        out <- system2(command = ogs6_command_str,
+                       args = c(prj_path, out_ref, log_ref))
+
+        ref_exit <- add_row(ref_exit,
+                            benchmark = prj,
+                            ref = out)
+    }
+
+    save(ref_exit, file = "ref_exit.rda")
+    dir.create("out_ref")
+    file.copy("/root/out_ref/logfiles", to = "out_ref", recursive = TRUE)
+}
+
+# test run with r2ogs6 ----------------------------------------------------
+if (commandArgs(trailingOnly=TRUE)[2] == "r2ogs6") {
+
+    test_exit <- tibble(benchmark = character(),
+                        test = numeric())
+    out_test <- "/root/out_test"
+    dir.create(out_test)
+    for (prj in prjs) {
+        print(paste0("Attempting to run benchmark ", prj))
+        prj_path <- paste0(ogs_repo, "Tests/Data/", prj)
+        out <- tryCatch({
+            r2ogs6:::run_benchmark(
+                prj_path = prj_path,
+                ogs6_bin_path = ogs6_container,
+                sim_path = out_test
+            )
+        },
+        error = function(e) {
+            message("\nrun_benchmark() failed!\nOriginal error message:")
+            message(e)
+        })
+
+        test_exit <- add_row(test_exit,
+                            benchmark = prj,
+                            test = out)
+    }
+    test_exit$test[which(is.na(test_exit$test))] <- 99
+    save(test_exit, file = "test_exit.rda")
+
+    dir.create("out_test")
+    file.copy("/root/out_test/logfiles", to = "out_test", recursive = TRUE)
+}
+
+if (commandArgs(trailingOnly=TRUE)[2] == "test") {
+
+    install.packages("dplyr")
+    load("ref_exit.rda")
+    load("test_exit.rda")
+    test_exit <- dplyr::full_join(ref_exit, test_exit, by = "benchmark")
+
+    print(test_exit, n = nrow(test_exit))
+
+    # benchmarks that passed with & without r2ogs6
+    r2ogs6_passed <- test_exit$test[which(test_exit$ref == 0)] == 0
+
+    if(!all(r2ogs6_passed)) {
+        cat(paste0(test_exit$benchmark[!r2ogs6_passed],
+                     collapse = "\n"))
+        stop("Exit codes for r2ogs6 were nonzero for above benchmarks!")
+    }
+}
+
+