[ci] custom docker container

[ci] try custom gitlab-runner

[ci] test privileged docker envinf1

[ci] bm-test for v6.4.0

[ci] parallel ref and test benchmark runs

[ci] run benchmark test for version 6.4.1

[ci] final changes

.Rbuildignore

@@ -10,4 +10,4 @@
 ^doc$
 ^Meta$
 ^data-raw$
-^.CI-setup.sh
+^ci$
\ No newline at end of file

.gitlab-ci.yml

@@ -4,7 +4,7 @@
 
 stages:
   - test
-  - benchmark-test
+  - test-benchmarks
 
 test:
   image: $CI_REGISTRY_IMAGE
@@ -15,17 +15,16 @@ test:
     R_LIBS_USER: "$CI_PROJECT_DIR/ci/lib/"
     RETICULATE_MINICONDA_PATH: "/root/.local/share/r-miniconda/" #default path
     RETICULATE_PYTHON_ENV: "/root/.local/share/r-miniconda/envs/r-reticulate/"
-    # RETICULATE_PYTHON: "$CI_PROJECT_DIR/ci/r-miniconda/envs/r-reticulate/bin/python3"
+#     # RETICULATE_PYTHON: "$CI_PROJECT_DIR/ci/r-miniconda/envs/r-reticulate/bin/python3"
   script:
     - if [ ! -d "$R_LIBS_USER" ]; then mkdir -p "$R_LIBS_USER"; fi
     - R -e 'withr::with_libpaths(new = Sys.getenv("R_LIBS_USER"), devtools::install_deps(dependencies = T))'
-    # checks if the packages from the Imports, Suggests, LinkinTo field
-    # are already installed in the cache
+#     # checks if the packages from the Imports, Suggests, LinkinTo field
+#     # are already installed in the cache
     - R -e '.libPaths(new = Sys.getenv("R_LIBS_USER"))'
-    # Build and Check r2ogs6
+#     # Build and Check r2ogs6
     - R CMD build . --no-build-vignettes --no-manual
     - R CMD check $(ls -1t *.tar.gz | head -n 1) --no-build-vignettes --no-manual
-    #echo First Stage!
   cache:
     key: "$CI_COMMIT_REF_SLUG"
     untracked: true
@@ -36,13 +35,70 @@ test:
   rules:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "web"'
 
-benchmark-test:
-  image: $CI_REGISTRY_IMAGE
-  stage: benchmark-test
+benchmark:ref:
+  stage: test-benchmarks
+  tags:
+    - envinf1
+  image: "$CI_REGISTRY_IMAGE"
+  variables:
+    R_LIBS_USER: "$CI_PROJECT_DIR/ci/lib/"
+    # change this as needed
+    OGS_VERSION: "6.4.1"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+      when: on_success
+  script:
+    - if [ ! -d "$R_LIBS_USER" ]; then mkdir -p "$R_LIBS_USER"; fi
+    - R -e 'withr::with_libpaths(new = Sys.getenv("R_LIBS_USER"), devtools::install_deps(dependencies = T))'
+    - R -e '.libPaths(new = Sys.getenv("R_LIBS_USER"))'
+    - Rscript ./ci/test_bm_script.R "$OGS_VERSION" "ref"
+  cache:
+    key: "$CI_COMMIT_REF_SLUG"
+    untracked: true
+    paths:
+      - "$R_LIBS_USER"
+  artifacts:
+    paths:
+      - ref_exit.rda
+
+benchmark:r2ogs6:
+  stage: test-benchmarks
+  tags:
+    - envinf1
+  image: "$CI_REGISTRY_IMAGE"
+  variables:
+    R_LIBS_USER: "$CI_PROJECT_DIR/ci/lib/"
+    # change this as needed
+    OGS_VERSION: "6.4.1"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+      when: on_success
+  script:
+    - if [ ! -d "$R_LIBS_USER" ]; then mkdir -p "$R_LIBS_USER"; fi
+    - R -e 'withr::with_libpaths(new = Sys.getenv("R_LIBS_USER"), devtools::install_deps(dependencies = T))'
+    - R -e '.libPaths(new = Sys.getenv("R_LIBS_USER"))'
+    - Rscript ./ci/test_bm_script.R "$OGS_VERSION" "r2ogs6"
+  cache:
+    key: "$CI_COMMIT_REF_SLUG"
+    untracked: true
+    paths:
+      - "$R_LIBS_USER"
+  artifacts:
+    paths:
+      - test_exit.rda
+
+compare-benchmarks:
+  needs: ["benchmark:ref", "benchmark:r2ogs6"]
+  stage: test-benchmarks
+  tags:
+    - envinf1
+  image: "$CI_REGISTRY_IMAGE"
   rules:
-    - if: '$CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '$CI_PIPELINE_SOURCE == "web"'
       when: on_success
+  dependencies:
+    - benchmark:ref
+    - benchmark:r2ogs6
   script:
-    - R -e 'print("HELLO R")'
-    - singularity --version
+    - Rscript ./ci/test_bm_script.R "$OGS_VERSION" "test"
 

ci/test_bm_script.R

+
+# git clone --depth 1 --branch 6.4.1 https://gitlab.opengeosys.org/ogs/ogs
+# wget https://ogsstorage.blob.core.windows.net/binaries/ogs6/6.4.1/ogs-6.4.1-serial.sif
+
+# Setup -------------------------------------------------------------------
+if (commandArgs(trailingOnly=TRUE)[2] == "ref" |
+    commandArgs(trailingOnly=TRUE)[2 ]== "r2ogs6") {
+
+    ogs_version <- commandArgs(trailingOnly=TRUE)[1]
+
+    if (!dir.exists("/root/ogs")) {
+        system2(
+            command = c(
+                "git",
+                "clone",
+                "--depth",
+                "1",
+                "--branch",
+                ogs_version,
+                "https://gitlab.opengeosys.org/ogs/ogs.git/",
+                "/root/ogs"
+            )
+        )
+    }
+    if (!file.exists(paste0("/root/ogs-", ogs_version, "-serial.sif"))) {
+        system2(
+            command = c(
+                "wget",
+                "-nv", # no verbose
+                "-P",
+                "/root",
+                paste0("https://ogsstorage.blob.core.windows.net/binaries/ogs6/",
+                       ogs_version, "/ogs-", ogs_version, "-serial.sif")
+            )
+        )
+    }
+    devtools::load_all(".")
+    library(dplyr)
+
+    ogs_repo <- "/root/ogs/"
+    prjs <- r2ogs6:::get_benchmark_paths(paste0(ogs_repo, "ProcessLib"))
+    prjs <- prjs[c(6, 45, 46, 47, 249, 250, 251, 252)] # for testing
+
+    #### Exceptions #####
+    # exclude some prj files
+    prjs <- prjs[-grep("\\.xml$", prjs)]
+    prjs <- prjs[-grep("TH2M/THM/slab/THM_1d_dirichlet.prj", prjs)]
+    prjs <- prjs[-grep("InvalidProjectFiles/", prjs)]
+    #####################
+
+    ogs6_container <- paste0("/root/ogs-", ogs_version, "-serial.sif")
+
+    # include call with --app ogs flag for ogs < 6.4.1
+    if (as.numeric(gsub("[.]", "", ogs_version)) <= 640) {
+        ogs6_command_str <- c("singularity", "exec", "--app", "ogs",
+                              ogs6_container, "ogs")
+    } else {
+        ogs6_command_str <- c("singularity", "exec", ogs6_container, "ogs")
+    }
+
+}
+
+# reference run -----------------------------------------------------------
+if (commandArgs(trailingOnly=TRUE)[2] == "ref") {
+
+    out_ref <- "-o /root/out_ref"
+    dir.create("/root/out_ref")
+    dir.create("/root/out_ref/logfiles/")
+
+    ref_exit <- tibble(benchmark = character(),
+                       ref = numeric())
+
+    for (prj in prjs) {
+
+        print(paste0("Running benchmark ", prj))
+        prj_path <- paste0(ogs_repo, "Tests/Data/", prj)
+
+        log_ref <- paste0("> /root/out_ref/logfiles/",
+                          sub(".prj",".txt", basename(prj)))
+
+        out <- system2(command = ogs6_command_str,
+                       args = c(prj_path, out_ref, log_ref))
+
+        ref_exit <- add_row(ref_exit,
+                            benchmark = prj,
+                            ref = out)
+    }
+
+    save(ref_exit, file = "ref_exit.rda")
+}
+
+# test run with r2ogs6 ----------------------------------------------------
+if (commandArgs(trailingOnly=TRUE)[2] == "r2ogs6") {
+
+    test_exit <- tibble(benchmark = character(),
+                        test = numeric())
+    out_test <- "/root/out_test"
+    dir.create(out_test)
+    for (prj in prjs) {
+        print(paste0("Attempting to run benchmark ", prj))
+        prj_path <- paste0(ogs_repo, "Tests/Data/", prj)
+        out <- tryCatch({
+            r2ogs6:::run_benchmark(
+                prj_path = prj_path,
+                ogs6_bin_path = ogs6_container,
+                sim_path = out_test
+            )
+        },
+        error = function(e) {
+            message("\nrun_benchmark() failed!\nOriginal error message:")
+            message(e)
+        })
+
+        test_exit <- add_row(test_exit,
+                            benchmark = prj,
+                            test = out)
+    }
+    test_exit$test[which(is.na(test_exit$test))] <- 99
+    save(test_exit, file = "test_exit.rda")
+}
+
+if (commandArgs(trailingOnly=TRUE)[2] == "test") {
+
+    install.packages("dplyr")
+    load("ref_exit.rda")
+    load("test_exit.rda")
+    test_exit <- dplyr::full_join(ref_exit, test_exit, by = "benchmark")
+
+    print(test_exit, n = nrow(test_exit))
+
+    # benchmarks that passed with & without r2ogs6
+    r2ogs6_passed <- test_exit$test[which(test_exit$ref == 0)] == 0
+
+    if(!all(r2ogs6_passed)) {
+        print(paste0(test_exit$benchmark[!r2ogs6_passed],
+                     collapse = "\n"))
+        stop("Exit codes for r2ogs6 were nonzero for above benchmarks!")
+    }
+}
+
+