[T] Added NotebookTest() as CMake integration for notebook testrunner.py.

CMakeLists.txt

@@ -55,6 +55,7 @@ set(OGS_PETSC_CONFIG_OPTIONS "" CACHE STRING
 option(OGS_USE_CVODE "Use the Sundials CVODE module?" OFF)
 option(OGS_BUILD_UTILS "Should the utilities programs be built?" ON)
 option(OGS_BUILD_TESTING "Should the tests be built?" ON)
+cmake_dependent_option(OGS_TEST_NOTEBOOKS "Should notebooks be tested?" ON "OGS_BUILD_TESTING" OFF)
 
 if(MSVC)
     set(CMD_COMMAND "cmd;/c")

Tests/Data/Notebooks/FailingNotebook.ci-skip.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Test Notebook assertion for testrunner\n",
+    "\n",
+    "A notebook test will fail with `assert False` or `raise SystemExit()`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert False\n",
+    "\n",
+    "# This line will not be reached but would also work:\n",
+    "raise SystemExit()"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

scripts/ci/extends/test-artifacts.yml

@@ -8,6 +8,9 @@
       - build/*/make.txt
       - build/*/*.zip
       - build/*/*.tar.gz
+      # Notebook testrunner:
+      - build/**/*.ipynb.html
+      - build/**/out.txt
     expire_in: 1 week
     reports:
       junit:

scripts/ci/jobs/build-linux.yml

@@ -70,6 +70,7 @@ build linux arch:
     CMAKE_ARGS: >-
       -DBUILD_SHARED_LIBS=ON
       -DOGS_USE_POETRY=OFF
+      -DOGS_TEST_NOTEBOOKS=OFF
 
 build linux ubuntu:
   extends:

scripts/cmake/test/NotebookTest.cmake

+# cmake-lint: disable=C0103
+function(NotebookTest)
+
+    if(NOT OGS_BUILD_CLI OR NOT OGS_BUILD_TESTING OR NOT OGS_TEST_NOTEBOOKS)
+        return()
+    endif()
+    set(options DISABLED)
+    set(oneValueArgs NOTEBOOKFILE RUNTIME)
+    set(multiValueArgs WRAPPER)
+    cmake_parse_arguments(
+        NotebookTest "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}
+    )
+
+    get_filename_component(
+        NotebookTest_DIR "${NotebookTest_NOTEBOOKFILE}" DIRECTORY
+    )
+    get_filename_component(
+        NotebookTest_NAME "${NotebookTest_NOTEBOOKFILE}" NAME
+    )
+    get_filename_component(
+        NotebookTest_NAME_WE "${NotebookTest_NOTEBOOKFILE}" NAME_WE
+    )
+
+    if(NotebookTest_UNPARSED_ARGUMENTS)
+        message(
+            FATAL_ERROR
+                "Unparsed argument(s) '${NotebookTest_UNPARSED_ARGUMENTS}' to NotebookTest call."
+        )
+    endif()
+
+    set(timeout ${ogs.ctest.large_runtime})
+    if(DEFINED NotebookTest_RUNTIME)
+        math(EXPR timeout "${NotebookTest_RUNTIME} * 3")
+    else()
+        set(NotebookTest_RUNTIME 1)
+    endif()
+
+    if(DEFINED OGS_CTEST_MAX_RUNTIME)
+        if(${NotebookTest_RUNTIME} GREATER ${OGS_CTEST_MAX_RUNTIME})
+            return()
+        endif()
+    endif()
+    if(${NotebookTest_RUNTIME} GREATER ${ogs.ctest.large_runtime})
+        string(APPEND NotebookTest_NAME_WE "-LARGE")
+    endif()
+
+    set(NotebookTest_SOURCE_DIR "${Data_SOURCE_DIR}/${NotebookTest_DIR}")
+    set(NotebookTest_BINARY_DIR "${Data_BINARY_DIR}/${NotebookTest_DIR}")
+    file(MAKE_DIRECTORY ${NotebookTest_BINARY_DIR})
+    file(TO_NATIVE_PATH "${NotebookTest_BINARY_DIR}"
+         NotebookTest_BINARY_DIR_NATIVE
+    )
+
+    set(TEST_NAME "nb-${NotebookTest_DIR}/${NotebookTest_NAME_WE}")
+
+    set(_exe_args Notebooks/testrunner.py --out ${Data_BINARY_DIR}
+                  ${NotebookTest_SOURCE_DIR}/${NotebookTest_NAME}
+    )
+    string(REPLACE "/" "_" TEST_NAME_UNDERSCORE ${TEST_NAME})
+    add_test(
+        NAME ${TEST_NAME}
+        COMMAND
+            ${CMAKE_COMMAND} -DEXECUTABLE=${Python3_EXECUTABLE}
+            "-DEXECUTABLE_ARGS=${_exe_args}"
+            -DWORKING_DIRECTORY=${Data_SOURCE_DIR} -DCAT_LOG=TRUE -P
+            ${PROJECT_SOURCE_DIR}/scripts/cmake/test/OgsTestWrapper.cmake
+    )
+
+    current_dir_as_list(ProcessLib labels)
+    list(APPEND labels Notebook)
+    if(${NotebookTest_RUNTIME} LESS_EQUAL ${ogs.ctest.large_runtime})
+        list(APPEND labels default)
+    else()
+        list(APPEND labels large)
+    endif()
+
+    set_tests_properties(
+        ${TEST_NAME}
+        PROPERTIES ENVIRONMENT
+                   PATH=$<TARGET_FILE_DIR:ogs>:$ENV{PATH}
+                   COST
+                   ${NotebookTest_RUNTIME}
+                   DISABLED
+                   ${NotebookTest_DISABLED}
+                   LABELS
+                   "${labels}"
+    )
+
+endfunction()

scripts/cmake/test/Test.cmake

@@ -48,6 +48,13 @@ enable_testing() # Enable CTest
 include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/AddTest.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/MeshTest.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/OgsTest.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/NotebookTest.cmake)
+
+# Check notebook testrunner
+NotebookTest(NOTEBOOKFILE Notebooks/FailingNotebook.ci-skip.ipynb RUNTIME 2)
+if(TEST nb-Notebooks/FailingNotebook)
+    set_tests_properties(nb-Notebooks/FailingNotebook PROPERTIES WILL_FAIL TRUE)
+endif()
 
 if(CMAKE_CONFIGURATION_TYPES)
     set(CONFIG_PARAMETER --build-config "$<CONFIGURATION>")